SPB Git

Repository Public License

TextSniffer.swift 31.4 KB 588 lines swift

1// ORIX Convert — Packages/ORIXConvertCore/Sources/ConvertCore/Detection/TextSniffer.swift2// Description: Text detection: encoding (BOMs, UTF-8 validation, UTF-16 without BOM, Latin-1 fallback) and3//              classification of textual formats from a bounded prefix (XML family, JSON family, YAML, TOML, CSV/TSV,4//              Markdown, LaTeX, RST, subtitles, OBJ, USDA…). All heuristics are documented next to their rule.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation78/// Text encoding recognised by the detector.9public enum TextEncoding: String, Sendable, Codable, CaseIterable {10    case ascii, utf8 = "utf-8", utf8BOM = "utf-8-bom", utf16LE = "utf-16le", utf16BE = "utf-16be"11    case utf32LE = "utf-32le", utf32BE = "utf-32be", latin1 = "windows-1252"1213    /// Foundation encoding to decode the file with.14    public var stringEncoding: String.Encoding {15        switch self {16        case .ascii, .utf8, .utf8BOM: .utf817        case .utf16LE: .utf16LittleEndian18        case .utf16BE: .utf16BigEndian19        case .utf32LE: .utf32LittleEndian20        case .utf32BE: .utf32BigEndian21        case .latin1: .windowsCP125222        }23    }24}2526struct SniffHit {27    let format: FormatID28    let confidence: DetectionResult.Confidence29    let evidence: String30    init(_ format: FormatID, _ confidence: DetectionResult.Confidence, _ evidence: String) {31        self.format = format; self.confidence = confidence; self.evidence = evidence32    }33}3435enum TextSniffer {36    // MARK: Encoding3738    /// Decodes a prefix as text. `truncated` tells that the bytes are only the beginning of the file, so an39    /// incomplete trailing sequence is tolerated. Returns `nil` for binary content.40    static func decode(_ p: [UInt8], truncated: Bool) -> (text: String, encoding: TextEncoding)? {41        guard !p.isEmpty else { return nil }42        if p.has([0xEF, 0xBB, 0xBF]) {43            return utf8(Array(p.dropFirst(3)), truncated: truncated).map { ($0, .utf8BOM) }44        }45        if p.has([0xFF, 0xFE, 0x00, 0x00]) { return wide(p, skip: 4, unit: 4, .utf32LE) }46        if p.has([0x00, 0x00, 0xFE, 0xFF]) { return wide(p, skip: 4, unit: 4, .utf32BE) }47        if p.has([0xFF, 0xFE]) { return wide(p, skip: 2, unit: 2, .utf16LE) }48        if p.has([0xFE, 0xFF]) { return wide(p, skip: 2, unit: 2, .utf16BE) }4950        if p.contains(0) {51            // UTF-16 without BOM: mostly-ASCII text puts the zero bytes on one side of every pair.52            let sample = Array(p.prefix(2048))53            guard sample.count >= 8 else { return nil }54            var evenZeros = 0, oddZeros = 055            for (i, b) in sample.enumerated() where b == 0 { if i % 2 == 0 { evenZeros += 1 } else { oddZeros += 1 } }56            let pairs = sample.count / 257            if oddZeros * 10 >= pairs * 4, evenZeros * 20 <= pairs { return wide(p, skip: 0, unit: 2, .utf16LE) }58            if evenZeros * 10 >= pairs * 4, oddZeros * 20 <= pairs { return wide(p, skip: 0, unit: 2, .utf16BE) }59            return nil60        }61        guard !hasBinaryControls(p) else { return nil }62        if let text = utf8(p, truncated: truncated) {63            return (text, p.allSatisfy { $0 < 0x80 } ? .ascii : .utf8)64        }65        // Not UTF-8, no control bytes: accept a single-byte Western encoding when high bytes are a minority.66        let high = p.reduce(0) { $0 + ($1 >= 0x80 ? 1 : 0) }67        guard high * 10 < p.count * 3, let text = String(bytes: p, encoding: .windowsCP1252) else { return nil }68        return (text, .latin1)69    }7071    private static func hasBinaryControls(_ p: [UInt8]) -> Bool {72        p.contains { $0 < 0x09 || ($0 > 0x0D && $0 < 0x20 && $0 != 0x1B) || $0 == 0x7F }73    }7475    private static func utf8(_ p: [UInt8], truncated: Bool) -> String? {76        if hasBinaryControls(p) { return nil }77        if let s = String(bytes: p, encoding: .utf8) { return s }78        guard truncated else { return nil }79        for drop in 1...3 where p.count > drop {80            if let s = String(bytes: p.dropLast(drop), encoding: .utf8) { return s }81        }82        return nil83    }8485    private static func wide(_ p: [UInt8], skip: Int, unit: Int, _ encoding: TextEncoding) -> (String, TextEncoding)? {86        var body = Array(p.dropFirst(skip))87        body.removeLast(body.count % unit)88        guard !body.isEmpty else { return ("", encoding) }89        guard let text = String(bytes: body, encoding: encoding.stringEncoding) else {90            // A surrogate pair cut by the prefix boundary: retry without the last unit.91            guard body.count > unit, let shorter = String(bytes: body.dropLast(unit), encoding: encoding.stringEncoding) else { return nil }92            return (shorter, encoding)93        }94        if text.unicodeScalars.contains(where: { $0.value < 9 || ($0.value > 13 && $0.value < 32 && $0.value != 27) }) { return nil }95        return (text, encoding)96    }9798    // MARK: Classification99100    /// Classifies decoded text. `complete` is true when `text` is the entire file.101    static func classify(_ text: String, complete: Bool) -> SniffHit {102        let trimmed = text.drop { $0.isWhitespace || $0 == "\u{FEFF}" }103        guard !trimmed.isEmpty else { return SniffHit(.txt, .likely, "whitespace-only text") }104        var lines = text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline).map(String.init)105        if !complete, lines.count > 1 { lines.removeLast() }106        if lines.count > 400 { lines = Array(lines.prefix(400)) }107        let content = lines.filter { !$0.allSatisfy(\.isWhitespace) }108109        if trimmed.hasPrefix("{\\rtf") { return SniffHit(.rtf, .certain, "RTF header") }110        if trimmed.hasPrefix("#usda") { return SniffHit(.usda, .certain, "#usda header") }111        if trimmed.hasPrefix("WEBVTT") { return SniffHit(.vtt, .certain, "WEBVTT header") }112        if trimmed.hasPrefix("<") { if let hit = xmlFamily(String(trimmed.prefix(8192))) { return hit } }113        if trimmed.hasPrefix("{") || trimmed.hasPrefix("[") {114            if let first = content.first, first.trimmingCharacters(in: .whitespaces).lowercased() == "[script info]" {115                return subStation(text)116            }117            if let hit = microDVD(content) { return hit }118            if let hit = jsonFamily(text, lines: content, complete: complete) { return hit }119        }120        if let hit = srt(content) { return hit }121        if text.contains("\\documentclass") || text.contains("\\begin{document}") {122            return SniffHit(.latex, .certain, "LaTeX document commands")123        }124        if trimmed.hasPrefix("ply"), content.count > 1, content[1].hasPrefix("format ascii") {125            return SniffHit(.ply, .certain, "ASCII PLY header")126        }127        if trimmed.hasPrefix("solid"), text.contains("facet") || text.contains("endsolid") {128            return SniffHit(.stl, .certain, "ASCII STL keywords")129        }130        if trimmed.hasPrefix("; FBX") { return SniffHit(.fbx, .likely, "ASCII FBX comment header") }131        if let hit = obj(content) { return hit }132        if let hit = toml(content) { return hit }133        if let hit = yaml(content) { return hit }134        // Lines that begin with a tab are indented code or text, not rows.135        if delimited(content, "\t"), content.prefix(30).filter({ $0.hasPrefix("\t") }).count * 2 < min(content.count, 30) {136            return SniffHit(.tsv, .likely, "consistent tab-separated columns")137        }138        if let hit = rstStrong(content) { return hit }139        if let hit = markdown(lines, strict: true) { return hit }140        if delimited(content, ",") { return SniffHit(.csv, .likely, "consistent comma-separated columns") }141        // Semicolons that end every line are statement terminators (source code), not separators.142        if delimited(content, ";"), !content.prefix(30).allSatisfy({ $0.trimmingCharacters(in: .whitespaces).hasSuffix(";") }) {143            return SniffHit(.csv, .likely, "consistent semicolon-separated columns")144        }145        if text.contains("\\section{") || text.contains("\\usepackage") { return SniffHit(.latex, .likely, "LaTeX commands") }146        if let hit = rstHeadings(lines) { return hit }147        if let hit = markdown(lines, strict: false) { return hit }148        return SniffHit(.txt, .likely, "plain text")149    }150151    // MARK: XML family152153    private static let htmlTags: Set<String> = [154        "html", "head", "body", "div", "p", "span", "table", "h1", "h2", "h3", "a", "ul", "ol", "script", "style", "meta",155        "title", "section", "article", "nav", "header", "footer", "main", "img", "br", "form", "link",156    ]157158    /// Root element name after skipping the XML declaration, processing instructions, comments and DOCTYPE.159    static func xmlRoot(_ s: String) -> (root: String?, hasDeclaration: Bool, doctype: String?) {160        var rest = Substring(s)161        var declaration = false162        var doctype: String?163        for _ in 0..<64 {164            rest = rest.drop { $0.isWhitespace || $0 == "\u{FEFF}" }165            if rest.hasPrefix("<?") {166                if rest.hasPrefix("<?xml") { declaration = true }167                guard let end = rest.range(of: "?>") else { return (nil, declaration, doctype) }168                rest = rest[end.upperBound...]169            } else if rest.hasPrefix("<!--") {170                guard let end = rest.range(of: "-->") else { return (nil, declaration, doctype) }171                rest = rest[end.upperBound...]172            } else if rest.hasPrefix("<!") {173                guard let end = rest.firstIndex(of: ">") else { return (nil, declaration, String(rest.prefix(64)).lowercased()) }174                doctype = String(rest[..<end]).lowercased()175                // An internal subset contains '>' characters: skip to the closing "]>" when there is one.176                if let open = rest[..<end].firstIndex(of: "["), let close = rest[open...].range(of: "]>") {177                    rest = rest[close.upperBound...]178                } else {179                    rest = rest[rest.index(after: end)...]180                }181            } else if rest.hasPrefix("<") {182                let name = rest.dropFirst().prefix { $0.isLetter || $0.isNumber || $0 == ":" || $0 == "_" || $0 == "-" || $0 == "." }183                guard !name.isEmpty else { return (nil, declaration, doctype) }184                let local = name.split(separator: ":").last.map(String.init) ?? String(name)185                return (local, declaration, doctype)186            } else {187                return (nil, declaration, doctype)188            }189        }190        return (nil, declaration, doctype)191    }192193    private static func xmlFamily(_ s: String) -> SniffHit? {194        let info = xmlRoot(s)195        if let doctype = info.doctype, doctype.hasPrefix("<!doctype html") { return SniffHit(.html, .certain, "HTML doctype") }196        guard let root = info.root else {197            return info.hasDeclaration ? SniffHit(.xml, .likely, "XML declaration") : nil198        }199        switch root {200        case "svg": return SniffHit(.svg, .certain, "root element <svg>")201        case "kml": return SniffHit(.kml, .certain, "root element <kml>")202        case "gpx": return SniffHit(.gpx, .certain, "root element <gpx>")203        case "COLLADA": return SniffHit(.dae, .certain, "root element <COLLADA>")204        case "plist": return SniffHit(.plist, .certain, "root element <plist>")205        default: break206        }207        let lower = root.lowercased()208        if lower == "html" { return SniffHit(.html, .certain, "root element <html>") }209        if info.hasDeclaration { return SniffHit(.xml, .certain, "XML declaration, root element <\(root)>") }210        if htmlTags.contains(lower) { return SniffHit(.html, .likely, "HTML tag <\(lower)> without XML declaration") }211        // A lone tag-like token is not enough to call arbitrary text XML: require a closing tag or self-closing root.212        if s.contains("</") || s.contains("/>") { return SniffHit(.xml, .likely, "root element <\(root)>") }213        return nil214    }215216    // MARK: JSON family217218    private static let geoTypes: Set<String> = [219        "Feature", "FeatureCollection", "Point", "LineString", "Polygon", "MultiPoint", "MultiLineString", "MultiPolygon",220        "GeometryCollection",221    ]222223    private static func parses(_ s: String) -> Any? {224        guard let data = s.data(using: .utf8) else { return nil }225        return try? JSONSerialization.jsonObject(with: data)226    }227228    private static func jsonFamily(_ text: String, lines: [String], complete: Bool) -> SniffHit? {229        // JSON Lines: at least two non-empty lines, each one a complete JSON object or array on its own.230        if lines.count >= 2 {231            let sample = lines.prefix(20)232            if sample.allSatisfy({ line in233                let t = line.trimmingCharacters(in: .whitespaces)234                return (t.hasPrefix("{") || t.hasPrefix("[")) && parses(t) != nil235            }) {236                return SniffHit(.jsonl, .certain, "\(sample.count) consecutive lines are complete JSON values")237            }238        }239        if complete {240            guard let value = parses(text) else { return nil }241            if let object = value as? [String: Any] {242                if let type = object["type"] as? String, geoTypes.contains(type),243                   type == "Feature" || type == "FeatureCollection" || object["coordinates"] != nil || object["geometries"] != nil {244                    return SniffHit(.geojson, .certain, "JSON with top-level type \"\(type)\"")245                }246                if let asset = object["asset"] as? [String: Any], asset["version"] != nil {247                    return SniffHit(.gltf, .certain, "JSON with glTF asset.version")248                }249            }250            return SniffHit(.json, .certain, "valid JSON document")251        }252        // Only a prefix is available: scan top-level keys without requiring the document to be complete.253        let scan = scanTopLevel(text)254        guard scan.plausible else { return nil }255        if let type = scan.type, geoTypes.contains(type) { return SniffHit(.geojson, .likely, "JSON prefix with top-level type \"\(type)\"") }256        if scan.keys.contains("asset"), scan.keys.contains("scenes") || scan.keys.contains("nodes") || scan.keys.contains("meshes") || scan.keys.contains("buffers") || scan.keys.count == 1 {257            return SniffHit(.gltf, .likely, "JSON prefix with glTF asset key")258        }259        return SniffHit(.json, .likely, "JSON syntax (prefix only)")260    }261262    /// Tolerant scan of the first level of a JSON text that may be cut anywhere.263    static func scanTopLevel(_ text: String) -> (plausible: Bool, keys: Set<String>, type: String?) {264        var depth = 0265        var keys = Set<String>()266        var type: String?267        var inString = false, escaped = false268        var current = ""269        var lastString: String?270        var pendingKey: String?271        var expectingKey = false272        var rootIsObject = false273        var structural = 0274        for ch in text {275            if inString {276                if escaped { escaped = false; current.append(ch); continue }277                if ch == "\\" { escaped = true; continue }278                if ch == "\"" {279                    inString = false280                    if depth == 1, rootIsObject {281                        if expectingKey { lastString = current } else if pendingKey == "type" { type = current }282                    }283                    continue284                }285                if current.count < 256 { current.append(ch) }286                continue287            }288            switch ch {289            case "\"": inString = true; current = ""290            case "{", "[":291                if depth == 0 { rootIsObject = ch == "{"; expectingKey = rootIsObject }292                depth += 1; structural += 1293            case "}", "]":294                depth -= 1; structural += 1295                if depth < 0 { return (false, keys, type) }296            case ":":297                if depth == 1, rootIsObject, let key = lastString { keys.insert(key); pendingKey = key; lastString = nil; expectingKey = false }298                structural += 1299            case ",":300                if depth == 1, rootIsObject { expectingKey = true; pendingKey = nil }301            default:302                if depth == 0, !ch.isWhitespace { return (false, keys, type) }303            }304        }305        // A JSON prefix must show real structure: a key, or nested values.306        let plausible = rootIsObject ? !keys.isEmpty : structural >= 2 || text.contains(",")307        return (plausible, keys, type)308    }309310    // MARK: Subtitles311312    private static func subStation(_ text: String) -> SniffHit {313        let lower = text.lowercased()314        if lower.contains("[v4+ styles]") || lower.contains("scripttype: v4.00+") { return SniffHit(.ass, .certain, "[Script Info] with V4+ styles") }315        if lower.contains("[v4 styles]") || lower.contains("scripttype: v4.00") { return SniffHit(.ssa, .certain, "[Script Info] with V4 styles") }316        return SniffHit(.ass, .likely, "[Script Info] section")317    }318319    private static func isTimestamp(_ s: Substring) -> Bool {320        // h:mm:ss,mmm or hh:mm:ss.mmm321        let parts = s.split(separator: ":")322        guard parts.count == 3, parts[0].count <= 3, parts[0].allSatisfy(\.isNumber), parts[1].count == 2, parts[1].allSatisfy(\.isNumber) else { return false }323        let seconds = parts[2].split(whereSeparator: { $0 == "," || $0 == "." })324        return seconds.count == 2 && seconds[0].count == 2 && seconds.allSatisfy { $0.allSatisfy(\.isNumber) }325    }326327    private static func srt(_ lines: [String]) -> SniffHit? {328        guard lines.count >= 2 else { return nil }329        let first = lines[0].trimmingCharacters(in: .whitespaces).replacingOccurrences(of: "\u{FEFF}", with: "")330        guard !first.isEmpty, first.count <= 9, first.allSatisfy(\.isNumber) else { return nil }331        let timing = lines[1].trimmingCharacters(in: .whitespaces)332        guard let arrow = timing.range(of: " --> ") else { return nil }333        let start = timing[..<arrow.lowerBound]334        let end = timing[arrow.upperBound...].split(separator: " ").first ?? ""335        guard isTimestamp(start), isTimestamp(end) else { return nil }336        return SniffHit(.srt, .certain, "cue number followed by a timing line")337    }338339    private static func microDVD(_ lines: [String]) -> SniffHit? {340        let sample = lines.prefix(5)341        guard !sample.isEmpty else { return nil }342        func isCue(_ line: String) -> Bool {343            var rest = Substring(line.trimmingCharacters(in: .whitespaces))344            for _ in 0..<2 {345                guard rest.hasPrefix("{"), let close = rest.firstIndex(of: "}") else { return false }346                let inner = rest[rest.index(after: rest.startIndex)..<close]347                guard !inner.isEmpty, inner.allSatisfy(\.isNumber) else { return false }348                rest = rest[rest.index(after: close)...]349            }350            return true351        }352        return sample.allSatisfy(isCue) ? SniffHit(.sub, .certain, "MicroDVD {start}{end} frame cues") : nil353    }354355    // MARK: 3D356357    private static let objKeywords: Set<String> = ["v", "vt", "vn", "vp", "f", "l", "o", "g", "s", "mtllib", "usemtl", "p"]358359    private static func obj(_ lines: [String]) -> SniffHit? {360        var vertices = 0, known = 0, total = 0361        for line in lines {362            let t = line.trimmingCharacters(in: .whitespaces)363            if t.hasPrefix("#") { continue }364            total += 1365            let tokens = t.split(separator: " ", omittingEmptySubsequences: true)366            guard let head = tokens.first, objKeywords.contains(String(head)) else { continue }367            known += 1368            if head == "v", tokens.count >= 4, tokens[1...3].allSatisfy({ Double($0) != nil }) { vertices += 1 }369        }370        guard vertices >= 1, total > 0, known * 10 >= total * 8 else { return nil }371        return SniffHit(.obj, .likely, "Wavefront statements (\(vertices) vertex lines in the prefix)")372    }373374    // MARK: Configuration formats375376    private static func isBareKey(_ s: Substring) -> Bool {377        !s.isEmpty && s.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" || $0 == "-" || $0 == "." || $0 == "\"" || $0 == "'" }378    }379380    /// TOML rule: at least 90 % of the meaningful lines are `[table]` headers or `key = value`; every assigned value381    /// has a TOML type (quoted string, number, boolean, array, inline table, date) — INI and `.env` files with bare382    /// values do not match — and there is a table header or at least two assignments.383    private static func toml(_ lines: [String]) -> SniffHit? {384        var matched = 0, total = 0, headers = 0, typed = 0, assignments = 0385        for line in lines {386            let t = line.trimmingCharacters(in: .whitespaces)387            if t.hasPrefix("#") { continue }388            total += 1389            if t.hasPrefix("["), let close = t.lastIndex(of: "]") {390                let inner = t[t.index(after: t.startIndex)..<close].trimmingCharacters(in: CharacterSet(charactersIn: "[] "))391                let tail = t[t.index(after: close)...].trimmingCharacters(in: .whitespaces)392                if !inner.isEmpty, inner.allSatisfy({ $0.isLetter || $0.isNumber || "_-.\"' ".contains($0) }), tail.isEmpty || tail.hasPrefix("#") {393                    matched += 1; headers += 1; continue394                }395            }396            if let eq = t.firstIndex(of: "=") {397                let key = t[..<eq].trimmingCharacters(in: .whitespaces)398                let value = t[t.index(after: eq)...].trimmingCharacters(in: .whitespaces)399                if isBareKey(Substring(key)), let first = value.first {400                    matched += 1401                    assignments += 1402                    if "\"'[{".contains(first) || first.isNumber || first == "-" || first == "+" || value.hasPrefix("true") || value.hasPrefix("false") { typed += 1 }403                }404            }405        }406        guard total >= 2, typed >= 1, typed == assignments, matched * 10 >= total * 9, headers >= 1 || typed >= 2 else { return nil }407        return SniffHit(.toml, .likely, "TOML tables and typed key = value lines")408    }409410    /// YAML rule: a `---`/`%YAML` start, or at least two lines, where every meaningful line is a mapping entry411    /// (`key: value`, `key:`), a sequence item (`- item`) or an indented continuation, with at least one mapping key.412    /// A `---` block closed by a second `---` and followed by prose is Markdown front matter, not YAML.413    private static func yaml(_ lines: [String]) -> SniffHit? {414        guard let first = lines.first else { return nil }415        let startsDocument = first.trimmingCharacters(in: .whitespaces) == "---" || first.hasPrefix("%YAML")416        if startsDocument, let closing = lines.dropFirst().firstIndex(where: { $0.trimmingCharacters(in: .whitespaces) == "---" }),417           closing + 1 < lines.count, lines[(closing + 1)...].contains(where: { !isYAMLLine($0).matches }) {418            return nil419        }420        var keys = 0, matched = 0, total = 0421        for line in lines {422            let t = line.trimmingCharacters(in: .whitespaces)423            if t.hasPrefix("#") || t == "---" || t == "..." || t.hasPrefix("%YAML") { continue }424            total += 1425            let result = isYAMLLine(line)426            if result.matches { matched += 1 }427            if result.isKey { keys += 1 }428        }429        guard keys >= 1, total >= (startsDocument ? 1 : 2), matched == total else { return nil }430        return SniffHit(.yaml, .likely, startsDocument ? "YAML document marker and mapping entries" : "YAML mapping entries on every line")431    }432433    private static func isYAMLLine(_ line: String) -> (matches: Bool, isKey: Bool) {434        var t = Substring(line.trimmingCharacters(in: .whitespaces))435        if t.isEmpty || t.hasPrefix("#") || t == "---" { return (true, false) }436        var isItem = false437        while t.hasPrefix("- ") { t = t.dropFirst(2).drop { $0 == " " }; isItem = true }438        if t == "-" { return (true, false) }439        if let colon = t.firstIndex(of: ":") {440            let key = t[..<colon]441            let after = t[t.index(after: colon)...]442            let keyOK = !key.isEmpty && key.count <= 80 && key.allSatisfy { $0.isLetter || $0.isNumber || "_-. \"'/".contains($0) } && !key.hasPrefix(" ")443            if keyOK, after.isEmpty || after.hasPrefix(" ") { return (true, true) }444        }445        if isItem { return (true, false) }446        // Indented continuation of a block scalar or flow collection.447        if line.hasPrefix("  ") || line.hasPrefix("\t") { return (true, false) }448        return (false, false)449    }450451    // MARK: Delimited data452453    /// CSV/TSV rule: at least two records and every record of the sample has the same number of fields, at least454    /// two. Quotes follow RFC 4180: a quote opens only at the start of a field, `""` is an escaped quote, a closing455    /// quote must be followed by the delimiter or the end of the record, and a quoted field may span lines (at most456    /// 20). Anything else is not delimited data.457    static func delimited(_ lines: [String], _ delimiter: Character) -> Bool {458        var counts: [Int] = []459        var fields = 1460        var inQuotes = false461        var spanned = 0462        for line in lines.prefix(120) {463            var atFieldStart = !inQuotes464            var justClosed = false465            var previousWasQuote = false466            for ch in line {467                if inQuotes {468                    if ch == "\"" {469                        if previousWasQuote { previousWasQuote = false } else { previousWasQuote = true }470                    } else if previousWasQuote {471                        // The quote before this character closed the field.472                        inQuotes = false473                        previousWasQuote = false474                        guard ch == delimiter else { return false }475                        fields += 1476                        atFieldStart = true477                    }478                    continue479                }480                if justClosed {481                    guard ch == delimiter else { return false }482                    justClosed = false483                }484                if ch == delimiter {485                    fields += 1486                    atFieldStart = true487                } else if ch == "\"", atFieldStart {488                    inQuotes = true489                    atFieldStart = false490                } else {491                    atFieldStart = false492                }493            }494            if inQuotes, previousWasQuote { inQuotes = false }495            if inQuotes {496                spanned += 1497                if spanned > 20 { return false }498                continue499            }500            spanned = 0501            counts.append(fields)502            fields = 1503            if counts.count >= 30 { break }504        }505        guard !inQuotes, counts.count >= 2, let first = counts.first, first >= 2 else { return false }506        return counts.allSatisfy { $0 == first }507    }508509    // MARK: Markup510511    /// reStructuredText strong signals: an explicit markup directive (`.. name::`) or an interpreted-text role.512    private static func rstStrong(_ lines: [String]) -> SniffHit? {513        for line in lines {514            let t = line.trimmingCharacters(in: .whitespaces)515            if t.hasPrefix(".. "), t.contains("::") { return SniffHit(.rst, .likely, "reStructuredText directive") }516            if t.contains(":ref:`") || t.contains(":doc:`") || t.contains(":class:`") || t.contains(":func:`") {517                return SniffHit(.rst, .likely, "reStructuredText role")518            }519        }520        return nil521    }522523    /// reStructuredText headings: at least two title lines (starting with a letter or digit) underlined by a524    /// punctuation run of the same length, give or take two characters.525    private static func rstHeadings(_ lines: [String]) -> SniffHit? {526        var headings = 0527        for i in 1..<max(lines.count, 1) {528            let under = lines[i].trimmingCharacters(in: .whitespaces)529            let title = lines[i - 1].trimmingCharacters(in: .whitespaces)530            guard under.count >= 3, let ch = under.first, "=-~^\"*+`".contains(ch), under.allSatisfy({ $0 == ch }),531                  !title.isEmpty, title.count <= under.count, under.count - title.count <= 2, title.contains(where: \.isLetter),532                  let lead = title.first, lead.isLetter || lead.isNumber else { continue }533            headings += 1534        }535        return headings >= 2 ? SniffHit(.rst, .likely, "\(headings) underlined section titles") : nil536    }537538    /// Markdown rule. Constructs are counted conservatively, because `#`, `*` and `_` are everywhere in source code539    /// and configuration files:540    ///541    /// - **heading**: `#`…`######` + space + text, with a blank line (or the file boundary) before and after. It542    ///   is ignored when the file starts with a `#!` shebang or when `#` lines that are not such headings outnumber543    ///   the headings three to one (a commented script, not a document);544    /// - **fence**: a line starting with ``` or ~~~;545    /// - **link**: `](` on a line;546    /// - **emphasis**: `**text**` closed on the same line (not `**kwargs`, not `__init__`);547    /// - **list**: at least two lines starting with `- `, `+ ` or `* ` at an even indentation (so ` * ` comment548    ///   blocks do not count);549    /// - **quote** (`> `) and **table** (`| … |`) lines.550    ///551    /// The strict pass (before CSV and RST) needs two of heading/fence/link/emphasis, or one of them plus another552    /// construct. The relaxed pass accepts a first-line heading followed by a blank line, or any two constructs.553    private static func markdown(_ lines: [String], strict: Bool) -> SniffHit? {554        var features = Set<String>()555        var headings = 0, otherHashLines = 0, listLines = 0556        let hasShebang = lines.first?.hasPrefix("#!") ?? false557        for (index, line) in lines.enumerated() {558            let t = line.trimmingCharacters(in: .whitespaces)559            if t.hasPrefix("#") {560                let hashes = t.prefix { $0 == "#" }561                let isHeading = hashes.count <= 6 && t.dropFirst(hashes.count).hasPrefix(" ") && line.hasPrefix("#")562                let blankBefore = index == 0 || lines[index - 1].allSatisfy(\.isWhitespace)563                let blankAfter = index == lines.count - 1 || lines[index + 1].allSatisfy(\.isWhitespace)564                if isHeading, blankBefore, blankAfter { headings += 1 } else { otherHashLines += 1 }565            }566            if t.hasPrefix("```") || t.hasPrefix("~~~") { features.insert("fence") }567            if t.hasPrefix("- ") || t.hasPrefix("* ") || t.hasPrefix("+ ") {568                let indent = line.prefix { $0 == " " }.count569                if indent % 2 == 0, !line.hasPrefix("\t") { listLines += 1 }570            }571            if t.hasPrefix("> ") { features.insert("quote") }572            if t.hasPrefix("|"), t.hasSuffix("|"), t.count > 2 { features.insert("table") }573            if t.contains("](") { features.insert("link") }574            if let open = t.range(of: "**"), let first = t[open.upperBound...].first, !first.isWhitespace, first != "*",575               let close = t[open.upperBound...].range(of: "**"), t[t.index(before: close.lowerBound)] != " " {576                features.insert("emphasis")577            }578        }579        if headings > 0, !hasShebang, otherHashLines < headings * 3 { features.insert("heading") }580        if listLines >= 2 { features.insert("list") }581        let strong = features.intersection(["heading", "fence", "link", "emphasis"]).count582        let firstIsHeading = features.contains("heading") && (lines.first.map { $0.hasPrefix("# ") || $0.hasPrefix("## ") } ?? false)583        let ok = strict ? strong >= 2 || (strong >= 1 && features.count >= 2) : firstIsHeading || features.count >= 2584        guard ok else { return nil }585        return SniffHit(.markdown, .likely, "Markdown constructs: " + features.sorted().joined(separator: ", "))586    }587}588
Files main
Menu

Theme