Repository Public License
1// ORIX Convert — Packages/ORIXConvertCore/Sources/ConvertCore/Detection/TextSniffer.swift2// Description: Text detection: encoding (BOMs, UTF-8 validation, UTF-16 without BOM, Latin-1 fallback) and3// classification of textual formats from a bounded prefix (XML family, JSON family, YAML, TOML, CSV/TSV,4// Markdown, LaTeX, RST, subtitles, OBJ, USDA…). All heuristics are documented next to their rule.5// Author: Simon-Pierre Boucher — contact@spboucher.ai6import Foundation78/// Text encoding recognised by the detector.9public enum TextEncoding: String, Sendable, Codable, CaseIterable {10 case ascii, utf8 = "utf-8", utf8BOM = "utf-8-bom", utf16LE = "utf-16le", utf16BE = "utf-16be"11 case utf32LE = "utf-32le", utf32BE = "utf-32be", latin1 = "windows-1252"1213 /// Foundation encoding to decode the file with.14 public var stringEncoding: String.Encoding {15 switch self {16 case .ascii, .utf8, .utf8BOM: .utf817 case .utf16LE: .utf16LittleEndian18 case .utf16BE: .utf16BigEndian19 case .utf32LE: .utf32LittleEndian20 case .utf32BE: .utf32BigEndian21 case .latin1: .windowsCP125222 }23 }24}2526struct SniffHit {27 let format: FormatID28 let confidence: DetectionResult.Confidence29 let evidence: String30 init(_ format: FormatID, _ confidence: DetectionResult.Confidence, _ evidence: String) {31 self.format = format; self.confidence = confidence; self.evidence = evidence32 }33}3435enum TextSniffer {36 // MARK: Encoding3738 /// Decodes a prefix as text. `truncated` tells that the bytes are only the beginning of the file, so an39 /// incomplete trailing sequence is tolerated. Returns `nil` for binary content.40 static func decode(_ p: [UInt8], truncated: Bool) -> (text: String, encoding: TextEncoding)? {41 guard !p.isEmpty else { return nil }42 if p.has([0xEF, 0xBB, 0xBF]) {43 return utf8(Array(p.dropFirst(3)), truncated: truncated).map { ($0, .utf8BOM) }44 }45 if p.has([0xFF, 0xFE, 0x00, 0x00]) { return wide(p, skip: 4, unit: 4, .utf32LE) }46 if p.has([0x00, 0x00, 0xFE, 0xFF]) { return wide(p, skip: 4, unit: 4, .utf32BE) }47 if p.has([0xFF, 0xFE]) { return wide(p, skip: 2, unit: 2, .utf16LE) }48 if p.has([0xFE, 0xFF]) { return wide(p, skip: 2, unit: 2, .utf16BE) }4950 if p.contains(0) {51 // UTF-16 without BOM: mostly-ASCII text puts the zero bytes on one side of every pair.52 let sample = Array(p.prefix(2048))53 guard sample.count >= 8 else { return nil }54 var evenZeros = 0, oddZeros = 055 for (i, b) in sample.enumerated() where b == 0 { if i % 2 == 0 { evenZeros += 1 } else { oddZeros += 1 } }56 let pairs = sample.count / 257 if oddZeros * 10 >= pairs * 4, evenZeros * 20 <= pairs { return wide(p, skip: 0, unit: 2, .utf16LE) }58 if evenZeros * 10 >= pairs * 4, oddZeros * 20 <= pairs { return wide(p, skip: 0, unit: 2, .utf16BE) }59 return nil60 }61 guard !hasBinaryControls(p) else { return nil }62 if let text = utf8(p, truncated: truncated) {63 return (text, p.allSatisfy { $0 < 0x80 } ? .ascii : .utf8)64 }65 // Not UTF-8, no control bytes: accept a single-byte Western encoding when high bytes are a minority.66 let high = p.reduce(0) { $0 + ($1 >= 0x80 ? 1 : 0) }67 guard high * 10 < p.count * 3, let text = String(bytes: p, encoding: .windowsCP1252) else { return nil }68 return (text, .latin1)69 }7071 private static func hasBinaryControls(_ p: [UInt8]) -> Bool {72 p.contains { $0 < 0x09 || ($0 > 0x0D && $0 < 0x20 && $0 != 0x1B) || $0 == 0x7F }73 }7475 private static func utf8(_ p: [UInt8], truncated: Bool) -> String? {76 if hasBinaryControls(p) { return nil }77 if let s = String(bytes: p, encoding: .utf8) { return s }78 guard truncated else { return nil }79 for drop in 1...3 where p.count > drop {80 if let s = String(bytes: p.dropLast(drop), encoding: .utf8) { return s }81 }82 return nil83 }8485 private static func wide(_ p: [UInt8], skip: Int, unit: Int, _ encoding: TextEncoding) -> (String, TextEncoding)? {86 var body = Array(p.dropFirst(skip))87 body.removeLast(body.count % unit)88 guard !body.isEmpty else { return ("", encoding) }89 guard let text = String(bytes: body, encoding: encoding.stringEncoding) else {90 // A surrogate pair cut by the prefix boundary: retry without the last unit.91 guard body.count > unit, let shorter = String(bytes: body.dropLast(unit), encoding: encoding.stringEncoding) else { return nil }92 return (shorter, encoding)93 }94 if text.unicodeScalars.contains(where: { $0.value < 9 || ($0.value > 13 && $0.value < 32 && $0.value != 27) }) { return nil }95 return (text, encoding)96 }9798 // MARK: Classification99100 /// Classifies decoded text. `complete` is true when `text` is the entire file.101 static func classify(_ text: String, complete: Bool) -> SniffHit {102 let trimmed = text.drop { $0.isWhitespace || $0 == "\u{FEFF}" }103 guard !trimmed.isEmpty else { return SniffHit(.txt, .likely, "whitespace-only text") }104 var lines = text.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline).map(String.init)105 if !complete, lines.count > 1 { lines.removeLast() }106 if lines.count > 400 { lines = Array(lines.prefix(400)) }107 let content = lines.filter { !$0.allSatisfy(\.isWhitespace) }108109 if trimmed.hasPrefix("{\\rtf") { return SniffHit(.rtf, .certain, "RTF header") }110 if trimmed.hasPrefix("#usda") { return SniffHit(.usda, .certain, "#usda header") }111 if trimmed.hasPrefix("WEBVTT") { return SniffHit(.vtt, .certain, "WEBVTT header") }112 if trimmed.hasPrefix("<") { if let hit = xmlFamily(String(trimmed.prefix(8192))) { return hit } }113 if trimmed.hasPrefix("{") || trimmed.hasPrefix("[") {114 if let first = content.first, first.trimmingCharacters(in: .whitespaces).lowercased() == "[script info]" {115 return subStation(text)116 }117 if let hit = microDVD(content) { return hit }118 if let hit = jsonFamily(text, lines: content, complete: complete) { return hit }119 }120 if let hit = srt(content) { return hit }121 if text.contains("\\documentclass") || text.contains("\\begin{document}") {122 return SniffHit(.latex, .certain, "LaTeX document commands")123 }124 if trimmed.hasPrefix("ply"), content.count > 1, content[1].hasPrefix("format ascii") {125 return SniffHit(.ply, .certain, "ASCII PLY header")126 }127 if trimmed.hasPrefix("solid"), text.contains("facet") || text.contains("endsolid") {128 return SniffHit(.stl, .certain, "ASCII STL keywords")129 }130 if trimmed.hasPrefix("; FBX") { return SniffHit(.fbx, .likely, "ASCII FBX comment header") }131 if let hit = obj(content) { return hit }132 if let hit = toml(content) { return hit }133 if let hit = yaml(content) { return hit }134 // Lines that begin with a tab are indented code or text, not rows.135 if delimited(content, "\t"), content.prefix(30).filter({ $0.hasPrefix("\t") }).count * 2 < min(content.count, 30) {136 return SniffHit(.tsv, .likely, "consistent tab-separated columns")137 }138 if let hit = rstStrong(content) { return hit }139 if let hit = markdown(lines, strict: true) { return hit }140 if delimited(content, ",") { return SniffHit(.csv, .likely, "consistent comma-separated columns") }141 // Semicolons that end every line are statement terminators (source code), not separators.142 if delimited(content, ";"), !content.prefix(30).allSatisfy({ $0.trimmingCharacters(in: .whitespaces).hasSuffix(";") }) {143 return SniffHit(.csv, .likely, "consistent semicolon-separated columns")144 }145 if text.contains("\\section{") || text.contains("\\usepackage") { return SniffHit(.latex, .likely, "LaTeX commands") }146 if let hit = rstHeadings(lines) { return hit }147 if let hit = markdown(lines, strict: false) { return hit }148 return SniffHit(.txt, .likely, "plain text")149 }150151 // MARK: XML family152153 private static let htmlTags: Set<String> = [154 "html", "head", "body", "div", "p", "span", "table", "h1", "h2", "h3", "a", "ul", "ol", "script", "style", "meta",155 "title", "section", "article", "nav", "header", "footer", "main", "img", "br", "form", "link",156 ]157158 /// Root element name after skipping the XML declaration, processing instructions, comments and DOCTYPE.159 static func xmlRoot(_ s: String) -> (root: String?, hasDeclaration: Bool, doctype: String?) {160 var rest = Substring(s)161 var declaration = false162 var doctype: String?163 for _ in 0..<64 {164 rest = rest.drop { $0.isWhitespace || $0 == "\u{FEFF}" }165 if rest.hasPrefix("<?") {166 if rest.hasPrefix("<?xml") { declaration = true }167 guard let end = rest.range(of: "?>") else { return (nil, declaration, doctype) }168 rest = rest[end.upperBound...]169 } else if rest.hasPrefix("<!--") {170 guard let end = rest.range(of: "-->") else { return (nil, declaration, doctype) }171 rest = rest[end.upperBound...]172 } else if rest.hasPrefix("<!") {173 guard let end = rest.firstIndex(of: ">") else { return (nil, declaration, String(rest.prefix(64)).lowercased()) }174 doctype = String(rest[..<end]).lowercased()175 // An internal subset contains '>' characters: skip to the closing "]>" when there is one.176 if let open = rest[..<end].firstIndex(of: "["), let close = rest[open...].range(of: "]>") {177 rest = rest[close.upperBound...]178 } else {179 rest = rest[rest.index(after: end)...]180 }181 } else if rest.hasPrefix("<") {182 let name = rest.dropFirst().prefix { $0.isLetter || $0.isNumber || $0 == ":" || $0 == "_" || $0 == "-" || $0 == "." }183 guard !name.isEmpty else { return (nil, declaration, doctype) }184 let local = name.split(separator: ":").last.map(String.init) ?? String(name)185 return (local, declaration, doctype)186 } else {187 return (nil, declaration, doctype)188 }189 }190 return (nil, declaration, doctype)191 }192193 private static func xmlFamily(_ s: String) -> SniffHit? {194 let info = xmlRoot(s)195 if let doctype = info.doctype, doctype.hasPrefix("<!doctype html") { return SniffHit(.html, .certain, "HTML doctype") }196 guard let root = info.root else {197 return info.hasDeclaration ? SniffHit(.xml, .likely, "XML declaration") : nil198 }199 switch root {200 case "svg": return SniffHit(.svg, .certain, "root element <svg>")201 case "kml": return SniffHit(.kml, .certain, "root element <kml>")202 case "gpx": return SniffHit(.gpx, .certain, "root element <gpx>")203 case "COLLADA": return SniffHit(.dae, .certain, "root element <COLLADA>")204 case "plist": return SniffHit(.plist, .certain, "root element <plist>")205 default: break206 }207 let lower = root.lowercased()208 if lower == "html" { return SniffHit(.html, .certain, "root element <html>") }209 if info.hasDeclaration { return SniffHit(.xml, .certain, "XML declaration, root element <\(root)>") }210 if htmlTags.contains(lower) { return SniffHit(.html, .likely, "HTML tag <\(lower)> without XML declaration") }211 // A lone tag-like token is not enough to call arbitrary text XML: require a closing tag or self-closing root.212 if s.contains("</") || s.contains("/>") { return SniffHit(.xml, .likely, "root element <\(root)>") }213 return nil214 }215216 // MARK: JSON family217218 private static let geoTypes: Set<String> = [219 "Feature", "FeatureCollection", "Point", "LineString", "Polygon", "MultiPoint", "MultiLineString", "MultiPolygon",220 "GeometryCollection",221 ]222223 private static func parses(_ s: String) -> Any? {224 guard let data = s.data(using: .utf8) else { return nil }225 return try? JSONSerialization.jsonObject(with: data)226 }227228 private static func jsonFamily(_ text: String, lines: [String], complete: Bool) -> SniffHit? {229 // JSON Lines: at least two non-empty lines, each one a complete JSON object or array on its own.230 if lines.count >= 2 {231 let sample = lines.prefix(20)232 if sample.allSatisfy({ line in233 let t = line.trimmingCharacters(in: .whitespaces)234 return (t.hasPrefix("{") || t.hasPrefix("[")) && parses(t) != nil235 }) {236 return SniffHit(.jsonl, .certain, "\(sample.count) consecutive lines are complete JSON values")237 }238 }239 if complete {240 guard let value = parses(text) else { return nil }241 if let object = value as? [String: Any] {242 if let type = object["type"] as? String, geoTypes.contains(type),243 type == "Feature" || type == "FeatureCollection" || object["coordinates"] != nil || object["geometries"] != nil {244 return SniffHit(.geojson, .certain, "JSON with top-level type \"\(type)\"")245 }246 if let asset = object["asset"] as? [String: Any], asset["version"] != nil {247 return SniffHit(.gltf, .certain, "JSON with glTF asset.version")248 }249 }250 return SniffHit(.json, .certain, "valid JSON document")251 }252 // Only a prefix is available: scan top-level keys without requiring the document to be complete.253 let scan = scanTopLevel(text)254 guard scan.plausible else { return nil }255 if let type = scan.type, geoTypes.contains(type) { return SniffHit(.geojson, .likely, "JSON prefix with top-level type \"\(type)\"") }256 if scan.keys.contains("asset"), scan.keys.contains("scenes") || scan.keys.contains("nodes") || scan.keys.contains("meshes") || scan.keys.contains("buffers") || scan.keys.count == 1 {257 return SniffHit(.gltf, .likely, "JSON prefix with glTF asset key")258 }259 return SniffHit(.json, .likely, "JSON syntax (prefix only)")260 }261262 /// Tolerant scan of the first level of a JSON text that may be cut anywhere.263 static func scanTopLevel(_ text: String) -> (plausible: Bool, keys: Set<String>, type: String?) {264 var depth = 0265 var keys = Set<String>()266 var type: String?267 var inString = false, escaped = false268 var current = ""269 var lastString: String?270 var pendingKey: String?271 var expectingKey = false272 var rootIsObject = false273 var structural = 0274 for ch in text {275 if inString {276 if escaped { escaped = false; current.append(ch); continue }277 if ch == "\\" { escaped = true; continue }278 if ch == "\"" {279 inString = false280 if depth == 1, rootIsObject {281 if expectingKey { lastString = current } else if pendingKey == "type" { type = current }282 }283 continue284 }285 if current.count < 256 { current.append(ch) }286 continue287 }288 switch ch {289 case "\"": inString = true; current = ""290 case "{", "[":291 if depth == 0 { rootIsObject = ch == "{"; expectingKey = rootIsObject }292 depth += 1; structural += 1293 case "}", "]":294 depth -= 1; structural += 1295 if depth < 0 { return (false, keys, type) }296 case ":":297 if depth == 1, rootIsObject, let key = lastString { keys.insert(key); pendingKey = key; lastString = nil; expectingKey = false }298 structural += 1299 case ",":300 if depth == 1, rootIsObject { expectingKey = true; pendingKey = nil }301 default:302 if depth == 0, !ch.isWhitespace { return (false, keys, type) }303 }304 }305 // A JSON prefix must show real structure: a key, or nested values.306 let plausible = rootIsObject ? !keys.isEmpty : structural >= 2 || text.contains(",")307 return (plausible, keys, type)308 }309310 // MARK: Subtitles311312 private static func subStation(_ text: String) -> SniffHit {313 let lower = text.lowercased()314 if lower.contains("[v4+ styles]") || lower.contains("scripttype: v4.00+") { return SniffHit(.ass, .certain, "[Script Info] with V4+ styles") }315 if lower.contains("[v4 styles]") || lower.contains("scripttype: v4.00") { return SniffHit(.ssa, .certain, "[Script Info] with V4 styles") }316 return SniffHit(.ass, .likely, "[Script Info] section")317 }318319 private static func isTimestamp(_ s: Substring) -> Bool {320 // h:mm:ss,mmm or hh:mm:ss.mmm321 let parts = s.split(separator: ":")322 guard parts.count == 3, parts[0].count <= 3, parts[0].allSatisfy(\.isNumber), parts[1].count == 2, parts[1].allSatisfy(\.isNumber) else { return false }323 let seconds = parts[2].split(whereSeparator: { $0 == "," || $0 == "." })324 return seconds.count == 2 && seconds[0].count == 2 && seconds.allSatisfy { $0.allSatisfy(\.isNumber) }325 }326327 private static func srt(_ lines: [String]) -> SniffHit? {328 guard lines.count >= 2 else { return nil }329 let first = lines[0].trimmingCharacters(in: .whitespaces).replacingOccurrences(of: "\u{FEFF}", with: "")330 guard !first.isEmpty, first.count <= 9, first.allSatisfy(\.isNumber) else { return nil }331 let timing = lines[1].trimmingCharacters(in: .whitespaces)332 guard let arrow = timing.range(of: " --> ") else { return nil }333 let start = timing[..<arrow.lowerBound]334 let end = timing[arrow.upperBound...].split(separator: " ").first ?? ""335 guard isTimestamp(start), isTimestamp(end) else { return nil }336 return SniffHit(.srt, .certain, "cue number followed by a timing line")337 }338339 private static func microDVD(_ lines: [String]) -> SniffHit? {340 let sample = lines.prefix(5)341 guard !sample.isEmpty else { return nil }342 func isCue(_ line: String) -> Bool {343 var rest = Substring(line.trimmingCharacters(in: .whitespaces))344 for _ in 0..<2 {345 guard rest.hasPrefix("{"), let close = rest.firstIndex(of: "}") else { return false }346 let inner = rest[rest.index(after: rest.startIndex)..<close]347 guard !inner.isEmpty, inner.allSatisfy(\.isNumber) else { return false }348 rest = rest[rest.index(after: close)...]349 }350 return true351 }352 return sample.allSatisfy(isCue) ? SniffHit(.sub, .certain, "MicroDVD {start}{end} frame cues") : nil353 }354355 // MARK: 3D356357 private static let objKeywords: Set<String> = ["v", "vt", "vn", "vp", "f", "l", "o", "g", "s", "mtllib", "usemtl", "p"]358359 private static func obj(_ lines: [String]) -> SniffHit? {360 var vertices = 0, known = 0, total = 0361 for line in lines {362 let t = line.trimmingCharacters(in: .whitespaces)363 if t.hasPrefix("#") { continue }364 total += 1365 let tokens = t.split(separator: " ", omittingEmptySubsequences: true)366 guard let head = tokens.first, objKeywords.contains(String(head)) else { continue }367 known += 1368 if head == "v", tokens.count >= 4, tokens[1...3].allSatisfy({ Double($0) != nil }) { vertices += 1 }369 }370 guard vertices >= 1, total > 0, known * 10 >= total * 8 else { return nil }371 return SniffHit(.obj, .likely, "Wavefront statements (\(vertices) vertex lines in the prefix)")372 }373374 // MARK: Configuration formats375376 private static func isBareKey(_ s: Substring) -> Bool {377 !s.isEmpty && s.allSatisfy { $0.isLetter || $0.isNumber || $0 == "_" || $0 == "-" || $0 == "." || $0 == "\"" || $0 == "'" }378 }379380 /// TOML rule: at least 90 % of the meaningful lines are `[table]` headers or `key = value`; every assigned value381 /// has a TOML type (quoted string, number, boolean, array, inline table, date) — INI and `.env` files with bare382 /// values do not match — and there is a table header or at least two assignments.383 private static func toml(_ lines: [String]) -> SniffHit? {384 var matched = 0, total = 0, headers = 0, typed = 0, assignments = 0385 for line in lines {386 let t = line.trimmingCharacters(in: .whitespaces)387 if t.hasPrefix("#") { continue }388 total += 1389 if t.hasPrefix("["), let close = t.lastIndex(of: "]") {390 let inner = t[t.index(after: t.startIndex)..<close].trimmingCharacters(in: CharacterSet(charactersIn: "[] "))391 let tail = t[t.index(after: close)...].trimmingCharacters(in: .whitespaces)392 if !inner.isEmpty, inner.allSatisfy({ $0.isLetter || $0.isNumber || "_-.\"' ".contains($0) }), tail.isEmpty || tail.hasPrefix("#") {393 matched += 1; headers += 1; continue394 }395 }396 if let eq = t.firstIndex(of: "=") {397 let key = t[..<eq].trimmingCharacters(in: .whitespaces)398 let value = t[t.index(after: eq)...].trimmingCharacters(in: .whitespaces)399 if isBareKey(Substring(key)), let first = value.first {400 matched += 1401 assignments += 1402 if "\"'[{".contains(first) || first.isNumber || first == "-" || first == "+" || value.hasPrefix("true") || value.hasPrefix("false") { typed += 1 }403 }404 }405 }406 guard total >= 2, typed >= 1, typed == assignments, matched * 10 >= total * 9, headers >= 1 || typed >= 2 else { return nil }407 return SniffHit(.toml, .likely, "TOML tables and typed key = value lines")408 }409410 /// YAML rule: a `---`/`%YAML` start, or at least two lines, where every meaningful line is a mapping entry411 /// (`key: value`, `key:`), a sequence item (`- item`) or an indented continuation, with at least one mapping key.412 /// A `---` block closed by a second `---` and followed by prose is Markdown front matter, not YAML.413 private static func yaml(_ lines: [String]) -> SniffHit? {414 guard let first = lines.first else { return nil }415 let startsDocument = first.trimmingCharacters(in: .whitespaces) == "---" || first.hasPrefix("%YAML")416 if startsDocument, let closing = lines.dropFirst().firstIndex(where: { $0.trimmingCharacters(in: .whitespaces) == "---" }),417 closing + 1 < lines.count, lines[(closing + 1)...].contains(where: { !isYAMLLine($0).matches }) {418 return nil419 }420 var keys = 0, matched = 0, total = 0421 for line in lines {422 let t = line.trimmingCharacters(in: .whitespaces)423 if t.hasPrefix("#") || t == "---" || t == "..." || t.hasPrefix("%YAML") { continue }424 total += 1425 let result = isYAMLLine(line)426 if result.matches { matched += 1 }427 if result.isKey { keys += 1 }428 }429 guard keys >= 1, total >= (startsDocument ? 1 : 2), matched == total else { return nil }430 return SniffHit(.yaml, .likely, startsDocument ? "YAML document marker and mapping entries" : "YAML mapping entries on every line")431 }432433 private static func isYAMLLine(_ line: String) -> (matches: Bool, isKey: Bool) {434 var t = Substring(line.trimmingCharacters(in: .whitespaces))435 if t.isEmpty || t.hasPrefix("#") || t == "---" { return (true, false) }436 var isItem = false437 while t.hasPrefix("- ") { t = t.dropFirst(2).drop { $0 == " " }; isItem = true }438 if t == "-" { return (true, false) }439 if let colon = t.firstIndex(of: ":") {440 let key = t[..<colon]441 let after = t[t.index(after: colon)...]442 let keyOK = !key.isEmpty && key.count <= 80 && key.allSatisfy { $0.isLetter || $0.isNumber || "_-. \"'/".contains($0) } && !key.hasPrefix(" ")443 if keyOK, after.isEmpty || after.hasPrefix(" ") { return (true, true) }444 }445 if isItem { return (true, false) }446 // Indented continuation of a block scalar or flow collection.447 if line.hasPrefix(" ") || line.hasPrefix("\t") { return (true, false) }448 return (false, false)449 }450451 // MARK: Delimited data452453 /// CSV/TSV rule: at least two records and every record of the sample has the same number of fields, at least454 /// two. Quotes follow RFC 4180: a quote opens only at the start of a field, `""` is an escaped quote, a closing455 /// quote must be followed by the delimiter or the end of the record, and a quoted field may span lines (at most456 /// 20). Anything else is not delimited data.457 static func delimited(_ lines: [String], _ delimiter: Character) -> Bool {458 var counts: [Int] = []459 var fields = 1460 var inQuotes = false461 var spanned = 0462 for line in lines.prefix(120) {463 var atFieldStart = !inQuotes464 var justClosed = false465 var previousWasQuote = false466 for ch in line {467 if inQuotes {468 if ch == "\"" {469 if previousWasQuote { previousWasQuote = false } else { previousWasQuote = true }470 } else if previousWasQuote {471 // The quote before this character closed the field.472 inQuotes = false473 previousWasQuote = false474 guard ch == delimiter else { return false }475 fields += 1476 atFieldStart = true477 }478 continue479 }480 if justClosed {481 guard ch == delimiter else { return false }482 justClosed = false483 }484 if ch == delimiter {485 fields += 1486 atFieldStart = true487 } else if ch == "\"", atFieldStart {488 inQuotes = true489 atFieldStart = false490 } else {491 atFieldStart = false492 }493 }494 if inQuotes, previousWasQuote { inQuotes = false }495 if inQuotes {496 spanned += 1497 if spanned > 20 { return false }498 continue499 }500 spanned = 0501 counts.append(fields)502 fields = 1503 if counts.count >= 30 { break }504 }505 guard !inQuotes, counts.count >= 2, let first = counts.first, first >= 2 else { return false }506 return counts.allSatisfy { $0 == first }507 }508509 // MARK: Markup510511 /// reStructuredText strong signals: an explicit markup directive (`.. name::`) or an interpreted-text role.512 private static func rstStrong(_ lines: [String]) -> SniffHit? {513 for line in lines {514 let t = line.trimmingCharacters(in: .whitespaces)515 if t.hasPrefix(".. "), t.contains("::") { return SniffHit(.rst, .likely, "reStructuredText directive") }516 if t.contains(":ref:`") || t.contains(":doc:`") || t.contains(":class:`") || t.contains(":func:`") {517 return SniffHit(.rst, .likely, "reStructuredText role")518 }519 }520 return nil521 }522523 /// reStructuredText headings: at least two title lines (starting with a letter or digit) underlined by a524 /// punctuation run of the same length, give or take two characters.525 private static func rstHeadings(_ lines: [String]) -> SniffHit? {526 var headings = 0527 for i in 1..<max(lines.count, 1) {528 let under = lines[i].trimmingCharacters(in: .whitespaces)529 let title = lines[i - 1].trimmingCharacters(in: .whitespaces)530 guard under.count >= 3, let ch = under.first, "=-~^\"*+`".contains(ch), under.allSatisfy({ $0 == ch }),531 !title.isEmpty, title.count <= under.count, under.count - title.count <= 2, title.contains(where: \.isLetter),532 let lead = title.first, lead.isLetter || lead.isNumber else { continue }533 headings += 1534 }535 return headings >= 2 ? SniffHit(.rst, .likely, "\(headings) underlined section titles") : nil536 }537538 /// Markdown rule. Constructs are counted conservatively, because `#`, `*` and `_` are everywhere in source code539 /// and configuration files:540 ///541 /// - **heading**: `#`…`######` + space + text, with a blank line (or the file boundary) before and after. It542 /// is ignored when the file starts with a `#!` shebang or when `#` lines that are not such headings outnumber543 /// the headings three to one (a commented script, not a document);544 /// - **fence**: a line starting with ``` or ~~~;545 /// - **link**: `](` on a line;546 /// - **emphasis**: `**text**` closed on the same line (not `**kwargs`, not `__init__`);547 /// - **list**: at least two lines starting with `- `, `+ ` or `* ` at an even indentation (so ` * ` comment548 /// blocks do not count);549 /// - **quote** (`> `) and **table** (`| … |`) lines.550 ///551 /// The strict pass (before CSV and RST) needs two of heading/fence/link/emphasis, or one of them plus another552 /// construct. The relaxed pass accepts a first-line heading followed by a blank line, or any two constructs.553 private static func markdown(_ lines: [String], strict: Bool) -> SniffHit? {554 var features = Set<String>()555 var headings = 0, otherHashLines = 0, listLines = 0556 let hasShebang = lines.first?.hasPrefix("#!") ?? false557 for (index, line) in lines.enumerated() {558 let t = line.trimmingCharacters(in: .whitespaces)559 if t.hasPrefix("#") {560 let hashes = t.prefix { $0 == "#" }561 let isHeading = hashes.count <= 6 && t.dropFirst(hashes.count).hasPrefix(" ") && line.hasPrefix("#")562 let blankBefore = index == 0 || lines[index - 1].allSatisfy(\.isWhitespace)563 let blankAfter = index == lines.count - 1 || lines[index + 1].allSatisfy(\.isWhitespace)564 if isHeading, blankBefore, blankAfter { headings += 1 } else { otherHashLines += 1 }565 }566 if t.hasPrefix("```") || t.hasPrefix("~~~") { features.insert("fence") }567 if t.hasPrefix("- ") || t.hasPrefix("* ") || t.hasPrefix("+ ") {568 let indent = line.prefix { $0 == " " }.count569 if indent % 2 == 0, !line.hasPrefix("\t") { listLines += 1 }570 }571 if t.hasPrefix("> ") { features.insert("quote") }572 if t.hasPrefix("|"), t.hasSuffix("|"), t.count > 2 { features.insert("table") }573 if t.contains("](") { features.insert("link") }574 if let open = t.range(of: "**"), let first = t[open.upperBound...].first, !first.isWhitespace, first != "*",575 let close = t[open.upperBound...].range(of: "**"), t[t.index(before: close.lowerBound)] != " " {576 features.insert("emphasis")577 }578 }579 if headings > 0, !hasShebang, otherHashLines < headings * 3 { features.insert("heading") }580 if listLines >= 2 { features.insert("list") }581 let strong = features.intersection(["heading", "fence", "link", "emphasis"]).count582 let firstIsHeading = features.contains("heading") && (lines.first.map { $0.hasPrefix("# ") || $0.hasPrefix("## ") } ?? false)583 let ok = strict ? strong >= 2 || (strong >= 1 && features.count >= 2) : firstIsHeading || features.count >= 2584 guard ok else { return nil }585 return SniffHit(.markdown, .likely, "Markdown constructs: " + features.sorted().joined(separator: ", "))586 }587}588