Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocAudit/Acronyms.swift3// Description: Detection of acronyms that are used but never spelled out, in English and French texts.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import DocModel910extension ResearchAssistant {11 /// Acronyms so common that they need no definition (general public, English and French).12 static let commonAcronyms: Set<String> = [13 "OK", "USA", "US", "UK", "EU", "UE", "UN", "ONU", "TV", "PDF", "URL", "HTML", "HTTP", "HTTPS", "WWW", "ID", "AM", "PM", "CEO", "PDG", "FAQ", "DIY",14 "GPS", "DNA", "ADN", "USB", "DVD", "PC", "IT", "TI", "AI", "IA", "HR", "RH", "VIP", "ATM", "NASA", "NATO", "OTAN", "FBI", "CIA", "ISBN", "ISSN", "DOI",15 "TODO", "FIXME", "TBD", "TBC", "NB", "PS", "RSVP", "SVP", "QC", "CA", "USD", "CAD", "EUR", "GBP", "JPG", "JPEG", "PNG", "GIF", "CSV", "XML", "JSON",16 "SMS", "WIFI", "ASAP", "ETA", "FYI", "AKA", "PHD", "MBA", "BA", "MA", "BSC", "MSC", "TPS", "TVQ", "TVA", "VAT", "COVID", "SARS", "HIV", "VIH", "AIDS", "SIDA",17 "UTC", "GMT", "EST", "PST", "BC", "AD", "CE", "BCE", "ISO", "IEEE", "ACM", "APA", "MLA", "RGB", "CMYK", "CPU", "GPU", "RAM", "SSD", "OS", "IOS", "API",18 "LLC", "INC", "LTD", "SA", "SARL", "PME", "SME", "ONG", "NGO", "AC", "DC",19 ]2021 /// Acronyms that are used but never expanded anywhere in the document, ordered by first use.22 ///23 /// An acronym is a word of 2 to 8 capital letters and digits starting with a letter and holding at least two capitals ("GDP",24 /// "CO2", "PIB"), optionally followed by a plural "s". Roman numerals, a built-in list of common acronyms, `known`, and words25 /// inside text typed in capitals are ignored. It counts as defined when the document contains, anywhere:26 /// "long form (ACR)", "ACR (long form)", or "ACR stands for / means / signifie / désigne / : long form" — where the letters of27 /// the acronym can be read in order in the long form, starting at a word initial and with at least half of them on word initials.28 public static func undefinedAcronyms(in doc: DocumentModel) -> [AcronymFinding] { undefinedAcronyms(in: doc, known: []) }2930 public static func undefinedAcronyms(in doc: DocumentModel, known: Set<String>) -> [AcronymFinding] {31 guard let pattern = try? NSRegularExpression(pattern: "(?<![\\p{L}\\p{N}_])(\\p{Lu}[\\p{Lu}\\p{N}]{1,7})(s)?(?![\\p{L}\\p{N}_])") else { return [] }32 let roman = try? NSRegularExpression(pattern: "^M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$")33 let ignore = commonAcronyms.union(known.map { $0.uppercased() })34 struct Use { var count = 0; var first: Hit?; var defined = false }35 var uses: [String: Use] = [:]36 var order: [String] = []37 let trails = HeadingTrails(doc)38 var levels = HeadingLevels(doc)3940 for ref in Scan.paragraphs(of: doc) {41 let p = ref.paragraph42 if p.style == StyleID.code { continue }43 let text = Scan.maskedText(p)44 let ns = text as NSString45 guard ns.length >= 2 else { continue }46 let matches = pattern.matches(in: text, range: NSRange(location: 0, length: ns.length))47 guard !matches.isEmpty else { continue }48 // Text typed in capitals (titles, shouting): its words are not acronyms. Definitions are still read from it.49 var upper = 0, letters = 050 for u in text.unicodeScalars where u.properties.isAlphabetic { letters += 1; if u.properties.isUppercase { upper += 1 } }51 let allCaps = letters >= 12 && Double(upper) / Double(letters) > 0.652 let isHeading = levels.level(p) > 053 for m in matches {54 let r = m.range(at: 1)55 let acronym = ns.substring(with: r)56 var capitals = 057 for u in acronym.unicodeScalars where u.properties.isUppercase { capitals += 1 }58 if capitals < 2 || ignore.contains(acronym.uppercased()) { continue }59 if let roman, roman.firstMatch(in: acronym, range: NSRange(location: 0, length: (acronym as NSString).length)) != nil { continue }60 var use = uses[acronym] ?? Use()61 if !use.defined && isDefinition(acronym, ns, m.range) { use.defined = true }62 if !allCaps {63 use.count += 164 if use.first == nil {65 let range = r.location..<(r.location + r.length)66 use.first = Hit(id: "acronym#\(acronym)", documentName: doc.meta.title, story: ref.story, path: ref.path, range: range,67 context: Scan.context(text, around: range, radius: 50),68 headingTrail: ref.story == .body && !isHeading ? trails.trail(ofBlock: ref.path.block) : [])69 order.append(acronym)70 }71 }72 uses[acronym] = use73 }74 }75 return order.compactMap { a in76 guard let u = uses[a], !u.defined, let first = u.first else { return nil }77 return AcronymFinding(acronym: a, count: u.count, firstUse: first)78 }79 }8081 /// True when the occurrence of `acronym` at `range` in `text` is accompanied by its long form.82 static func isDefinition(_ acronym: String, _ text: NSString, _ range: NSRange) -> Bool {83 let start = range.location, end = range.location + range.length84 let limit = min(14, 2 * acronym.count + 4)8586 // "long form (ACR)", "long form (the ACR)", "long form (ci-après « ACR »)": the acronym sits in short parentheses.87 let before = text.substring(to: start)88 if let open = before.range(of: "(", options: .backwards), before[open.upperBound...].range(of: ")") == nil {89 let after = text.substring(from: end)90 if let close = after.firstIndex(where: { $0 == ")" || $0 == "(" }), after[close] == ")" {91 let inner = String(before[open.upperBound...]) + String(after[..<close])92 if Scan.wordCount(inner) <= 3 {93 let words = longFormWords(String(before[..<open.lowerBound]))94 if matchesLongForm(acronym, Array(words.suffix(limit)), anchoredAtEnd: true) { return true }95 }96 }97 }98 let after = text.substring(from: end)99 let trimmed = after.drop(while: { $0 == " " || $0 == "\u{00A0}" || $0 == "\u{202F}" })100 // "ACR (long form)".101 if trimmed.first == "(", let close = trimmed.firstIndex(of: ")") {102 let words = longFormWords(String(trimmed[trimmed.index(after: trimmed.startIndex)..<close]))103 if words.count >= 2 && matchesLongForm(acronym, words, anchoredAtEnd: false) { return true }104 }105 // "ACR stands for …", "ACR signifie …", "ACR : …", "ACR — …", "ACR, or …".106 let lowered = String(trimmed.prefix(24)).lowercased()107 let markers = ["stands for ", "is short for ", "means ", "signifie ", "désigne ", "veut dire ", ": ", "— ", "– ", "= ", ", or ", ", ou ", ", soit ", "is the ", "est le ", "est la ", "est l'"]108 for marker in markers where lowered.hasPrefix(marker) {109 let rest = String(trimmed.dropFirst(marker.count))110 let words = longFormWords(String(rest.prefix(160)))111 if words.count >= 2 && matchesLongForm(acronym, Array(words.prefix(limit)), anchoredAtEnd: false) { return true }112 }113 return false114 }115116 /// Words of a candidate long form: folded to lower case without diacritics, split on anything that is not a letter or digit.117 static func longFormWords(_ s: String) -> [String] {118 let folded = s.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: nil)119 return folded.split(whereSeparator: { !($0.isLetter || $0.isNumber) }).map(String.init)120 }121122 /// Can the letters of `acronym` be read in order in `words`, starting on a word initial, with at least half of the letters on123 /// word initials? With `anchoredAtEnd` the long form must run up to the last word (the one just before the parenthesis).124 static func matchesLongForm(_ acronym: String, _ words: [String], anchoredAtEnd: Bool) -> Bool {125 let letters = Array(acronym.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: nil).filter(\.isLetter))126 guard letters.count >= 2, !words.isEmpty else { return false }127 let chars = words.map(Array.init)128 // Try the closest possible starting words first.129 for s in stride(from: words.count - 1, through: 0, by: -1) where chars[s].first == letters[0] {130 var w = s, c = 1, initials = 1, matched = 1131 var lastWord = s132 while matched < letters.count {133 let target = letters[matched]134 // Prefer the initial of one of the next three words, then a letter further inside the current word.135 var found = false136 for next in (w + 1)..<min(words.count, w + 4) where chars[next].first == target {137 w = next; c = 1; initials += 1; found = true; break138 }139 if !found, let inner = chars[w][min(c, chars[w].count)...].firstIndex(of: target) {140 c = inner + 1; found = true141 }142 if !found { break }143 matched += 1144 lastWord = w145 }146 if matched == letters.count && initials * 2 >= letters.count && (!anchoredAtEnd || lastWord >= words.count - 2) { return true }147 }148 return false149 }150}151