SPB Git

Repository Public

Acronyms.swift 9.5 KB 151 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocAudit/Acronyms.swift3// Description: Detection of acronyms that are used but never spelled out, in English and French texts.4//5// Author: Simon-Pierre Boucher6// Contact: contact@spboucher.ai7import Foundation8import DocModel910extension ResearchAssistant {11    /// Acronyms so common that they need no definition (general public, English and French).12    static let commonAcronyms: Set<String> = [13        "OK", "USA", "US", "UK", "EU", "UE", "UN", "ONU", "TV", "PDF", "URL", "HTML", "HTTP", "HTTPS", "WWW", "ID", "AM", "PM", "CEO", "PDG", "FAQ", "DIY",14        "GPS", "DNA", "ADN", "USB", "DVD", "PC", "IT", "TI", "AI", "IA", "HR", "RH", "VIP", "ATM", "NASA", "NATO", "OTAN", "FBI", "CIA", "ISBN", "ISSN", "DOI",15        "TODO", "FIXME", "TBD", "TBC", "NB", "PS", "RSVP", "SVP", "QC", "CA", "USD", "CAD", "EUR", "GBP", "JPG", "JPEG", "PNG", "GIF", "CSV", "XML", "JSON",16        "SMS", "WIFI", "ASAP", "ETA", "FYI", "AKA", "PHD", "MBA", "BA", "MA", "BSC", "MSC", "TPS", "TVQ", "TVA", "VAT", "COVID", "SARS", "HIV", "VIH", "AIDS", "SIDA",17        "UTC", "GMT", "EST", "PST", "BC", "AD", "CE", "BCE", "ISO", "IEEE", "ACM", "APA", "MLA", "RGB", "CMYK", "CPU", "GPU", "RAM", "SSD", "OS", "IOS", "API",18        "LLC", "INC", "LTD", "SA", "SARL", "PME", "SME", "ONG", "NGO", "AC", "DC",19    ]2021    /// Acronyms that are used but never expanded anywhere in the document, ordered by first use.22    ///23    /// An acronym is a word of 2 to 8 capital letters and digits starting with a letter and holding at least two capitals ("GDP",24    /// "CO2", "PIB"), optionally followed by a plural "s". Roman numerals, a built-in list of common acronyms, `known`, and words25    /// inside text typed in capitals are ignored. It counts as defined when the document contains, anywhere:26    /// "long form (ACR)", "ACR (long form)", or "ACR stands for / means / signifie / désigne / : long form" — where the letters of27    /// the acronym can be read in order in the long form, starting at a word initial and with at least half of them on word initials.28    public static func undefinedAcronyms(in doc: DocumentModel) -> [AcronymFinding] { undefinedAcronyms(in: doc, known: []) }2930    public static func undefinedAcronyms(in doc: DocumentModel, known: Set<String>) -> [AcronymFinding] {31        guard let pattern = try? NSRegularExpression(pattern: "(?<![\\p{L}\\p{N}_])(\\p{Lu}[\\p{Lu}\\p{N}]{1,7})(s)?(?![\\p{L}\\p{N}_])") else { return [] }32        let roman = try? NSRegularExpression(pattern: "^M{0,3}(?:CM|CD|D?C{0,3})(?:XC|XL|L?X{0,3})(?:IX|IV|V?I{0,3})$")33        let ignore = commonAcronyms.union(known.map { $0.uppercased() })34        struct Use { var count = 0; var first: Hit?; var defined = false }35        var uses: [String: Use] = [:]36        var order: [String] = []37        let trails = HeadingTrails(doc)38        var levels = HeadingLevels(doc)3940        for ref in Scan.paragraphs(of: doc) {41            let p = ref.paragraph42            if p.style == StyleID.code { continue }43            let text = Scan.maskedText(p)44            let ns = text as NSString45            guard ns.length >= 2 else { continue }46            let matches = pattern.matches(in: text, range: NSRange(location: 0, length: ns.length))47            guard !matches.isEmpty else { continue }48            // Text typed in capitals (titles, shouting): its words are not acronyms. Definitions are still read from it.49            var upper = 0, letters = 050            for u in text.unicodeScalars where u.properties.isAlphabetic { letters += 1; if u.properties.isUppercase { upper += 1 } }51            let allCaps = letters >= 12 && Double(upper) / Double(letters) > 0.652            let isHeading = levels.level(p) > 053            for m in matches {54                let r = m.range(at: 1)55                let acronym = ns.substring(with: r)56                var capitals = 057                for u in acronym.unicodeScalars where u.properties.isUppercase { capitals += 1 }58                if capitals < 2 || ignore.contains(acronym.uppercased()) { continue }59                if let roman, roman.firstMatch(in: acronym, range: NSRange(location: 0, length: (acronym as NSString).length)) != nil { continue }60                var use = uses[acronym] ?? Use()61                if !use.defined && isDefinition(acronym, ns, m.range) { use.defined = true }62                if !allCaps {63                    use.count += 164                    if use.first == nil {65                        let range = r.location..<(r.location + r.length)66                        use.first = Hit(id: "acronym#\(acronym)", documentName: doc.meta.title, story: ref.story, path: ref.path, range: range,67                                        context: Scan.context(text, around: range, radius: 50),68                                        headingTrail: ref.story == .body && !isHeading ? trails.trail(ofBlock: ref.path.block) : [])69                        order.append(acronym)70                    }71                }72                uses[acronym] = use73            }74        }75        return order.compactMap { a in76            guard let u = uses[a], !u.defined, let first = u.first else { return nil }77            return AcronymFinding(acronym: a, count: u.count, firstUse: first)78        }79    }8081    /// True when the occurrence of `acronym` at `range` in `text` is accompanied by its long form.82    static func isDefinition(_ acronym: String, _ text: NSString, _ range: NSRange) -> Bool {83        let start = range.location, end = range.location + range.length84        let limit = min(14, 2 * acronym.count + 4)8586        // "long form (ACR)", "long form (the ACR)", "long form (ci-après « ACR »)": the acronym sits in short parentheses.87        let before = text.substring(to: start)88        if let open = before.range(of: "(", options: .backwards), before[open.upperBound...].range(of: ")") == nil {89            let after = text.substring(from: end)90            if let close = after.firstIndex(where: { $0 == ")" || $0 == "(" }), after[close] == ")" {91                let inner = String(before[open.upperBound...]) + String(after[..<close])92                if Scan.wordCount(inner) <= 3 {93                    let words = longFormWords(String(before[..<open.lowerBound]))94                    if matchesLongForm(acronym, Array(words.suffix(limit)), anchoredAtEnd: true) { return true }95                }96            }97        }98        let after = text.substring(from: end)99        let trimmed = after.drop(while: { $0 == " " || $0 == "\u{00A0}" || $0 == "\u{202F}" })100        // "ACR (long form)".101        if trimmed.first == "(", let close = trimmed.firstIndex(of: ")") {102            let words = longFormWords(String(trimmed[trimmed.index(after: trimmed.startIndex)..<close]))103            if words.count >= 2 && matchesLongForm(acronym, words, anchoredAtEnd: false) { return true }104        }105        // "ACR stands for …", "ACR signifie …", "ACR : …", "ACR — …", "ACR, or …".106        let lowered = String(trimmed.prefix(24)).lowercased()107        let markers = ["stands for ", "is short for ", "means ", "signifie ", "désigne ", "veut dire ", ": ", "— ", "– ", "= ", ", or ", ", ou ", ", soit ", "is the ", "est le ", "est la ", "est l'"]108        for marker in markers where lowered.hasPrefix(marker) {109            let rest = String(trimmed.dropFirst(marker.count))110            let words = longFormWords(String(rest.prefix(160)))111            if words.count >= 2 && matchesLongForm(acronym, Array(words.prefix(limit)), anchoredAtEnd: false) { return true }112        }113        return false114    }115116    /// Words of a candidate long form: folded to lower case without diacritics, split on anything that is not a letter or digit.117    static func longFormWords(_ s: String) -> [String] {118        let folded = s.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: nil)119        return folded.split(whereSeparator: { !($0.isLetter || $0.isNumber) }).map(String.init)120    }121122    /// Can the letters of `acronym` be read in order in `words`, starting on a word initial, with at least half of the letters on123    /// word initials? With `anchoredAtEnd` the long form must run up to the last word (the one just before the parenthesis).124    static func matchesLongForm(_ acronym: String, _ words: [String], anchoredAtEnd: Bool) -> Bool {125        let letters = Array(acronym.folding(options: [.caseInsensitive, .diacriticInsensitive], locale: nil).filter(\.isLetter))126        guard letters.count >= 2, !words.isEmpty else { return false }127        let chars = words.map(Array.init)128        // Try the closest possible starting words first.129        for s in stride(from: words.count - 1, through: 0, by: -1) where chars[s].first == letters[0] {130            var w = s, c = 1, initials = 1, matched = 1131            var lastWord = s132            while matched < letters.count {133                let target = letters[matched]134                // Prefer the initial of one of the next three words, then a letter further inside the current word.135                var found = false136                for next in (w + 1)..<min(words.count, w + 4) where chars[next].first == target {137                    w = next; c = 1; initials += 1; found = true; break138                }139                if !found, let inner = chars[w][min(c, chars[w].count)...].firstIndex(of: target) {140                    c = inner + 1; found = true141                }142                if !found { break }143                matched += 1144                lastWord = w145            }146            if matched == letters.count && initials * 2 >= letters.count && (!anchoredAtEnd || lastWord >= words.count - 2) { return true }147        }148        return false149    }150}151
Files main
Menu

Theme