SPB Git

Repository Public

HTMLTokenizer.swift 11.6 KB 221 lines swift

1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocFormats/HTMLTokenizer.swift3// Description: Character references (named and numeric) and a tolerant HTML tokenizer. Nothing here interprets scripts or styles:4//              the content of <script> and <style> is skipped as raw text.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation910enum HTMLEntities {11    /// The 252 named character references of HTML 4 plus `apos`.12    static let named: [String: UInt32] = {13        var t: [String: UInt32] = ["quot": 34, "amp": 38, "lt": 60, "gt": 62, "apos": 39]14        let latin1 = "nbsp iexcl cent pound curren yen brvbar sect uml copy ordf laquo not shy reg macr deg plusmn sup2 sup3 acute micro para middot cedil sup1 ordm raquo frac14 frac12 frac34 iquest Agrave Aacute Acirc Atilde Auml Aring AElig Ccedil Egrave Eacute Ecirc Euml Igrave Iacute Icirc Iuml ETH Ntilde Ograve Oacute Ocirc Otilde Ouml times Oslash Ugrave Uacute Ucirc Uuml Yacute THORN szlig agrave aacute acirc atilde auml aring aelig ccedil egrave eacute ecirc euml igrave iacute icirc iuml eth ntilde ograve oacute ocirc otilde ouml divide oslash ugrave uacute ucirc uuml yacute thorn yuml"15        for (i, name) in latin1.split(separator: " ").enumerated() { t[String(name)] = UInt32(160 + i) }16        let upperGreek = "Alpha Beta Gamma Delta Epsilon Zeta Eta Theta Iota Kappa Lambda Mu Nu Xi Omicron Pi Rho - Sigma Tau Upsilon Phi Chi Psi Omega"17        for (i, name) in upperGreek.split(separator: " ").enumerated() where name != "-" { t[String(name)] = UInt32(913 + i) }18        let lowerGreek = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigmaf sigma tau upsilon phi chi psi omega"19        for (i, name) in lowerGreek.split(separator: " ").enumerated() { t[String(name)] = UInt32(945 + i) }20        let others = "thetasym:977 upsih:978 piv:982 OElig:338 oelig:339 Scaron:352 scaron:353 Yuml:376 fnof:402 circ:710 tilde:732 ensp:8194 emsp:8195 thinsp:8201 zwnj:8204 zwj:8205 lrm:8206 rlm:8207 ndash:8211 mdash:8212 lsquo:8216 rsquo:8217 sbquo:8218 ldquo:8220 rdquo:8221 bdquo:8222 dagger:8224 Dagger:8225 bull:8226 hellip:8230 permil:8240 prime:8242 Prime:8243 lsaquo:8249 rsaquo:8250 oline:8254 frasl:8260 euro:8364 image:8465 weierp:8472 real:8476 trade:8482 alefsym:8501 larr:8592 uarr:8593 rarr:8594 darr:8595 harr:8596 crarr:8629 lArr:8656 uArr:8657 rArr:8658 dArr:8659 hArr:8660 forall:8704 part:8706 exist:8707 empty:8709 nabla:8711 isin:8712 notin:8713 ni:8715 prod:8719 sum:8721 minus:8722 lowast:8727 radic:8730 prop:8733 infin:8734 ang:8736 and:8743 or:8744 cap:8745 cup:8746 int:8747 there4:8756 sim:8764 cong:8773 asymp:8776 ne:8800 equiv:8801 le:8804 ge:8805 sub:8834 sup:8835 nsub:8836 sube:8838 supe:8839 oplus:8853 otimes:8855 perp:8869 sdot:8901 lceil:8968 rceil:8969 lfloor:8970 rfloor:8971 lang:9001 rang:9002 loz:9674 spades:9824 clubs:9827 hearts:9829 diams:9830"21        for pair in others.split(separator: " ") {22            let kv = pair.split(separator: ":")23            t[String(kv[0])] = UInt32(kv[1])!24        }25        return t26    }()2728    /// Code points 128...159 in numeric references mean Windows-1252 characters (HTML parsing rule).29    private static let cp1252: [UInt32: UInt32] = [128: 0x20AC, 130: 0x201A, 131: 0x0192, 132: 0x201E, 133: 0x2026, 134: 0x2020, 135: 0x2021, 136: 0x02C6,30                                                   137: 0x2030, 138: 0x0160, 139: 0x2039, 140: 0x0152, 142: 0x017D, 145: 0x2018, 146: 0x2019, 147: 0x201C,31                                                   148: 0x201D, 149: 0x2022, 150: 0x2013, 151: 0x2014, 152: 0x02DC, 153: 0x2122, 154: 0x0161, 155: 0x203A,32                                                   156: 0x0153, 158: 0x017E, 159: 0x0178]3334    /// Decodes the reference starting at `s[i]` (an ampersand). Returns the character and the number of scalars consumed.35    static func decode(_ s: [Unicode.Scalar], at i: Int) -> (scalar: Unicode.Scalar, length: Int)? {36        guard i + 2 < s.count, s[i] == "&" else { return nil }37        var j = i + 138        if s[j] == "#" {39            j += 140            var hex = false41            if j < s.count, s[j] == "x" || s[j] == "X" { hex = true; j += 1 }42            var value: UInt32 = 043            var digits = 044            while j < s.count, let d = hex ? hexValue(s[j]) : decValue(s[j]) {45                value = min(0x0FFF_FFFF, value * (hex ? 16 : 10) + d)      // saturates: anything past U+10FFFF is invalid anyway46                digits += 1; j += 147            }48            guard digits > 0 else { return nil }49            if j < s.count, s[j] == ";" { j += 1 }50            if let mapped = cp1252[value] { value = mapped }51            let scalar = value == 0 ? Unicode.Scalar(0xFFFD)! : (Unicode.Scalar(value) ?? Unicode.Scalar(0xFFFD)!)52            return (scalar, j - i)53        }54        var name = ""55        while j < s.count, name.count < 10, s[j].isASCII, s[j].properties.isAlphabetic || (("0"..."9").contains(Character(s[j])) && !name.isEmpty) {56            name.unicodeScalars.append(s[j]); j += 157        }58        guard j < s.count, s[j] == ";", let value = named[name], let scalar = Unicode.Scalar(value) else { return nil }59        return (scalar, j + 1 - i)60    }6162    static func decodeAll(_ text: String) -> String {63        guard text.contains("&") else { return text }64        let s = Array(text.unicodeScalars)65        var out = String.UnicodeScalarView()66        var i = 067        while i < s.count {68            if s[i] == "&", let (scalar, length) = decode(s, at: i) { out.append(scalar); i += length } else { out.append(s[i]); i += 1 }69        }70        return String(out)71    }7273    private static func hexValue(_ u: Unicode.Scalar) -> UInt32? {74        switch u.value {75        case 48...57: return u.value - 4876        case 65...70: return u.value - 5577        case 97...102: return u.value - 8778        default: return nil79        }80    }81    private static func decValue(_ u: Unicode.Scalar) -> UInt32? { (48...57).contains(u.value) ? u.value - 48 : nil }8283    static func escapeText(_ s: String) -> String {84        var out = ""85        out.reserveCapacity(s.utf8.count)86        for ch in s.unicodeScalars {87            switch ch {88            case "&": out += "&amp;"89            case "<": out += "&lt;"90            case ">": out += "&gt;"91            case "\u{A0}": out += "&nbsp;"92            default: out.unicodeScalars.append(ch)93            }94        }95        return out96    }97    static func escapeAttribute(_ s: String) -> String { escapeText(s).replacingOccurrences(of: "\"", with: "&quot;") }98}99100enum HTMLToken {101    case start(name: String, attributes: [String: String], selfClosing: Bool)102    case end(String)103    case text(String)104    /// Content of a raw-text element (script, style): never interpreted.105    case rawText(element: String)106    case comment107    case doctype108}109110struct HTMLTokenizer {111    private let s: [Unicode.Scalar]112    private var i = 0113    private var rawElement: String?114115    init(_ html: String) { s = Array(html.unicodeScalars) }116117    private func isSpace(_ u: Unicode.Scalar) -> Bool { u == " " || u == "\n" || u == "\t" || u == "\r" || u == "\u{0C}" }118    private func isLetter(_ u: Unicode.Scalar) -> Bool { (u.value >= 65 && u.value <= 90) || (u.value >= 97 && u.value <= 122) }119120    private func matches(_ text: String, at p: Int) -> Bool {121        var k = p122        for u in text.unicodeScalars {123            guard k < s.count, String(s[k]).lowercased() == String(u) else { return false }124            k += 1125        }126        return true127    }128129    mutating func next() -> HTMLToken? {130        guard i < s.count else { return nil }131        if let raw = rawElement {132            rawElement = nil133            // Skip to the matching end tag without looking at the content.134            var j = i135            while j < s.count, !(s[j] == "<" && j + 1 < s.count && s[j + 1] == "/" && matches(raw, at: j + 2)) { j += 1 }136            let hadContent = j > i137            i = j138            if hadContent { return .rawText(element: raw) }139            guard i < s.count else { return nil }140        }141        if s[i] == "<" {142            if i + 1 < s.count {143                let n = s[i + 1]144                if isLetter(n) { return tag(end: false) }145                if n == "/", i + 2 < s.count, isLetter(s[i + 2]) { return tag(end: true) }146                if n == "!" {147                    if matches("--", at: i + 2) {148                        var j = i + 4149                        while j < s.count, !(s[j] == "-" && j + 2 < s.count && s[j + 1] == "-" && s[j + 2] == ">") { j += 1 }150                        i = min(s.count, j + 3)151                        return .comment152                    }153                    if matches("[cdata[", at: i + 2) {154                        var j = i + 9155                        var text = String.UnicodeScalarView()156                        while j < s.count, !(s[j] == "]" && j + 2 < s.count && s[j + 1] == "]" && s[j + 2] == ">") { text.append(s[j]); j += 1 }157                        i = min(s.count, j + 3)158                        return .text(String(text))159                    }160                    skipPast(">")161                    return .doctype162                }163                if n == "?" { skipPast(">"); return .comment }164            }165            i += 1166            return .text("<")167        }168        var text = String.UnicodeScalarView()169        while i < s.count, s[i] != "<" {170            if s[i] == "&", let (scalar, length) = HTMLEntities.decode(s, at: i) { text.append(scalar); i += length } else { text.append(s[i]); i += 1 }171        }172        return .text(String(text))173    }174175    private mutating func skipPast(_ u: Unicode.Scalar) {176        while i < s.count, s[i] != u { i += 1 }177        i = min(s.count, i + 1)178    }179180    private mutating func tag(end: Bool) -> HTMLToken {181        i += end ? 2 : 1182        var name = ""183        while i < s.count, !isSpace(s[i]), s[i] != ">", s[i] != "/" { name.unicodeScalars.append(s[i]); i += 1 }184        name = name.lowercased()185        var attributes: [String: String] = [:]186        var selfClosing = false187        while i < s.count {188            while i < s.count, isSpace(s[i]) { i += 1 }189            guard i < s.count else { break }190            if s[i] == ">" { i += 1; break }191            if s[i] == "/" { selfClosing = i + 1 < s.count && s[i + 1] == ">"; i += 1; continue }192            var key = ""193            while i < s.count, !isSpace(s[i]), s[i] != "=", s[i] != ">", !(s[i] == "/" && i + 1 < s.count && s[i + 1] == ">") { key.unicodeScalars.append(s[i]); i += 1 }194            while i < s.count, isSpace(s[i]) { i += 1 }195            var value = ""196            if i < s.count, s[i] == "=" {197                i += 1198                while i < s.count, isSpace(s[i]) { i += 1 }199                if i < s.count, s[i] == "\"" || s[i] == "'" {200                    let quote = s[i]201                    i += 1202                    var v = String.UnicodeScalarView()203                    while i < s.count, s[i] != quote { v.append(s[i]); i += 1 }204                    i = min(s.count, i + 1)205                    value = String(v)206                } else {207                    var v = String.UnicodeScalarView()208                    while i < s.count, !isSpace(s[i]), s[i] != ">" { v.append(s[i]); i += 1 }209                    value = String(v)210                }211            }212            key = key.lowercased()213            if !key.isEmpty, attributes[key] == nil, attributes.count < 256 { attributes[key] = HTMLEntities.decodeAll(value) }214            if key.isEmpty, i < s.count, s[i] != ">" { i += 1 }   // stray character: never loop in place215        }216        if end { return .end(name) }217        if name == "script" || name == "style" { rawElement = name }218        return .start(name: name, attributes: attributes, selfClosing: selfClosing)219    }220}221
Files main
Menu

Theme