Repository Public
1// ORIX Docs2// File: Packages/ORIXDocsKit/Sources/DocFormats/HTMLTokenizer.swift3// Description: Character references (named and numeric) and a tolerant HTML tokenizer. Nothing here interprets scripts or styles:4// the content of <script> and <style> is skipped as raw text.5//6// Author: Simon-Pierre Boucher7// Contact: contact@spboucher.ai8import Foundation910enum HTMLEntities {11 /// The 252 named character references of HTML 4 plus `apos`.12 static let named: [String: UInt32] = {13 var t: [String: UInt32] = ["quot": 34, "amp": 38, "lt": 60, "gt": 62, "apos": 39]14 let latin1 = "nbsp iexcl cent pound curren yen brvbar sect uml copy ordf laquo not shy reg macr deg plusmn sup2 sup3 acute micro para middot cedil sup1 ordm raquo frac14 frac12 frac34 iquest Agrave Aacute Acirc Atilde Auml Aring AElig Ccedil Egrave Eacute Ecirc Euml Igrave Iacute Icirc Iuml ETH Ntilde Ograve Oacute Ocirc Otilde Ouml times Oslash Ugrave Uacute Ucirc Uuml Yacute THORN szlig agrave aacute acirc atilde auml aring aelig ccedil egrave eacute ecirc euml igrave iacute icirc iuml eth ntilde ograve oacute ocirc otilde ouml divide oslash ugrave uacute ucirc uuml yacute thorn yuml"15 for (i, name) in latin1.split(separator: " ").enumerated() { t[String(name)] = UInt32(160 + i) }16 let upperGreek = "Alpha Beta Gamma Delta Epsilon Zeta Eta Theta Iota Kappa Lambda Mu Nu Xi Omicron Pi Rho - Sigma Tau Upsilon Phi Chi Psi Omega"17 for (i, name) in upperGreek.split(separator: " ").enumerated() where name != "-" { t[String(name)] = UInt32(913 + i) }18 let lowerGreek = "alpha beta gamma delta epsilon zeta eta theta iota kappa lambda mu nu xi omicron pi rho sigmaf sigma tau upsilon phi chi psi omega"19 for (i, name) in lowerGreek.split(separator: " ").enumerated() { t[String(name)] = UInt32(945 + i) }20 let others = "thetasym:977 upsih:978 piv:982 OElig:338 oelig:339 Scaron:352 scaron:353 Yuml:376 fnof:402 circ:710 tilde:732 ensp:8194 emsp:8195 thinsp:8201 zwnj:8204 zwj:8205 lrm:8206 rlm:8207 ndash:8211 mdash:8212 lsquo:8216 rsquo:8217 sbquo:8218 ldquo:8220 rdquo:8221 bdquo:8222 dagger:8224 Dagger:8225 bull:8226 hellip:8230 permil:8240 prime:8242 Prime:8243 lsaquo:8249 rsaquo:8250 oline:8254 frasl:8260 euro:8364 image:8465 weierp:8472 real:8476 trade:8482 alefsym:8501 larr:8592 uarr:8593 rarr:8594 darr:8595 harr:8596 crarr:8629 lArr:8656 uArr:8657 rArr:8658 dArr:8659 hArr:8660 forall:8704 part:8706 exist:8707 empty:8709 nabla:8711 isin:8712 notin:8713 ni:8715 prod:8719 sum:8721 minus:8722 lowast:8727 radic:8730 prop:8733 infin:8734 ang:8736 and:8743 or:8744 cap:8745 cup:8746 int:8747 there4:8756 sim:8764 cong:8773 asymp:8776 ne:8800 equiv:8801 le:8804 ge:8805 sub:8834 sup:8835 nsub:8836 sube:8838 supe:8839 oplus:8853 otimes:8855 perp:8869 sdot:8901 lceil:8968 rceil:8969 lfloor:8970 rfloor:8971 lang:9001 rang:9002 loz:9674 spades:9824 clubs:9827 hearts:9829 diams:9830"21 for pair in others.split(separator: " ") {22 let kv = pair.split(separator: ":")23 t[String(kv[0])] = UInt32(kv[1])!24 }25 return t26 }()2728 /// Code points 128...159 in numeric references mean Windows-1252 characters (HTML parsing rule).29 private static let cp1252: [UInt32: UInt32] = [128: 0x20AC, 130: 0x201A, 131: 0x0192, 132: 0x201E, 133: 0x2026, 134: 0x2020, 135: 0x2021, 136: 0x02C6,30 137: 0x2030, 138: 0x0160, 139: 0x2039, 140: 0x0152, 142: 0x017D, 145: 0x2018, 146: 0x2019, 147: 0x201C,31 148: 0x201D, 149: 0x2022, 150: 0x2013, 151: 0x2014, 152: 0x02DC, 153: 0x2122, 154: 0x0161, 155: 0x203A,32 156: 0x0153, 158: 0x017E, 159: 0x0178]3334 /// Decodes the reference starting at `s[i]` (an ampersand). Returns the character and the number of scalars consumed.35 static func decode(_ s: [Unicode.Scalar], at i: Int) -> (scalar: Unicode.Scalar, length: Int)? {36 guard i + 2 < s.count, s[i] == "&" else { return nil }37 var j = i + 138 if s[j] == "#" {39 j += 140 var hex = false41 if j < s.count, s[j] == "x" || s[j] == "X" { hex = true; j += 1 }42 var value: UInt32 = 043 var digits = 044 while j < s.count, let d = hex ? hexValue(s[j]) : decValue(s[j]) {45 value = min(0x0FFF_FFFF, value * (hex ? 16 : 10) + d) // saturates: anything past U+10FFFF is invalid anyway46 digits += 1; j += 147 }48 guard digits > 0 else { return nil }49 if j < s.count, s[j] == ";" { j += 1 }50 if let mapped = cp1252[value] { value = mapped }51 let scalar = value == 0 ? Unicode.Scalar(0xFFFD)! : (Unicode.Scalar(value) ?? Unicode.Scalar(0xFFFD)!)52 return (scalar, j - i)53 }54 var name = ""55 while j < s.count, name.count < 10, s[j].isASCII, s[j].properties.isAlphabetic || (("0"..."9").contains(Character(s[j])) && !name.isEmpty) {56 name.unicodeScalars.append(s[j]); j += 157 }58 guard j < s.count, s[j] == ";", let value = named[name], let scalar = Unicode.Scalar(value) else { return nil }59 return (scalar, j + 1 - i)60 }6162 static func decodeAll(_ text: String) -> String {63 guard text.contains("&") else { return text }64 let s = Array(text.unicodeScalars)65 var out = String.UnicodeScalarView()66 var i = 067 while i < s.count {68 if s[i] == "&", let (scalar, length) = decode(s, at: i) { out.append(scalar); i += length } else { out.append(s[i]); i += 1 }69 }70 return String(out)71 }7273 private static func hexValue(_ u: Unicode.Scalar) -> UInt32? {74 switch u.value {75 case 48...57: return u.value - 4876 case 65...70: return u.value - 5577 case 97...102: return u.value - 8778 default: return nil79 }80 }81 private static func decValue(_ u: Unicode.Scalar) -> UInt32? { (48...57).contains(u.value) ? u.value - 48 : nil }8283 static func escapeText(_ s: String) -> String {84 var out = ""85 out.reserveCapacity(s.utf8.count)86 for ch in s.unicodeScalars {87 switch ch {88 case "&": out += "&"89 case "<": out += "<"90 case ">": out += ">"91 case "\u{A0}": out += " "92 default: out.unicodeScalars.append(ch)93 }94 }95 return out96 }97 static func escapeAttribute(_ s: String) -> String { escapeText(s).replacingOccurrences(of: "\"", with: """) }98}99100enum HTMLToken {101 case start(name: String, attributes: [String: String], selfClosing: Bool)102 case end(String)103 case text(String)104 /// Content of a raw-text element (script, style): never interpreted.105 case rawText(element: String)106 case comment107 case doctype108}109110struct HTMLTokenizer {111 private let s: [Unicode.Scalar]112 private var i = 0113 private var rawElement: String?114115 init(_ html: String) { s = Array(html.unicodeScalars) }116117 private func isSpace(_ u: Unicode.Scalar) -> Bool { u == " " || u == "\n" || u == "\t" || u == "\r" || u == "\u{0C}" }118 private func isLetter(_ u: Unicode.Scalar) -> Bool { (u.value >= 65 && u.value <= 90) || (u.value >= 97 && u.value <= 122) }119120 private func matches(_ text: String, at p: Int) -> Bool {121 var k = p122 for u in text.unicodeScalars {123 guard k < s.count, String(s[k]).lowercased() == String(u) else { return false }124 k += 1125 }126 return true127 }128129 mutating func next() -> HTMLToken? {130 guard i < s.count else { return nil }131 if let raw = rawElement {132 rawElement = nil133 // Skip to the matching end tag without looking at the content.134 var j = i135 while j < s.count, !(s[j] == "<" && j + 1 < s.count && s[j + 1] == "/" && matches(raw, at: j + 2)) { j += 1 }136 let hadContent = j > i137 i = j138 if hadContent { return .rawText(element: raw) }139 guard i < s.count else { return nil }140 }141 if s[i] == "<" {142 if i + 1 < s.count {143 let n = s[i + 1]144 if isLetter(n) { return tag(end: false) }145 if n == "/", i + 2 < s.count, isLetter(s[i + 2]) { return tag(end: true) }146 if n == "!" {147 if matches("--", at: i + 2) {148 var j = i + 4149 while j < s.count, !(s[j] == "-" && j + 2 < s.count && s[j + 1] == "-" && s[j + 2] == ">") { j += 1 }150 i = min(s.count, j + 3)151 return .comment152 }153 if matches("[cdata[", at: i + 2) {154 var j = i + 9155 var text = String.UnicodeScalarView()156 while j < s.count, !(s[j] == "]" && j + 2 < s.count && s[j + 1] == "]" && s[j + 2] == ">") { text.append(s[j]); j += 1 }157 i = min(s.count, j + 3)158 return .text(String(text))159 }160 skipPast(">")161 return .doctype162 }163 if n == "?" { skipPast(">"); return .comment }164 }165 i += 1166 return .text("<")167 }168 var text = String.UnicodeScalarView()169 while i < s.count, s[i] != "<" {170 if s[i] == "&", let (scalar, length) = HTMLEntities.decode(s, at: i) { text.append(scalar); i += length } else { text.append(s[i]); i += 1 }171 }172 return .text(String(text))173 }174175 private mutating func skipPast(_ u: Unicode.Scalar) {176 while i < s.count, s[i] != u { i += 1 }177 i = min(s.count, i + 1)178 }179180 private mutating func tag(end: Bool) -> HTMLToken {181 i += end ? 2 : 1182 var name = ""183 while i < s.count, !isSpace(s[i]), s[i] != ">", s[i] != "/" { name.unicodeScalars.append(s[i]); i += 1 }184 name = name.lowercased()185 var attributes: [String: String] = [:]186 var selfClosing = false187 while i < s.count {188 while i < s.count, isSpace(s[i]) { i += 1 }189 guard i < s.count else { break }190 if s[i] == ">" { i += 1; break }191 if s[i] == "/" { selfClosing = i + 1 < s.count && s[i + 1] == ">"; i += 1; continue }192 var key = ""193 while i < s.count, !isSpace(s[i]), s[i] != "=", s[i] != ">", !(s[i] == "/" && i + 1 < s.count && s[i + 1] == ">") { key.unicodeScalars.append(s[i]); i += 1 }194 while i < s.count, isSpace(s[i]) { i += 1 }195 var value = ""196 if i < s.count, s[i] == "=" {197 i += 1198 while i < s.count, isSpace(s[i]) { i += 1 }199 if i < s.count, s[i] == "\"" || s[i] == "'" {200 let quote = s[i]201 i += 1202 var v = String.UnicodeScalarView()203 while i < s.count, s[i] != quote { v.append(s[i]); i += 1 }204 i = min(s.count, i + 1)205 value = String(v)206 } else {207 var v = String.UnicodeScalarView()208 while i < s.count, !isSpace(s[i]), s[i] != ">" { v.append(s[i]); i += 1 }209 value = String(v)210 }211 }212 key = key.lowercased()213 if !key.isEmpty, attributes[key] == nil, attributes.count < 256 { attributes[key] = HTMLEntities.decodeAll(value) }214 if key.isEmpty, i < s.count, s[i] != ">" { i += 1 } // stray character: never loop in place215 }216 if end { return .end(name) }217 if name == "script" || name == "style" { rawElement = name }218 return .start(name: name, attributes: attributes, selfClosing: selfClosing)219 }220}221