phase3.A: content extraction — Mozilla Readability (Apache-2.0) + AtlasExtractor driver in isolated WKContentWorld, PageContext, heading-aware Chunker; visible-text-only (injection-safe)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
9 changed files +3,417 −1
modified
Package.swift
+7 −1
@@ -23,7 +23,13 @@ let package = Package( | ||
| 23 | 23 | dependencies: [ |
| 24 | 24 | .product(name: "Markdown", package: "swift-markdown") |
| 25 | 25 | ], |
| 26 | − path: "Sources/ZyquoAtlas" | |
| 26 | + path: "Sources/ZyquoAtlas", | |
| 27 | + resources: [ | |
| 28 | + .copy("Content/Readability.js"), | |
| 29 | + .copy("Content/Readability-readerable.js"), | |
| 30 | + .copy("Content/AtlasExtractor.js"), | |
| 31 | + .copy("Content/LICENSE-Readability.txt"), | |
| 32 | + ] | |
| 27 | 33 | ), |
| 28 | 34 | .executableTarget( |
| 29 | 35 | name: "zyquo-verify", |
modified
Sources/ZyquoAtlas/Browser/ProfileStore.swift
+3 −0
@@ -48,6 +48,9 @@ final class ProfileStore { | ||
| 48 | 48 | config.websiteDataStore = dataStore(for: profile) |
| 49 | 49 | config.defaultWebpagePreferences.allowsContentJavaScript = true |
| 50 | 50 | config.preferences.isElementFullscreenEnabled = true |
| 51 | + // Inject the content-extraction scripts into their isolated world so | |
| 52 | + // every tab can produce a PageContext for AI actions. | |
| 53 | + ContentExtractor.install(into: config) | |
| 51 | 54 | return config |
| 52 | 55 | } |
| 53 | 56 | } |
added
Sources/ZyquoAtlas/Content/AtlasExtractor.js
+200 −0
@@ -0,0 +1,200 @@ | ||
| 1 | +// | |
| 2 | +// AtlasExtractor.js | |
| 3 | +// Zyquo Atlas | |
| 4 | +// | |
| 5 | +// Author: Simon-Pierre Boucher | |
| 6 | +// Mail: contact@spboucher.ai | |
| 7 | +// | |
| 8 | +// Content-extraction driver injected into an isolated WKContentWorld alongside | |
| 9 | +// Mozilla Readability.js + Readability-readerable.js. Turns the live page into | |
| 10 | +// clean, model-ready markdown with metadata and the user's selection, using | |
| 11 | +// the fallback ladder from docs/AI-BROWSER-RESEARCH.md §2: | |
| 12 | +// 1. Readability (article) → markdown | |
| 13 | +// 2. visible structured-text walk (app pages) | |
| 14 | +// 3. title + url + meta description | |
| 15 | +// SECURITY: only visible, rendered content is emitted — hidden/off-screen and | |
| 16 | +// aria-hidden nodes are dropped so injected instructions in invisible text | |
| 17 | +// never reach the model. Every entry point returns a value (never undefined) | |
| 18 | +// so the Swift bridge never crashes. | |
| 19 | +// | |
| 20 | + | |
| 21 | +(function () { | |
| 22 | + "use strict"; | |
| 23 | + if (window.__zyquoAtlas) return; | |
| 24 | + | |
| 25 | + const MAX_BYTES_DEFAULT = 600000; | |
| 26 | + | |
| 27 | + function meta(name) { | |
| 28 | + const el = document.querySelector( | |
| 29 | + 'meta[property="' + name + '"],meta[name="' + name + '"]' | |
| 30 | + ); | |
| 31 | + return el ? el.getAttribute("content") : null; | |
| 32 | + } | |
| 33 | + | |
| 34 | + function canonicalURL() { | |
| 35 | + const l = document.querySelector('link[rel="canonical"]'); | |
| 36 | + return (l && l.href) || location.href; | |
| 37 | + } | |
| 38 | + | |
| 39 | + function faviconURL() { | |
| 40 | + const l = document.querySelector('link[rel~="icon"],link[rel="apple-touch-icon"]'); | |
| 41 | + try { | |
| 42 | + return l ? new URL(l.getAttribute("href"), location.href).href | |
| 43 | + : new URL("/favicon.ico", location.origin).href; | |
| 44 | + } catch (_) { return null; } | |
| 45 | + } | |
| 46 | + | |
| 47 | + // Is a node visible to the user (drops the injection surface)? | |
| 48 | + function isVisible(el) { | |
| 49 | + if (!el || el.nodeType !== 1) return false; | |
| 50 | + if (el.getAttribute && el.getAttribute("aria-hidden") === "true") return false; | |
| 51 | + const s = window.getComputedStyle(el); | |
| 52 | + if (!s || s.display === "none" || s.visibility === "hidden" || parseFloat(s.opacity) === 0) { | |
| 53 | + return false; | |
| 54 | + } | |
| 55 | + const r = el.getBoundingClientRect(); | |
| 56 | + if (r.width === 0 && r.height === 0) return false; | |
| 57 | + return true; | |
| 58 | + } | |
| 59 | + | |
| 60 | + // Walk visible DOM emitting lightweight markdown (headings, lists, links, | |
| 61 | + // code, paragraphs). Used both to render a Readability article DOM and as the | |
| 62 | + // app-page fallback. | |
| 63 | + function toMarkdown(root) { | |
| 64 | + const out = []; | |
| 65 | + const SKIP = new Set(["SCRIPT", "STYLE", "NOSCRIPT", "TEMPLATE", "SVG", "CANVAS", "IFRAME"]); | |
| 66 | + function walk(node, listDepth) { | |
| 67 | + for (const child of node.childNodes) { | |
| 68 | + if (child.nodeType === 3) { | |
| 69 | + const t = child.textContent.replace(/\s+/g, " "); | |
| 70 | + if (t.trim()) out.push(t); | |
| 71 | + continue; | |
| 72 | + } | |
| 73 | + if (child.nodeType !== 1) continue; | |
| 74 | + const tag = child.tagName; | |
| 75 | + if (SKIP.has(tag)) continue; | |
| 76 | + if (!isVisible(child)) continue; | |
| 77 | + if (/^H[1-6]$/.test(tag)) { | |
| 78 | + const level = "#".repeat(parseInt(tag[1], 10)); | |
| 79 | + out.push("\n" + level + " " + child.textContent.trim() + "\n"); | |
| 80 | + } else if (tag === "P") { | |
| 81 | + out.push("\n" + child.textContent.trim() + "\n"); | |
| 82 | + } else if (tag === "LI") { | |
| 83 | + out.push("\n" + " ".repeat(listDepth) + "- " + child.textContent.trim()); | |
| 84 | + } else if (tag === "PRE") { | |
| 85 | + out.push("\n```\n" + child.textContent.replace(/\n+$/, "") + "\n```\n"); | |
| 86 | + } else if (tag === "A" && child.getAttribute("href")) { | |
| 87 | + const txt = child.textContent.trim(); | |
| 88 | + if (txt) out.push("[" + txt + "](" + child.href + ")"); | |
| 89 | + } else if (tag === "BR") { | |
| 90 | + out.push("\n"); | |
| 91 | + } else if (tag === "UL" || tag === "OL") { | |
| 92 | + walk(child, listDepth + 1); | |
| 93 | + out.push("\n"); | |
| 94 | + } else { | |
| 95 | + walk(child, listDepth); | |
| 96 | + } | |
| 97 | + } | |
| 98 | + } | |
| 99 | + walk(root, 0); | |
| 100 | + return out.join(" ").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").trim(); | |
| 101 | + } | |
| 102 | + | |
| 103 | + // Index of heading offsets in the produced markdown for chunking + citations. | |
| 104 | + function headingIndex(markdown) { | |
| 105 | + const headings = []; | |
| 106 | + const re = /^(#{1,6})\s+(.*)$/gm; | |
| 107 | + let m; | |
| 108 | + while ((m = re.exec(markdown)) !== null) { | |
| 109 | + headings.push({ level: m[1].length, title: m[2].trim(), offset: m.index }); | |
| 110 | + } | |
| 111 | + return headings; | |
| 112 | + } | |
| 113 | + | |
| 114 | + function currentSelection() { | |
| 115 | + const sel = window.getSelection(); | |
| 116 | + const text = sel ? String(sel).trim() : ""; | |
| 117 | + if (!text) return null; | |
| 118 | + let context = null; | |
| 119 | + try { | |
| 120 | + const node = sel.anchorNode && sel.anchorNode.parentElement; | |
| 121 | + const block = node && node.closest("p,li,section,article,div"); | |
| 122 | + if (block) context = block.textContent.replace(/\s+/g, " ").trim().slice(0, 800); | |
| 123 | + } catch (_) {} | |
| 124 | + return { text: text, context: context }; | |
| 125 | + } | |
| 126 | + | |
| 127 | + function truncateAtBoundary(s, maxBytes) { | |
| 128 | + if (s.length <= maxBytes) return s; | |
| 129 | + const cut = s.lastIndexOf("\n", maxBytes); | |
| 130 | + return s.slice(0, cut > maxBytes * 0.5 ? cut : maxBytes) + "\n\n…[truncated]"; | |
| 131 | + } | |
| 132 | + | |
| 133 | + function extract(mode, maxBytes) { | |
| 134 | + maxBytes = maxBytes || MAX_BYTES_DEFAULT; | |
| 135 | + try { | |
| 136 | + let markdown = ""; | |
| 137 | + let quality = "reader"; | |
| 138 | + let title = document.title; | |
| 139 | + let byline = null; | |
| 140 | + let published = null; | |
| 141 | + let readerable = false; | |
| 142 | + | |
| 143 | + try { | |
| 144 | + readerable = typeof isProbablyReaderable === "function" && | |
| 145 | + isProbablyReaderable(document); | |
| 146 | + } catch (_) {} | |
| 147 | + | |
| 148 | + if (mode !== "rawText" && readerable && typeof Readability === "function") { | |
| 149 | + try { | |
| 150 | + const clone = document.cloneNode(true); // never mutate the live page | |
| 151 | + const article = new Readability(clone).parse(); | |
| 152 | + if (article) { | |
| 153 | + title = article.title || title; | |
| 154 | + byline = article.byline || null; | |
| 155 | + published = article.publishedTime || null; | |
| 156 | + const holder = document.createElement("div"); | |
| 157 | + holder.innerHTML = article.content || ""; | |
| 158 | + markdown = toMarkdown(holder); | |
| 159 | + } | |
| 160 | + } catch (_) {} | |
| 161 | + } | |
| 162 | + | |
| 163 | + if (!markdown || markdown.length < 400) { // fallback ladder | |
| 164 | + markdown = toMarkdown(document.body || document.documentElement); | |
| 165 | + quality = readerable ? "reader" : "rawText"; | |
| 166 | + } | |
| 167 | + if (!markdown) { // last resort | |
| 168 | + markdown = (document.body ? document.body.innerText : "").trim(); | |
| 169 | + quality = "rawText"; | |
| 170 | + } | |
| 171 | + | |
| 172 | + markdown = truncateAtBoundary(markdown, maxBytes); | |
| 173 | + | |
| 174 | + return { | |
| 175 | + ok: true, | |
| 176 | + context: { | |
| 177 | + url: location.href, | |
| 178 | + canonical: canonicalURL(), | |
| 179 | + title: (title || location.host || "").trim(), | |
| 180 | + byline: byline, | |
| 181 | + description: meta("description") || meta("og:description"), | |
| 182 | + siteName: meta("og:site_name"), | |
| 183 | + lang: document.documentElement.lang || null, | |
| 184 | + published: published, | |
| 185 | + favicon: faviconURL(), | |
| 186 | + markdown: markdown, | |
| 187 | + quality: quality, | |
| 188 | + headings: headingIndex(markdown), | |
| 189 | + selection: currentSelection(), | |
| 190 | + wordCount: markdown ? markdown.split(/\s+/).length : 0, | |
| 191 | + truncated: markdown.indexOf("…[truncated]") !== -1 | |
| 192 | + } | |
| 193 | + }; | |
| 194 | + } catch (e) { | |
| 195 | + return { ok: false, error: String((e && e.stack) || e) }; | |
| 196 | + } | |
| 197 | + } | |
| 198 | + | |
| 199 | + window.__zyquoAtlas = { extract: extract }; | |
| 200 | +})(); | |
added
Sources/ZyquoAtlas/Content/Chunker.swift
+73 −0
@@ -0,0 +1,73 @@ | ||
| 1 | +// | |
| 2 | +// Chunker.swift | |
| 3 | +// Zyquo Atlas | |
| 4 | +// | |
| 5 | +// Author: Simon-Pierre Boucher | |
| 6 | +// Mail: contact@spboucher.ai | |
| 7 | +// | |
| 8 | +// Splits a PageContext's markdown into heading-aware chunks for long-page | |
| 9 | +// summarization (map-reduce) and chat-with-page retrieval (docs/ | |
| 10 | +// AI-BROWSER-RESEARCH.md §3). Each chunk carries its heading path and | |
| 11 | +// character range so answers can cite and scroll-to-highlight the source. | |
| 12 | +// | |
| 13 | + | |
| 14 | +import Foundation | |
| 15 | + | |
| 16 | +struct Chunk: Identifiable, Hashable { | |
| 17 | + let id: Int | |
| 18 | + /// e.g. "Setup > macOS". | |
| 19 | + let headingPath: String | |
| 20 | + let text: String | |
| 21 | + let charRange: Range<Int> | |
| 22 | + var estimatedTokens: Int { text.count / 4 } | |
| 23 | +} | |
| 24 | + | |
| 25 | +enum Chunker { | |
| 26 | + /// ~600-token Q&A chunks by default; pass a larger target for summarization | |
| 27 | + /// (map-reduce tolerates coarse chunks). | |
| 28 | + static func chunk(_ context: PageContext, targetTokens: Int = 600, overlapTokens: Int = 50) -> [Chunk] { | |
| 29 | + let md = context.markdown | |
| 30 | + guard !md.isEmpty else { return [] } | |
| 31 | + | |
| 32 | + let targetChars = targetTokens * 4 | |
| 33 | + let overlapChars = overlapTokens * 4 | |
| 34 | + let scalars = Array(md) | |
| 35 | + | |
| 36 | + // Section boundaries from the heading index (fall back to the whole doc). | |
| 37 | + var boundaries = context.headings.map { $0.offset }.sorted() | |
| 38 | + if boundaries.first != 0 { boundaries.insert(0, at: 0) } | |
| 39 | + boundaries.append(scalars.count) | |
| 40 | + | |
| 41 | + var chunks: [Chunk] = [] | |
| 42 | + var id = 0 | |
| 43 | + var headingStack: [(level: Int, title: String)] = [] | |
| 44 | + | |
| 45 | + for i in 0..<(boundaries.count - 1) { | |
| 46 | + let start = boundaries[i] | |
| 47 | + let end = boundaries[i + 1] | |
| 48 | + guard start < end else { continue } | |
| 49 | + | |
| 50 | + // Track heading path from the heading that opens this section. | |
| 51 | + if let h = context.headings.first(where: { $0.offset == start }) { | |
| 52 | + while let last = headingStack.last, last.level >= h.level { headingStack.removeLast() } | |
| 53 | + headingStack.append((h.level, h.title)) | |
| 54 | + } | |
| 55 | + let path = headingStack.map(\.title).joined(separator: " > ") | |
| 56 | + | |
| 57 | + // Sub-split large sections with overlap. | |
| 58 | + var cursor = start | |
| 59 | + while cursor < end { | |
| 60 | + let sliceEnd = min(cursor + targetChars, end) | |
| 61 | + let text = String(scalars[cursor..<sliceEnd]).trimmingCharacters(in: .whitespacesAndNewlines) | |
| 62 | + if !text.isEmpty { | |
| 63 | + chunks.append(Chunk(id: id, headingPath: path, text: text, | |
| 64 | + charRange: cursor..<sliceEnd)) | |
| 65 | + id += 1 | |
| 66 | + } | |
| 67 | + if sliceEnd >= end { break } | |
| 68 | + cursor = max(cursor + targetChars - overlapChars, cursor + 1) | |
| 69 | + } | |
| 70 | + } | |
| 71 | + return chunks | |
| 72 | + } | |
| 73 | +} | |
added
Sources/ZyquoAtlas/Content/ContentExtractor.swift
+97 −0
@@ -0,0 +1,97 @@ | ||
| 1 | +// | |
| 2 | +// ContentExtractor.swift | |
| 3 | +// Zyquo Atlas | |
| 4 | +// | |
| 5 | +// Author: Simon-Pierre Boucher | |
| 6 | +// Mail: contact@spboucher.ai | |
| 7 | +// | |
| 8 | +// Injects the extraction scripts (Mozilla Readability + AtlasExtractor driver) | |
| 9 | +// into an isolated WKContentWorld and pulls a normalized PageContext out of a | |
| 10 | +// live tab via callAsyncJavaScript. Isolation keeps the extractor from | |
| 11 | +// colliding with or being observed by page JS (docs/AI-BROWSER-RESEARCH.md §2). | |
| 12 | +// All page content is treated as untrusted downstream (Phase 3.C prompt | |
| 13 | +// hygiene). | |
| 14 | +// | |
| 15 | + | |
| 16 | +import Foundation | |
| 17 | +import WebKit | |
| 18 | + | |
| 19 | +enum ContentExtractionError: LocalizedError { | |
| 20 | + case scriptsUnavailable | |
| 21 | + case scriptFailure(String) | |
| 22 | + case decodeFailure(String) | |
| 23 | + | |
| 24 | + var errorDescription: String? { | |
| 25 | + switch self { | |
| 26 | + case .scriptsUnavailable: | |
| 27 | + return "Content extraction scripts are missing from the app bundle." | |
| 28 | + case .scriptFailure(let detail): | |
| 29 | + return "Couldn't read this page's content (\(detail))." | |
| 30 | + case .decodeFailure(let detail): | |
| 31 | + return "Couldn't parse the extracted page content (\(detail))." | |
| 32 | + } | |
| 33 | + } | |
| 34 | +} | |
| 35 | + | |
| 36 | +@MainActor | |
| 37 | +final class ContentExtractor { | |
| 38 | + /// Dedicated world so extraction never touches the page's JS globals. | |
| 39 | + static let world = WKContentWorld.world(name: "ZyquoAtlasContent") | |
| 40 | + | |
| 41 | + /// Installs the extraction user scripts into a configuration. Call once per | |
| 42 | + /// WKWebViewConfiguration (ProfileStore) before any tab loads. | |
| 43 | + static func install(into config: WKWebViewConfiguration) { | |
| 44 | + let ucc = config.userContentController | |
| 45 | + for name in ["Readability", "Readability-readerable", "AtlasExtractor"] { | |
| 46 | + guard let source = script(name) else { continue } | |
| 47 | + ucc.addUserScript(WKUserScript( | |
| 48 | + source: source, | |
| 49 | + injectionTime: .atDocumentEnd, | |
| 50 | + forMainFrameOnly: true, | |
| 51 | + in: world | |
| 52 | + )) | |
| 53 | + } | |
| 54 | + } | |
| 55 | + | |
| 56 | + /// Extracts a PageContext from a tab's web view. `mode` "auto" tries | |
| 57 | + /// Readability first; "rawText" forces the visible-text fallback. | |
| 58 | + static func extract(from webView: WKWebView, mode: String = "auto") async throws -> PageContext { | |
| 59 | + let result: Any? | |
| 60 | + do { | |
| 61 | + result = try await webView.callAsyncJavaScript( | |
| 62 | + "return window.__zyquoAtlas ? window.__zyquoAtlas.extract(mode, maxBytes) : { ok:false, error:'driver-missing' };", | |
| 63 | + arguments: ["mode": mode, "maxBytes": 600_000], | |
| 64 | + in: nil, | |
| 65 | + contentWorld: world | |
| 66 | + ) | |
| 67 | + } catch { | |
| 68 | + throw ContentExtractionError.scriptFailure(error.localizedDescription) | |
| 69 | + } | |
| 70 | + | |
| 71 | + guard let dict = result as? [String: Any] else { | |
| 72 | + throw ContentExtractionError.scriptFailure("no result") | |
| 73 | + } | |
| 74 | + if dict["ok"] as? Bool != true { | |
| 75 | + throw ContentExtractionError.scriptFailure((dict["error"] as? String) ?? "unknown") | |
| 76 | + } | |
| 77 | + guard let context = dict["context"] else { | |
| 78 | + throw ContentExtractionError.scriptFailure("empty context") | |
| 79 | + } | |
| 80 | + do { | |
| 81 | + let data = try JSONSerialization.data(withJSONObject: context) | |
| 82 | + return try JSONDecoder().decode(PageContext.self, from: data) | |
| 83 | + } catch { | |
| 84 | + throw ContentExtractionError.decodeFailure(error.localizedDescription) | |
| 85 | + } | |
| 86 | + } | |
| 87 | + | |
| 88 | + // MARK: - Script loading | |
| 89 | + | |
| 90 | + private static func script(_ name: String) -> String? { | |
| 91 | + guard let url = Bundle.module.url(forResource: name, withExtension: "js"), | |
| 92 | + let source = try? String(contentsOf: url, encoding: .utf8) else { | |
| 93 | + return nil | |
| 94 | + } | |
| 95 | + return source | |
| 96 | + } | |
| 97 | +} | |
added
Sources/ZyquoAtlas/Content/LICENSE-Readability.txt
+16 −0
@@ -0,0 +1,16 @@ | ||
| 1 | +Readability.js and Readability-readerable.js in this directory are from the | |
| 2 | +Mozilla Readability project (https://github.com/mozilla/readability), used | |
| 3 | +unmodified. | |
| 4 | + | |
| 5 | + Copyright (c) 2010 Arc90 Inc | |
| 6 | + Licensed under the Apache License, Version 2.0 (the "License"); | |
| 7 | + you may not use this file except in compliance with the License. | |
| 8 | + You may obtain a copy of the License at | |
| 9 | + | |
| 10 | + http://www.apache.org/licenses/LICENSE-2.0 | |
| 11 | + | |
| 12 | + Unless required by applicable law or agreed to in writing, software | |
| 13 | + distributed under the License is distributed on an "AS IS" BASIS, | |
| 14 | + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 15 | + See the License for the specific language governing permissions and | |
| 16 | + limitations under the License. | |
added
Sources/ZyquoAtlas/Content/PageContext.swift
+87 −0
@@ -0,0 +1,87 @@ | ||
| 1 | +// | |
| 2 | +// PageContext.swift | |
| 3 | +// Zyquo Atlas | |
| 4 | +// | |
| 5 | +// Author: Simon-Pierre Boucher | |
| 6 | +// Mail: contact@spboucher.ai | |
| 7 | +// | |
| 8 | +// Normalized, model-ready representation of a web page: clean markdown, key | |
| 9 | +// metadata, the user's current selection, and a heading index for chunking and | |
| 10 | +// citations. Produced by ContentExtractor from the injected AtlasExtractor.js; | |
| 11 | +// chunked by Chunker for long pages / multi-tab reasoning. This is the ONLY | |
| 12 | +// page representation the AI layer sees — WKWebView/DOM guts never leak past it. | |
| 13 | +// | |
| 14 | + | |
| 15 | +import Foundation | |
| 16 | + | |
| 17 | +/// Extraction fidelity, so the UI can be honest when a page couldn't be read | |
| 18 | +/// cleanly (docs/AI-BROWSER-RESEARCH.md §2: extraction failure is AI failure). | |
| 19 | +enum ExtractionQuality: String, Codable { | |
| 20 | + case reader // Readability main-content | |
| 21 | + case rawText // visible structured-text fallback (app pages) | |
| 22 | +} | |
| 23 | + | |
| 24 | +struct PageHeading: Codable, Hashable { | |
| 25 | + let level: Int | |
| 26 | + let title: String | |
| 27 | + /// Character offset into `markdown`. | |
| 28 | + let offset: Int | |
| 29 | +} | |
| 30 | + | |
| 31 | +struct PageSelection: Codable, Hashable { | |
| 32 | + let text: String | |
| 33 | + /// Surrounding block text for grounding "explain this". | |
| 34 | + let context: String? | |
| 35 | +} | |
| 36 | + | |
| 37 | +struct PageContext: Codable, Hashable, Identifiable { | |
| 38 | + var id: String { canonical.isEmpty ? url : canonical } | |
| 39 | + | |
| 40 | + let url: String | |
| 41 | + let canonical: String | |
| 42 | + let title: String | |
| 43 | + let byline: String? | |
| 44 | + let description: String? | |
| 45 | + let siteName: String? | |
| 46 | + let lang: String? | |
| 47 | + let published: String? | |
| 48 | + let favicon: String? | |
| 49 | + /// Canonical model-facing representation. | |
| 50 | + let markdown: String | |
| 51 | + let quality: ExtractionQuality | |
| 52 | + let headings: [PageHeading] | |
| 53 | + let selection: PageSelection? | |
| 54 | + let wordCount: Int | |
| 55 | + let truncated: Bool | |
| 56 | + | |
| 57 | + /// Rough token estimate (chars/4) for the stuff-vs-map-reduce budget rule. | |
| 58 | + var estimatedTokens: Int { markdown.count / 4 } | |
| 59 | + | |
| 60 | + private enum CodingKeys: String, CodingKey { | |
| 61 | + case url, canonical, title, byline, description, siteName, lang | |
| 62 | + case published, favicon, markdown, quality, headings, selection | |
| 63 | + case wordCount, truncated | |
| 64 | + } | |
| 65 | + | |
| 66 | + // Tolerate a missing/unknown `quality` from JS by defaulting to rawText. | |
| 67 | + init(from decoder: Decoder) throws { | |
| 68 | + let c = try decoder.container(keyedBy: CodingKeys.self) | |
| 69 | + url = try c.decode(String.self, forKey: .url) | |
| 70 | + canonical = (try? c.decode(String.self, forKey: .canonical)) ?? url | |
| 71 | + title = (try? c.decode(String.self, forKey: .title)) ?? "" | |
| 72 | + byline = try? c.decodeIfPresent(String.self, forKey: .byline) | |
| 73 | + description = try? c.decodeIfPresent(String.self, forKey: .description) | |
| 74 | + siteName = try? c.decodeIfPresent(String.self, forKey: .siteName) | |
| 75 | + lang = try? c.decodeIfPresent(String.self, forKey: .lang) | |
| 76 | + published = try? c.decodeIfPresent(String.self, forKey: .published) | |
| 77 | + favicon = try? c.decodeIfPresent(String.self, forKey: .favicon) | |
| 78 | + markdown = (try? c.decode(String.self, forKey: .markdown)) ?? "" | |
| 79 | + quality = ExtractionQuality( | |
| 80 | + rawValue: (try? c.decode(String.self, forKey: .quality)) ?? "rawText" | |
| 81 | + ) ?? .rawText | |
| 82 | + headings = (try? c.decode([PageHeading].self, forKey: .headings)) ?? [] | |
| 83 | + selection = try? c.decodeIfPresent(PageSelection.self, forKey: .selection) | |
| 84 | + wordCount = (try? c.decode(Int.self, forKey: .wordCount)) ?? 0 | |
| 85 | + truncated = (try? c.decode(Bool.self, forKey: .truncated)) ?? false | |
| 86 | + } | |
| 87 | +} | |
added
Sources/ZyquoAtlas/Content/Readability-readerable.js
+122 −0
@@ -0,0 +1,122 @@ | ||
| 1 | +/* | |
| 2 | + * Copyright (c) 2010 Arc90 Inc | |
| 3 | + * | |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); | |
| 5 | + * you may not use this file except in compliance with the License. | |
| 6 | + * You may obtain a copy of the License at | |
| 7 | + * | |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 | |
| 9 | + * | |
| 10 | + * Unless required by applicable law or agreed to in writing, software | |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, | |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| 13 | + * See the License for the specific language governing permissions and | |
| 14 | + * limitations under the License. | |
| 15 | + */ | |
| 16 | + | |
| 17 | +/* | |
| 18 | + * This code is heavily based on Arc90's readability.js (1.7.1) script | |
| 19 | + * available at: http://code.google.com/p/arc90labs-readability | |
| 20 | + */ | |
| 21 | + | |
| 22 | +var REGEXPS = { | |
| 23 | + // NOTE: These two regular expressions are duplicated in | |
| 24 | + // Readability.js. Please keep both copies in sync. | |
| 25 | + unlikelyCandidates: | |
| 26 | + /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i, | |
| 27 | + okMaybeItsACandidate: /and|article|body|column|content|main|mathjax|shadow/i, | |
| 28 | +}; | |
| 29 | + | |
| 30 | +function isNodeVisible(node) { | |
| 31 | + // Have to null-check node.style and node.className.includes to deal with SVG and MathML nodes. | |
| 32 | + return ( | |
| 33 | + (!node.style || node.style.display != "none") && | |
| 34 | + !node.hasAttribute("hidden") && | |
| 35 | + //check for "fallback-image" so that wikimedia math images are displayed | |
| 36 | + (!node.hasAttribute("aria-hidden") || | |
| 37 | + node.getAttribute("aria-hidden") != "true" || | |
| 38 | + (node.className && | |
| 39 | + node.className.includes && | |
| 40 | + node.className.includes("fallback-image"))) | |
| 41 | + ); | |
| 42 | +} | |
| 43 | + | |
| 44 | +/** | |
| 45 | + * Decides whether or not the document is reader-able without parsing the whole thing. | |
| 46 | + * @param {Object} options Configuration object. | |
| 47 | + * @param {number} [options.minContentLength=140] The minimum node content length used to decide if the document is readerable. | |
| 48 | + * @param {number} [options.minScore=20] The minumum cumulated 'score' used to determine if the document is readerable. | |
| 49 | + * @param {Function} [options.visibilityChecker=isNodeVisible] The function used to determine if a node is visible. | |
| 50 | + * @return {boolean} Whether or not we suspect Readability.parse() will suceeed at returning an article object. | |
| 51 | + */ | |
| 52 | +function isProbablyReaderable(doc, options = {}) { | |
| 53 | + // For backward compatibility reasons 'options' can either be a configuration object or the function used | |
| 54 | + // to determine if a node is visible. | |
| 55 | + if (typeof options == "function") { | |
| 56 | + options = { visibilityChecker: options }; | |
| 57 | + } | |
| 58 | + | |
| 59 | + var defaultOptions = { | |
| 60 | + minScore: 20, | |
| 61 | + minContentLength: 140, | |
| 62 | + visibilityChecker: isNodeVisible, | |
| 63 | + }; | |
| 64 | + options = Object.assign(defaultOptions, options); | |
| 65 | + | |
| 66 | + var nodes = doc.querySelectorAll("p, pre, article"); | |
| 67 | + | |
| 68 | + // Get <div> nodes which have <br> node(s) and append them into the `nodes` variable. | |
| 69 | + // Some articles' DOM structures might look like | |
| 70 | + // <div> | |
| 71 | + // Sentences<br> | |
| 72 | + // <br> | |
| 73 | + // Sentences<br> | |
| 74 | + // </div> | |
| 75 | + var brNodes = doc.querySelectorAll("div > br"); | |
| 76 | + if (brNodes.length) { | |
| 77 | + var set = new Set(nodes); | |
| 78 | + [].forEach.call(brNodes, function (node) { | |
| 79 | + set.add(node.parentNode); | |
| 80 | + }); | |
| 81 | + nodes = Array.from(set); | |
| 82 | + } | |
| 83 | + | |
| 84 | + var score = 0; | |
| 85 | + // This is a little cheeky, we use the accumulator 'score' to decide what to return from | |
| 86 | + // this callback: | |
| 87 | + return [].some.call(nodes, function (node) { | |
| 88 | + if (!options.visibilityChecker(node)) { | |
| 89 | + return false; | |
| 90 | + } | |
| 91 | + | |
| 92 | + var matchString = node.className + " " + node.id; | |
| 93 | + if ( | |
| 94 | + REGEXPS.unlikelyCandidates.test(matchString) && | |
| 95 | + !REGEXPS.okMaybeItsACandidate.test(matchString) | |
| 96 | + ) { | |
| 97 | + return false; | |
| 98 | + } | |
| 99 | + | |
| 100 | + if (node.matches("li p")) { | |
| 101 | + return false; | |
| 102 | + } | |
| 103 | + | |
| 104 | + var textContentLength = node.textContent.trim().length; | |
| 105 | + if (textContentLength < options.minContentLength) { | |
| 106 | + return false; | |
| 107 | + } | |
| 108 | + | |
| 109 | + score += Math.sqrt(textContentLength - options.minContentLength); | |
| 110 | + | |
| 111 | + if (score > options.minScore) { | |
| 112 | + return true; | |
| 113 | + } | |
| 114 | + return false; | |
| 115 | + }); | |
| 116 | +} | |
| 117 | + | |
| 118 | +if (typeof module === "object") { | |
| 119 | + /* eslint-disable-next-line no-redeclare */ | |
| 120 | + /* global module */ | |
| 121 | + module.exports = isProbablyReaderable; | |
| 122 | +} | |
added
Sources/ZyquoAtlas/Content/Readability.js
+2812 −0
Large diff (2,813 lines) is not loaded by default to keep the page fast.Load diff