SPB Git

spb/zyquo-atlas Public License

The AI-native macOS web browser — every surface, intelligent.

macosswiftaibrowser
Swift 75.2% JavaScript 22% Shell 2% Makefile 0.9%

phase3.A: content extraction — Mozilla Readability (Apache-2.0) + AtlasExtractor driver in isolated WKContentWorld, PageContext, heading-aware Chunker; visible-text-only (injection-safe)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
simon-pierre boucher committed 11 days ago (Jul 30, 2026) parent 57b8819

Showing 9 changed files with +3,417 and −1

modified Package.swift +7 −1
@@ -23,7 +23,13 @@ let package = Package(
23 23 dependencies: [
24 24 .product(name: "Markdown", package: "swift-markdown")
25 25 ],
26 − path: "Sources/ZyquoAtlas"
26 + path: "Sources/ZyquoAtlas",
27 + resources: [
28 + .copy("Content/Readability.js"),
29 + .copy("Content/Readability-readerable.js"),
30 + .copy("Content/AtlasExtractor.js"),
31 + .copy("Content/LICENSE-Readability.txt"),
32 + ]
27 33 ),
28 34 .executableTarget(
29 35 name: "zyquo-verify",
modified Sources/ZyquoAtlas/Browser/ProfileStore.swift +3 −0
@@ -48,6 +48,9 @@ final class ProfileStore {
48 48 config.websiteDataStore = dataStore(for: profile)
49 49 config.defaultWebpagePreferences.allowsContentJavaScript = true
50 50 config.preferences.isElementFullscreenEnabled = true
51 + // Inject the content-extraction scripts into their isolated world so
52 + // every tab can produce a PageContext for AI actions.
53 + ContentExtractor.install(into: config)
51 54 return config
52 55 }
53 56 }
added Sources/ZyquoAtlas/Content/AtlasExtractor.js +200 −0
@@ -0,0 +1,200 @@
1 +//
2 +// AtlasExtractor.js
3 +// Zyquo Atlas
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Mail: contact@spboucher.ai
7 +//
8 +// Content-extraction driver injected into an isolated WKContentWorld alongside
9 +// Mozilla Readability.js + Readability-readerable.js. Turns the live page into
10 +// clean, model-ready markdown with metadata and the user's selection, using
11 +// the fallback ladder from docs/AI-BROWSER-RESEARCH.md §2:
12 +// 1. Readability (article) → markdown
13 +// 2. visible structured-text walk (app pages)
14 +// 3. title + url + meta description
15 +// SECURITY: only visible, rendered content is emitted — hidden/off-screen and
16 +// aria-hidden nodes are dropped so injected instructions in invisible text
17 +// never reach the model. Every entry point returns a value (never undefined)
18 +// so the Swift bridge never crashes.
19 +//
20 +
21 +(function () {
22 + "use strict";
23 + if (window.__zyquoAtlas) return;
24 +
25 + const MAX_BYTES_DEFAULT = 600000;
26 +
27 + function meta(name) {
28 + const el = document.querySelector(
29 + 'meta[property="' + name + '"],meta[name="' + name + '"]'
30 + );
31 + return el ? el.getAttribute("content") : null;
32 + }
33 +
34 + function canonicalURL() {
35 + const l = document.querySelector('link[rel="canonical"]');
36 + return (l && l.href) || location.href;
37 + }
38 +
39 + function faviconURL() {
40 + const l = document.querySelector('link[rel~="icon"],link[rel="apple-touch-icon"]');
41 + try {
42 + return l ? new URL(l.getAttribute("href"), location.href).href
43 + : new URL("/favicon.ico", location.origin).href;
44 + } catch (_) { return null; }
45 + }
46 +
47 + // Is a node visible to the user (drops the injection surface)?
48 + function isVisible(el) {
49 + if (!el || el.nodeType !== 1) return false;
50 + if (el.getAttribute && el.getAttribute("aria-hidden") === "true") return false;
51 + const s = window.getComputedStyle(el);
52 + if (!s || s.display === "none" || s.visibility === "hidden" || parseFloat(s.opacity) === 0) {
53 + return false;
54 + }
55 + const r = el.getBoundingClientRect();
56 + if (r.width === 0 && r.height === 0) return false;
57 + return true;
58 + }
59 +
60 + // Walk visible DOM emitting lightweight markdown (headings, lists, links,
61 + // code, paragraphs). Used both to render a Readability article DOM and as the
62 + // app-page fallback.
63 + function toMarkdown(root) {
64 + const out = [];
65 + const SKIP = new Set(["SCRIPT", "STYLE", "NOSCRIPT", "TEMPLATE", "SVG", "CANVAS", "IFRAME"]);
66 + function walk(node, listDepth) {
67 + for (const child of node.childNodes) {
68 + if (child.nodeType === 3) {
69 + const t = child.textContent.replace(/\s+/g, " ");
70 + if (t.trim()) out.push(t);
71 + continue;
72 + }
73 + if (child.nodeType !== 1) continue;
74 + const tag = child.tagName;
75 + if (SKIP.has(tag)) continue;
76 + if (!isVisible(child)) continue;
77 + if (/^H[1-6]$/.test(tag)) {
78 + const level = "#".repeat(parseInt(tag[1], 10));
79 + out.push("\n" + level + " " + child.textContent.trim() + "\n");
80 + } else if (tag === "P") {
81 + out.push("\n" + child.textContent.trim() + "\n");
82 + } else if (tag === "LI") {
83 + out.push("\n" + " ".repeat(listDepth) + "- " + child.textContent.trim());
84 + } else if (tag === "PRE") {
85 + out.push("\n```\n" + child.textContent.replace(/\n+$/, "") + "\n```\n");
86 + } else if (tag === "A" && child.getAttribute("href")) {
87 + const txt = child.textContent.trim();
88 + if (txt) out.push("[" + txt + "](" + child.href + ")");
89 + } else if (tag === "BR") {
90 + out.push("\n");
91 + } else if (tag === "UL" || tag === "OL") {
92 + walk(child, listDepth + 1);
93 + out.push("\n");
94 + } else {
95 + walk(child, listDepth);
96 + }
97 + }
98 + }
99 + walk(root, 0);
100 + return out.join(" ").replace(/[ \t]+/g, " ").replace(/\n{3,}/g, "\n\n").trim();
101 + }
102 +
103 + // Index of heading offsets in the produced markdown for chunking + citations.
104 + function headingIndex(markdown) {
105 + const headings = [];
106 + const re = /^(#{1,6})\s+(.*)$/gm;
107 + let m;
108 + while ((m = re.exec(markdown)) !== null) {
109 + headings.push({ level: m[1].length, title: m[2].trim(), offset: m.index });
110 + }
111 + return headings;
112 + }
113 +
114 + function currentSelection() {
115 + const sel = window.getSelection();
116 + const text = sel ? String(sel).trim() : "";
117 + if (!text) return null;
118 + let context = null;
119 + try {
120 + const node = sel.anchorNode && sel.anchorNode.parentElement;
121 + const block = node && node.closest("p,li,section,article,div");
122 + if (block) context = block.textContent.replace(/\s+/g, " ").trim().slice(0, 800);
123 + } catch (_) {}
124 + return { text: text, context: context };
125 + }
126 +
127 + function truncateAtBoundary(s, maxBytes) {
128 + if (s.length <= maxBytes) return s;
129 + const cut = s.lastIndexOf("\n", maxBytes);
130 + return s.slice(0, cut > maxBytes * 0.5 ? cut : maxBytes) + "\n\n…[truncated]";
131 + }
132 +
133 + function extract(mode, maxBytes) {
134 + maxBytes = maxBytes || MAX_BYTES_DEFAULT;
135 + try {
136 + let markdown = "";
137 + let quality = "reader";
138 + let title = document.title;
139 + let byline = null;
140 + let published = null;
141 + let readerable = false;
142 +
143 + try {
144 + readerable = typeof isProbablyReaderable === "function" &&
145 + isProbablyReaderable(document);
146 + } catch (_) {}
147 +
148 + if (mode !== "rawText" && readerable && typeof Readability === "function") {
149 + try {
150 + const clone = document.cloneNode(true); // never mutate the live page
151 + const article = new Readability(clone).parse();
152 + if (article) {
153 + title = article.title || title;
154 + byline = article.byline || null;
155 + published = article.publishedTime || null;
156 + const holder = document.createElement("div");
157 + holder.innerHTML = article.content || "";
158 + markdown = toMarkdown(holder);
159 + }
160 + } catch (_) {}
161 + }
162 +
163 + if (!markdown || markdown.length < 400) { // fallback ladder
164 + markdown = toMarkdown(document.body || document.documentElement);
165 + quality = readerable ? "reader" : "rawText";
166 + }
167 + if (!markdown) { // last resort
168 + markdown = (document.body ? document.body.innerText : "").trim();
169 + quality = "rawText";
170 + }
171 +
172 + markdown = truncateAtBoundary(markdown, maxBytes);
173 +
174 + return {
175 + ok: true,
176 + context: {
177 + url: location.href,
178 + canonical: canonicalURL(),
179 + title: (title || location.host || "").trim(),
180 + byline: byline,
181 + description: meta("description") || meta("og:description"),
182 + siteName: meta("og:site_name"),
183 + lang: document.documentElement.lang || null,
184 + published: published,
185 + favicon: faviconURL(),
186 + markdown: markdown,
187 + quality: quality,
188 + headings: headingIndex(markdown),
189 + selection: currentSelection(),
190 + wordCount: markdown ? markdown.split(/\s+/).length : 0,
191 + truncated: markdown.indexOf("…[truncated]") !== -1
192 + }
193 + };
194 + } catch (e) {
195 + return { ok: false, error: String((e && e.stack) || e) };
196 + }
197 + }
198 +
199 + window.__zyquoAtlas = { extract: extract };
200 +})();
added Sources/ZyquoAtlas/Content/Chunker.swift +73 −0
@@ -0,0 +1,73 @@
1 +//
2 +// Chunker.swift
3 +// Zyquo Atlas
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Mail: contact@spboucher.ai
7 +//
8 +// Splits a PageContext's markdown into heading-aware chunks for long-page
9 +// summarization (map-reduce) and chat-with-page retrieval (docs/
10 +// AI-BROWSER-RESEARCH.md §3). Each chunk carries its heading path and
11 +// character range so answers can cite and scroll-to-highlight the source.
12 +//
13 +
14 +import Foundation
15 +
16 +struct Chunk: Identifiable, Hashable {
17 + let id: Int
18 + /// e.g. "Setup > macOS".
19 + let headingPath: String
20 + let text: String
21 + let charRange: Range<Int>
22 + var estimatedTokens: Int { text.count / 4 }
23 +}
24 +
25 +enum Chunker {
26 + /// ~600-token Q&A chunks by default; pass a larger target for summarization
27 + /// (map-reduce tolerates coarse chunks).
28 + static func chunk(_ context: PageContext, targetTokens: Int = 600, overlapTokens: Int = 50) -> [Chunk] {
29 + let md = context.markdown
30 + guard !md.isEmpty else { return [] }
31 +
32 + let targetChars = targetTokens * 4
33 + let overlapChars = overlapTokens * 4
34 + let scalars = Array(md)
35 +
36 + // Section boundaries from the heading index (fall back to the whole doc).
37 + var boundaries = context.headings.map { $0.offset }.sorted()
38 + if boundaries.first != 0 { boundaries.insert(0, at: 0) }
39 + boundaries.append(scalars.count)
40 +
41 + var chunks: [Chunk] = []
42 + var id = 0
43 + var headingStack: [(level: Int, title: String)] = []
44 +
45 + for i in 0..<(boundaries.count - 1) {
46 + let start = boundaries[i]
47 + let end = boundaries[i + 1]
48 + guard start < end else { continue }
49 +
50 + // Track heading path from the heading that opens this section.
51 + if let h = context.headings.first(where: { $0.offset == start }) {
52 + while let last = headingStack.last, last.level >= h.level { headingStack.removeLast() }
53 + headingStack.append((h.level, h.title))
54 + }
55 + let path = headingStack.map(\.title).joined(separator: " > ")
56 +
57 + // Sub-split large sections with overlap.
58 + var cursor = start
59 + while cursor < end {
60 + let sliceEnd = min(cursor + targetChars, end)
61 + let text = String(scalars[cursor..<sliceEnd]).trimmingCharacters(in: .whitespacesAndNewlines)
62 + if !text.isEmpty {
63 + chunks.append(Chunk(id: id, headingPath: path, text: text,
64 + charRange: cursor..<sliceEnd))
65 + id += 1
66 + }
67 + if sliceEnd >= end { break }
68 + cursor = max(cursor + targetChars - overlapChars, cursor + 1)
69 + }
70 + }
71 + return chunks
72 + }
73 +}
added Sources/ZyquoAtlas/Content/ContentExtractor.swift +97 −0
@@ -0,0 +1,97 @@
1 +//
2 +// ContentExtractor.swift
3 +// Zyquo Atlas
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Mail: contact@spboucher.ai
7 +//
8 +// Injects the extraction scripts (Mozilla Readability + AtlasExtractor driver)
9 +// into an isolated WKContentWorld and pulls a normalized PageContext out of a
10 +// live tab via callAsyncJavaScript. Isolation keeps the extractor from
11 +// colliding with or being observed by page JS (docs/AI-BROWSER-RESEARCH.md §2).
12 +// All page content is treated as untrusted downstream (Phase 3.C prompt
13 +// hygiene).
14 +//
15 +
16 +import Foundation
17 +import WebKit
18 +
19 +enum ContentExtractionError: LocalizedError {
20 + case scriptsUnavailable
21 + case scriptFailure(String)
22 + case decodeFailure(String)
23 +
24 + var errorDescription: String? {
25 + switch self {
26 + case .scriptsUnavailable:
27 + return "Content extraction scripts are missing from the app bundle."
28 + case .scriptFailure(let detail):
29 + return "Couldn't read this page's content (\(detail))."
30 + case .decodeFailure(let detail):
31 + return "Couldn't parse the extracted page content (\(detail))."
32 + }
33 + }
34 +}
35 +
36 +@MainActor
37 +final class ContentExtractor {
38 + /// Dedicated world so extraction never touches the page's JS globals.
39 + static let world = WKContentWorld.world(name: "ZyquoAtlasContent")
40 +
41 + /// Installs the extraction user scripts into a configuration. Call once per
42 + /// WKWebViewConfiguration (ProfileStore) before any tab loads.
43 + static func install(into config: WKWebViewConfiguration) {
44 + let ucc = config.userContentController
45 + for name in ["Readability", "Readability-readerable", "AtlasExtractor"] {
46 + guard let source = script(name) else { continue }
47 + ucc.addUserScript(WKUserScript(
48 + source: source,
49 + injectionTime: .atDocumentEnd,
50 + forMainFrameOnly: true,
51 + in: world
52 + ))
53 + }
54 + }
55 +
56 + /// Extracts a PageContext from a tab's web view. `mode` "auto" tries
57 + /// Readability first; "rawText" forces the visible-text fallback.
58 + static func extract(from webView: WKWebView, mode: String = "auto") async throws -> PageContext {
59 + let result: Any?
60 + do {
61 + result = try await webView.callAsyncJavaScript(
62 + "return window.__zyquoAtlas ? window.__zyquoAtlas.extract(mode, maxBytes) : { ok:false, error:'driver-missing' };",
63 + arguments: ["mode": mode, "maxBytes": 600_000],
64 + in: nil,
65 + contentWorld: world
66 + )
67 + } catch {
68 + throw ContentExtractionError.scriptFailure(error.localizedDescription)
69 + }
70 +
71 + guard let dict = result as? [String: Any] else {
72 + throw ContentExtractionError.scriptFailure("no result")
73 + }
74 + if dict["ok"] as? Bool != true {
75 + throw ContentExtractionError.scriptFailure((dict["error"] as? String) ?? "unknown")
76 + }
77 + guard let context = dict["context"] else {
78 + throw ContentExtractionError.scriptFailure("empty context")
79 + }
80 + do {
81 + let data = try JSONSerialization.data(withJSONObject: context)
82 + return try JSONDecoder().decode(PageContext.self, from: data)
83 + } catch {
84 + throw ContentExtractionError.decodeFailure(error.localizedDescription)
85 + }
86 + }
87 +
88 + // MARK: - Script loading
89 +
90 + private static func script(_ name: String) -> String? {
91 + guard let url = Bundle.module.url(forResource: name, withExtension: "js"),
92 + let source = try? String(contentsOf: url, encoding: .utf8) else {
93 + return nil
94 + }
95 + return source
96 + }
97 +}
added Sources/ZyquoAtlas/Content/LICENSE-Readability.txt +16 −0
@@ -0,0 +1,16 @@
1 +Readability.js and Readability-readerable.js in this directory are from the
2 +Mozilla Readability project (https://github.com/mozilla/readability), used
3 +unmodified.
4 +
5 + Copyright (c) 2010 Arc90 Inc
6 + Licensed under the Apache License, Version 2.0 (the "License");
7 + you may not use this file except in compliance with the License.
8 + You may obtain a copy of the License at
9 +
10 + http://www.apache.org/licenses/LICENSE-2.0
11 +
12 + Unless required by applicable law or agreed to in writing, software
13 + distributed under the License is distributed on an "AS IS" BASIS,
14 + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 + See the License for the specific language governing permissions and
16 + limitations under the License.
added Sources/ZyquoAtlas/Content/PageContext.swift +87 −0
@@ -0,0 +1,87 @@
1 +//
2 +// PageContext.swift
3 +// Zyquo Atlas
4 +//
5 +// Author: Simon-Pierre Boucher
6 +// Mail: contact@spboucher.ai
7 +//
8 +// Normalized, model-ready representation of a web page: clean markdown, key
9 +// metadata, the user's current selection, and a heading index for chunking and
10 +// citations. Produced by ContentExtractor from the injected AtlasExtractor.js;
11 +// chunked by Chunker for long pages / multi-tab reasoning. This is the ONLY
12 +// page representation the AI layer sees — WKWebView/DOM guts never leak past it.
13 +//
14 +
15 +import Foundation
16 +
17 +/// Extraction fidelity, so the UI can be honest when a page couldn't be read
18 +/// cleanly (docs/AI-BROWSER-RESEARCH.md §2: extraction failure is AI failure).
19 +enum ExtractionQuality: String, Codable {
20 + case reader // Readability main-content
21 + case rawText // visible structured-text fallback (app pages)
22 +}
23 +
24 +struct PageHeading: Codable, Hashable {
25 + let level: Int
26 + let title: String
27 + /// Character offset into `markdown`.
28 + let offset: Int
29 +}
30 +
31 +struct PageSelection: Codable, Hashable {
32 + let text: String
33 + /// Surrounding block text for grounding "explain this".
34 + let context: String?
35 +}
36 +
37 +struct PageContext: Codable, Hashable, Identifiable {
38 + var id: String { canonical.isEmpty ? url : canonical }
39 +
40 + let url: String
41 + let canonical: String
42 + let title: String
43 + let byline: String?
44 + let description: String?
45 + let siteName: String?
46 + let lang: String?
47 + let published: String?
48 + let favicon: String?
49 + /// Canonical model-facing representation.
50 + let markdown: String
51 + let quality: ExtractionQuality
52 + let headings: [PageHeading]
53 + let selection: PageSelection?
54 + let wordCount: Int
55 + let truncated: Bool
56 +
57 + /// Rough token estimate (chars/4) for the stuff-vs-map-reduce budget rule.
58 + var estimatedTokens: Int { markdown.count / 4 }
59 +
60 + private enum CodingKeys: String, CodingKey {
61 + case url, canonical, title, byline, description, siteName, lang
62 + case published, favicon, markdown, quality, headings, selection
63 + case wordCount, truncated
64 + }
65 +
66 + // Tolerate a missing/unknown `quality` from JS by defaulting to rawText.
67 + init(from decoder: Decoder) throws {
68 + let c = try decoder.container(keyedBy: CodingKeys.self)
69 + url = try c.decode(String.self, forKey: .url)
70 + canonical = (try? c.decode(String.self, forKey: .canonical)) ?? url
71 + title = (try? c.decode(String.self, forKey: .title)) ?? ""
72 + byline = try? c.decodeIfPresent(String.self, forKey: .byline)
73 + description = try? c.decodeIfPresent(String.self, forKey: .description)
74 + siteName = try? c.decodeIfPresent(String.self, forKey: .siteName)
75 + lang = try? c.decodeIfPresent(String.self, forKey: .lang)
76 + published = try? c.decodeIfPresent(String.self, forKey: .published)
77 + favicon = try? c.decodeIfPresent(String.self, forKey: .favicon)
78 + markdown = (try? c.decode(String.self, forKey: .markdown)) ?? ""
79 + quality = ExtractionQuality(
80 + rawValue: (try? c.decode(String.self, forKey: .quality)) ?? "rawText"
81 + ) ?? .rawText
82 + headings = (try? c.decode([PageHeading].self, forKey: .headings)) ?? []
83 + selection = try? c.decodeIfPresent(PageSelection.self, forKey: .selection)
84 + wordCount = (try? c.decode(Int.self, forKey: .wordCount)) ?? 0
85 + truncated = (try? c.decode(Bool.self, forKey: .truncated)) ?? false
86 + }
87 +}
added Sources/ZyquoAtlas/Content/Readability-readerable.js +122 −0
@@ -0,0 +1,122 @@
1 +/*
2 + * Copyright (c) 2010 Arc90 Inc
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +
17 +/*
18 + * This code is heavily based on Arc90's readability.js (1.7.1) script
19 + * available at: http://code.google.com/p/arc90labs-readability
20 + */
21 +
22 +var REGEXPS = {
23 + // NOTE: These two regular expressions are duplicated in
24 + // Readability.js. Please keep both copies in sync.
25 + unlikelyCandidates:
26 + /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
27 + okMaybeItsACandidate: /and|article|body|column|content|main|mathjax|shadow/i,
28 +};
29 +
30 +function isNodeVisible(node) {
31 + // Have to null-check node.style and node.className.includes to deal with SVG and MathML nodes.
32 + return (
33 + (!node.style || node.style.display != "none") &&
34 + !node.hasAttribute("hidden") &&
35 + //check for "fallback-image" so that wikimedia math images are displayed
36 + (!node.hasAttribute("aria-hidden") ||
37 + node.getAttribute("aria-hidden") != "true" ||
38 + (node.className &&
39 + node.className.includes &&
40 + node.className.includes("fallback-image")))
41 + );
42 +}
43 +
44 +/**
45 + * Decides whether or not the document is reader-able without parsing the whole thing.
46 + * @param {Object} options Configuration object.
47 + * @param {number} [options.minContentLength=140] The minimum node content length used to decide if the document is readerable.
48 + * @param {number} [options.minScore=20] The minumum cumulated 'score' used to determine if the document is readerable.
49 + * @param {Function} [options.visibilityChecker=isNodeVisible] The function used to determine if a node is visible.
50 + * @return {boolean} Whether or not we suspect Readability.parse() will suceeed at returning an article object.
51 + */
52 +function isProbablyReaderable(doc, options = {}) {
53 + // For backward compatibility reasons 'options' can either be a configuration object or the function used
54 + // to determine if a node is visible.
55 + if (typeof options == "function") {
56 + options = { visibilityChecker: options };
57 + }
58 +
59 + var defaultOptions = {
60 + minScore: 20,
61 + minContentLength: 140,
62 + visibilityChecker: isNodeVisible,
63 + };
64 + options = Object.assign(defaultOptions, options);
65 +
66 + var nodes = doc.querySelectorAll("p, pre, article");
67 +
68 + // Get <div> nodes which have <br> node(s) and append them into the `nodes` variable.
69 + // Some articles' DOM structures might look like
70 + // <div>
71 + // Sentences<br>
72 + // <br>
73 + // Sentences<br>
74 + // </div>
75 + var brNodes = doc.querySelectorAll("div > br");
76 + if (brNodes.length) {
77 + var set = new Set(nodes);
78 + [].forEach.call(brNodes, function (node) {
79 + set.add(node.parentNode);
80 + });
81 + nodes = Array.from(set);
82 + }
83 +
84 + var score = 0;
85 + // This is a little cheeky, we use the accumulator 'score' to decide what to return from
86 + // this callback:
87 + return [].some.call(nodes, function (node) {
88 + if (!options.visibilityChecker(node)) {
89 + return false;
90 + }
91 +
92 + var matchString = node.className + " " + node.id;
93 + if (
94 + REGEXPS.unlikelyCandidates.test(matchString) &&
95 + !REGEXPS.okMaybeItsACandidate.test(matchString)
96 + ) {
97 + return false;
98 + }
99 +
100 + if (node.matches("li p")) {
101 + return false;
102 + }
103 +
104 + var textContentLength = node.textContent.trim().length;
105 + if (textContentLength < options.minContentLength) {
106 + return false;
107 + }
108 +
109 + score += Math.sqrt(textContentLength - options.minContentLength);
110 +
111 + if (score > options.minScore) {
112 + return true;
113 + }
114 + return false;
115 + });
116 +}
117 +
118 +if (typeof module === "object") {
119 + /* eslint-disable-next-line no-redeclare */
120 + /* global module */
121 + module.exports = isProbablyReaderable;
122 +}
added Sources/ZyquoAtlas/Content/Readability.js +2812 −0
@@ -0,0 +1,2812 @@
1 +/*
2 + * Copyright (c) 2010 Arc90 Inc
3 + *
4 + * Licensed under the Apache License, Version 2.0 (the "License");
5 + * you may not use this file except in compliance with the License.
6 + * You may obtain a copy of the License at
7 + *
8 + * http://www.apache.org/licenses/LICENSE-2.0
9 + *
10 + * Unless required by applicable law or agreed to in writing, software
11 + * distributed under the License is distributed on an "AS IS" BASIS,
12 + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 + * See the License for the specific language governing permissions and
14 + * limitations under the License.
15 + */
16 +
17 +/*
18 + * This code is heavily based on Arc90's readability.js (1.7.1) script
19 + * available at: http://code.google.com/p/arc90labs-readability
20 + */
21 +
22 +/**
23 + * Public constructor.
24 + * @param {HTMLDocument} doc The document to parse.
25 + * @param {Object} options The options object.
26 + */
27 +function Readability(doc, options) {
28 + // In some older versions, people passed a URI as the first argument. Cope:
29 + if (options && options.documentElement) {
30 + doc = options;
31 + options = arguments[2];
32 + } else if (!doc || !doc.documentElement) {
33 + throw new Error(
34 + "First argument to Readability constructor should be a document object."
35 + );
36 + }
37 + options = options || {};
38 +
39 + this._doc = doc;
40 + this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__;
41 + this._articleTitle = null;
42 + this._articleByline = null;
43 + this._articleDir = null;
44 + this._articleSiteName = null;
45 + this._attempts = [];
46 + this._metadata = {};
47 +
48 + // Configurable options
49 + this._debug = !!options.debug;
50 + this._maxElemsToParse =
51 + options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE;
52 + this._nbTopCandidates =
53 + options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES;
54 + this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD;
55 + this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat(
56 + options.classesToPreserve || []
57 + );
58 + this._keepClasses = !!options.keepClasses;
59 + this._serializer =
60 + options.serializer ||
61 + function (el) {
62 + return el.innerHTML;
63 + };
64 + this._disableJSONLD = !!options.disableJSONLD;
65 + this._allowedVideoRegex = options.allowedVideoRegex || this.REGEXPS.videos;
66 + this._linkDensityModifier = options.linkDensityModifier || 0;
67 +
68 + // Start with all flags set
69 + this._flags =
70 + this.FLAG_STRIP_UNLIKELYS |
71 + this.FLAG_WEIGHT_CLASSES |
72 + this.FLAG_CLEAN_CONDITIONALLY;
73 +
74 + // Control whether log messages are sent to the console
75 + if (this._debug) {
76 + let logNode = function (node) {
77 + if (node.nodeType == node.TEXT_NODE) {
78 + return `${node.nodeName} ("${node.textContent}")`;
79 + }
80 + let attrPairs = Array.from(node.attributes || [], function (attr) {
81 + return `${attr.name}="${attr.value}"`;
82 + }).join(" ");
83 + return `<${node.localName} ${attrPairs}>`;
84 + };
85 + this.log = function () {
86 + if (typeof console !== "undefined") {
87 + let args = Array.from(arguments, arg => {
88 + if (arg && arg.nodeType == this.ELEMENT_NODE) {
89 + return logNode(arg);
90 + }
91 + return arg;
92 + });
93 + args.unshift("Reader: (Readability)");
94 + // eslint-disable-next-line no-console
95 + console.log(...args);
96 + } else if (typeof dump !== "undefined") {
97 + /* global dump */
98 + var msg = Array.prototype.map
99 + .call(arguments, function (x) {
100 + return x && x.nodeName ? logNode(x) : x;
101 + })
102 + .join(" ");
103 + dump("Reader: (Readability) " + msg + "\n");
104 + }
105 + };
106 + } else {
107 + this.log = function () {};
108 + }
109 +}
110 +
111 +Readability.prototype = {
112 + FLAG_STRIP_UNLIKELYS: 0x1,
113 + FLAG_WEIGHT_CLASSES: 0x2,
114 + FLAG_CLEAN_CONDITIONALLY: 0x4,
115 +
116 + // https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType
117 + ELEMENT_NODE: 1,
118 + TEXT_NODE: 3,
119 +
120 + // Max number of nodes supported by this parser. Default: 0 (no limit)
121 + DEFAULT_MAX_ELEMS_TO_PARSE: 0,
122 +
123 + // The number of top candidates to consider when analysing how
124 + // tight the competition is among candidates.
125 + DEFAULT_N_TOP_CANDIDATES: 5,
126 +
127 + // Element tags to score by default.
128 + DEFAULT_TAGS_TO_SCORE: "section,h2,h3,h4,h5,h6,p,td,pre"
129 + .toUpperCase()
130 + .split(","),
131 +
132 + // The default number of chars an article must have in order to return a result
133 + DEFAULT_CHAR_THRESHOLD: 500,
134 +
135 + // All of the regular expressions in use within readability.
136 + // Defined up here so we don't instantiate them repeatedly in loops.
137 + REGEXPS: {
138 + // NOTE: These two regular expressions are duplicated in
139 + // Readability-readerable.js. Please keep both copies in sync.
140 + unlikelyCandidates:
141 + /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,
142 + okMaybeItsACandidate:
143 + /and|article|body|column|content|main|mathjax|shadow/i,
144 +
145 + positive:
146 + /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,
147 + negative:
148 + /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i,
149 + extraneous:
150 + /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,
151 + byline: /byline|author|dateline|writtenby|p-author/i,
152 + replaceFonts: /<(\/?)font[^>]*>/gi,
153 + normalize: /\s{2,}/g,
154 + videos:
155 + /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq|bilibili|live.bilibili)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,
156 + shareElements: /(\b|_)(share|sharedaddy)(\b|_)/i,
157 + nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,
158 + prevLink: /(prev|earl|old|new|<|«)/i,
159 + tokenize: /\W+/g,
160 + whitespace: /^\s*$/,
161 + hasContent: /\S$/,
162 + hashUrl: /^#.+/,
163 + srcsetUrl: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g,
164 + b64DataUrl: /^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i,
165 + // Commas as used in Latin, Sindhi, Chinese and various other scripts.
166 + // see: https://en.wikipedia.org/wiki/Comma#Comma_variants
167 + commas: /\u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C/g,
168 + // See: https://schema.org/Article
169 + jsonLdArticleTypes:
170 + /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/,
171 + // used to see if a node's content matches words commonly used for ad blocks or loading indicators
172 + adWords:
173 + /^(ad(vertising|vertisement)?|pub(licité)?|werb(ung)?|广告|Реклама|Anuncio)$/iu,
174 + loadingWords:
175 + /^((loading|正在加载|Загрузка|chargement|cargando)(…|\.\.\.)?)$/iu,
176 + },
177 +
178 + UNLIKELY_ROLES: [
179 + "menu",
180 + "menubar",
181 + "complementary",
182 + "navigation",
183 + "alert",
184 + "alertdialog",
185 + "dialog",
186 + ],
187 +
188 + DIV_TO_P_ELEMS: new Set([
189 + "BLOCKQUOTE",
190 + "DL",
191 + "DIV",
192 + "IMG",
193 + "OL",
194 + "P",
195 + "PRE",
196 + "TABLE",
197 + "UL",
198 + ]),
199 +
200 + ALTER_TO_DIV_EXCEPTIONS: ["DIV", "ARTICLE", "SECTION", "P", "OL", "UL"],
201 +
202 + PRESENTATIONAL_ATTRIBUTES: [
203 + "align",
204 + "background",
205 + "bgcolor",
206 + "border",
207 + "cellpadding",
208 + "cellspacing",
209 + "frame",
210 + "hspace",
211 + "rules",
212 + "style",
213 + "valign",
214 + "vspace",
215 + ],
216 +
217 + DEPRECATED_SIZE_ATTRIBUTE_ELEMS: ["TABLE", "TH", "TD", "HR", "PRE"],
218 +
219 + // The commented out elements qualify as phrasing content but tend to be
220 + // removed by readability when put into paragraphs, so we ignore them here.
221 + PHRASING_ELEMS: [
222 + // "CANVAS", "IFRAME", "SVG", "VIDEO",
223 + "ABBR",
224 + "AUDIO",
225 + "B",
226 + "BDO",
227 + "BR",
228 + "BUTTON",
229 + "CITE",
230 + "CODE",
231 + "DATA",
232 + "DATALIST",
233 + "DFN",
234 + "EM",
235 + "EMBED",
236 + "I",
237 + "IMG",
238 + "INPUT",
239 + "KBD",
240 + "LABEL",
241 + "MARK",
242 + "MATH",
243 + "METER",
244 + "NOSCRIPT",
245 + "OBJECT",
246 + "OUTPUT",
247 + "PROGRESS",
248 + "Q",
249 + "RUBY",
250 + "SAMP",
251 + "SCRIPT",
252 + "SELECT",
253 + "SMALL",
254 + "SPAN",
255 + "STRONG",
256 + "SUB",
257 + "SUP",
258 + "TEXTAREA",
259 + "TIME",
260 + "VAR",
261 + "WBR",
262 + ],
263 +
264 + // These are the classes that readability sets itself.
265 + CLASSES_TO_PRESERVE: ["page"],
266 +
267 + // These are the list of HTML entities that need to be escaped.
268 + HTML_ESCAPE_MAP: {
269 + lt: "<",
270 + gt: ">",
271 + amp: "&",
272 + quot: '"',
273 + apos: "'",
274 + },
275 +
276 + /**
277 + * Run any post-process modifications to article content as necessary.
278 + *
279 + * @param Element
280 + * @return void
281 + **/
282 + _postProcessContent(articleContent) {
283 + // Readability cannot open relative uris so we convert them to absolute uris.
284 + this._fixRelativeUris(articleContent);
285 +
286 + this._simplifyNestedElements(articleContent);
287 +
288 + if (!this._keepClasses) {
289 + // Remove classes.
290 + this._cleanClasses(articleContent);
291 + }
292 + },
293 +
294 + /**
295 + * Iterates over a NodeList, calls `filterFn` for each node and removes node
296 + * if function returned `true`.
297 + *
298 + * If function is not passed, removes all the nodes in node list.
299 + *
300 + * @param NodeList nodeList The nodes to operate on
301 + * @param Function filterFn the function to use as a filter
302 + * @return void
303 + */
304 + _removeNodes(nodeList, filterFn) {
305 + // Avoid ever operating on live node lists.
306 + if (this._docJSDOMParser && nodeList._isLiveNodeList) {
307 + throw new Error("Do not pass live node lists to _removeNodes");
308 + }
309 + for (var i = nodeList.length - 1; i >= 0; i--) {
310 + var node = nodeList[i];
311 + var parentNode = node.parentNode;
312 + if (parentNode) {
313 + if (!filterFn || filterFn.call(this, node, i, nodeList)) {
314 + parentNode.removeChild(node);
315 + }
316 + }
317 + }
318 + },
319 +
320 + /**
321 + * Iterates over a NodeList, and calls _setNodeTag for each node.
322 + *
323 + * @param NodeList nodeList The nodes to operate on
324 + * @param String newTagName the new tag name to use
325 + * @return void
326 + */
327 + _replaceNodeTags(nodeList, newTagName) {
328 + // Avoid ever operating on live node lists.
329 + if (this._docJSDOMParser && nodeList._isLiveNodeList) {
330 + throw new Error("Do not pass live node lists to _replaceNodeTags");
331 + }
332 + for (const node of nodeList) {
333 + this._setNodeTag(node, newTagName);
334 + }
335 + },
336 +
337 + /**
338 + * Iterate over a NodeList, which doesn't natively fully implement the Array
339 + * interface.
340 + *
341 + * For convenience, the current object context is applied to the provided
342 + * iterate function.
343 + *
344 + * @param NodeList nodeList The NodeList.
345 + * @param Function fn The iterate function.
346 + * @return void
347 + */
348 + _forEachNode(nodeList, fn) {
349 + Array.prototype.forEach.call(nodeList, fn, this);
350 + },
351 +
352 + /**
353 + * Iterate over a NodeList, and return the first node that passes
354 + * the supplied test function
355 + *
356 + * For convenience, the current object context is applied to the provided
357 + * test function.
358 + *
359 + * @param NodeList nodeList The NodeList.
360 + * @param Function fn The test function.
361 + * @return void
362 + */
363 + _findNode(nodeList, fn) {
364 + return Array.prototype.find.call(nodeList, fn, this);
365 + },
366 +
367 + /**
368 + * Iterate over a NodeList, return true if any of the provided iterate
369 + * function calls returns true, false otherwise.
370 + *
371 + * For convenience, the current object context is applied to the
372 + * provided iterate function.
373 + *
374 + * @param NodeList nodeList The NodeList.
375 + * @param Function fn The iterate function.
376 + * @return Boolean
377 + */
378 + _someNode(nodeList, fn) {
379 + return Array.prototype.some.call(nodeList, fn, this);
380 + },
381 +
382 + /**
383 + * Iterate over a NodeList, return true if all of the provided iterate
384 + * function calls return true, false otherwise.
385 + *
386 + * For convenience, the current object context is applied to the
387 + * provided iterate function.
388 + *
389 + * @param NodeList nodeList The NodeList.
390 + * @param Function fn The iterate function.
391 + * @return Boolean
392 + */
393 + _everyNode(nodeList, fn) {
394 + return Array.prototype.every.call(nodeList, fn, this);
395 + },
396 +
397 + _getAllNodesWithTag(node, tagNames) {
398 + if (node.querySelectorAll) {
399 + return node.querySelectorAll(tagNames.join(","));
400 + }
401 + return [].concat.apply(
402 + [],
403 + tagNames.map(function (tag) {
404 + var collection = node.getElementsByTagName(tag);
405 + return Array.isArray(collection) ? collection : Array.from(collection);
406 + })
407 + );
408 + },
409 +
410 + /**
411 + * Removes the class="" attribute from every element in the given
412 + * subtree, except those that match CLASSES_TO_PRESERVE and
413 + * the classesToPreserve array from the options object.
414 + *
415 + * @param Element
416 + * @return void
417 + */
418 + _cleanClasses(node) {
419 + var classesToPreserve = this._classesToPreserve;
420 + var className = (node.getAttribute("class") || "")
421 + .split(/\s+/)
422 + .filter(cls => classesToPreserve.includes(cls))
423 + .join(" ");
424 +
425 + if (className) {
426 + node.setAttribute("class", className);
427 + } else {
428 + node.removeAttribute("class");
429 + }
430 +
431 + for (node = node.firstElementChild; node; node = node.nextElementSibling) {
432 + this._cleanClasses(node);
433 + }
434 + },
435 +
436 + /**
437 + * Tests whether a string is a URL or not.
438 + *
439 + * @param {string} str The string to test
440 + * @return {boolean} true if str is a URL, false if not
441 + */
442 + _isUrl(str) {
443 + try {
444 + new URL(str);
445 + return true;
446 + } catch {
447 + return false;
448 + }
449 + },
450 + /**
451 + * Converts each <a> and <img> uri in the given element to an absolute URI,
452 + * ignoring #ref URIs.
453 + *
454 + * @param Element
455 + * @return void
456 + */
457 + _fixRelativeUris(articleContent) {
458 + var baseURI = this._doc.baseURI;
459 + var documentURI = this._doc.documentURI;
460 + function toAbsoluteURI(uri) {
461 + // Leave hash links alone if the base URI matches the document URI:
462 + if (baseURI == documentURI && uri.charAt(0) == "#") {
463 + return uri;
464 + }
465 +
466 + // Otherwise, resolve against base URI:
467 + try {
468 + return new URL(uri, baseURI).href;
469 + } catch (ex) {
470 + // Something went wrong, just return the original:
471 + }
472 + return uri;
473 + }
474 +
475 + var links = this._getAllNodesWithTag(articleContent, ["a"]);
476 + this._forEachNode(links, function (link) {
477 + var href = link.getAttribute("href");
478 + if (href) {
479 + // Remove links with javascript: URIs, since
480 + // they won't work after scripts have been removed from the page.
481 + if (href.indexOf("javascript:") === 0) {
482 + // if the link only contains simple text content, it can be converted to a text node
483 + if (
484 + link.childNodes.length === 1 &&
485 + link.childNodes[0].nodeType === this.TEXT_NODE
486 + ) {
487 + var text = this._doc.createTextNode(link.textContent);
488 + link.parentNode.replaceChild(text, link);
489 + } else {
490 + // if the link has multiple children, they should all be preserved
491 + var container = this._doc.createElement("span");
492 + while (link.firstChild) {
493 + container.appendChild(link.firstChild);
494 + }
495 + link.parentNode.replaceChild(container, link);
496 + }
497 + } else {
498 + link.setAttribute("href", toAbsoluteURI(href));
499 + }
500 + }
501 + });
502 +
503 + var medias = this._getAllNodesWithTag(articleContent, [
504 + "img",
505 + "picture",
506 + "figure",
507 + "video",
508 + "audio",
509 + "source",
510 + ]);
511 +
512 + this._forEachNode(medias, function (media) {
513 + var src = media.getAttribute("src");
514 + var poster = media.getAttribute("poster");
515 + var srcset = media.getAttribute("srcset");
516 +
517 + if (src) {
518 + media.setAttribute("src", toAbsoluteURI(src));
519 + }
520 +
521 + if (poster) {
522 + media.setAttribute("poster", toAbsoluteURI(poster));
523 + }
524 +
525 + if (srcset) {
526 + var newSrcset = srcset.replace(
527 + this.REGEXPS.srcsetUrl,
528 + function (_, p1, p2, p3) {
529 + return toAbsoluteURI(p1) + (p2 || "") + p3;
530 + }
531 + );
532 +
533 + media.setAttribute("srcset", newSrcset);
534 + }
535 + });
536 + },
537 +
538 + _simplifyNestedElements(articleContent) {
539 + var node = articleContent;
540 +
541 + while (node) {
542 + if (
543 + node.parentNode &&
544 + ["DIV", "SECTION"].includes(node.tagName) &&
545 + !(node.id && node.id.startsWith("readability"))
546 + ) {
547 + if (this._isElementWithoutContent(node)) {
548 + node = this._removeAndGetNext(node);
549 + continue;
550 + } else if (
551 + this._hasSingleTagInsideElement(node, "DIV") ||
552 + this._hasSingleTagInsideElement(node, "SECTION")
553 + ) {
554 + var child = node.children[0];
555 + for (var i = 0; i < node.attributes.length; i++) {
556 + child.setAttributeNode(node.attributes[i].cloneNode());
557 + }
558 + node.parentNode.replaceChild(child, node);
559 + node = child;
560 + continue;
561 + }
562 + }
563 +
564 + node = this._getNextNode(node);
565 + }
566 + },
567 +
568 + /**
569 + * Get the article title as an H1.
570 + *
571 + * @return string
572 + **/
573 + _getArticleTitle() {
574 + var doc = this._doc;
575 + var curTitle = "";
576 + var origTitle = "";
577 +
578 + try {
579 + curTitle = origTitle = doc.title.trim();
580 +
581 + // If they had an element with id "title" in their HTML
582 + if (typeof curTitle !== "string") {
583 + curTitle = origTitle = this._getInnerText(
584 + doc.getElementsByTagName("title")[0]
585 + );
586 + }
587 + } catch (e) {
588 + /* ignore exceptions setting the title. */
589 + }
590 +
591 + var titleHadHierarchicalSeparators = false;
592 + function wordCount(str) {
593 + return str.split(/\s+/).length;
594 + }
595 +
596 + // If there's a separator in the title, first remove the final part
597 + const titleSeparators = /\|\-–—\\\/>»/.source;
598 + if (new RegExp(`\\s[${titleSeparators}]\\s`).test(curTitle)) {
599 + titleHadHierarchicalSeparators = /\s[\\\/>»]\s/.test(curTitle);
600 + let allSeparators = Array.from(
601 + origTitle.matchAll(new RegExp(`\\s[${titleSeparators}]\\s`, "gi"))
602 + );
603 + curTitle = origTitle.substring(0, allSeparators.pop().index);
604 +
605 + // If the resulting title is too short, remove the first part instead:
606 + if (wordCount(curTitle) < 3) {
607 + curTitle = origTitle.replace(
608 + new RegExp(`^[^${titleSeparators}]*[${titleSeparators}]`, "gi"),
609 + ""
610 + );
611 + }
612 + } else if (curTitle.includes(": ")) {
613 + // Check if we have an heading containing this exact string, so we
614 + // could assume it's the full title.
615 + var headings = this._getAllNodesWithTag(doc, ["h1", "h2"]);
616 + var trimmedTitle = curTitle.trim();
617 + var match = this._someNode(headings, function (heading) {
618 + return heading.textContent.trim() === trimmedTitle;
619 + });
620 +
621 + // If we don't, let's extract the title out of the original title string.
622 + if (!match) {
623 + curTitle = origTitle.substring(origTitle.lastIndexOf(":") + 1);
624 +
625 + // If the title is now too short, try the first colon instead:
626 + if (wordCount(curTitle) < 3) {
627 + curTitle = origTitle.substring(origTitle.indexOf(":") + 1);
628 + // But if we have too many words before the colon there's something weird
629 + // with the titles and the H tags so let's just use the original title instead
630 + } else if (wordCount(origTitle.substr(0, origTitle.indexOf(":"))) > 5) {
631 + curTitle = origTitle;
632 + }
633 + }
634 + } else if (curTitle.length > 150 || curTitle.length < 15) {
635 + var hOnes = doc.getElementsByTagName("h1");
636 +
637 + if (hOnes.length === 1) {
638 + curTitle = this._getInnerText(hOnes[0]);
639 + }
640 + }
641 +
642 + curTitle = curTitle.trim().replace(this.REGEXPS.normalize, " ");
643 + // If we now have 4 words or fewer as our title, and either no
644 + // 'hierarchical' separators (\, /, > or ») were found in the original
645 + // title or we decreased the number of words by more than 1 word, use
646 + // the original title.
647 + var curTitleWordCount = wordCount(curTitle);
648 + if (
649 + curTitleWordCount <= 4 &&
650 + (!titleHadHierarchicalSeparators ||
651 + curTitleWordCount !=
652 + wordCount(
653 + origTitle.replace(new RegExp(`\\s[${titleSeparators}]\\s`, "g"), "")
654 + ) -
655 + 1)
656 + ) {
657 + curTitle = origTitle;
658 + }
659 +
660 + return curTitle;
661 + },
662 +
663 + /**
664 + * Prepare the HTML document for readability to scrape it.
665 + * This includes things like stripping javascript, CSS, and handling terrible markup.
666 + *
667 + * @return void
668 + **/
669 + _prepDocument() {
670 + var doc = this._doc;
671 +
672 + // Remove all style tags in head
673 + this._removeNodes(this._getAllNodesWithTag(doc, ["style"]));
674 +
675 + if (doc.body) {
676 + this._replaceBrs(doc.body);
677 + }
678 +
679 + this._replaceNodeTags(this._getAllNodesWithTag(doc, ["font"]), "SPAN");
680 + },
681 +
682 + /**
683 + * Finds the next node, starting from the given node, and ignoring
684 + * whitespace in between. If the given node is an element, the same node is
685 + * returned.
686 + */
687 + _nextNode(node) {
688 + var next = node;
689 + while (
690 + next &&
691 + next.nodeType != this.ELEMENT_NODE &&
692 + this.REGEXPS.whitespace.test(next.textContent)
693 + ) {
694 + next = next.nextSibling;
695 + }
696 + return next;
697 + },
698 +
699 + /**
700 + * Replaces 2 or more successive <br> elements with a single <p>.
701 + * Whitespace between <br> elements are ignored. For example:
702 + * <div>foo<br>bar<br> <br><br>abc</div>
703 + * will become:
704 + * <div>foo<br>bar<p>abc</p></div>
705 + */
706 + _replaceBrs(elem) {
707 + this._forEachNode(this._getAllNodesWithTag(elem, ["br"]), function (br) {
708 + var next = br.nextSibling;
709 +
710 + // Whether 2 or more <br> elements have been found and replaced with a
711 + // <p> block.
712 + var replaced = false;
713 +
714 + // If we find a <br> chain, remove the <br>s until we hit another node
715 + // or non-whitespace. This leaves behind the first <br> in the chain
716 + // (which will be replaced with a <p> later).
717 + while ((next = this._nextNode(next)) && next.tagName == "BR") {
718 + replaced = true;
719 + var brSibling = next.nextSibling;
720 + next.remove();
721 + next = brSibling;
722 + }
723 +
724 + // If we removed a <br> chain, replace the remaining <br> with a <p>. Add
725 + // all sibling nodes as children of the <p> until we hit another <br>
726 + // chain.
727 + if (replaced) {
728 + var p = this._doc.createElement("p");
729 + br.parentNode.replaceChild(p, br);
730 +
731 + next = p.nextSibling;
732 + while (next) {
733 + // If we've hit another <br><br>, we're done adding children to this <p>.
734 + if (next.tagName == "BR") {
735 + var nextElem = this._nextNode(next.nextSibling);
736 + if (nextElem && nextElem.tagName == "BR") {
737 + break;
738 + }
739 + }
740 +
741 + if (!this._isPhrasingContent(next)) {
742 + break;
743 + }
744 +
745 + // Otherwise, make this node a child of the new <p>.
746 + var sibling = next.nextSibling;
747 + p.appendChild(next);
748 + next = sibling;
749 + }
750 +
751 + while (p.lastChild && this._isWhitespace(p.lastChild)) {
752 + p.lastChild.remove();
753 + }
754 +
755 + if (p.parentNode.tagName === "P") {
756 + this._setNodeTag(p.parentNode, "DIV");
757 + }
758 + }
759 + });
760 + },
761 +
762 + _setNodeTag(node, tag) {
763 + this.log("_setNodeTag", node, tag);
764 + if (this._docJSDOMParser) {
765 + node.localName = tag.toLowerCase();
766 + node.tagName = tag.toUpperCase();
767 + return node;
768 + }
769 +
770 + var replacement = node.ownerDocument.createElement(tag);
771 + while (node.firstChild) {
772 + replacement.appendChild(node.firstChild);
773 + }
774 + node.parentNode.replaceChild(replacement, node);
775 + if (node.readability) {
776 + replacement.readability = node.readability;
777 + }
778 +
779 + for (var i = 0; i < node.attributes.length; i++) {
780 + replacement.setAttributeNode(node.attributes[i].cloneNode());
781 + }
782 + return replacement;
783 + },
784 +
785 + /**
786 + * Prepare the article node for display. Clean out any inline styles,
787 + * iframes, forms, strip extraneous <p> tags, etc.
788 + *
789 + * @param Element
790 + * @return void
791 + **/
792 + _prepArticle(articleContent) {
793 + this._cleanStyles(articleContent);
794 +
795 + // Check for data tables before we continue, to avoid removing items in
796 + // those tables, which will often be isolated even though they're
797 + // visually linked to other content-ful elements (text, images, etc.).
798 + this._markDataTables(articleContent);
799 +
800 + this._fixLazyImages(articleContent);
801 +
802 + // Clean out junk from the article content
803 + this._cleanConditionally(articleContent, "form");
804 + this._cleanConditionally(articleContent, "fieldset");
805 + this._clean(articleContent, "object");
806 + this._clean(articleContent, "embed");
807 + this._clean(articleContent, "footer");
808 + this._clean(articleContent, "link");
809 + this._clean(articleContent, "aside");
810 +
811 + // Clean out elements with little content that have "share" in their id/class combinations from final top candidates,
812 + // which means we don't remove the top candidates even they have "share".
813 +
814 + var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD;
815 +
816 + this._forEachNode(articleContent.children, function (topCandidate) {
817 + this._cleanMatchedNodes(topCandidate, function (node, matchString) {
818 + return (
819 + this.REGEXPS.shareElements.test(matchString) &&
820 + node.textContent.length < shareElementThreshold
821 + );
822 + });
823 + });
824 +
825 + this._clean(articleContent, "iframe");
826 + this._clean(articleContent, "input");
827 + this._clean(articleContent, "textarea");
828 + this._clean(articleContent, "select");
829 + this._clean(articleContent, "button");
830 + this._cleanHeaders(articleContent);
831 +
832 + // Do these last as the previous stuff may have removed junk
833 + // that will affect these
834 + this._cleanConditionally(articleContent, "table");
835 + this._cleanConditionally(articleContent, "ul");
836 + this._cleanConditionally(articleContent, "div");
837 +
838 + // replace H1 with H2 as H1 should be only title that is displayed separately
839 + this._replaceNodeTags(
840 + this._getAllNodesWithTag(articleContent, ["h1"]),
841 + "h2"
842 + );
843 +
844 + // Remove extra paragraphs
845 + this._removeNodes(
846 + this._getAllNodesWithTag(articleContent, ["p"]),
847 + function (paragraph) {
848 + // At this point, nasty iframes have been removed; only embedded video
849 + // ones remain.
850 + var contentElementCount = this._getAllNodesWithTag(paragraph, [
851 + "img",
852 + "embed",
853 + "object",
854 + "iframe",
855 + ]).length;
856 + return (
857 + contentElementCount === 0 && !this._getInnerText(paragraph, false)
858 + );
859 + }
860 + );
861 +
862 + this._forEachNode(
863 + this._getAllNodesWithTag(articleContent, ["br"]),
864 + function (br) {
865 + var next = this._nextNode(br.nextSibling);
866 + if (next && next.tagName == "P") {
867 + br.remove();
868 + }
869 + }
870 + );
871 +
872 + // Remove single-cell tables
873 + this._forEachNode(
874 + this._getAllNodesWithTag(articleContent, ["table"]),
875 + function (table) {
876 + var tbody = this._hasSingleTagInsideElement(table, "TBODY")
877 + ? table.firstElementChild
878 + : table;
879 + if (this._hasSingleTagInsideElement(tbody, "TR")) {
880 + var row = tbody.firstElementChild;
881 + if (this._hasSingleTagInsideElement(row, "TD")) {
882 + var cell = row.firstElementChild;
883 + cell = this._setNodeTag(
884 + cell,
885 + this._everyNode(cell.childNodes, this._isPhrasingContent)
886 + ? "P"
887 + : "DIV"
888 + );
889 + table.parentNode.replaceChild(cell, table);
890 + }
891 + }
892 + }
893 + );
894 + },
895 +
896 + /**
897 + * Initialize a node with the readability object. Also checks the
898 + * className/id for special names to add to its score.
899 + *
900 + * @param Element
901 + * @return void
902 + **/
903 + _initializeNode(node) {
904 + node.readability = { contentScore: 0 };
905 +
906 + switch (node.tagName) {
907 + case "DIV":
908 + node.readability.contentScore += 5;
909 + break;
910 +
911 + case "PRE":
912 + case "TD":
913 + case "BLOCKQUOTE":
914 + node.readability.contentScore += 3;
915 + break;
916 +
917 + case "ADDRESS":
918 + case "OL":
919 + case "UL":
920 + case "DL":
921 + case "DD":
922 + case "DT":
923 + case "LI":
924 + case "FORM":
925 + node.readability.contentScore -= 3;
926 + break;
927 +
928 + case "H1":
929 + case "H2":
930 + case "H3":
931 + case "H4":
932 + case "H5":
933 + case "H6":
934 + case "TH":
935 + node.readability.contentScore -= 5;
936 + break;
937 + }
938 +
939 + node.readability.contentScore += this._getClassWeight(node);
940 + },
941 +
942 + _removeAndGetNext(node) {
943 + var nextNode = this._getNextNode(node, true);
944 + node.remove();
945 + return nextNode;
946 + },
947 +
948 + /**
949 + * Traverse the DOM from node to node, starting at the node passed in.
950 + * Pass true for the second parameter to indicate this node itself
951 + * (and its kids) are going away, and we want the next node over.
952 + *
953 + * Calling this in a loop will traverse the DOM depth-first.
954 + *
955 + * @param {Element} node
956 + * @param {boolean} ignoreSelfAndKids
957 + * @return {Element}
958 + */
959 + _getNextNode(node, ignoreSelfAndKids) {
960 + // First check for kids if those aren't being ignored
961 + if (!ignoreSelfAndKids && node.firstElementChild) {
962 + return node.firstElementChild;
963 + }
964 + // Then for siblings...
965 + if (node.nextElementSibling) {
966 + return node.nextElementSibling;
967 + }
968 + // And finally, move up the parent chain *and* find a sibling
969 + // (because this is depth-first traversal, we will have already
970 + // seen the parent nodes themselves).
971 + do {
972 + node = node.parentNode;
973 + } while (node && !node.nextElementSibling);
974 + return node && node.nextElementSibling;
975 + },
976 +
977 + // compares second text to first one
978 + // 1 = same text, 0 = completely different text
979 + // works the way that it splits both texts into words and then finds words that are unique in second text
980 + // the result is given by the lower length of unique parts
981 + _textSimilarity(textA, textB) {
982 + var tokensA = textA
983 + .toLowerCase()
984 + .split(this.REGEXPS.tokenize)
985 + .filter(Boolean);
986 + var tokensB = textB
987 + .toLowerCase()
988 + .split(this.REGEXPS.tokenize)
989 + .filter(Boolean);
990 + if (!tokensA.length || !tokensB.length) {
991 + return 0;
992 + }
993 + var uniqTokensB = tokensB.filter(token => !tokensA.includes(token));
994 + var distanceB = uniqTokensB.join(" ").length / tokensB.join(" ").length;
995 + return 1 - distanceB;
996 + },
997 +
998 + /**
999 + * Checks whether an element node contains a valid byline
1000 + *
1001 + * @param node {Element}
1002 + * @param matchString {string}
1003 + * @return boolean
1004 + */
1005 + _isValidByline(node, matchString) {
1006 + var rel = node.getAttribute("rel");
1007 + var itemprop = node.getAttribute("itemprop");
1008 + var bylineLength = node.textContent.trim().length;
1009 +
1010 + return (
1011 + (rel === "author" ||
1012 + (itemprop && itemprop.includes("author")) ||
1013 + this.REGEXPS.byline.test(matchString)) &&
1014 + !!bylineLength &&
1015 + bylineLength < 100
1016 + );
1017 + },
1018 +
1019 + _getNodeAncestors(node, maxDepth) {
1020 + maxDepth = maxDepth || 0;
1021 + var i = 0,
1022 + ancestors = [];
1023 + while (node.parentNode) {
1024 + ancestors.push(node.parentNode);
1025 + if (maxDepth && ++i === maxDepth) {
1026 + break;
1027 + }
1028 + node = node.parentNode;
1029 + }
1030 + return ancestors;
1031 + },
1032 +
1033 + /***
1034 + * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is
1035 + * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.
1036 + *
1037 + * @param page a document to run upon. Needs to be a full document, complete with body.
1038 + * @return Element
1039 + **/
1040 + /* eslint-disable-next-line complexity */
1041 + _grabArticle(page) {
1042 + this.log("**** grabArticle ****");
1043 + var doc = this._doc;
1044 + var isPaging = page !== null;
1045 + page = page ? page : this._doc.body;
1046 +
1047 + // We can't grab an article if we don't have a page!
1048 + if (!page) {
1049 + this.log("No body found in document. Abort.");
1050 + return null;
1051 + }
1052 +
1053 + var pageCacheHtml = page.innerHTML;
1054 +
1055 + while (true) {
1056 + this.log("Starting grabArticle loop");
1057 + var stripUnlikelyCandidates = this._flagIsActive(
1058 + this.FLAG_STRIP_UNLIKELYS
1059 + );
1060 +
1061 + // First, node prepping. Trash nodes that look cruddy (like ones with the
1062 + // class name "comment", etc), and turn divs into P tags where they have been
1063 + // used inappropriately (as in, where they contain no other block level elements.)
1064 + var elementsToScore = [];
1065 + var node = this._doc.documentElement;
1066 +
1067 + let shouldRemoveTitleHeader = true;
1068 +
1069 + while (node) {
1070 + if (node.tagName === "HTML") {
1071 + this._articleLang = node.getAttribute("lang");
1072 + }
1073 +
1074 + var matchString = node.className + " " + node.id;
1075 +
1076 + if (!this._isProbablyVisible(node)) {
1077 + this.log("Removing hidden node - " + matchString);
1078 + node = this._removeAndGetNext(node);
1079 + continue;
1080 + }
1081 +
1082 + // User is not able to see elements applied with both "aria-modal = true" and "role = dialog"
1083 + if (
1084 + node.getAttribute("aria-modal") == "true" &&
1085 + node.getAttribute("role") == "dialog"
1086 + ) {
1087 + node = this._removeAndGetNext(node);
1088 + continue;
1089 + }
1090 +
1091 + // If we don't have a byline yet check to see if this node is a byline; if it is store the byline and remove the node.
1092 + if (
1093 + !this._articleByline &&
1094 + !this._metadata.byline &&
1095 + this._isValidByline(node, matchString)
1096 + ) {
1097 + // Find child node matching [itemprop="name"] and use that if it exists for a more accurate author name byline
1098 + var endOfSearchMarkerNode = this._getNextNode(node, true);
1099 + var next = this._getNextNode(node);
1100 + var itemPropNameNode = null;
1101 + while (next && next != endOfSearchMarkerNode) {
1102 + var itemprop = next.getAttribute("itemprop");
1103 + if (itemprop && itemprop.includes("name")) {
1104 + itemPropNameNode = next;
1105 + break;
1106 + } else {
1107 + next = this._getNextNode(next);
1108 + }
1109 + }
1110 + this._articleByline = (itemPropNameNode ?? node).textContent.trim();
1111 + node = this._removeAndGetNext(node);
1112 + continue;
1113 + }
1114 +
1115 + if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) {
1116 + this.log(
1117 + "Removing header: ",
1118 + node.textContent.trim(),
1119 + this._articleTitle.trim()
1120 + );
1121 + shouldRemoveTitleHeader = false;
1122 + node = this._removeAndGetNext(node);
1123 + continue;
1124 + }
1125 +
1126 + // Remove unlikely candidates
1127 + if (stripUnlikelyCandidates) {
1128 + if (
1129 + this.REGEXPS.unlikelyCandidates.test(matchString) &&
1130 + !this.REGEXPS.okMaybeItsACandidate.test(matchString) &&
1131 + !this._hasAncestorTag(node, "table") &&
1132 + !this._hasAncestorTag(node, "code") &&
1133 + node.tagName !== "BODY" &&
1134 + node.tagName !== "A"
1135 + ) {
1136 + this.log("Removing unlikely candidate - " + matchString);
1137 + node = this._removeAndGetNext(node);
1138 + continue;
1139 + }
1140 +
1141 + if (this.UNLIKELY_ROLES.includes(node.getAttribute("role"))) {
1142 + this.log(
1143 + "Removing content with role " +
1144 + node.getAttribute("role") +
1145 + " - " +
1146 + matchString
1147 + );
1148 + node = this._removeAndGetNext(node);
1149 + continue;
1150 + }
1151 + }
1152 +
1153 + // Remove DIV, SECTION, and HEADER nodes without any content(e.g. text, image, video, or iframe).
1154 + if (
1155 + (node.tagName === "DIV" ||
1156 + node.tagName === "SECTION" ||
1157 + node.tagName === "HEADER" ||
1158 + node.tagName === "H1" ||
1159 + node.tagName === "H2" ||
1160 + node.tagName === "H3" ||
1161 + node.tagName === "H4" ||
1162 + node.tagName === "H5" ||
1163 + node.tagName === "H6") &&
1164 + this._isElementWithoutContent(node)
1165 + ) {
1166 + node = this._removeAndGetNext(node);
1167 + continue;
1168 + }
1169 +
1170 + if (this.DEFAULT_TAGS_TO_SCORE.includes(node.tagName)) {
1171 + elementsToScore.push(node);
1172 + }
1173 +
1174 + // Turn all divs that don't have children block level elements into p's
1175 + if (node.tagName === "DIV") {
1176 + // Put phrasing content into paragraphs.
1177 + var childNode = node.firstChild;
1178 + while (childNode) {
1179 + var nextSibling = childNode.nextSibling;
1180 + if (this._isPhrasingContent(childNode)) {
1181 + var fragment = doc.createDocumentFragment();
1182 + // Collect all consecutive phrasing content into a fragment.
1183 + do {
1184 + nextSibling = childNode.nextSibling;
1185 + fragment.appendChild(childNode);
1186 + childNode = nextSibling;
1187 + } while (childNode && this._isPhrasingContent(childNode));
1188 +
1189 + // Trim leading and trailing whitespace from the fragment.
1190 + while (
1191 + fragment.firstChild &&
1192 + this._isWhitespace(fragment.firstChild)
1193 + ) {
1194 + fragment.firstChild.remove();
1195 + }
1196 + while (
1197 + fragment.lastChild &&
1198 + this._isWhitespace(fragment.lastChild)
1199 + ) {
1200 + fragment.lastChild.remove();
1201 + }
1202 +
1203 + // If the fragment contains anything, wrap it in a paragraph and
1204 + // insert it before the next non-phrasing node.
1205 + if (fragment.firstChild) {
1206 + var p = doc.createElement("p");
1207 + p.appendChild(fragment);
1208 + node.insertBefore(p, nextSibling);
1209 + }
1210 + }
1211 + childNode = nextSibling;
1212 + }
1213 +
1214 + // Sites like http://mobile.slate.com encloses each paragraph with a DIV
1215 + // element. DIVs with only a P element inside and no text content can be
1216 + // safely converted into plain P elements to avoid confusing the scoring
1217 + // algorithm with DIVs with are, in practice, paragraphs.
1218 + if (
1219 + this._hasSingleTagInsideElement(node, "P") &&
1220 + this._getLinkDensity(node) < 0.25
1221 + ) {
1222 + var newNode = node.children[0];
1223 + node.parentNode.replaceChild(newNode, node);
1224 + node = newNode;
1225 + elementsToScore.push(node);
1226 + } else if (!this._hasChildBlockElement(node)) {
1227 + node = this._setNodeTag(node, "P");
1228 + elementsToScore.push(node);
1229 + }
1230 + }
1231 + node = this._getNextNode(node);
1232 + }
1233 +
1234 + /**
1235 + * Loop through all paragraphs, and assign a score to them based on how content-y they look.
1236 + * Then add their score to their parent node.
1237 + *
1238 + * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.
1239 + **/
1240 + var candidates = [];
1241 + this._forEachNode(elementsToScore, function (elementToScore) {
1242 + if (
1243 + !elementToScore.parentNode ||
1244 + typeof elementToScore.parentNode.tagName === "undefined"
1245 + ) {
1246 + return;
1247 + }
1248 +
1249 + // If this paragraph is less than 25 characters, don't even count it.
1250 + var innerText = this._getInnerText(elementToScore);
1251 + if (innerText.length < 25) {
1252 + return;
1253 + }
1254 +
1255 + // Exclude nodes with no ancestor.
1256 + var ancestors = this._getNodeAncestors(elementToScore, 5);
1257 + if (ancestors.length === 0) {
1258 + return;
1259 + }
1260 +
1261 + var contentScore = 0;
1262 +
1263 + // Add a point for the paragraph itself as a base.
1264 + contentScore += 1;
1265 +
1266 + // Add points for any commas within this paragraph.
1267 + contentScore += innerText.split(this.REGEXPS.commas).length;
1268 +
1269 + // For every 100 characters in this paragraph, add another point. Up to 3 points.
1270 + contentScore += Math.min(Math.floor(innerText.length / 100), 3);
1271 +
1272 + // Initialize and score ancestors.
1273 + this._forEachNode(ancestors, function (ancestor, level) {
1274 + if (
1275 + !ancestor.tagName ||
1276 + !ancestor.parentNode ||
1277 + typeof ancestor.parentNode.tagName === "undefined"
1278 + ) {
1279 + return;
1280 + }
1281 +
1282 + if (typeof ancestor.readability === "undefined") {
1283 + this._initializeNode(ancestor);
1284 + candidates.push(ancestor);
1285 + }
1286 +
1287 + // Node score divider:
1288 + // - parent: 1 (no division)
1289 + // - grandparent: 2
1290 + // - great grandparent+: ancestor level * 3
1291 + if (level === 0) {
1292 + var scoreDivider = 1;
1293 + } else if (level === 1) {
1294 + scoreDivider = 2;
1295 + } else {
1296 + scoreDivider = level * 3;
1297 + }
1298 + ancestor.readability.contentScore += contentScore / scoreDivider;
1299 + });
1300 + });
1301 +
1302 + // After we've calculated scores, loop through all of the possible
1303 + // candidate nodes we found and find the one with the highest score.
1304 + var topCandidates = [];
1305 + for (var c = 0, cl = candidates.length; c < cl; c += 1) {
1306 + var candidate = candidates[c];
1307 +
1308 + // Scale the final candidates score based on link density. Good content
1309 + // should have a relatively small link density (5% or less) and be mostly
1310 + // unaffected by this operation.
1311 + var candidateScore =
1312 + candidate.readability.contentScore *
1313 + (1 - this._getLinkDensity(candidate));
1314 + candidate.readability.contentScore = candidateScore;
1315 +
1316 + this.log("Candidate:", candidate, "with score " + candidateScore);
1317 +
1318 + for (var t = 0; t < this._nbTopCandidates; t++) {
1319 + var aTopCandidate = topCandidates[t];
1320 +
1321 + if (
1322 + !aTopCandidate ||
1323 + candidateScore > aTopCandidate.readability.contentScore
1324 + ) {
1325 + topCandidates.splice(t, 0, candidate);
1326 + if (topCandidates.length > this._nbTopCandidates) {
1327 + topCandidates.pop();
1328 + }
1329 + break;
1330 + }
1331 + }
1332 + }
1333 +
1334 + var topCandidate = topCandidates[0] || null;
1335 + var neededToCreateTopCandidate = false;
1336 + var parentOfTopCandidate;
1337 +
1338 + // If we still have no top candidate, just use the body as a last resort.
1339 + // We also have to copy the body node so it is something we can modify.
1340 + if (topCandidate === null || topCandidate.tagName === "BODY") {
1341 + // Move all of the page's children into topCandidate
1342 + topCandidate = doc.createElement("DIV");
1343 + neededToCreateTopCandidate = true;
1344 + // Move everything (not just elements, also text nodes etc.) into the container
1345 + // so we even include text directly in the body:
1346 + while (page.firstChild) {
1347 + this.log("Moving child out:", page.firstChild);
1348 + topCandidate.appendChild(page.firstChild);
1349 + }
1350 +
1351 + page.appendChild(topCandidate);
1352 +
1353 + this._initializeNode(topCandidate);
1354 + } else if (topCandidate) {
1355 + // Find a better top candidate node if it contains (at least three) nodes which belong to `topCandidates` array
1356 + // and whose scores are quite closed with current `topCandidate` node.
1357 + var alternativeCandidateAncestors = [];
1358 + for (var i = 1; i < topCandidates.length; i++) {
1359 + if (
1360 + topCandidates[i].readability.contentScore /
1361 + topCandidate.readability.contentScore >=
1362 + 0.75
1363 + ) {
1364 + alternativeCandidateAncestors.push(
1365 + this._getNodeAncestors(topCandidates[i])
1366 + );
1367 + }
1368 + }
1369 + var MINIMUM_TOPCANDIDATES = 3;
1370 + if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) {
1371 + parentOfTopCandidate = topCandidate.parentNode;
1372 + while (parentOfTopCandidate.tagName !== "BODY") {
1373 + var listsContainingThisAncestor = 0;
1374 + for (
1375 + var ancestorIndex = 0;
1376 + ancestorIndex < alternativeCandidateAncestors.length &&
1377 + listsContainingThisAncestor < MINIMUM_TOPCANDIDATES;
1378 + ancestorIndex++
1379 + ) {
1380 + listsContainingThisAncestor += Number(
1381 + alternativeCandidateAncestors[ancestorIndex].includes(
1382 + parentOfTopCandidate
1383 + )
1384 + );
1385 + }
1386 + if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) {
1387 + topCandidate = parentOfTopCandidate;
1388 + break;
1389 + }
1390 + parentOfTopCandidate = parentOfTopCandidate.parentNode;
1391 + }
1392 + }
1393 + if (!topCandidate.readability) {
1394 + this._initializeNode(topCandidate);
1395 + }
1396 +
1397 + // Because of our bonus system, parents of candidates might have scores
1398 + // themselves. They get half of the node. There won't be nodes with higher
1399 + // scores than our topCandidate, but if we see the score going *up* in the first
1400 + // few steps up the tree, that's a decent sign that there might be more content
1401 + // lurking in other places that we want to unify in. The sibling stuff
1402 + // below does some of that - but only if we've looked high enough up the DOM
1403 + // tree.
1404 + parentOfTopCandidate = topCandidate.parentNode;
1405 + var lastScore = topCandidate.readability.contentScore;
1406 + // The scores shouldn't get too low.
1407 + var scoreThreshold = lastScore / 3;
1408 + while (parentOfTopCandidate.tagName !== "BODY") {
1409 + if (!parentOfTopCandidate.readability) {
1410 + parentOfTopCandidate = parentOfTopCandidate.parentNode;
1411 + continue;
1412 + }
1413 + var parentScore = parentOfTopCandidate.readability.contentScore;
1414 + if (parentScore < scoreThreshold) {
1415 + break;
1416 + }
1417 + if (parentScore > lastScore) {
1418 + // Alright! We found a better parent to use.
1419 + topCandidate = parentOfTopCandidate;
1420 + break;
1421 + }
1422 + lastScore = parentOfTopCandidate.readability.contentScore;
1423 + parentOfTopCandidate = parentOfTopCandidate.parentNode;
1424 + }
1425 +
1426 + // If the top candidate is the only child, use parent instead. This will help sibling
1427 + // joining logic when adjacent content is actually located in parent's sibling node.
1428 + parentOfTopCandidate = topCandidate.parentNode;
1429 + while (
1430 + parentOfTopCandidate.tagName != "BODY" &&
1431 + parentOfTopCandidate.children.length == 1
1432 + ) {
1433 + topCandidate = parentOfTopCandidate;
1434 + parentOfTopCandidate = topCandidate.parentNode;
1435 + }
1436 + if (!topCandidate.readability) {
1437 + this._initializeNode(topCandidate);
1438 + }
1439 + }
1440 +
1441 + // Now that we have the top candidate, look through its siblings for content
1442 + // that might also be related. Things like preambles, content split by ads
1443 + // that we removed, etc.
1444 + var articleContent = doc.createElement("DIV");
1445 + if (isPaging) {
1446 + articleContent.id = "readability-content";
1447 + }
1448 +
1449 + var siblingScoreThreshold = Math.max(
1450 + 10,
1451 + topCandidate.readability.contentScore * 0.2
1452 + );
1453 + // Keep potential top candidate's parent node to try to get text direction of it later.
1454 + parentOfTopCandidate = topCandidate.parentNode;
1455 + var siblings = parentOfTopCandidate.children;
1456 +
1457 + for (var s = 0, sl = siblings.length; s < sl; s++) {
1458 + var sibling = siblings[s];
1459 + var append = false;
1460 +
1461 + this.log(
1462 + "Looking at sibling node:",
1463 + sibling,
1464 + sibling.readability
1465 + ? "with score " + sibling.readability.contentScore
1466 + : ""
1467 + );
1468 + this.log(
1469 + "Sibling has score",
1470 + sibling.readability ? sibling.readability.contentScore : "Unknown"
1471 + );
1472 +
1473 + if (sibling === topCandidate) {
1474 + append = true;
1475 + } else {
1476 + var contentBonus = 0;
1477 +
1478 + // Give a bonus if sibling nodes and top candidates have the example same classname
1479 + if (
1480 + sibling.className === topCandidate.className &&
1481 + topCandidate.className !== ""
1482 + ) {
1483 + contentBonus += topCandidate.readability.contentScore * 0.2;
1484 + }
1485 +
1486 + if (
1487 + sibling.readability &&
1488 + sibling.readability.contentScore + contentBonus >=
1489 + siblingScoreThreshold
1490 + ) {
1491 + append = true;
1492 + } else if (sibling.nodeName === "P") {
1493 + var linkDensity = this._getLinkDensity(sibling);
1494 + var nodeContent = this._getInnerText(sibling);
1495 + var nodeLength = nodeContent.length;
1496 +
1497 + if (nodeLength > 80 && linkDensity < 0.25) {
1498 + append = true;
1499 + } else if (
1500 + nodeLength < 80 &&
1501 + nodeLength > 0 &&
1502 + linkDensity === 0 &&
1503 + nodeContent.search(/\.( |$)/) !== -1
1504 + ) {
1505 + append = true;
1506 + }
1507 + }
1508 + }
1509 +
1510 + if (append) {
1511 + this.log("Appending node:", sibling);
1512 +
1513 + if (!this.ALTER_TO_DIV_EXCEPTIONS.includes(sibling.nodeName)) {
1514 + // We have a node that isn't a common block level element, like a form or td tag.
1515 + // Turn it into a div so it doesn't get filtered out later by accident.
1516 + this.log("Altering sibling:", sibling, "to div.");
1517 +
1518 + sibling = this._setNodeTag(sibling, "DIV");
1519 + }
1520 +
1521 + articleContent.appendChild(sibling);
1522 + // Fetch children again to make it compatible
1523 + // with DOM parsers without live collection support.
1524 + siblings = parentOfTopCandidate.children;
1525 + // siblings is a reference to the children array, and
1526 + // sibling is removed from the array when we call appendChild().
1527 + // As a result, we must revisit this index since the nodes
1528 + // have been shifted.
1529 + s -= 1;
1530 + sl -= 1;
1531 + }
1532 + }
1533 +
1534 + if (this._debug) {
1535 + this.log("Article content pre-prep: " + articleContent.innerHTML);
1536 + }
1537 + // So we have all of the content that we need. Now we clean it up for presentation.
1538 + this._prepArticle(articleContent);
1539 + if (this._debug) {
1540 + this.log("Article content post-prep: " + articleContent.innerHTML);
1541 + }
1542 +
1543 + if (neededToCreateTopCandidate) {
1544 + // We already created a fake div thing, and there wouldn't have been any siblings left
1545 + // for the previous loop, so there's no point trying to create a new div, and then
1546 + // move all the children over. Just assign IDs and class names here. No need to append
1547 + // because that already happened anyway.
1548 + topCandidate.id = "readability-page-1";
1549 + topCandidate.className = "page";
1550 + } else {
1551 + var div = doc.createElement("DIV");
1552 + div.id = "readability-page-1";
1553 + div.className = "page";
1554 + while (articleContent.firstChild) {
1555 + div.appendChild(articleContent.firstChild);
1556 + }
1557 + articleContent.appendChild(div);
1558 + }
1559 +
1560 + if (this._debug) {
1561 + this.log("Article content after paging: " + articleContent.innerHTML);
1562 + }
1563 +
1564 + var parseSuccessful = true;
1565 +
1566 + // Now that we've gone through the full algorithm, check to see if
1567 + // we got any meaningful content. If we didn't, we may need to re-run
1568 + // grabArticle with different flags set. This gives us a higher likelihood of
1569 + // finding the content, and the sieve approach gives us a higher likelihood of
1570 + // finding the -right- content.
1571 + var textLength = this._getInnerText(articleContent, true).length;
1572 + if (textLength < this._charThreshold) {
1573 + parseSuccessful = false;
1574 + // eslint-disable-next-line no-unsanitized/property
1575 + page.innerHTML = pageCacheHtml;
1576 +
1577 + this._attempts.push({
1578 + articleContent,
1579 + textLength,
1580 + });
1581 +
1582 + if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) {
1583 + this._removeFlag(this.FLAG_STRIP_UNLIKELYS);
1584 + } else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {
1585 + this._removeFlag(this.FLAG_WEIGHT_CLASSES);
1586 + } else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {
1587 + this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);
1588 + } else {
1589 + // No luck after removing flags, just return the longest text we found during the different loops
1590 + this._attempts.sort(function (a, b) {
1591 + return b.textLength - a.textLength;
1592 + });
1593 +
1594 + // But first check if we actually have something
1595 + if (!this._attempts[0].textLength) {
1596 + return null;
1597 + }
1598 +
1599 + articleContent = this._attempts[0].articleContent;
1600 + parseSuccessful = true;
1601 + }
1602 + }
1603 +
1604 + if (parseSuccessful) {
1605 + // Find out text direction from ancestors of final top candidate.
1606 + var ancestors = [parentOfTopCandidate, topCandidate].concat(
1607 + this._getNodeAncestors(parentOfTopCandidate)
1608 + );
1609 + this._someNode(ancestors, function (ancestor) {
1610 + if (!ancestor.tagName) {
1611 + return false;
1612 + }
1613 + var articleDir = ancestor.getAttribute("dir");
1614 + if (articleDir) {
1615 + this._articleDir = articleDir;
1616 + return true;
1617 + }
1618 + return false;
1619 + });
1620 + return articleContent;
1621 + }
1622 + }
1623 + },
1624 +
1625 + /**
1626 + * Converts some of the common HTML entities in string to their corresponding characters.
1627 + *
1628 + * @param str {string} - a string to unescape.
1629 + * @return string without HTML entity.
1630 + */
1631 + _unescapeHtmlEntities(str) {
1632 + if (!str) {
1633 + return str;
1634 + }
1635 +
1636 + var htmlEscapeMap = this.HTML_ESCAPE_MAP;
1637 + return str
1638 + .replace(/&(quot|amp|apos|lt|gt);/g, function (_, tag) {
1639 + return htmlEscapeMap[tag];
1640 + })
1641 + .replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi, function (_, hex, numStr) {
1642 + var num = parseInt(hex || numStr, hex ? 16 : 10);
1643 +
1644 + // these character references are replaced by a conforming HTML parser
1645 + if (num == 0 || num > 0x10ffff || (num >= 0xd800 && num <= 0xdfff)) {
1646 + num = 0xfffd;
1647 + }
1648 +
1649 + return String.fromCodePoint(num);
1650 + });
1651 + },
1652 +
1653 + /**
1654 + * Try to extract metadata from JSON-LD object.
1655 + * For now, only Schema.org objects of type Article or its subtypes are supported.
1656 + * @return Object with any metadata that could be extracted (possibly none)
1657 + */
1658 + _getJSONLD(doc) {
1659 + var scripts = this._getAllNodesWithTag(doc, ["script"]);
1660 +
1661 + var metadata;
1662 +
1663 + this._forEachNode(scripts, function (jsonLdElement) {
1664 + if (
1665 + !metadata &&
1666 + jsonLdElement.getAttribute("type") === "application/ld+json"
1667 + ) {
1668 + try {
1669 + // Strip CDATA markers if present
1670 + var content = jsonLdElement.textContent.replace(
1671 + /^\s*<!\[CDATA\[|\]\]>\s*$/g,
1672 + ""
1673 + );
1674 + var parsed = JSON.parse(content);
1675 +
1676 + if (Array.isArray(parsed)) {
1677 + parsed = parsed.find(it => {
1678 + return (
1679 + it["@type"] &&
1680 + it["@type"].match(this.REGEXPS.jsonLdArticleTypes)
1681 + );
1682 + });
1683 + if (!parsed) {
1684 + return;
1685 + }
1686 + }
1687 +
1688 + var schemaDotOrgRegex = /^https?\:\/\/schema\.org\/?$/;
1689 + var matches =
1690 + (typeof parsed["@context"] === "string" &&
1691 + parsed["@context"].match(schemaDotOrgRegex)) ||
1692 + (typeof parsed["@context"] === "object" &&
1693 + typeof parsed["@context"]["@vocab"] == "string" &&
1694 + parsed["@context"]["@vocab"].match(schemaDotOrgRegex));
1695 +
1696 + if (!matches) {
1697 + return;
1698 + }
1699 +
1700 + if (!parsed["@type"] && Array.isArray(parsed["@graph"])) {
1701 + parsed = parsed["@graph"].find(it => {
1702 + return (it["@type"] || "").match(this.REGEXPS.jsonLdArticleTypes);
1703 + });
1704 + }
1705 +
1706 + if (
1707 + !parsed ||
1708 + !parsed["@type"] ||
1709 + !parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes)
1710 + ) {
1711 + return;
1712 + }
1713 +
1714 + metadata = {};
1715 +
1716 + if (
1717 + typeof parsed.name === "string" &&
1718 + typeof parsed.headline === "string" &&
1719 + parsed.name !== parsed.headline
1720 + ) {
1721 + // we have both name and headline element in the JSON-LD. They should both be the same but some websites like aktualne.cz
1722 + // put their own name into "name" and the article title to "headline" which confuses Readability. So we try to check if either
1723 + // "name" or "headline" closely matches the html title, and if so, use that one. If not, then we use "name" by default.
1724 +
1725 + var title = this._getArticleTitle();
1726 + var nameMatches = this._textSimilarity(parsed.name, title) > 0.75;
1727 + var headlineMatches =
1728 + this._textSimilarity(parsed.headline, title) > 0.75;
1729 +
1730 + if (headlineMatches && !nameMatches) {
1731 + metadata.title = parsed.headline;
1732 + } else {
1733 + metadata.title = parsed.name;
1734 + }
1735 + } else if (typeof parsed.name === "string") {
1736 + metadata.title = parsed.name.trim();
1737 + } else if (typeof parsed.headline === "string") {
1738 + metadata.title = parsed.headline.trim();
1739 + }
1740 + if (parsed.author) {
1741 + if (typeof parsed.author.name === "string") {
1742 + metadata.byline = parsed.author.name.trim();
1743 + } else if (
1744 + Array.isArray(parsed.author) &&
1745 + parsed.author[0] &&
1746 + typeof parsed.author[0].name === "string"
1747 + ) {
1748 + metadata.byline = parsed.author
1749 + .filter(function (author) {
1750 + return author && typeof author.name === "string";
1751 + })
1752 + .map(function (author) {
1753 + return author.name.trim();
1754 + })
1755 + .join(", ");
1756 + }
1757 + }
1758 + if (typeof parsed.description === "string") {
1759 + metadata.excerpt = parsed.description.trim();
1760 + }
1761 + if (parsed.publisher && typeof parsed.publisher.name === "string") {
1762 + metadata.siteName = parsed.publisher.name.trim();
1763 + }
1764 + if (typeof parsed.datePublished === "string") {
1765 + metadata.datePublished = parsed.datePublished.trim();
1766 + }
1767 + } catch (err) {
1768 + this.log(err.message);
1769 + }
1770 + }
1771 + });
1772 + return metadata ? metadata : {};
1773 + },
1774 +
1775 + /**
1776 + * Attempts to get excerpt and byline metadata for the article.
1777 + *
1778 + * @param {Object} jsonld — object containing any metadata that
1779 + * could be extracted from JSON-LD object.
1780 + *
1781 + * @return Object with optional "excerpt" and "byline" properties
1782 + */
1783 + _getArticleMetadata(jsonld) {
1784 + var metadata = {};
1785 + var values = {};
1786 + var metaElements = this._doc.getElementsByTagName("meta");
1787 +
1788 + // property is a space-separated list of values
1789 + var propertyPattern =
1790 + /\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi;
1791 +
1792 + // name is a single value
1793 + var namePattern =
1794 + /^\s*(?:(dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-\.:]\s*)?(author|creator|pub-date|description|title|site_name)\s*$/i;
1795 +
1796 + // Find description tags.
1797 + this._forEachNode(metaElements, function (element) {
1798 + var elementName = element.getAttribute("name");
1799 + var elementProperty = element.getAttribute("property");
1800 + var content = element.getAttribute("content");
1801 + if (!content) {
1802 + return;
1803 + }
1804 + var matches = null;
1805 + var name = null;
1806 +
1807 + if (elementProperty) {
1808 + matches = elementProperty.match(propertyPattern);
1809 + if (matches) {
1810 + // Convert to lowercase, and remove any whitespace
1811 + // so we can match below.
1812 + name = matches[0].toLowerCase().replace(/\s/g, "");
1813 + // multiple authors
1814 + values[name] = content.trim();
1815 + }
1816 + }
1817 + if (!matches && elementName && namePattern.test(elementName)) {
1818 + name = elementName;
1819 + if (content) {
1820 + // Convert to lowercase, remove any whitespace, and convert dots
1821 + // to colons so we can match below.
1822 + name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":");
1823 + values[name] = content.trim();
1824 + }
1825 + }
1826 + });
1827 +
1828 + // get title
1829 + metadata.title =
1830 + jsonld.title ||
1831 + values["dc:title"] ||
1832 + values["dcterm:title"] ||
1833 + values["og:title"] ||
1834 + values["weibo:article:title"] ||
1835 + values["weibo:webpage:title"] ||
1836 + values.title ||
1837 + values["twitter:title"] ||
1838 + values["parsely-title"];
1839 +
1840 + if (!metadata.title) {
1841 + metadata.title = this._getArticleTitle();
1842 + }
1843 +
1844 + const articleAuthor =
1845 + typeof values["article:author"] === "string" &&
1846 + !this._isUrl(values["article:author"])
1847 + ? values["article:author"]
1848 + : undefined;
1849 +
1850 + // get author
1851 + metadata.byline =
1852 + jsonld.byline ||
1853 + values["dc:creator"] ||
1854 + values["dcterm:creator"] ||
1855 + values.author ||
1856 + values["parsely-author"] ||
1857 + articleAuthor;
1858 +
1859 + // get description
1860 + metadata.excerpt =
1861 + jsonld.excerpt ||
1862 + values["dc:description"] ||
1863 + values["dcterm:description"] ||
1864 + values["og:description"] ||
1865 + values["weibo:article:description"] ||
1866 + values["weibo:webpage:description"] ||
1867 + values.description ||
1868 + values["twitter:description"];
1869 +
1870 + // get site name
1871 + metadata.siteName = jsonld.siteName || values["og:site_name"];
1872 +
1873 + // get article published time
1874 + metadata.publishedTime =
1875 + jsonld.datePublished ||
1876 + values["article:published_time"] ||
1877 + values["parsely-pub-date"] ||
1878 + null;
1879 +
1880 + // in many sites the meta value is escaped with HTML entities,
1881 + // so here we need to unescape it
1882 + metadata.title = this._unescapeHtmlEntities(metadata.title);
1883 + metadata.byline = this._unescapeHtmlEntities(metadata.byline);
1884 + metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);
1885 + metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);
1886 + metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime);
1887 +
1888 + return metadata;
1889 + },
1890 +
1891 + /**
1892 + * Check if node is image, or if node contains exactly only one image
1893 + * whether as a direct child or as its descendants.
1894 + *
1895 + * @param Element
1896 + **/
1897 + _isSingleImage(node) {
1898 + while (node) {
1899 + if (node.tagName === "IMG") {
1900 + return true;
1901 + }
1902 + if (node.children.length !== 1 || node.textContent.trim() !== "") {
1903 + return false;
1904 + }
1905 + node = node.children[0];
1906 + }
1907 + return false;
1908 + },
1909 +
1910 + /**
1911 + * Find all <noscript> that are located after <img> nodes, and which contain only one
1912 + * <img> element. Replace the first image with the image from inside the <noscript> tag,
1913 + * and remove the <noscript> tag. This improves the quality of the images we use on
1914 + * some sites (e.g. Medium).
1915 + *
1916 + * @param Element
1917 + **/
1918 + _unwrapNoscriptImages(doc) {
1919 + // Find img without source or attributes that might contains image, and remove it.
1920 + // This is done to prevent a placeholder img is replaced by img from noscript in next step.
1921 + var imgs = Array.from(doc.getElementsByTagName("img"));
1922 + this._forEachNode(imgs, function (img) {
1923 + for (var i = 0; i < img.attributes.length; i++) {
1924 + var attr = img.attributes[i];
1925 + switch (attr.name) {
1926 + case "src":
1927 + case "srcset":
1928 + case "data-src":
1929 + case "data-srcset":
1930 + return;
1931 + }
1932 +
1933 + if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
1934 + return;
1935 + }
1936 + }
1937 +
1938 + img.remove();
1939 + });
1940 +
1941 + // Next find noscript and try to extract its image
1942 + var noscripts = Array.from(doc.getElementsByTagName("noscript"));
1943 + this._forEachNode(noscripts, function (noscript) {
1944 + // Parse content of noscript and make sure it only contains image
1945 + if (!this._isSingleImage(noscript)) {
1946 + return;
1947 + }
1948 + var tmp = doc.createElement("div");
1949 + // We're running in the document context, and using unmodified
1950 + // document contents, so doing this should be safe.
1951 + // (Also we heavily discourage people from allowing script to
1952 + // run at all in this document...)
1953 + // eslint-disable-next-line no-unsanitized/property
1954 + tmp.innerHTML = noscript.innerHTML;
1955 +
1956 + // If noscript has previous sibling and it only contains image,
1957 + // replace it with noscript content. However we also keep old
1958 + // attributes that might contains image.
1959 + var prevElement = noscript.previousElementSibling;
1960 + if (prevElement && this._isSingleImage(prevElement)) {
1961 + var prevImg = prevElement;
1962 + if (prevImg.tagName !== "IMG") {
1963 + prevImg = prevElement.getElementsByTagName("img")[0];
1964 + }
1965 +
1966 + var newImg = tmp.getElementsByTagName("img")[0];
1967 + for (var i = 0; i < prevImg.attributes.length; i++) {
1968 + var attr = prevImg.attributes[i];
1969 + if (attr.value === "") {
1970 + continue;
1971 + }
1972 +
1973 + if (
1974 + attr.name === "src" ||
1975 + attr.name === "srcset" ||
1976 + /\.(jpg|jpeg|png|webp)/i.test(attr.value)
1977 + ) {
1978 + if (newImg.getAttribute(attr.name) === attr.value) {
1979 + continue;
1980 + }
1981 +
1982 + var attrName = attr.name;
1983 + if (newImg.hasAttribute(attrName)) {
1984 + attrName = "data-old-" + attrName;
1985 + }
1986 +
1987 + newImg.setAttribute(attrName, attr.value);
1988 + }
1989 + }
1990 +
1991 + noscript.parentNode.replaceChild(tmp.firstElementChild, prevElement);
1992 + }
1993 + });
1994 + },
1995 +
1996 + /**
1997 + * Removes script tags from the document.
1998 + *
1999 + * @param Element
2000 + **/
2001 + _removeScripts(doc) {
2002 + this._removeNodes(this._getAllNodesWithTag(doc, ["script", "noscript"]));
2003 + },
2004 +
2005 + /**
2006 + * Check if this node has only whitespace and a single element with given tag
2007 + * Returns false if the DIV node contains non-empty text nodes
2008 + * or if it contains no element with given tag or more than 1 element.
2009 + *
2010 + * @param Element
2011 + * @param string tag of child element
2012 + **/
2013 + _hasSingleTagInsideElement(element, tag) {
2014 + // There should be exactly 1 element child with given tag
2015 + if (element.children.length != 1 || element.children[0].tagName !== tag) {
2016 + return false;
2017 + }
2018 +
2019 + // And there should be no text nodes with real content
2020 + return !this._someNode(element.childNodes, function (node) {
2021 + return (
2022 + node.nodeType === this.TEXT_NODE &&
2023 + this.REGEXPS.hasContent.test(node.textContent)
2024 + );
2025 + });
2026 + },
2027 +
2028 + _isElementWithoutContent(node) {
2029 + return (
2030 + node.nodeType === this.ELEMENT_NODE &&
2031 + !node.textContent.trim().length &&
2032 + (!node.children.length ||
2033 + node.children.length ==
2034 + node.getElementsByTagName("br").length +
2035 + node.getElementsByTagName("hr").length)
2036 + );
2037 + },
2038 +
2039 + /**
2040 + * Determine whether element has any children block level elements.
2041 + *
2042 + * @param Element
2043 + */
2044 + _hasChildBlockElement(element) {
2045 + return this._someNode(element.childNodes, function (node) {
2046 + return (
2047 + this.DIV_TO_P_ELEMS.has(node.tagName) ||
2048 + this._hasChildBlockElement(node)
2049 + );
2050 + });
2051 + },
2052 +
2053 + /***
2054 + * Determine if a node qualifies as phrasing content.
2055 + * https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Content_categories#Phrasing_content
2056 + **/
2057 + _isPhrasingContent(node) {
2058 + return (
2059 + node.nodeType === this.TEXT_NODE ||
2060 + this.PHRASING_ELEMS.includes(node.tagName) ||
2061 + ((node.tagName === "A" ||
2062 + node.tagName === "DEL" ||
2063 + node.tagName === "INS") &&
2064 + this._everyNode(node.childNodes, this._isPhrasingContent))
2065 + );
2066 + },
2067 +
2068 + _isWhitespace(node) {
2069 + return (
2070 + (node.nodeType === this.TEXT_NODE &&
2071 + node.textContent.trim().length === 0) ||
2072 + (node.nodeType === this.ELEMENT_NODE && node.tagName === "BR")
2073 + );
2074 + },
2075 +
2076 + /**
2077 + * Get the inner text of a node - cross browser compatibly.
2078 + * This also strips out any excess whitespace to be found.
2079 + *
2080 + * @param Element
2081 + * @param Boolean normalizeSpaces (default: true)
2082 + * @return string
2083 + **/
2084 + _getInnerText(e, normalizeSpaces) {
2085 + normalizeSpaces =
2086 + typeof normalizeSpaces === "undefined" ? true : normalizeSpaces;
2087 + var textContent = e.textContent.trim();
2088 +
2089 + if (normalizeSpaces) {
2090 + return textContent.replace(this.REGEXPS.normalize, " ");
2091 + }
2092 + return textContent;
2093 + },
2094 +
2095 + /**
2096 + * Get the number of times a string s appears in the node e.
2097 + *
2098 + * @param Element
2099 + * @param string - what to split on. Default is ","
2100 + * @return number (integer)
2101 + **/
2102 + _getCharCount(e, s) {
2103 + s = s || ",";
2104 + return this._getInnerText(e).split(s).length - 1;
2105 + },
2106 +
2107 + /**
2108 + * Remove the style attribute on every e and under.
2109 + * TODO: Test if getElementsByTagName(*) is faster.
2110 + *
2111 + * @param Element
2112 + * @return void
2113 + **/
2114 + _cleanStyles(e) {
2115 + if (!e || e.tagName.toLowerCase() === "svg") {
2116 + return;
2117 + }
2118 +
2119 + // Remove `style` and deprecated presentational attributes
2120 + for (var i = 0; i < this.PRESENTATIONAL_ATTRIBUTES.length; i++) {
2121 + e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[i]);
2122 + }
2123 +
2124 + if (this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.includes(e.tagName)) {
2125 + e.removeAttribute("width");
2126 + e.removeAttribute("height");
2127 + }
2128 +
2129 + var cur = e.firstElementChild;
2130 + while (cur !== null) {
2131 + this._cleanStyles(cur);
2132 + cur = cur.nextElementSibling;
2133 + }
2134 + },
2135 +
2136 + /**
2137 + * Get the density of links as a percentage of the content
2138 + * This is the amount of text that is inside a link divided by the total text in the node.
2139 + *
2140 + * @param Element
2141 + * @return number (float)
2142 + **/
2143 + _getLinkDensity(element) {
2144 + var textLength = this._getInnerText(element).length;
2145 + if (textLength === 0) {
2146 + return 0;
2147 + }
2148 +
2149 + var linkLength = 0;
2150 +
2151 + // XXX implement _reduceNodeList?
2152 + this._forEachNode(element.getElementsByTagName("a"), function (linkNode) {
2153 + var href = linkNode.getAttribute("href");
2154 + var coefficient = href && this.REGEXPS.hashUrl.test(href) ? 0.3 : 1;
2155 + linkLength += this._getInnerText(linkNode).length * coefficient;
2156 + });
2157 +
2158 + return linkLength / textLength;
2159 + },
2160 +
2161 + /**
2162 + * Get an elements class/id weight. Uses regular expressions to tell if this
2163 + * element looks good or bad.
2164 + *
2165 + * @param Element
2166 + * @return number (Integer)
2167 + **/
2168 + _getClassWeight(e) {
2169 + if (!this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {
2170 + return 0;
2171 + }
2172 +
2173 + var weight = 0;
2174 +
2175 + // Look for a special classname
2176 + if (typeof e.className === "string" && e.className !== "") {
2177 + if (this.REGEXPS.negative.test(e.className)) {
2178 + weight -= 25;
2179 + }
2180 +
2181 + if (this.REGEXPS.positive.test(e.className)) {
2182 + weight += 25;
2183 + }
2184 + }
2185 +
2186 + // Look for a special ID
2187 + if (typeof e.id === "string" && e.id !== "") {
2188 + if (this.REGEXPS.negative.test(e.id)) {
2189 + weight -= 25;
2190 + }
2191 +
2192 + if (this.REGEXPS.positive.test(e.id)) {
2193 + weight += 25;
2194 + }
2195 + }
2196 +
2197 + return weight;
2198 + },
2199 +
2200 + /**
2201 + * Clean a node of all elements of type "tag".
2202 + * (Unless it's a youtube/vimeo video. People love movies.)
2203 + *
2204 + * @param Element
2205 + * @param string tag to clean
2206 + * @return void
2207 + **/
2208 + _clean(e, tag) {
2209 + var isEmbed = ["object", "embed", "iframe"].includes(tag);
2210 +
2211 + this._removeNodes(this._getAllNodesWithTag(e, [tag]), function (element) {
2212 + // Allow youtube and vimeo videos through as people usually want to see those.
2213 + if (isEmbed) {
2214 + // First, check the elements attributes to see if any of them contain youtube or vimeo
2215 + for (var i = 0; i < element.attributes.length; i++) {
2216 + if (this._allowedVideoRegex.test(element.attributes[i].value)) {
2217 + return false;
2218 + }
2219 + }
2220 +
2221 + // For embed with <object> tag, check inner HTML as well.
2222 + if (
2223 + element.tagName === "object" &&
2224 + this._allowedVideoRegex.test(element.innerHTML)
2225 + ) {
2226 + return false;
2227 + }
2228 + }
2229 +
2230 + return true;
2231 + });
2232 + },
2233 +
2234 + /**
2235 + * Check if a given node has one of its ancestor tag name matching the
2236 + * provided one.
2237 + * @param HTMLElement node
2238 + * @param String tagName
2239 + * @param Number maxDepth
2240 + * @param Function filterFn a filter to invoke to determine whether this node 'counts'
2241 + * @return Boolean
2242 + */
2243 + _hasAncestorTag(node, tagName, maxDepth, filterFn) {
2244 + maxDepth = maxDepth || 3;
2245 + tagName = tagName.toUpperCase();
2246 + var depth = 0;
2247 + while (node.parentNode) {
2248 + if (maxDepth > 0 && depth > maxDepth) {
2249 + return false;
2250 + }
2251 + if (
2252 + node.parentNode.tagName === tagName &&
2253 + (!filterFn || filterFn(node.parentNode))
2254 + ) {
2255 + return true;
2256 + }
2257 + node = node.parentNode;
2258 + depth++;
2259 + }
2260 + return false;
2261 + },
2262 +
2263 + /**
2264 + * Return an object indicating how many rows and columns this table has.
2265 + */
2266 + _getRowAndColumnCount(table) {
2267 + var rows = 0;
2268 + var columns = 0;
2269 + var trs = table.getElementsByTagName("tr");
2270 + for (var i = 0; i < trs.length; i++) {
2271 + var rowspan = trs[i].getAttribute("rowspan") || 0;
2272 + if (rowspan) {
2273 + rowspan = parseInt(rowspan, 10);
2274 + }
2275 + rows += rowspan || 1;
2276 +
2277 + // Now look for column-related info
2278 + var columnsInThisRow = 0;
2279 + var cells = trs[i].getElementsByTagName("td");
2280 + for (var j = 0; j < cells.length; j++) {
2281 + var colspan = cells[j].getAttribute("colspan") || 0;
2282 + if (colspan) {
2283 + colspan = parseInt(colspan, 10);
2284 + }
2285 + columnsInThisRow += colspan || 1;
2286 + }
2287 + columns = Math.max(columns, columnsInThisRow);
2288 + }
2289 + return { rows, columns };
2290 + },
2291 +
2292 + /**
2293 + * Look for 'data' (as opposed to 'layout') tables, for which we use
2294 + * similar checks as
2295 + * https://searchfox.org/mozilla-central/rev/f82d5c549f046cb64ce5602bfd894b7ae807c8f8/accessible/generic/TableAccessible.cpp#19
2296 + */
2297 + _markDataTables(root) {
2298 + var tables = root.getElementsByTagName("table");
2299 + for (var i = 0; i < tables.length; i++) {
2300 + var table = tables[i];
2301 + var role = table.getAttribute("role");
2302 + if (role == "presentation") {
2303 + table._readabilityDataTable = false;
2304 + continue;
2305 + }
2306 + var datatable = table.getAttribute("datatable");
2307 + if (datatable == "0") {
2308 + table._readabilityDataTable = false;
2309 + continue;
2310 + }
2311 + var summary = table.getAttribute("summary");
2312 + if (summary) {
2313 + table._readabilityDataTable = true;
2314 + continue;
2315 + }
2316 +
2317 + var caption = table.getElementsByTagName("caption")[0];
2318 + if (caption && caption.childNodes.length) {
2319 + table._readabilityDataTable = true;
2320 + continue;
2321 + }
2322 +
2323 + // If the table has a descendant with any of these tags, consider a data table:
2324 + var dataTableDescendants = ["col", "colgroup", "tfoot", "thead", "th"];
2325 + var descendantExists = function (tag) {
2326 + return !!table.getElementsByTagName(tag)[0];
2327 + };
2328 + if (dataTableDescendants.some(descendantExists)) {
2329 + this.log("Data table because found data-y descendant");
2330 + table._readabilityDataTable = true;
2331 + continue;
2332 + }
2333 +
2334 + // Nested tables indicate a layout table:
2335 + if (table.getElementsByTagName("table")[0]) {
2336 + table._readabilityDataTable = false;
2337 + continue;
2338 + }
2339 +
2340 + var sizeInfo = this._getRowAndColumnCount(table);
2341 +
2342 + if (sizeInfo.columns == 1 || sizeInfo.rows == 1) {
2343 + // single colum/row tables are commonly used for page layout purposes.
2344 + table._readabilityDataTable = false;
2345 + continue;
2346 + }
2347 +
2348 + if (sizeInfo.rows >= 10 || sizeInfo.columns > 4) {
2349 + table._readabilityDataTable = true;
2350 + continue;
2351 + }
2352 + // Now just go by size entirely:
2353 + table._readabilityDataTable = sizeInfo.rows * sizeInfo.columns > 10;
2354 + }
2355 + },
2356 +
2357 + /* convert images and figures that have properties like data-src into images that can be loaded without JS */
2358 + _fixLazyImages(root) {
2359 + this._forEachNode(
2360 + this._getAllNodesWithTag(root, ["img", "picture", "figure"]),
2361 + function (elem) {
2362 + // In some sites (e.g. Kotaku), they put 1px square image as base64 data uri in the src attribute.
2363 + // So, here we check if the data uri is too short, just might as well remove it.
2364 + if (elem.src && this.REGEXPS.b64DataUrl.test(elem.src)) {
2365 + // Make sure it's not SVG, because SVG can have a meaningful image in under 133 bytes.
2366 + var parts = this.REGEXPS.b64DataUrl.exec(elem.src);
2367 + if (parts[1] === "image/svg+xml") {
2368 + return;
2369 + }
2370 +
2371 + // Make sure this element has other attributes which contains image.
2372 + // If it doesn't, then this src is important and shouldn't be removed.
2373 + var srcCouldBeRemoved = false;
2374 + for (var i = 0; i < elem.attributes.length; i++) {
2375 + var attr = elem.attributes[i];
2376 + if (attr.name === "src") {
2377 + continue;
2378 + }
2379 +
2380 + if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) {
2381 + srcCouldBeRemoved = true;
2382 + break;
2383 + }
2384 + }
2385 +
2386 + // Here we assume if image is less than 100 bytes (or 133 after encoded to base64)
2387 + // it will be too small, therefore it might be placeholder image.
2388 + if (srcCouldBeRemoved) {
2389 + var b64starts = parts[0].length;
2390 + var b64length = elem.src.length - b64starts;
2391 + if (b64length < 133) {
2392 + elem.removeAttribute("src");
2393 + }
2394 + }
2395 + }
2396 +
2397 + // also check for "null" to work around https://github.com/jsdom/jsdom/issues/2580
2398 + if (
2399 + (elem.src || (elem.srcset && elem.srcset != "null")) &&
2400 + !elem.className.toLowerCase().includes("lazy")
2401 + ) {
2402 + return;
2403 + }
2404 +
2405 + for (var j = 0; j < elem.attributes.length; j++) {
2406 + attr = elem.attributes[j];
2407 + if (
2408 + attr.name === "src" ||
2409 + attr.name === "srcset" ||
2410 + attr.name === "alt"
2411 + ) {
2412 + continue;
2413 + }
2414 + var copyTo = null;
2415 + if (/\.(jpg|jpeg|png|webp)\s+\d/.test(attr.value)) {
2416 + copyTo = "srcset";
2417 + } else if (/^\s*\S+\.(jpg|jpeg|png|webp)\S*\s*$/.test(attr.value)) {
2418 + copyTo = "src";
2419 + }
2420 + if (copyTo) {
2421 + //if this is an img or picture, set the attribute directly
2422 + if (elem.tagName === "IMG" || elem.tagName === "PICTURE") {
2423 + elem.setAttribute(copyTo, attr.value);
2424 + } else if (
2425 + elem.tagName === "FIGURE" &&
2426 + !this._getAllNodesWithTag(elem, ["img", "picture"]).length
2427 + ) {
2428 + //if the item is a <figure> that does not contain an image or picture, create one and place it inside the figure
2429 + //see the nytimes-3 testcase for an example
2430 + var img = this._doc.createElement("img");
2431 + img.setAttribute(copyTo, attr.value);
2432 + elem.appendChild(img);
2433 + }
2434 + }
2435 + }
2436 + }
2437 + );
2438 + },
2439 +
2440 + _getTextDensity(e, tags) {
2441 + var textLength = this._getInnerText(e, true).length;
2442 + if (textLength === 0) {
2443 + return 0;
2444 + }
2445 + var childrenLength = 0;
2446 + var children = this._getAllNodesWithTag(e, tags);
2447 + this._forEachNode(
2448 + children,
2449 + child => (childrenLength += this._getInnerText(child, true).length)
2450 + );
2451 + return childrenLength / textLength;
2452 + },
2453 +
2454 + /**
2455 + * Clean an element of all tags of type "tag" if they look fishy.
2456 + * "Fishy" is an algorithm based on content length, classnames, link density, number of images & embeds, etc.
2457 + *
2458 + * @return void
2459 + **/
2460 + _cleanConditionally(e, tag) {
2461 + if (!this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {
2462 + return;
2463 + }
2464 +
2465 + // Gather counts for other typical elements embedded within.
2466 + // Traverse backwards so we can remove nodes at the same time
2467 + // without effecting the traversal.
2468 + //
2469 + // TODO: Consider taking into account original contentScore here.
2470 + this._removeNodes(this._getAllNodesWithTag(e, [tag]), function (node) {
2471 + // First check if this node IS data table, in which case don't remove it.
2472 + var isDataTable = function (t) {
2473 + return t._readabilityDataTable;
2474 + };
2475 +
2476 + var isList = tag === "ul" || tag === "ol";
2477 + if (!isList) {
2478 + var listLength = 0;
2479 + var listNodes = this._getAllNodesWithTag(node, ["ul", "ol"]);
2480 + this._forEachNode(
2481 + listNodes,
2482 + list => (listLength += this._getInnerText(list).length)
2483 + );
2484 + isList = listLength / this._getInnerText(node).length > 0.9;
2485 + }
2486 +
2487 + if (tag === "table" && isDataTable(node)) {
2488 + return false;
2489 + }
2490 +
2491 + // Next check if we're inside a data table, in which case don't remove it as well.
2492 + if (this._hasAncestorTag(node, "table", -1, isDataTable)) {
2493 + return false;
2494 + }
2495 +
2496 + if (this._hasAncestorTag(node, "code")) {
2497 + return false;
2498 + }
2499 +
2500 + // keep element if it has a data tables
2501 + if (
2502 + [...node.getElementsByTagName("table")].some(
2503 + tbl => tbl._readabilityDataTable
2504 + )
2505 + ) {
2506 + return false;
2507 + }
2508 +
2509 + var weight = this._getClassWeight(node);
2510 +
2511 + this.log("Cleaning Conditionally", node);
2512 +
2513 + var contentScore = 0;
2514 +
2515 + if (weight + contentScore < 0) {
2516 + return true;
2517 + }
2518 +
2519 + if (this._getCharCount(node, ",") < 10) {
2520 + // If there are not very many commas, and the number of
2521 + // non-paragraph elements is more than paragraphs or other
2522 + // ominous signs, remove the element.
2523 + var p = node.getElementsByTagName("p").length;
2524 + var img = node.getElementsByTagName("img").length;
2525 + var li = node.getElementsByTagName("li").length - 100;
2526 + var input = node.getElementsByTagName("input").length;
2527 + var headingDensity = this._getTextDensity(node, [
2528 + "h1",
2529 + "h2",
2530 + "h3",
2531 + "h4",
2532 + "h5",
2533 + "h6",
2534 + ]);
2535 +
2536 + var embedCount = 0;
2537 + var embeds = this._getAllNodesWithTag(node, [
2538 + "object",
2539 + "embed",
2540 + "iframe",
2541 + ]);
2542 +
2543 + for (var i = 0; i < embeds.length; i++) {
2544 + // If this embed has attribute that matches video regex, don't delete it.
2545 + for (var j = 0; j < embeds[i].attributes.length; j++) {
2546 + if (this._allowedVideoRegex.test(embeds[i].attributes[j].value)) {
2547 + return false;
2548 + }
2549 + }
2550 +
2551 + // For embed with <object> tag, check inner HTML as well.
2552 + if (
2553 + embeds[i].tagName === "object" &&
2554 + this._allowedVideoRegex.test(embeds[i].innerHTML)
2555 + ) {
2556 + return false;
2557 + }
2558 +
2559 + embedCount++;
2560 + }
2561 +
2562 + var innerText = this._getInnerText(node);
2563 +
2564 + // toss any node whose inner text contains nothing but suspicious words
2565 + if (
2566 + this.REGEXPS.adWords.test(innerText) ||
2567 + this.REGEXPS.loadingWords.test(innerText)
2568 + ) {
2569 + return true;
2570 + }
2571 +
2572 + var contentLength = innerText.length;
2573 + var linkDensity = this._getLinkDensity(node);
2574 + var textishTags = ["SPAN", "LI", "TD"].concat(
2575 + Array.from(this.DIV_TO_P_ELEMS)
2576 + );
2577 + var textDensity = this._getTextDensity(node, textishTags);
2578 + var isFigureChild = this._hasAncestorTag(node, "figure");
2579 +
2580 + // apply shadiness checks, then check for exceptions
2581 + const shouldRemoveNode = () => {
2582 + const errs = [];
2583 + if (!isFigureChild && img > 1 && p / img < 0.5) {
2584 + errs.push(`Bad p to img ratio (img=${img}, p=${p})`);
2585 + }
2586 + if (!isList && li > p) {
2587 + errs.push(`Too many li's outside of a list. (li=${li} > p=${p})`);
2588 + }
2589 + if (input > Math.floor(p / 3)) {
2590 + errs.push(`Too many inputs per p. (input=${input}, p=${p})`);
2591 + }
2592 + if (
2593 + !isList &&
2594 + !isFigureChild &&
2595 + headingDensity < 0.9 &&
2596 + contentLength < 25 &&
2597 + (img === 0 || img > 2) &&
2598 + linkDensity > 0
2599 + ) {
2600 + errs.push(
2601 + `Suspiciously short. (headingDensity=${headingDensity}, img=${img}, linkDensity=${linkDensity})`
2602 + );
2603 + }
2604 + if (
2605 + !isList &&
2606 + weight < 25 &&
2607 + linkDensity > 0.2 + this._linkDensityModifier
2608 + ) {
2609 + errs.push(
2610 + `Low weight and a little linky. (linkDensity=${linkDensity})`
2611 + );
2612 + }
2613 + if (weight >= 25 && linkDensity > 0.5 + this._linkDensityModifier) {
2614 + errs.push(
2615 + `High weight and mostly links. (linkDensity=${linkDensity})`
2616 + );
2617 + }
2618 + if ((embedCount === 1 && contentLength < 75) || embedCount > 1) {
2619 + errs.push(
2620 + `Suspicious embed. (embedCount=${embedCount}, contentLength=${contentLength})`
2621 + );
2622 + }
2623 + if (img === 0 && textDensity === 0) {
2624 + errs.push(
2625 + `No useful content. (img=${img}, textDensity=${textDensity})`
2626 + );
2627 + }
2628 +
2629 + if (errs.length) {
2630 + this.log("Checks failed", errs);
2631 + return true;
2632 + }
2633 +
2634 + return false;
2635 + };
2636 +
2637 + var haveToRemove = shouldRemoveNode();
2638 +
2639 + // Allow simple lists of images to remain in pages
2640 + if (isList && haveToRemove) {
2641 + for (var x = 0; x < node.children.length; x++) {
2642 + let child = node.children[x];
2643 + // Don't filter in lists with li's that contain more than one child
2644 + if (child.children.length > 1) {
2645 + return haveToRemove;
2646 + }
2647 + }
2648 + let li_count = node.getElementsByTagName("li").length;
2649 + // Only allow the list to remain if every li contains an image
2650 + if (img == li_count) {
2651 + return false;
2652 + }
2653 + }
2654 + return haveToRemove;
2655 + }
2656 + return false;
2657 + });
2658 + },
2659 +
2660 + /**
2661 + * Clean out elements that match the specified conditions
2662 + *
2663 + * @param Element
2664 + * @param Function determines whether a node should be removed
2665 + * @return void
2666 + **/
2667 + _cleanMatchedNodes(e, filter) {
2668 + var endOfSearchMarkerNode = this._getNextNode(e, true);
2669 + var next = this._getNextNode(e);
2670 + while (next && next != endOfSearchMarkerNode) {
2671 + if (filter.call(this, next, next.className + " " + next.id)) {
2672 + next = this._removeAndGetNext(next);
2673 + } else {
2674 + next = this._getNextNode(next);
2675 + }
2676 + }
2677 + },
2678 +
2679 + /**
2680 + * Clean out spurious headers from an Element.
2681 + *
2682 + * @param Element
2683 + * @return void
2684 + **/
2685 + _cleanHeaders(e) {
2686 + let headingNodes = this._getAllNodesWithTag(e, ["h1", "h2"]);
2687 + this._removeNodes(headingNodes, function (node) {
2688 + let shouldRemove = this._getClassWeight(node) < 0;
2689 + if (shouldRemove) {
2690 + this.log("Removing header with low class weight:", node);
2691 + }
2692 + return shouldRemove;
2693 + });
2694 + },
2695 +
2696 + /**
2697 + * Check if this node is an H1 or H2 element whose content is mostly
2698 + * the same as the article title.
2699 + *
2700 + * @param Element the node to check.
2701 + * @return boolean indicating whether this is a title-like header.
2702 + */
2703 + _headerDuplicatesTitle(node) {
2704 + if (node.tagName != "H1" && node.tagName != "H2") {
2705 + return false;
2706 + }
2707 + var heading = this._getInnerText(node, false);
2708 + this.log("Evaluating similarity of header:", heading, this._articleTitle);
2709 + return this._textSimilarity(this._articleTitle, heading) > 0.75;
2710 + },
2711 +
2712 + _flagIsActive(flag) {
2713 + return (this._flags & flag) > 0;
2714 + },
2715 +
2716 + _removeFlag(flag) {
2717 + this._flags = this._flags & ~flag;
2718 + },
2719 +
2720 + _isProbablyVisible(node) {
2721 + // Have to null-check node.style and node.className.includes to deal with SVG and MathML nodes.
2722 + return (
2723 + (!node.style || node.style.display != "none") &&
2724 + (!node.style || node.style.visibility != "hidden") &&
2725 + !node.hasAttribute("hidden") &&
2726 + //check for "fallback-image" so that wikimedia math images are displayed
2727 + (!node.hasAttribute("aria-hidden") ||
2728 + node.getAttribute("aria-hidden") != "true" ||
2729 + (node.className &&
2730 + node.className.includes &&
2731 + node.className.includes("fallback-image")))
2732 + );
2733 + },
2734 +
2735 + /**
2736 + * Runs readability.
2737 + *
2738 + * Workflow:
2739 + * 1. Prep the document by removing script tags, css, etc.
2740 + * 2. Build readability's DOM tree.
2741 + * 3. Grab the article content from the current dom tree.
2742 + * 4. Replace the current DOM tree with the new one.
2743 + * 5. Read peacefully.
2744 + *
2745 + * @return void
2746 + **/
2747 + parse() {
2748 + // Avoid parsing too large documents, as per configuration option
2749 + if (this._maxElemsToParse > 0) {
2750 + var numTags = this._doc.getElementsByTagName("*").length;
2751 + if (numTags > this._maxElemsToParse) {
2752 + throw new Error(
2753 + "Aborting parsing document; " + numTags + " elements found"
2754 + );
2755 + }
2756 + }
2757 +
2758 + // Unwrap image from noscript
2759 + this._unwrapNoscriptImages(this._doc);
2760 +
2761 + // Extract JSON-LD metadata before removing scripts
2762 + var jsonLd = this._disableJSONLD ? {} : this._getJSONLD(this._doc);
2763 +
2764 + // Remove script tags from the document.
2765 + this._removeScripts(this._doc);
2766 +
2767 + this._prepDocument();
2768 +
2769 + var metadata = this._getArticleMetadata(jsonLd);
2770 + this._metadata = metadata;
2771 + this._articleTitle = metadata.title;
2772 +
2773 + var articleContent = this._grabArticle();
2774 + if (!articleContent) {
2775 + return null;
2776 + }
2777 +
2778 + this.log("Grabbed: " + articleContent.innerHTML);
2779 +
2780 + this._postProcessContent(articleContent);
2781 +
2782 + // If we haven't found an excerpt in the article's metadata, use the article's
2783 + // first paragraph as the excerpt. This is used for displaying a preview of
2784 + // the article's content.
2785 + if (!metadata.excerpt) {
2786 + var paragraphs = articleContent.getElementsByTagName("p");
2787 + if (paragraphs.length) {
2788 + metadata.excerpt = paragraphs[0].textContent.trim();
2789 + }
2790 + }
2791 +
2792 + var textContent = articleContent.textContent;
2793 + return {
2794 + title: this._articleTitle,
2795 + byline: metadata.byline || this._articleByline,
2796 + dir: this._articleDir,
2797 + lang: this._articleLang,
2798 + content: this._serializer(articleContent),
2799 + textContent,
2800 + length: textContent.length,
2801 + excerpt: metadata.excerpt,
2802 + siteName: metadata.siteName || this._articleSiteName,
2803 + publishedTime: metadata.publishedTime,
2804 + };
2805 + },
2806 +};
2807 +
2808 +if (typeof module === "object") {
2809 + /* eslint-disable-next-line no-redeclare */
2810 + /* global module */
2811 + module.exports = Readability;
2812 +}
2813