spb/zyquo-atlas Public License
The AI-native macOS web browser — every surface, intelligent.
Swift 75.2%
JavaScript 22%
Shell 2%
Makefile 0.9%
1/*2 * Copyright (c) 2010 Arc90 Inc3 *4 * Licensed under the Apache License, Version 2.0 (the "License");5 * you may not use this file except in compliance with the License.6 * You may obtain a copy of the License at7 *8 * http://www.apache.org/licenses/LICENSE-2.09 *10 * Unless required by applicable law or agreed to in writing, software11 * distributed under the License is distributed on an "AS IS" BASIS,12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13 * See the License for the specific language governing permissions and14 * limitations under the License.15 */1617/*18 * This code is heavily based on Arc90's readability.js (1.7.1) script19 * available at: http://code.google.com/p/arc90labs-readability20 */2122/**23 * Public constructor.24 * @param {HTMLDocument} doc The document to parse.25 * @param {Object} options The options object.26 */27function Readability(doc, options) {28 // In some older versions, people passed a URI as the first argument. Cope:29 if (options && options.documentElement) {30 doc = options;31 options = arguments[2];32 } else if (!doc || !doc.documentElement) {33 throw new Error(34 "First argument to Readability constructor should be a document object."35 );36 }37 options = options || {};3839 this._doc = doc;40 this._docJSDOMParser = this._doc.firstChild.__JSDOMParser__;41 this._articleTitle = null;42 this._articleByline = null;43 this._articleDir = null;44 this._articleSiteName = null;45 this._attempts = [];46 this._metadata = {};4748 // Configurable options49 this._debug = !!options.debug;50 this._maxElemsToParse =51 options.maxElemsToParse || this.DEFAULT_MAX_ELEMS_TO_PARSE;52 this._nbTopCandidates =53 options.nbTopCandidates || this.DEFAULT_N_TOP_CANDIDATES;54 this._charThreshold = options.charThreshold || this.DEFAULT_CHAR_THRESHOLD;55 this._classesToPreserve = this.CLASSES_TO_PRESERVE.concat(56 options.classesToPreserve || []57 );58 this._keepClasses = !!options.keepClasses;59 this._serializer =60 options.serializer ||61 function (el) {62 return el.innerHTML;63 };64 this._disableJSONLD = !!options.disableJSONLD;65 this._allowedVideoRegex = options.allowedVideoRegex || this.REGEXPS.videos;66 this._linkDensityModifier = options.linkDensityModifier || 0;6768 // Start with all flags set69 this._flags =70 this.FLAG_STRIP_UNLIKELYS |71 this.FLAG_WEIGHT_CLASSES |72 this.FLAG_CLEAN_CONDITIONALLY;7374 // Control whether log messages are sent to the console75 if (this._debug) {76 let logNode = function (node) {77 if (node.nodeType == node.TEXT_NODE) {78 return `${node.nodeName} ("${node.textContent}")`;79 }80 let attrPairs = Array.from(node.attributes || [], function (attr) {81 return `${attr.name}="${attr.value}"`;82 }).join(" ");83 return `<${node.localName} ${attrPairs}>`;84 };85 this.log = function () {86 if (typeof console !== "undefined") {87 let args = Array.from(arguments, arg => {88 if (arg && arg.nodeType == this.ELEMENT_NODE) {89 return logNode(arg);90 }91 return arg;92 });93 args.unshift("Reader: (Readability)");94 // eslint-disable-next-line no-console95 console.log(...args);96 } else if (typeof dump !== "undefined") {97 /* global dump */98 var msg = Array.prototype.map99 .call(arguments, function (x) {100 return x && x.nodeName ? logNode(x) : x;101 })102 .join(" ");103 dump("Reader: (Readability) " + msg + "\n");104 }105 };106 } else {107 this.log = function () {};108 }109}110111Readability.prototype = {112 FLAG_STRIP_UNLIKELYS: 0x1,113 FLAG_WEIGHT_CLASSES: 0x2,114 FLAG_CLEAN_CONDITIONALLY: 0x4,115116 // https://developer.mozilla.org/en-US/docs/Web/API/Node/nodeType117 ELEMENT_NODE: 1,118 TEXT_NODE: 3,119120 // Max number of nodes supported by this parser. Default: 0 (no limit)121 DEFAULT_MAX_ELEMS_TO_PARSE: 0,122123 // The number of top candidates to consider when analysing how124 // tight the competition is among candidates.125 DEFAULT_N_TOP_CANDIDATES: 5,126127 // Element tags to score by default.128 DEFAULT_TAGS_TO_SCORE: "section,h2,h3,h4,h5,h6,p,td,pre"129 .toUpperCase()130 .split(","),131132 // The default number of chars an article must have in order to return a result133 DEFAULT_CHAR_THRESHOLD: 500,134135 // All of the regular expressions in use within readability.136 // Defined up here so we don't instantiate them repeatedly in loops.137 REGEXPS: {138 // NOTE: These two regular expressions are duplicated in139 // Readability-readerable.js. Please keep both copies in sync.140 unlikelyCandidates:141 /-ad-|ai2html|banner|breadcrumbs|combx|comment|community|cover-wrap|disqus|extra|footer|gdpr|header|legends|menu|related|remark|replies|rss|shoutbox|sidebar|skyscraper|social|sponsor|supplemental|ad-break|agegate|pagination|pager|popup|yom-remote/i,142 okMaybeItsACandidate:143 /and|article|body|column|content|main|mathjax|shadow/i,144145 positive:146 /article|body|content|entry|hentry|h-entry|main|page|pagination|post|text|blog|story/i,147 negative:148 /-ad-|hidden|^hid$| hid$| hid |^hid |banner|combx|comment|com-|contact|footer|gdpr|masthead|media|meta|outbrain|promo|related|scroll|share|shoutbox|sidebar|skyscraper|sponsor|shopping|tags|widget/i,149 extraneous:150 /print|archive|comment|discuss|e[\-]?mail|share|reply|all|login|sign|single|utility/i,151 byline: /byline|author|dateline|writtenby|p-author/i,152 replaceFonts: /<(\/?)font[^>]*>/gi,153 normalize: /\s{2,}/g,154 videos:155 /\/\/(www\.)?((dailymotion|youtube|youtube-nocookie|player\.vimeo|v\.qq|bilibili|live.bilibili)\.com|(archive|upload\.wikimedia)\.org|player\.twitch\.tv)/i,156 shareElements: /(\b|_)(share|sharedaddy)(\b|_)/i,157 nextLink: /(next|weiter|continue|>([^\|]|$)|»([^\|]|$))/i,158 prevLink: /(prev|earl|old|new|<|«)/i,159 tokenize: /\W+/g,160 whitespace: /^\s*$/,161 hasContent: /\S$/,162 hashUrl: /^#.+/,163 srcsetUrl: /(\S+)(\s+[\d.]+[xw])?(\s*(?:,|$))/g,164 b64DataUrl: /^data:\s*([^\s;,]+)\s*;\s*base64\s*,/i,165 // Commas as used in Latin, Sindhi, Chinese and various other scripts.166 // see: https://en.wikipedia.org/wiki/Comma#Comma_variants167 commas: /\u002C|\u060C|\uFE50|\uFE10|\uFE11|\u2E41|\u2E34|\u2E32|\uFF0C/g,168 // See: https://schema.org/Article169 jsonLdArticleTypes:170 /^Article|AdvertiserContentArticle|NewsArticle|AnalysisNewsArticle|AskPublicNewsArticle|BackgroundNewsArticle|OpinionNewsArticle|ReportageNewsArticle|ReviewNewsArticle|Report|SatiricalArticle|ScholarlyArticle|MedicalScholarlyArticle|SocialMediaPosting|BlogPosting|LiveBlogPosting|DiscussionForumPosting|TechArticle|APIReference$/,171 // used to see if a node's content matches words commonly used for ad blocks or loading indicators172 adWords:173 /^(ad(vertising|vertisement)?|pub(licité)?|werb(ung)?|广告|Реклама|Anuncio)$/iu,174 loadingWords:175 /^((loading|正在加载|Загрузка|chargement|cargando)(…|\.\.\.)?)$/iu,176 },177178 UNLIKELY_ROLES: [179 "menu",180 "menubar",181 "complementary",182 "navigation",183 "alert",184 "alertdialog",185 "dialog",186 ],187188 DIV_TO_P_ELEMS: new Set([189 "BLOCKQUOTE",190 "DL",191 "DIV",192 "IMG",193 "OL",194 "P",195 "PRE",196 "TABLE",197 "UL",198 ]),199200 ALTER_TO_DIV_EXCEPTIONS: ["DIV", "ARTICLE", "SECTION", "P", "OL", "UL"],201202 PRESENTATIONAL_ATTRIBUTES: [203 "align",204 "background",205 "bgcolor",206 "border",207 "cellpadding",208 "cellspacing",209 "frame",210 "hspace",211 "rules",212 "style",213 "valign",214 "vspace",215 ],216217 DEPRECATED_SIZE_ATTRIBUTE_ELEMS: ["TABLE", "TH", "TD", "HR", "PRE"],218219 // The commented out elements qualify as phrasing content but tend to be220 // removed by readability when put into paragraphs, so we ignore them here.221 PHRASING_ELEMS: [222 // "CANVAS", "IFRAME", "SVG", "VIDEO",223 "ABBR",224 "AUDIO",225 "B",226 "BDO",227 "BR",228 "BUTTON",229 "CITE",230 "CODE",231 "DATA",232 "DATALIST",233 "DFN",234 "EM",235 "EMBED",236 "I",237 "IMG",238 "INPUT",239 "KBD",240 "LABEL",241 "MARK",242 "MATH",243 "METER",244 "NOSCRIPT",245 "OBJECT",246 "OUTPUT",247 "PROGRESS",248 "Q",249 "RUBY",250 "SAMP",251 "SCRIPT",252 "SELECT",253 "SMALL",254 "SPAN",255 "STRONG",256 "SUB",257 "SUP",258 "TEXTAREA",259 "TIME",260 "VAR",261 "WBR",262 ],263264 // These are the classes that readability sets itself.265 CLASSES_TO_PRESERVE: ["page"],266267 // These are the list of HTML entities that need to be escaped.268 HTML_ESCAPE_MAP: {269 lt: "<",270 gt: ">",271 amp: "&",272 quot: '"',273 apos: "'",274 },275276 /**277 * Run any post-process modifications to article content as necessary.278 *279 * @param Element280 * @return void281 **/282 _postProcessContent(articleContent) {283 // Readability cannot open relative uris so we convert them to absolute uris.284 this._fixRelativeUris(articleContent);285286 this._simplifyNestedElements(articleContent);287288 if (!this._keepClasses) {289 // Remove classes.290 this._cleanClasses(articleContent);291 }292 },293294 /**295 * Iterates over a NodeList, calls `filterFn` for each node and removes node296 * if function returned `true`.297 *298 * If function is not passed, removes all the nodes in node list.299 *300 * @param NodeList nodeList The nodes to operate on301 * @param Function filterFn the function to use as a filter302 * @return void303 */304 _removeNodes(nodeList, filterFn) {305 // Avoid ever operating on live node lists.306 if (this._docJSDOMParser && nodeList._isLiveNodeList) {307 throw new Error("Do not pass live node lists to _removeNodes");308 }309 for (var i = nodeList.length - 1; i >= 0; i--) {310 var node = nodeList[i];311 var parentNode = node.parentNode;312 if (parentNode) {313 if (!filterFn || filterFn.call(this, node, i, nodeList)) {314 parentNode.removeChild(node);315 }316 }317 }318 },319320 /**321 * Iterates over a NodeList, and calls _setNodeTag for each node.322 *323 * @param NodeList nodeList The nodes to operate on324 * @param String newTagName the new tag name to use325 * @return void326 */327 _replaceNodeTags(nodeList, newTagName) {328 // Avoid ever operating on live node lists.329 if (this._docJSDOMParser && nodeList._isLiveNodeList) {330 throw new Error("Do not pass live node lists to _replaceNodeTags");331 }332 for (const node of nodeList) {333 this._setNodeTag(node, newTagName);334 }335 },336337 /**338 * Iterate over a NodeList, which doesn't natively fully implement the Array339 * interface.340 *341 * For convenience, the current object context is applied to the provided342 * iterate function.343 *344 * @param NodeList nodeList The NodeList.345 * @param Function fn The iterate function.346 * @return void347 */348 _forEachNode(nodeList, fn) {349 Array.prototype.forEach.call(nodeList, fn, this);350 },351352 /**353 * Iterate over a NodeList, and return the first node that passes354 * the supplied test function355 *356 * For convenience, the current object context is applied to the provided357 * test function.358 *359 * @param NodeList nodeList The NodeList.360 * @param Function fn The test function.361 * @return void362 */363 _findNode(nodeList, fn) {364 return Array.prototype.find.call(nodeList, fn, this);365 },366367 /**368 * Iterate over a NodeList, return true if any of the provided iterate369 * function calls returns true, false otherwise.370 *371 * For convenience, the current object context is applied to the372 * provided iterate function.373 *374 * @param NodeList nodeList The NodeList.375 * @param Function fn The iterate function.376 * @return Boolean377 */378 _someNode(nodeList, fn) {379 return Array.prototype.some.call(nodeList, fn, this);380 },381382 /**383 * Iterate over a NodeList, return true if all of the provided iterate384 * function calls return true, false otherwise.385 *386 * For convenience, the current object context is applied to the387 * provided iterate function.388 *389 * @param NodeList nodeList The NodeList.390 * @param Function fn The iterate function.391 * @return Boolean392 */393 _everyNode(nodeList, fn) {394 return Array.prototype.every.call(nodeList, fn, this);395 },396397 _getAllNodesWithTag(node, tagNames) {398 if (node.querySelectorAll) {399 return node.querySelectorAll(tagNames.join(","));400 }401 return [].concat.apply(402 [],403 tagNames.map(function (tag) {404 var collection = node.getElementsByTagName(tag);405 return Array.isArray(collection) ? collection : Array.from(collection);406 })407 );408 },409410 /**411 * Removes the class="" attribute from every element in the given412 * subtree, except those that match CLASSES_TO_PRESERVE and413 * the classesToPreserve array from the options object.414 *415 * @param Element416 * @return void417 */418 _cleanClasses(node) {419 var classesToPreserve = this._classesToPreserve;420 var className = (node.getAttribute("class") || "")421 .split(/\s+/)422 .filter(cls => classesToPreserve.includes(cls))423 .join(" ");424425 if (className) {426 node.setAttribute("class", className);427 } else {428 node.removeAttribute("class");429 }430431 for (node = node.firstElementChild; node; node = node.nextElementSibling) {432 this._cleanClasses(node);433 }434 },435436 /**437 * Tests whether a string is a URL or not.438 *439 * @param {string} str The string to test440 * @return {boolean} true if str is a URL, false if not441 */442 _isUrl(str) {443 try {444 new URL(str);445 return true;446 } catch {447 return false;448 }449 },450 /**451 * Converts each <a> and <img> uri in the given element to an absolute URI,452 * ignoring #ref URIs.453 *454 * @param Element455 * @return void456 */457 _fixRelativeUris(articleContent) {458 var baseURI = this._doc.baseURI;459 var documentURI = this._doc.documentURI;460 function toAbsoluteURI(uri) {461 // Leave hash links alone if the base URI matches the document URI:462 if (baseURI == documentURI && uri.charAt(0) == "#") {463 return uri;464 }465466 // Otherwise, resolve against base URI:467 try {468 return new URL(uri, baseURI).href;469 } catch (ex) {470 // Something went wrong, just return the original:471 }472 return uri;473 }474475 var links = this._getAllNodesWithTag(articleContent, ["a"]);476 this._forEachNode(links, function (link) {477 var href = link.getAttribute("href");478 if (href) {479 // Remove links with javascript: URIs, since480 // they won't work after scripts have been removed from the page.481 if (href.indexOf("javascript:") === 0) {482 // if the link only contains simple text content, it can be converted to a text node483 if (484 link.childNodes.length === 1 &&485 link.childNodes[0].nodeType === this.TEXT_NODE486 ) {487 var text = this._doc.createTextNode(link.textContent);488 link.parentNode.replaceChild(text, link);489 } else {490 // if the link has multiple children, they should all be preserved491 var container = this._doc.createElement("span");492 while (link.firstChild) {493 container.appendChild(link.firstChild);494 }495 link.parentNode.replaceChild(container, link);496 }497 } else {498 link.setAttribute("href", toAbsoluteURI(href));499 }500 }501 });502503 var medias = this._getAllNodesWithTag(articleContent, [504 "img",505 "picture",506 "figure",507 "video",508 "audio",509 "source",510 ]);511512 this._forEachNode(medias, function (media) {513 var src = media.getAttribute("src");514 var poster = media.getAttribute("poster");515 var srcset = media.getAttribute("srcset");516517 if (src) {518 media.setAttribute("src", toAbsoluteURI(src));519 }520521 if (poster) {522 media.setAttribute("poster", toAbsoluteURI(poster));523 }524525 if (srcset) {526 var newSrcset = srcset.replace(527 this.REGEXPS.srcsetUrl,528 function (_, p1, p2, p3) {529 return toAbsoluteURI(p1) + (p2 || "") + p3;530 }531 );532533 media.setAttribute("srcset", newSrcset);534 }535 });536 },537538 _simplifyNestedElements(articleContent) {539 var node = articleContent;540541 while (node) {542 if (543 node.parentNode &&544 ["DIV", "SECTION"].includes(node.tagName) &&545 !(node.id && node.id.startsWith("readability"))546 ) {547 if (this._isElementWithoutContent(node)) {548 node = this._removeAndGetNext(node);549 continue;550 } else if (551 this._hasSingleTagInsideElement(node, "DIV") ||552 this._hasSingleTagInsideElement(node, "SECTION")553 ) {554 var child = node.children[0];555 for (var i = 0; i < node.attributes.length; i++) {556 child.setAttributeNode(node.attributes[i].cloneNode());557 }558 node.parentNode.replaceChild(child, node);559 node = child;560 continue;561 }562 }563564 node = this._getNextNode(node);565 }566 },567568 /**569 * Get the article title as an H1.570 *571 * @return string572 **/573 _getArticleTitle() {574 var doc = this._doc;575 var curTitle = "";576 var origTitle = "";577578 try {579 curTitle = origTitle = doc.title.trim();580581 // If they had an element with id "title" in their HTML582 if (typeof curTitle !== "string") {583 curTitle = origTitle = this._getInnerText(584 doc.getElementsByTagName("title")[0]585 );586 }587 } catch (e) {588 /* ignore exceptions setting the title. */589 }590591 var titleHadHierarchicalSeparators = false;592 function wordCount(str) {593 return str.split(/\s+/).length;594 }595596 // If there's a separator in the title, first remove the final part597 const titleSeparators = /\|\-–—\\\/>»/.source;598 if (new RegExp(`\\s[${titleSeparators}]\\s`).test(curTitle)) {599 titleHadHierarchicalSeparators = /\s[\\\/>»]\s/.test(curTitle);600 let allSeparators = Array.from(601 origTitle.matchAll(new RegExp(`\\s[${titleSeparators}]\\s`, "gi"))602 );603 curTitle = origTitle.substring(0, allSeparators.pop().index);604605 // If the resulting title is too short, remove the first part instead:606 if (wordCount(curTitle) < 3) {607 curTitle = origTitle.replace(608 new RegExp(`^[^${titleSeparators}]*[${titleSeparators}]`, "gi"),609 ""610 );611 }612 } else if (curTitle.includes(": ")) {613 // Check if we have an heading containing this exact string, so we614 // could assume it's the full title.615 var headings = this._getAllNodesWithTag(doc, ["h1", "h2"]);616 var trimmedTitle = curTitle.trim();617 var match = this._someNode(headings, function (heading) {618 return heading.textContent.trim() === trimmedTitle;619 });620621 // If we don't, let's extract the title out of the original title string.622 if (!match) {623 curTitle = origTitle.substring(origTitle.lastIndexOf(":") + 1);624625 // If the title is now too short, try the first colon instead:626 if (wordCount(curTitle) < 3) {627 curTitle = origTitle.substring(origTitle.indexOf(":") + 1);628 // But if we have too many words before the colon there's something weird629 // with the titles and the H tags so let's just use the original title instead630 } else if (wordCount(origTitle.substr(0, origTitle.indexOf(":"))) > 5) {631 curTitle = origTitle;632 }633 }634 } else if (curTitle.length > 150 || curTitle.length < 15) {635 var hOnes = doc.getElementsByTagName("h1");636637 if (hOnes.length === 1) {638 curTitle = this._getInnerText(hOnes[0]);639 }640 }641642 curTitle = curTitle.trim().replace(this.REGEXPS.normalize, " ");643 // If we now have 4 words or fewer as our title, and either no644 // 'hierarchical' separators (\, /, > or ») were found in the original645 // title or we decreased the number of words by more than 1 word, use646 // the original title.647 var curTitleWordCount = wordCount(curTitle);648 if (649 curTitleWordCount <= 4 &&650 (!titleHadHierarchicalSeparators ||651 curTitleWordCount !=652 wordCount(653 origTitle.replace(new RegExp(`\\s[${titleSeparators}]\\s`, "g"), "")654 ) -655 1)656 ) {657 curTitle = origTitle;658 }659660 return curTitle;661 },662663 /**664 * Prepare the HTML document for readability to scrape it.665 * This includes things like stripping javascript, CSS, and handling terrible markup.666 *667 * @return void668 **/669 _prepDocument() {670 var doc = this._doc;671672 // Remove all style tags in head673 this._removeNodes(this._getAllNodesWithTag(doc, ["style"]));674675 if (doc.body) {676 this._replaceBrs(doc.body);677 }678679 this._replaceNodeTags(this._getAllNodesWithTag(doc, ["font"]), "SPAN");680 },681682 /**683 * Finds the next node, starting from the given node, and ignoring684 * whitespace in between. If the given node is an element, the same node is685 * returned.686 */687 _nextNode(node) {688 var next = node;689 while (690 next &&691 next.nodeType != this.ELEMENT_NODE &&692 this.REGEXPS.whitespace.test(next.textContent)693 ) {694 next = next.nextSibling;695 }696 return next;697 },698699 /**700 * Replaces 2 or more successive <br> elements with a single <p>.701 * Whitespace between <br> elements are ignored. For example:702 * <div>foo<br>bar<br> <br><br>abc</div>703 * will become:704 * <div>foo<br>bar<p>abc</p></div>705 */706 _replaceBrs(elem) {707 this._forEachNode(this._getAllNodesWithTag(elem, ["br"]), function (br) {708 var next = br.nextSibling;709710 // Whether 2 or more <br> elements have been found and replaced with a711 // <p> block.712 var replaced = false;713714 // If we find a <br> chain, remove the <br>s until we hit another node715 // or non-whitespace. This leaves behind the first <br> in the chain716 // (which will be replaced with a <p> later).717 while ((next = this._nextNode(next)) && next.tagName == "BR") {718 replaced = true;719 var brSibling = next.nextSibling;720 next.remove();721 next = brSibling;722 }723724 // If we removed a <br> chain, replace the remaining <br> with a <p>. Add725 // all sibling nodes as children of the <p> until we hit another <br>726 // chain.727 if (replaced) {728 var p = this._doc.createElement("p");729 br.parentNode.replaceChild(p, br);730731 next = p.nextSibling;732 while (next) {733 // If we've hit another <br><br>, we're done adding children to this <p>.734 if (next.tagName == "BR") {735 var nextElem = this._nextNode(next.nextSibling);736 if (nextElem && nextElem.tagName == "BR") {737 break;738 }739 }740741 if (!this._isPhrasingContent(next)) {742 break;743 }744745 // Otherwise, make this node a child of the new <p>.746 var sibling = next.nextSibling;747 p.appendChild(next);748 next = sibling;749 }750751 while (p.lastChild && this._isWhitespace(p.lastChild)) {752 p.lastChild.remove();753 }754755 if (p.parentNode.tagName === "P") {756 this._setNodeTag(p.parentNode, "DIV");757 }758 }759 });760 },761762 _setNodeTag(node, tag) {763 this.log("_setNodeTag", node, tag);764 if (this._docJSDOMParser) {765 node.localName = tag.toLowerCase();766 node.tagName = tag.toUpperCase();767 return node;768 }769770 var replacement = node.ownerDocument.createElement(tag);771 while (node.firstChild) {772 replacement.appendChild(node.firstChild);773 }774 node.parentNode.replaceChild(replacement, node);775 if (node.readability) {776 replacement.readability = node.readability;777 }778779 for (var i = 0; i < node.attributes.length; i++) {780 replacement.setAttributeNode(node.attributes[i].cloneNode());781 }782 return replacement;783 },784785 /**786 * Prepare the article node for display. Clean out any inline styles,787 * iframes, forms, strip extraneous <p> tags, etc.788 *789 * @param Element790 * @return void791 **/792 _prepArticle(articleContent) {793 this._cleanStyles(articleContent);794795 // Check for data tables before we continue, to avoid removing items in796 // those tables, which will often be isolated even though they're797 // visually linked to other content-ful elements (text, images, etc.).798 this._markDataTables(articleContent);799800 this._fixLazyImages(articleContent);801802 // Clean out junk from the article content803 this._cleanConditionally(articleContent, "form");804 this._cleanConditionally(articleContent, "fieldset");805 this._clean(articleContent, "object");806 this._clean(articleContent, "embed");807 this._clean(articleContent, "footer");808 this._clean(articleContent, "link");809 this._clean(articleContent, "aside");810811 // Clean out elements with little content that have "share" in their id/class combinations from final top candidates,812 // which means we don't remove the top candidates even they have "share".813814 var shareElementThreshold = this.DEFAULT_CHAR_THRESHOLD;815816 this._forEachNode(articleContent.children, function (topCandidate) {817 this._cleanMatchedNodes(topCandidate, function (node, matchString) {818 return (819 this.REGEXPS.shareElements.test(matchString) &&820 node.textContent.length < shareElementThreshold821 );822 });823 });824825 this._clean(articleContent, "iframe");826 this._clean(articleContent, "input");827 this._clean(articleContent, "textarea");828 this._clean(articleContent, "select");829 this._clean(articleContent, "button");830 this._cleanHeaders(articleContent);831832 // Do these last as the previous stuff may have removed junk833 // that will affect these834 this._cleanConditionally(articleContent, "table");835 this._cleanConditionally(articleContent, "ul");836 this._cleanConditionally(articleContent, "div");837838 // replace H1 with H2 as H1 should be only title that is displayed separately839 this._replaceNodeTags(840 this._getAllNodesWithTag(articleContent, ["h1"]),841 "h2"842 );843844 // Remove extra paragraphs845 this._removeNodes(846 this._getAllNodesWithTag(articleContent, ["p"]),847 function (paragraph) {848 // At this point, nasty iframes have been removed; only embedded video849 // ones remain.850 var contentElementCount = this._getAllNodesWithTag(paragraph, [851 "img",852 "embed",853 "object",854 "iframe",855 ]).length;856 return (857 contentElementCount === 0 && !this._getInnerText(paragraph, false)858 );859 }860 );861862 this._forEachNode(863 this._getAllNodesWithTag(articleContent, ["br"]),864 function (br) {865 var next = this._nextNode(br.nextSibling);866 if (next && next.tagName == "P") {867 br.remove();868 }869 }870 );871872 // Remove single-cell tables873 this._forEachNode(874 this._getAllNodesWithTag(articleContent, ["table"]),875 function (table) {876 var tbody = this._hasSingleTagInsideElement(table, "TBODY")877 ? table.firstElementChild878 : table;879 if (this._hasSingleTagInsideElement(tbody, "TR")) {880 var row = tbody.firstElementChild;881 if (this._hasSingleTagInsideElement(row, "TD")) {882 var cell = row.firstElementChild;883 cell = this._setNodeTag(884 cell,885 this._everyNode(cell.childNodes, this._isPhrasingContent)886 ? "P"887 : "DIV"888 );889 table.parentNode.replaceChild(cell, table);890 }891 }892 }893 );894 },895896 /**897 * Initialize a node with the readability object. Also checks the898 * className/id for special names to add to its score.899 *900 * @param Element901 * @return void902 **/903 _initializeNode(node) {904 node.readability = { contentScore: 0 };905906 switch (node.tagName) {907 case "DIV":908 node.readability.contentScore += 5;909 break;910911 case "PRE":912 case "TD":913 case "BLOCKQUOTE":914 node.readability.contentScore += 3;915 break;916917 case "ADDRESS":918 case "OL":919 case "UL":920 case "DL":921 case "DD":922 case "DT":923 case "LI":924 case "FORM":925 node.readability.contentScore -= 3;926 break;927928 case "H1":929 case "H2":930 case "H3":931 case "H4":932 case "H5":933 case "H6":934 case "TH":935 node.readability.contentScore -= 5;936 break;937 }938939 node.readability.contentScore += this._getClassWeight(node);940 },941942 _removeAndGetNext(node) {943 var nextNode = this._getNextNode(node, true);944 node.remove();945 return nextNode;946 },947948 /**949 * Traverse the DOM from node to node, starting at the node passed in.950 * Pass true for the second parameter to indicate this node itself951 * (and its kids) are going away, and we want the next node over.952 *953 * Calling this in a loop will traverse the DOM depth-first.954 *955 * @param {Element} node956 * @param {boolean} ignoreSelfAndKids957 * @return {Element}958 */959 _getNextNode(node, ignoreSelfAndKids) {960 // First check for kids if those aren't being ignored961 if (!ignoreSelfAndKids && node.firstElementChild) {962 return node.firstElementChild;963 }964 // Then for siblings...965 if (node.nextElementSibling) {966 return node.nextElementSibling;967 }968 // And finally, move up the parent chain *and* find a sibling969 // (because this is depth-first traversal, we will have already970 // seen the parent nodes themselves).971 do {972 node = node.parentNode;973 } while (node && !node.nextElementSibling);974 return node && node.nextElementSibling;975 },976977 // compares second text to first one978 // 1 = same text, 0 = completely different text979 // works the way that it splits both texts into words and then finds words that are unique in second text980 // the result is given by the lower length of unique parts981 _textSimilarity(textA, textB) {982 var tokensA = textA983 .toLowerCase()984 .split(this.REGEXPS.tokenize)985 .filter(Boolean);986 var tokensB = textB987 .toLowerCase()988 .split(this.REGEXPS.tokenize)989 .filter(Boolean);990 if (!tokensA.length || !tokensB.length) {991 return 0;992 }993 var uniqTokensB = tokensB.filter(token => !tokensA.includes(token));994 var distanceB = uniqTokensB.join(" ").length / tokensB.join(" ").length;995 return 1 - distanceB;996 },997998 /**999 * Checks whether an element node contains a valid byline1000 *1001 * @param node {Element}1002 * @param matchString {string}1003 * @return boolean1004 */1005 _isValidByline(node, matchString) {1006 var rel = node.getAttribute("rel");1007 var itemprop = node.getAttribute("itemprop");1008 var bylineLength = node.textContent.trim().length;10091010 return (1011 (rel === "author" ||1012 (itemprop && itemprop.includes("author")) ||1013 this.REGEXPS.byline.test(matchString)) &&1014 !!bylineLength &&1015 bylineLength < 1001016 );1017 },10181019 _getNodeAncestors(node, maxDepth) {1020 maxDepth = maxDepth || 0;1021 var i = 0,1022 ancestors = [];1023 while (node.parentNode) {1024 ancestors.push(node.parentNode);1025 if (maxDepth && ++i === maxDepth) {1026 break;1027 }1028 node = node.parentNode;1029 }1030 return ancestors;1031 },10321033 /***1034 * grabArticle - Using a variety of metrics (content score, classname, element types), find the content that is1035 * most likely to be the stuff a user wants to read. Then return it wrapped up in a div.1036 *1037 * @param page a document to run upon. Needs to be a full document, complete with body.1038 * @return Element1039 **/1040 /* eslint-disable-next-line complexity */1041 _grabArticle(page) {1042 this.log("**** grabArticle ****");1043 var doc = this._doc;1044 var isPaging = page !== null;1045 page = page ? page : this._doc.body;10461047 // We can't grab an article if we don't have a page!1048 if (!page) {1049 this.log("No body found in document. Abort.");1050 return null;1051 }10521053 var pageCacheHtml = page.innerHTML;10541055 while (true) {1056 this.log("Starting grabArticle loop");1057 var stripUnlikelyCandidates = this._flagIsActive(1058 this.FLAG_STRIP_UNLIKELYS1059 );10601061 // First, node prepping. Trash nodes that look cruddy (like ones with the1062 // class name "comment", etc), and turn divs into P tags where they have been1063 // used inappropriately (as in, where they contain no other block level elements.)1064 var elementsToScore = [];1065 var node = this._doc.documentElement;10661067 let shouldRemoveTitleHeader = true;10681069 while (node) {1070 if (node.tagName === "HTML") {1071 this._articleLang = node.getAttribute("lang");1072 }10731074 var matchString = node.className + " " + node.id;10751076 if (!this._isProbablyVisible(node)) {1077 this.log("Removing hidden node - " + matchString);1078 node = this._removeAndGetNext(node);1079 continue;1080 }10811082 // User is not able to see elements applied with both "aria-modal = true" and "role = dialog"1083 if (1084 node.getAttribute("aria-modal") == "true" &&1085 node.getAttribute("role") == "dialog"1086 ) {1087 node = this._removeAndGetNext(node);1088 continue;1089 }10901091 // If we don't have a byline yet check to see if this node is a byline; if it is store the byline and remove the node.1092 if (1093 !this._articleByline &&1094 !this._metadata.byline &&1095 this._isValidByline(node, matchString)1096 ) {1097 // Find child node matching [itemprop="name"] and use that if it exists for a more accurate author name byline1098 var endOfSearchMarkerNode = this._getNextNode(node, true);1099 var next = this._getNextNode(node);1100 var itemPropNameNode = null;1101 while (next && next != endOfSearchMarkerNode) {1102 var itemprop = next.getAttribute("itemprop");1103 if (itemprop && itemprop.includes("name")) {1104 itemPropNameNode = next;1105 break;1106 } else {1107 next = this._getNextNode(next);1108 }1109 }1110 this._articleByline = (itemPropNameNode ?? node).textContent.trim();1111 node = this._removeAndGetNext(node);1112 continue;1113 }11141115 if (shouldRemoveTitleHeader && this._headerDuplicatesTitle(node)) {1116 this.log(1117 "Removing header: ",1118 node.textContent.trim(),1119 this._articleTitle.trim()1120 );1121 shouldRemoveTitleHeader = false;1122 node = this._removeAndGetNext(node);1123 continue;1124 }11251126 // Remove unlikely candidates1127 if (stripUnlikelyCandidates) {1128 if (1129 this.REGEXPS.unlikelyCandidates.test(matchString) &&1130 !this.REGEXPS.okMaybeItsACandidate.test(matchString) &&1131 !this._hasAncestorTag(node, "table") &&1132 !this._hasAncestorTag(node, "code") &&1133 node.tagName !== "BODY" &&1134 node.tagName !== "A"1135 ) {1136 this.log("Removing unlikely candidate - " + matchString);1137 node = this._removeAndGetNext(node);1138 continue;1139 }11401141 if (this.UNLIKELY_ROLES.includes(node.getAttribute("role"))) {1142 this.log(1143 "Removing content with role " +1144 node.getAttribute("role") +1145 " - " +1146 matchString1147 );1148 node = this._removeAndGetNext(node);1149 continue;1150 }1151 }11521153 // Remove DIV, SECTION, and HEADER nodes without any content(e.g. text, image, video, or iframe).1154 if (1155 (node.tagName === "DIV" ||1156 node.tagName === "SECTION" ||1157 node.tagName === "HEADER" ||1158 node.tagName === "H1" ||1159 node.tagName === "H2" ||1160 node.tagName === "H3" ||1161 node.tagName === "H4" ||1162 node.tagName === "H5" ||1163 node.tagName === "H6") &&1164 this._isElementWithoutContent(node)1165 ) {1166 node = this._removeAndGetNext(node);1167 continue;1168 }11691170 if (this.DEFAULT_TAGS_TO_SCORE.includes(node.tagName)) {1171 elementsToScore.push(node);1172 }11731174 // Turn all divs that don't have children block level elements into p's1175 if (node.tagName === "DIV") {1176 // Put phrasing content into paragraphs.1177 var childNode = node.firstChild;1178 while (childNode) {1179 var nextSibling = childNode.nextSibling;1180 if (this._isPhrasingContent(childNode)) {1181 var fragment = doc.createDocumentFragment();1182 // Collect all consecutive phrasing content into a fragment.1183 do {1184 nextSibling = childNode.nextSibling;1185 fragment.appendChild(childNode);1186 childNode = nextSibling;1187 } while (childNode && this._isPhrasingContent(childNode));11881189 // Trim leading and trailing whitespace from the fragment.1190 while (1191 fragment.firstChild &&1192 this._isWhitespace(fragment.firstChild)1193 ) {1194 fragment.firstChild.remove();1195 }1196 while (1197 fragment.lastChild &&1198 this._isWhitespace(fragment.lastChild)1199 ) {1200 fragment.lastChild.remove();1201 }12021203 // If the fragment contains anything, wrap it in a paragraph and1204 // insert it before the next non-phrasing node.1205 if (fragment.firstChild) {1206 var p = doc.createElement("p");1207 p.appendChild(fragment);1208 node.insertBefore(p, nextSibling);1209 }1210 }1211 childNode = nextSibling;1212 }12131214 // Sites like http://mobile.slate.com encloses each paragraph with a DIV1215 // element. DIVs with only a P element inside and no text content can be1216 // safely converted into plain P elements to avoid confusing the scoring1217 // algorithm with DIVs with are, in practice, paragraphs.1218 if (1219 this._hasSingleTagInsideElement(node, "P") &&1220 this._getLinkDensity(node) < 0.251221 ) {1222 var newNode = node.children[0];1223 node.parentNode.replaceChild(newNode, node);1224 node = newNode;1225 elementsToScore.push(node);1226 } else if (!this._hasChildBlockElement(node)) {1227 node = this._setNodeTag(node, "P");1228 elementsToScore.push(node);1229 }1230 }1231 node = this._getNextNode(node);1232 }12331234 /**1235 * Loop through all paragraphs, and assign a score to them based on how content-y they look.1236 * Then add their score to their parent node.1237 *1238 * A score is determined by things like number of commas, class names, etc. Maybe eventually link density.1239 **/1240 var candidates = [];1241 this._forEachNode(elementsToScore, function (elementToScore) {1242 if (1243 !elementToScore.parentNode ||1244 typeof elementToScore.parentNode.tagName === "undefined"1245 ) {1246 return;1247 }12481249 // If this paragraph is less than 25 characters, don't even count it.1250 var innerText = this._getInnerText(elementToScore);1251 if (innerText.length < 25) {1252 return;1253 }12541255 // Exclude nodes with no ancestor.1256 var ancestors = this._getNodeAncestors(elementToScore, 5);1257 if (ancestors.length === 0) {1258 return;1259 }12601261 var contentScore = 0;12621263 // Add a point for the paragraph itself as a base.1264 contentScore += 1;12651266 // Add points for any commas within this paragraph.1267 contentScore += innerText.split(this.REGEXPS.commas).length;12681269 // For every 100 characters in this paragraph, add another point. Up to 3 points.1270 contentScore += Math.min(Math.floor(innerText.length / 100), 3);12711272 // Initialize and score ancestors.1273 this._forEachNode(ancestors, function (ancestor, level) {1274 if (1275 !ancestor.tagName ||1276 !ancestor.parentNode ||1277 typeof ancestor.parentNode.tagName === "undefined"1278 ) {1279 return;1280 }12811282 if (typeof ancestor.readability === "undefined") {1283 this._initializeNode(ancestor);1284 candidates.push(ancestor);1285 }12861287 // Node score divider:1288 // - parent: 1 (no division)1289 // - grandparent: 21290 // - great grandparent+: ancestor level * 31291 if (level === 0) {1292 var scoreDivider = 1;1293 } else if (level === 1) {1294 scoreDivider = 2;1295 } else {1296 scoreDivider = level * 3;1297 }1298 ancestor.readability.contentScore += contentScore / scoreDivider;1299 });1300 });13011302 // After we've calculated scores, loop through all of the possible1303 // candidate nodes we found and find the one with the highest score.1304 var topCandidates = [];1305 for (var c = 0, cl = candidates.length; c < cl; c += 1) {1306 var candidate = candidates[c];13071308 // Scale the final candidates score based on link density. Good content1309 // should have a relatively small link density (5% or less) and be mostly1310 // unaffected by this operation.1311 var candidateScore =1312 candidate.readability.contentScore *1313 (1 - this._getLinkDensity(candidate));1314 candidate.readability.contentScore = candidateScore;13151316 this.log("Candidate:", candidate, "with score " + candidateScore);13171318 for (var t = 0; t < this._nbTopCandidates; t++) {1319 var aTopCandidate = topCandidates[t];13201321 if (1322 !aTopCandidate ||1323 candidateScore > aTopCandidate.readability.contentScore1324 ) {1325 topCandidates.splice(t, 0, candidate);1326 if (topCandidates.length > this._nbTopCandidates) {1327 topCandidates.pop();1328 }1329 break;1330 }1331 }1332 }13331334 var topCandidate = topCandidates[0] || null;1335 var neededToCreateTopCandidate = false;1336 var parentOfTopCandidate;13371338 // If we still have no top candidate, just use the body as a last resort.1339 // We also have to copy the body node so it is something we can modify.1340 if (topCandidate === null || topCandidate.tagName === "BODY") {1341 // Move all of the page's children into topCandidate1342 topCandidate = doc.createElement("DIV");1343 neededToCreateTopCandidate = true;1344 // Move everything (not just elements, also text nodes etc.) into the container1345 // so we even include text directly in the body:1346 while (page.firstChild) {1347 this.log("Moving child out:", page.firstChild);1348 topCandidate.appendChild(page.firstChild);1349 }13501351 page.appendChild(topCandidate);13521353 this._initializeNode(topCandidate);1354 } else if (topCandidate) {1355 // Find a better top candidate node if it contains (at least three) nodes which belong to `topCandidates` array1356 // and whose scores are quite closed with current `topCandidate` node.1357 var alternativeCandidateAncestors = [];1358 for (var i = 1; i < topCandidates.length; i++) {1359 if (1360 topCandidates[i].readability.contentScore /1361 topCandidate.readability.contentScore >=1362 0.751363 ) {1364 alternativeCandidateAncestors.push(1365 this._getNodeAncestors(topCandidates[i])1366 );1367 }1368 }1369 var MINIMUM_TOPCANDIDATES = 3;1370 if (alternativeCandidateAncestors.length >= MINIMUM_TOPCANDIDATES) {1371 parentOfTopCandidate = topCandidate.parentNode;1372 while (parentOfTopCandidate.tagName !== "BODY") {1373 var listsContainingThisAncestor = 0;1374 for (1375 var ancestorIndex = 0;1376 ancestorIndex < alternativeCandidateAncestors.length &&1377 listsContainingThisAncestor < MINIMUM_TOPCANDIDATES;1378 ancestorIndex++1379 ) {1380 listsContainingThisAncestor += Number(1381 alternativeCandidateAncestors[ancestorIndex].includes(1382 parentOfTopCandidate1383 )1384 );1385 }1386 if (listsContainingThisAncestor >= MINIMUM_TOPCANDIDATES) {1387 topCandidate = parentOfTopCandidate;1388 break;1389 }1390 parentOfTopCandidate = parentOfTopCandidate.parentNode;1391 }1392 }1393 if (!topCandidate.readability) {1394 this._initializeNode(topCandidate);1395 }13961397 // Because of our bonus system, parents of candidates might have scores1398 // themselves. They get half of the node. There won't be nodes with higher1399 // scores than our topCandidate, but if we see the score going *up* in the first1400 // few steps up the tree, that's a decent sign that there might be more content1401 // lurking in other places that we want to unify in. The sibling stuff1402 // below does some of that - but only if we've looked high enough up the DOM1403 // tree.1404 parentOfTopCandidate = topCandidate.parentNode;1405 var lastScore = topCandidate.readability.contentScore;1406 // The scores shouldn't get too low.1407 var scoreThreshold = lastScore / 3;1408 while (parentOfTopCandidate.tagName !== "BODY") {1409 if (!parentOfTopCandidate.readability) {1410 parentOfTopCandidate = parentOfTopCandidate.parentNode;1411 continue;1412 }1413 var parentScore = parentOfTopCandidate.readability.contentScore;1414 if (parentScore < scoreThreshold) {1415 break;1416 }1417 if (parentScore > lastScore) {1418 // Alright! We found a better parent to use.1419 topCandidate = parentOfTopCandidate;1420 break;1421 }1422 lastScore = parentOfTopCandidate.readability.contentScore;1423 parentOfTopCandidate = parentOfTopCandidate.parentNode;1424 }14251426 // If the top candidate is the only child, use parent instead. This will help sibling1427 // joining logic when adjacent content is actually located in parent's sibling node.1428 parentOfTopCandidate = topCandidate.parentNode;1429 while (1430 parentOfTopCandidate.tagName != "BODY" &&1431 parentOfTopCandidate.children.length == 11432 ) {1433 topCandidate = parentOfTopCandidate;1434 parentOfTopCandidate = topCandidate.parentNode;1435 }1436 if (!topCandidate.readability) {1437 this._initializeNode(topCandidate);1438 }1439 }14401441 // Now that we have the top candidate, look through its siblings for content1442 // that might also be related. Things like preambles, content split by ads1443 // that we removed, etc.1444 var articleContent = doc.createElement("DIV");1445 if (isPaging) {1446 articleContent.id = "readability-content";1447 }14481449 var siblingScoreThreshold = Math.max(1450 10,1451 topCandidate.readability.contentScore * 0.21452 );1453 // Keep potential top candidate's parent node to try to get text direction of it later.1454 parentOfTopCandidate = topCandidate.parentNode;1455 var siblings = parentOfTopCandidate.children;14561457 for (var s = 0, sl = siblings.length; s < sl; s++) {1458 var sibling = siblings[s];1459 var append = false;14601461 this.log(1462 "Looking at sibling node:",1463 sibling,1464 sibling.readability1465 ? "with score " + sibling.readability.contentScore1466 : ""1467 );1468 this.log(1469 "Sibling has score",1470 sibling.readability ? sibling.readability.contentScore : "Unknown"1471 );14721473 if (sibling === topCandidate) {1474 append = true;1475 } else {1476 var contentBonus = 0;14771478 // Give a bonus if sibling nodes and top candidates have the example same classname1479 if (1480 sibling.className === topCandidate.className &&1481 topCandidate.className !== ""1482 ) {1483 contentBonus += topCandidate.readability.contentScore * 0.2;1484 }14851486 if (1487 sibling.readability &&1488 sibling.readability.contentScore + contentBonus >=1489 siblingScoreThreshold1490 ) {1491 append = true;1492 } else if (sibling.nodeName === "P") {1493 var linkDensity = this._getLinkDensity(sibling);1494 var nodeContent = this._getInnerText(sibling);1495 var nodeLength = nodeContent.length;14961497 if (nodeLength > 80 && linkDensity < 0.25) {1498 append = true;1499 } else if (1500 nodeLength < 80 &&1501 nodeLength > 0 &&1502 linkDensity === 0 &&1503 nodeContent.search(/\.( |$)/) !== -11504 ) {1505 append = true;1506 }1507 }1508 }15091510 if (append) {1511 this.log("Appending node:", sibling);15121513 if (!this.ALTER_TO_DIV_EXCEPTIONS.includes(sibling.nodeName)) {1514 // We have a node that isn't a common block level element, like a form or td tag.1515 // Turn it into a div so it doesn't get filtered out later by accident.1516 this.log("Altering sibling:", sibling, "to div.");15171518 sibling = this._setNodeTag(sibling, "DIV");1519 }15201521 articleContent.appendChild(sibling);1522 // Fetch children again to make it compatible1523 // with DOM parsers without live collection support.1524 siblings = parentOfTopCandidate.children;1525 // siblings is a reference to the children array, and1526 // sibling is removed from the array when we call appendChild().1527 // As a result, we must revisit this index since the nodes1528 // have been shifted.1529 s -= 1;1530 sl -= 1;1531 }1532 }15331534 if (this._debug) {1535 this.log("Article content pre-prep: " + articleContent.innerHTML);1536 }1537 // So we have all of the content that we need. Now we clean it up for presentation.1538 this._prepArticle(articleContent);1539 if (this._debug) {1540 this.log("Article content post-prep: " + articleContent.innerHTML);1541 }15421543 if (neededToCreateTopCandidate) {1544 // We already created a fake div thing, and there wouldn't have been any siblings left1545 // for the previous loop, so there's no point trying to create a new div, and then1546 // move all the children over. Just assign IDs and class names here. No need to append1547 // because that already happened anyway.1548 topCandidate.id = "readability-page-1";1549 topCandidate.className = "page";1550 } else {1551 var div = doc.createElement("DIV");1552 div.id = "readability-page-1";1553 div.className = "page";1554 while (articleContent.firstChild) {1555 div.appendChild(articleContent.firstChild);1556 }1557 articleContent.appendChild(div);1558 }15591560 if (this._debug) {1561 this.log("Article content after paging: " + articleContent.innerHTML);1562 }15631564 var parseSuccessful = true;15651566 // Now that we've gone through the full algorithm, check to see if1567 // we got any meaningful content. If we didn't, we may need to re-run1568 // grabArticle with different flags set. This gives us a higher likelihood of1569 // finding the content, and the sieve approach gives us a higher likelihood of1570 // finding the -right- content.1571 var textLength = this._getInnerText(articleContent, true).length;1572 if (textLength < this._charThreshold) {1573 parseSuccessful = false;1574 // eslint-disable-next-line no-unsanitized/property1575 page.innerHTML = pageCacheHtml;15761577 this._attempts.push({1578 articleContent,1579 textLength,1580 });15811582 if (this._flagIsActive(this.FLAG_STRIP_UNLIKELYS)) {1583 this._removeFlag(this.FLAG_STRIP_UNLIKELYS);1584 } else if (this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {1585 this._removeFlag(this.FLAG_WEIGHT_CLASSES);1586 } else if (this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {1587 this._removeFlag(this.FLAG_CLEAN_CONDITIONALLY);1588 } else {1589 // No luck after removing flags, just return the longest text we found during the different loops1590 this._attempts.sort(function (a, b) {1591 return b.textLength - a.textLength;1592 });15931594 // But first check if we actually have something1595 if (!this._attempts[0].textLength) {1596 return null;1597 }15981599 articleContent = this._attempts[0].articleContent;1600 parseSuccessful = true;1601 }1602 }16031604 if (parseSuccessful) {1605 // Find out text direction from ancestors of final top candidate.1606 var ancestors = [parentOfTopCandidate, topCandidate].concat(1607 this._getNodeAncestors(parentOfTopCandidate)1608 );1609 this._someNode(ancestors, function (ancestor) {1610 if (!ancestor.tagName) {1611 return false;1612 }1613 var articleDir = ancestor.getAttribute("dir");1614 if (articleDir) {1615 this._articleDir = articleDir;1616 return true;1617 }1618 return false;1619 });1620 return articleContent;1621 }1622 }1623 },16241625 /**1626 * Converts some of the common HTML entities in string to their corresponding characters.1627 *1628 * @param str {string} - a string to unescape.1629 * @return string without HTML entity.1630 */1631 _unescapeHtmlEntities(str) {1632 if (!str) {1633 return str;1634 }16351636 var htmlEscapeMap = this.HTML_ESCAPE_MAP;1637 return str1638 .replace(/&(quot|amp|apos|lt|gt);/g, function (_, tag) {1639 return htmlEscapeMap[tag];1640 })1641 .replace(/&#(?:x([0-9a-f]+)|([0-9]+));/gi, function (_, hex, numStr) {1642 var num = parseInt(hex || numStr, hex ? 16 : 10);16431644 // these character references are replaced by a conforming HTML parser1645 if (num == 0 || num > 0x10ffff || (num >= 0xd800 && num <= 0xdfff)) {1646 num = 0xfffd;1647 }16481649 return String.fromCodePoint(num);1650 });1651 },16521653 /**1654 * Try to extract metadata from JSON-LD object.1655 * For now, only Schema.org objects of type Article or its subtypes are supported.1656 * @return Object with any metadata that could be extracted (possibly none)1657 */1658 _getJSONLD(doc) {1659 var scripts = this._getAllNodesWithTag(doc, ["script"]);16601661 var metadata;16621663 this._forEachNode(scripts, function (jsonLdElement) {1664 if (1665 !metadata &&1666 jsonLdElement.getAttribute("type") === "application/ld+json"1667 ) {1668 try {1669 // Strip CDATA markers if present1670 var content = jsonLdElement.textContent.replace(1671 /^\s*<!\[CDATA\[|\]\]>\s*$/g,1672 ""1673 );1674 var parsed = JSON.parse(content);16751676 if (Array.isArray(parsed)) {1677 parsed = parsed.find(it => {1678 return (1679 it["@type"] &&1680 it["@type"].match(this.REGEXPS.jsonLdArticleTypes)1681 );1682 });1683 if (!parsed) {1684 return;1685 }1686 }16871688 var schemaDotOrgRegex = /^https?\:\/\/schema\.org\/?$/;1689 var matches =1690 (typeof parsed["@context"] === "string" &&1691 parsed["@context"].match(schemaDotOrgRegex)) ||1692 (typeof parsed["@context"] === "object" &&1693 typeof parsed["@context"]["@vocab"] == "string" &&1694 parsed["@context"]["@vocab"].match(schemaDotOrgRegex));16951696 if (!matches) {1697 return;1698 }16991700 if (!parsed["@type"] && Array.isArray(parsed["@graph"])) {1701 parsed = parsed["@graph"].find(it => {1702 return (it["@type"] || "").match(this.REGEXPS.jsonLdArticleTypes);1703 });1704 }17051706 if (1707 !parsed ||1708 !parsed["@type"] ||1709 !parsed["@type"].match(this.REGEXPS.jsonLdArticleTypes)1710 ) {1711 return;1712 }17131714 metadata = {};17151716 if (1717 typeof parsed.name === "string" &&1718 typeof parsed.headline === "string" &&1719 parsed.name !== parsed.headline1720 ) {1721 // we have both name and headline element in the JSON-LD. They should both be the same but some websites like aktualne.cz1722 // put their own name into "name" and the article title to "headline" which confuses Readability. So we try to check if either1723 // "name" or "headline" closely matches the html title, and if so, use that one. If not, then we use "name" by default.17241725 var title = this._getArticleTitle();1726 var nameMatches = this._textSimilarity(parsed.name, title) > 0.75;1727 var headlineMatches =1728 this._textSimilarity(parsed.headline, title) > 0.75;17291730 if (headlineMatches && !nameMatches) {1731 metadata.title = parsed.headline;1732 } else {1733 metadata.title = parsed.name;1734 }1735 } else if (typeof parsed.name === "string") {1736 metadata.title = parsed.name.trim();1737 } else if (typeof parsed.headline === "string") {1738 metadata.title = parsed.headline.trim();1739 }1740 if (parsed.author) {1741 if (typeof parsed.author.name === "string") {1742 metadata.byline = parsed.author.name.trim();1743 } else if (1744 Array.isArray(parsed.author) &&1745 parsed.author[0] &&1746 typeof parsed.author[0].name === "string"1747 ) {1748 metadata.byline = parsed.author1749 .filter(function (author) {1750 return author && typeof author.name === "string";1751 })1752 .map(function (author) {1753 return author.name.trim();1754 })1755 .join(", ");1756 }1757 }1758 if (typeof parsed.description === "string") {1759 metadata.excerpt = parsed.description.trim();1760 }1761 if (parsed.publisher && typeof parsed.publisher.name === "string") {1762 metadata.siteName = parsed.publisher.name.trim();1763 }1764 if (typeof parsed.datePublished === "string") {1765 metadata.datePublished = parsed.datePublished.trim();1766 }1767 } catch (err) {1768 this.log(err.message);1769 }1770 }1771 });1772 return metadata ? metadata : {};1773 },17741775 /**1776 * Attempts to get excerpt and byline metadata for the article.1777 *1778 * @param {Object} jsonld — object containing any metadata that1779 * could be extracted from JSON-LD object.1780 *1781 * @return Object with optional "excerpt" and "byline" properties1782 */1783 _getArticleMetadata(jsonld) {1784 var metadata = {};1785 var values = {};1786 var metaElements = this._doc.getElementsByTagName("meta");17871788 // property is a space-separated list of values1789 var propertyPattern =1790 /\s*(article|dc|dcterm|og|twitter)\s*:\s*(author|creator|description|published_time|title|site_name)\s*/gi;17911792 // name is a single value1793 var namePattern =1794 /^\s*(?:(dc|dcterm|og|twitter|parsely|weibo:(article|webpage))\s*[-\.:]\s*)?(author|creator|pub-date|description|title|site_name)\s*$/i;17951796 // Find description tags.1797 this._forEachNode(metaElements, function (element) {1798 var elementName = element.getAttribute("name");1799 var elementProperty = element.getAttribute("property");1800 var content = element.getAttribute("content");1801 if (!content) {1802 return;1803 }1804 var matches = null;1805 var name = null;18061807 if (elementProperty) {1808 matches = elementProperty.match(propertyPattern);1809 if (matches) {1810 // Convert to lowercase, and remove any whitespace1811 // so we can match below.1812 name = matches[0].toLowerCase().replace(/\s/g, "");1813 // multiple authors1814 values[name] = content.trim();1815 }1816 }1817 if (!matches && elementName && namePattern.test(elementName)) {1818 name = elementName;1819 if (content) {1820 // Convert to lowercase, remove any whitespace, and convert dots1821 // to colons so we can match below.1822 name = name.toLowerCase().replace(/\s/g, "").replace(/\./g, ":");1823 values[name] = content.trim();1824 }1825 }1826 });18271828 // get title1829 metadata.title =1830 jsonld.title ||1831 values["dc:title"] ||1832 values["dcterm:title"] ||1833 values["og:title"] ||1834 values["weibo:article:title"] ||1835 values["weibo:webpage:title"] ||1836 values.title ||1837 values["twitter:title"] ||1838 values["parsely-title"];18391840 if (!metadata.title) {1841 metadata.title = this._getArticleTitle();1842 }18431844 const articleAuthor =1845 typeof values["article:author"] === "string" &&1846 !this._isUrl(values["article:author"])1847 ? values["article:author"]1848 : undefined;18491850 // get author1851 metadata.byline =1852 jsonld.byline ||1853 values["dc:creator"] ||1854 values["dcterm:creator"] ||1855 values.author ||1856 values["parsely-author"] ||1857 articleAuthor;18581859 // get description1860 metadata.excerpt =1861 jsonld.excerpt ||1862 values["dc:description"] ||1863 values["dcterm:description"] ||1864 values["og:description"] ||1865 values["weibo:article:description"] ||1866 values["weibo:webpage:description"] ||1867 values.description ||1868 values["twitter:description"];18691870 // get site name1871 metadata.siteName = jsonld.siteName || values["og:site_name"];18721873 // get article published time1874 metadata.publishedTime =1875 jsonld.datePublished ||1876 values["article:published_time"] ||1877 values["parsely-pub-date"] ||1878 null;18791880 // in many sites the meta value is escaped with HTML entities,1881 // so here we need to unescape it1882 metadata.title = this._unescapeHtmlEntities(metadata.title);1883 metadata.byline = this._unescapeHtmlEntities(metadata.byline);1884 metadata.excerpt = this._unescapeHtmlEntities(metadata.excerpt);1885 metadata.siteName = this._unescapeHtmlEntities(metadata.siteName);1886 metadata.publishedTime = this._unescapeHtmlEntities(metadata.publishedTime);18871888 return metadata;1889 },18901891 /**1892 * Check if node is image, or if node contains exactly only one image1893 * whether as a direct child or as its descendants.1894 *1895 * @param Element1896 **/1897 _isSingleImage(node) {1898 while (node) {1899 if (node.tagName === "IMG") {1900 return true;1901 }1902 if (node.children.length !== 1 || node.textContent.trim() !== "") {1903 return false;1904 }1905 node = node.children[0];1906 }1907 return false;1908 },19091910 /**1911 * Find all <noscript> that are located after <img> nodes, and which contain only one1912 * <img> element. Replace the first image with the image from inside the <noscript> tag,1913 * and remove the <noscript> tag. This improves the quality of the images we use on1914 * some sites (e.g. Medium).1915 *1916 * @param Element1917 **/1918 _unwrapNoscriptImages(doc) {1919 // Find img without source or attributes that might contains image, and remove it.1920 // This is done to prevent a placeholder img is replaced by img from noscript in next step.1921 var imgs = Array.from(doc.getElementsByTagName("img"));1922 this._forEachNode(imgs, function (img) {1923 for (var i = 0; i < img.attributes.length; i++) {1924 var attr = img.attributes[i];1925 switch (attr.name) {1926 case "src":1927 case "srcset":1928 case "data-src":1929 case "data-srcset":1930 return;1931 }19321933 if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) {1934 return;1935 }1936 }19371938 img.remove();1939 });19401941 // Next find noscript and try to extract its image1942 var noscripts = Array.from(doc.getElementsByTagName("noscript"));1943 this._forEachNode(noscripts, function (noscript) {1944 // Parse content of noscript and make sure it only contains image1945 if (!this._isSingleImage(noscript)) {1946 return;1947 }1948 var tmp = doc.createElement("div");1949 // We're running in the document context, and using unmodified1950 // document contents, so doing this should be safe.1951 // (Also we heavily discourage people from allowing script to1952 // run at all in this document...)1953 // eslint-disable-next-line no-unsanitized/property1954 tmp.innerHTML = noscript.innerHTML;19551956 // If noscript has previous sibling and it only contains image,1957 // replace it with noscript content. However we also keep old1958 // attributes that might contains image.1959 var prevElement = noscript.previousElementSibling;1960 if (prevElement && this._isSingleImage(prevElement)) {1961 var prevImg = prevElement;1962 if (prevImg.tagName !== "IMG") {1963 prevImg = prevElement.getElementsByTagName("img")[0];1964 }19651966 var newImg = tmp.getElementsByTagName("img")[0];1967 for (var i = 0; i < prevImg.attributes.length; i++) {1968 var attr = prevImg.attributes[i];1969 if (attr.value === "") {1970 continue;1971 }19721973 if (1974 attr.name === "src" ||1975 attr.name === "srcset" ||1976 /\.(jpg|jpeg|png|webp)/i.test(attr.value)1977 ) {1978 if (newImg.getAttribute(attr.name) === attr.value) {1979 continue;1980 }19811982 var attrName = attr.name;1983 if (newImg.hasAttribute(attrName)) {1984 attrName = "data-old-" + attrName;1985 }19861987 newImg.setAttribute(attrName, attr.value);1988 }1989 }19901991 noscript.parentNode.replaceChild(tmp.firstElementChild, prevElement);1992 }1993 });1994 },19951996 /**1997 * Removes script tags from the document.1998 *1999 * @param Element2000 **/2001 _removeScripts(doc) {2002 this._removeNodes(this._getAllNodesWithTag(doc, ["script", "noscript"]));2003 },20042005 /**2006 * Check if this node has only whitespace and a single element with given tag2007 * Returns false if the DIV node contains non-empty text nodes2008 * or if it contains no element with given tag or more than 1 element.2009 *2010 * @param Element2011 * @param string tag of child element2012 **/2013 _hasSingleTagInsideElement(element, tag) {2014 // There should be exactly 1 element child with given tag2015 if (element.children.length != 1 || element.children[0].tagName !== tag) {2016 return false;2017 }20182019 // And there should be no text nodes with real content2020 return !this._someNode(element.childNodes, function (node) {2021 return (2022 node.nodeType === this.TEXT_NODE &&2023 this.REGEXPS.hasContent.test(node.textContent)2024 );2025 });2026 },20272028 _isElementWithoutContent(node) {2029 return (2030 node.nodeType === this.ELEMENT_NODE &&2031 !node.textContent.trim().length &&2032 (!node.children.length ||2033 node.children.length ==2034 node.getElementsByTagName("br").length +2035 node.getElementsByTagName("hr").length)2036 );2037 },20382039 /**2040 * Determine whether element has any children block level elements.2041 *2042 * @param Element2043 */2044 _hasChildBlockElement(element) {2045 return this._someNode(element.childNodes, function (node) {2046 return (2047 this.DIV_TO_P_ELEMS.has(node.tagName) ||2048 this._hasChildBlockElement(node)2049 );2050 });2051 },20522053 /***2054 * Determine if a node qualifies as phrasing content.2055 * https://developer.mozilla.org/en-US/docs/Web/Guide/HTML/Content_categories#Phrasing_content2056 **/2057 _isPhrasingContent(node) {2058 return (2059 node.nodeType === this.TEXT_NODE ||2060 this.PHRASING_ELEMS.includes(node.tagName) ||2061 ((node.tagName === "A" ||2062 node.tagName === "DEL" ||2063 node.tagName === "INS") &&2064 this._everyNode(node.childNodes, this._isPhrasingContent))2065 );2066 },20672068 _isWhitespace(node) {2069 return (2070 (node.nodeType === this.TEXT_NODE &&2071 node.textContent.trim().length === 0) ||2072 (node.nodeType === this.ELEMENT_NODE && node.tagName === "BR")2073 );2074 },20752076 /**2077 * Get the inner text of a node - cross browser compatibly.2078 * This also strips out any excess whitespace to be found.2079 *2080 * @param Element2081 * @param Boolean normalizeSpaces (default: true)2082 * @return string2083 **/2084 _getInnerText(e, normalizeSpaces) {2085 normalizeSpaces =2086 typeof normalizeSpaces === "undefined" ? true : normalizeSpaces;2087 var textContent = e.textContent.trim();20882089 if (normalizeSpaces) {2090 return textContent.replace(this.REGEXPS.normalize, " ");2091 }2092 return textContent;2093 },20942095 /**2096 * Get the number of times a string s appears in the node e.2097 *2098 * @param Element2099 * @param string - what to split on. Default is ","2100 * @return number (integer)2101 **/2102 _getCharCount(e, s) {2103 s = s || ",";2104 return this._getInnerText(e).split(s).length - 1;2105 },21062107 /**2108 * Remove the style attribute on every e and under.2109 * TODO: Test if getElementsByTagName(*) is faster.2110 *2111 * @param Element2112 * @return void2113 **/2114 _cleanStyles(e) {2115 if (!e || e.tagName.toLowerCase() === "svg") {2116 return;2117 }21182119 // Remove `style` and deprecated presentational attributes2120 for (var i = 0; i < this.PRESENTATIONAL_ATTRIBUTES.length; i++) {2121 e.removeAttribute(this.PRESENTATIONAL_ATTRIBUTES[i]);2122 }21232124 if (this.DEPRECATED_SIZE_ATTRIBUTE_ELEMS.includes(e.tagName)) {2125 e.removeAttribute("width");2126 e.removeAttribute("height");2127 }21282129 var cur = e.firstElementChild;2130 while (cur !== null) {2131 this._cleanStyles(cur);2132 cur = cur.nextElementSibling;2133 }2134 },21352136 /**2137 * Get the density of links as a percentage of the content2138 * This is the amount of text that is inside a link divided by the total text in the node.2139 *2140 * @param Element2141 * @return number (float)2142 **/2143 _getLinkDensity(element) {2144 var textLength = this._getInnerText(element).length;2145 if (textLength === 0) {2146 return 0;2147 }21482149 var linkLength = 0;21502151 // XXX implement _reduceNodeList?2152 this._forEachNode(element.getElementsByTagName("a"), function (linkNode) {2153 var href = linkNode.getAttribute("href");2154 var coefficient = href && this.REGEXPS.hashUrl.test(href) ? 0.3 : 1;2155 linkLength += this._getInnerText(linkNode).length * coefficient;2156 });21572158 return linkLength / textLength;2159 },21602161 /**2162 * Get an elements class/id weight. Uses regular expressions to tell if this2163 * element looks good or bad.2164 *2165 * @param Element2166 * @return number (Integer)2167 **/2168 _getClassWeight(e) {2169 if (!this._flagIsActive(this.FLAG_WEIGHT_CLASSES)) {2170 return 0;2171 }21722173 var weight = 0;21742175 // Look for a special classname2176 if (typeof e.className === "string" && e.className !== "") {2177 if (this.REGEXPS.negative.test(e.className)) {2178 weight -= 25;2179 }21802181 if (this.REGEXPS.positive.test(e.className)) {2182 weight += 25;2183 }2184 }21852186 // Look for a special ID2187 if (typeof e.id === "string" && e.id !== "") {2188 if (this.REGEXPS.negative.test(e.id)) {2189 weight -= 25;2190 }21912192 if (this.REGEXPS.positive.test(e.id)) {2193 weight += 25;2194 }2195 }21962197 return weight;2198 },21992200 /**2201 * Clean a node of all elements of type "tag".2202 * (Unless it's a youtube/vimeo video. People love movies.)2203 *2204 * @param Element2205 * @param string tag to clean2206 * @return void2207 **/2208 _clean(e, tag) {2209 var isEmbed = ["object", "embed", "iframe"].includes(tag);22102211 this._removeNodes(this._getAllNodesWithTag(e, [tag]), function (element) {2212 // Allow youtube and vimeo videos through as people usually want to see those.2213 if (isEmbed) {2214 // First, check the elements attributes to see if any of them contain youtube or vimeo2215 for (var i = 0; i < element.attributes.length; i++) {2216 if (this._allowedVideoRegex.test(element.attributes[i].value)) {2217 return false;2218 }2219 }22202221 // For embed with <object> tag, check inner HTML as well.2222 if (2223 element.tagName === "object" &&2224 this._allowedVideoRegex.test(element.innerHTML)2225 ) {2226 return false;2227 }2228 }22292230 return true;2231 });2232 },22332234 /**2235 * Check if a given node has one of its ancestor tag name matching the2236 * provided one.2237 * @param HTMLElement node2238 * @param String tagName2239 * @param Number maxDepth2240 * @param Function filterFn a filter to invoke to determine whether this node 'counts'2241 * @return Boolean2242 */2243 _hasAncestorTag(node, tagName, maxDepth, filterFn) {2244 maxDepth = maxDepth || 3;2245 tagName = tagName.toUpperCase();2246 var depth = 0;2247 while (node.parentNode) {2248 if (maxDepth > 0 && depth > maxDepth) {2249 return false;2250 }2251 if (2252 node.parentNode.tagName === tagName &&2253 (!filterFn || filterFn(node.parentNode))2254 ) {2255 return true;2256 }2257 node = node.parentNode;2258 depth++;2259 }2260 return false;2261 },22622263 /**2264 * Return an object indicating how many rows and columns this table has.2265 */2266 _getRowAndColumnCount(table) {2267 var rows = 0;2268 var columns = 0;2269 var trs = table.getElementsByTagName("tr");2270 for (var i = 0; i < trs.length; i++) {2271 var rowspan = trs[i].getAttribute("rowspan") || 0;2272 if (rowspan) {2273 rowspan = parseInt(rowspan, 10);2274 }2275 rows += rowspan || 1;22762277 // Now look for column-related info2278 var columnsInThisRow = 0;2279 var cells = trs[i].getElementsByTagName("td");2280 for (var j = 0; j < cells.length; j++) {2281 var colspan = cells[j].getAttribute("colspan") || 0;2282 if (colspan) {2283 colspan = parseInt(colspan, 10);2284 }2285 columnsInThisRow += colspan || 1;2286 }2287 columns = Math.max(columns, columnsInThisRow);2288 }2289 return { rows, columns };2290 },22912292 /**2293 * Look for 'data' (as opposed to 'layout') tables, for which we use2294 * similar checks as2295 * https://searchfox.org/mozilla-central/rev/f82d5c549f046cb64ce5602bfd894b7ae807c8f8/accessible/generic/TableAccessible.cpp#192296 */2297 _markDataTables(root) {2298 var tables = root.getElementsByTagName("table");2299 for (var i = 0; i < tables.length; i++) {2300 var table = tables[i];2301 var role = table.getAttribute("role");2302 if (role == "presentation") {2303 table._readabilityDataTable = false;2304 continue;2305 }2306 var datatable = table.getAttribute("datatable");2307 if (datatable == "0") {2308 table._readabilityDataTable = false;2309 continue;2310 }2311 var summary = table.getAttribute("summary");2312 if (summary) {2313 table._readabilityDataTable = true;2314 continue;2315 }23162317 var caption = table.getElementsByTagName("caption")[0];2318 if (caption && caption.childNodes.length) {2319 table._readabilityDataTable = true;2320 continue;2321 }23222323 // If the table has a descendant with any of these tags, consider a data table:2324 var dataTableDescendants = ["col", "colgroup", "tfoot", "thead", "th"];2325 var descendantExists = function (tag) {2326 return !!table.getElementsByTagName(tag)[0];2327 };2328 if (dataTableDescendants.some(descendantExists)) {2329 this.log("Data table because found data-y descendant");2330 table._readabilityDataTable = true;2331 continue;2332 }23332334 // Nested tables indicate a layout table:2335 if (table.getElementsByTagName("table")[0]) {2336 table._readabilityDataTable = false;2337 continue;2338 }23392340 var sizeInfo = this._getRowAndColumnCount(table);23412342 if (sizeInfo.columns == 1 || sizeInfo.rows == 1) {2343 // single colum/row tables are commonly used for page layout purposes.2344 table._readabilityDataTable = false;2345 continue;2346 }23472348 if (sizeInfo.rows >= 10 || sizeInfo.columns > 4) {2349 table._readabilityDataTable = true;2350 continue;2351 }2352 // Now just go by size entirely:2353 table._readabilityDataTable = sizeInfo.rows * sizeInfo.columns > 10;2354 }2355 },23562357 /* convert images and figures that have properties like data-src into images that can be loaded without JS */2358 _fixLazyImages(root) {2359 this._forEachNode(2360 this._getAllNodesWithTag(root, ["img", "picture", "figure"]),2361 function (elem) {2362 // In some sites (e.g. Kotaku), they put 1px square image as base64 data uri in the src attribute.2363 // So, here we check if the data uri is too short, just might as well remove it.2364 if (elem.src && this.REGEXPS.b64DataUrl.test(elem.src)) {2365 // Make sure it's not SVG, because SVG can have a meaningful image in under 133 bytes.2366 var parts = this.REGEXPS.b64DataUrl.exec(elem.src);2367 if (parts[1] === "image/svg+xml") {2368 return;2369 }23702371 // Make sure this element has other attributes which contains image.2372 // If it doesn't, then this src is important and shouldn't be removed.2373 var srcCouldBeRemoved = false;2374 for (var i = 0; i < elem.attributes.length; i++) {2375 var attr = elem.attributes[i];2376 if (attr.name === "src") {2377 continue;2378 }23792380 if (/\.(jpg|jpeg|png|webp)/i.test(attr.value)) {2381 srcCouldBeRemoved = true;2382 break;2383 }2384 }23852386 // Here we assume if image is less than 100 bytes (or 133 after encoded to base64)2387 // it will be too small, therefore it might be placeholder image.2388 if (srcCouldBeRemoved) {2389 var b64starts = parts[0].length;2390 var b64length = elem.src.length - b64starts;2391 if (b64length < 133) {2392 elem.removeAttribute("src");2393 }2394 }2395 }23962397 // also check for "null" to work around https://github.com/jsdom/jsdom/issues/25802398 if (2399 (elem.src || (elem.srcset && elem.srcset != "null")) &&2400 !elem.className.toLowerCase().includes("lazy")2401 ) {2402 return;2403 }24042405 for (var j = 0; j < elem.attributes.length; j++) {2406 attr = elem.attributes[j];2407 if (2408 attr.name === "src" ||2409 attr.name === "srcset" ||2410 attr.name === "alt"2411 ) {2412 continue;2413 }2414 var copyTo = null;2415 if (/\.(jpg|jpeg|png|webp)\s+\d/.test(attr.value)) {2416 copyTo = "srcset";2417 } else if (/^\s*\S+\.(jpg|jpeg|png|webp)\S*\s*$/.test(attr.value)) {2418 copyTo = "src";2419 }2420 if (copyTo) {2421 //if this is an img or picture, set the attribute directly2422 if (elem.tagName === "IMG" || elem.tagName === "PICTURE") {2423 elem.setAttribute(copyTo, attr.value);2424 } else if (2425 elem.tagName === "FIGURE" &&2426 !this._getAllNodesWithTag(elem, ["img", "picture"]).length2427 ) {2428 //if the item is a <figure> that does not contain an image or picture, create one and place it inside the figure2429 //see the nytimes-3 testcase for an example2430 var img = this._doc.createElement("img");2431 img.setAttribute(copyTo, attr.value);2432 elem.appendChild(img);2433 }2434 }2435 }2436 }2437 );2438 },24392440 _getTextDensity(e, tags) {2441 var textLength = this._getInnerText(e, true).length;2442 if (textLength === 0) {2443 return 0;2444 }2445 var childrenLength = 0;2446 var children = this._getAllNodesWithTag(e, tags);2447 this._forEachNode(2448 children,2449 child => (childrenLength += this._getInnerText(child, true).length)2450 );2451 return childrenLength / textLength;2452 },24532454 /**2455 * Clean an element of all tags of type "tag" if they look fishy.2456 * "Fishy" is an algorithm based on content length, classnames, link density, number of images & embeds, etc.2457 *2458 * @return void2459 **/2460 _cleanConditionally(e, tag) {2461 if (!this._flagIsActive(this.FLAG_CLEAN_CONDITIONALLY)) {2462 return;2463 }24642465 // Gather counts for other typical elements embedded within.2466 // Traverse backwards so we can remove nodes at the same time2467 // without effecting the traversal.2468 //2469 // TODO: Consider taking into account original contentScore here.2470 this._removeNodes(this._getAllNodesWithTag(e, [tag]), function (node) {2471 // First check if this node IS data table, in which case don't remove it.2472 var isDataTable = function (t) {2473 return t._readabilityDataTable;2474 };24752476 var isList = tag === "ul" || tag === "ol";2477 if (!isList) {2478 var listLength = 0;2479 var listNodes = this._getAllNodesWithTag(node, ["ul", "ol"]);2480 this._forEachNode(2481 listNodes,2482 list => (listLength += this._getInnerText(list).length)2483 );2484 isList = listLength / this._getInnerText(node).length > 0.9;2485 }24862487 if (tag === "table" && isDataTable(node)) {2488 return false;2489 }24902491 // Next check if we're inside a data table, in which case don't remove it as well.2492 if (this._hasAncestorTag(node, "table", -1, isDataTable)) {2493 return false;2494 }24952496 if (this._hasAncestorTag(node, "code")) {2497 return false;2498 }24992500 // keep element if it has a data tables2501 if (2502 [...node.getElementsByTagName("table")].some(2503 tbl => tbl._readabilityDataTable2504 )2505 ) {2506 return false;2507 }25082509 var weight = this._getClassWeight(node);25102511 this.log("Cleaning Conditionally", node);25122513 var contentScore = 0;25142515 if (weight + contentScore < 0) {2516 return true;2517 }25182519 if (this._getCharCount(node, ",") < 10) {2520 // If there are not very many commas, and the number of2521 // non-paragraph elements is more than paragraphs or other2522 // ominous signs, remove the element.2523 var p = node.getElementsByTagName("p").length;2524 var img = node.getElementsByTagName("img").length;2525 var li = node.getElementsByTagName("li").length - 100;2526 var input = node.getElementsByTagName("input").length;2527 var headingDensity = this._getTextDensity(node, [2528 "h1",2529 "h2",2530 "h3",2531 "h4",2532 "h5",2533 "h6",2534 ]);25352536 var embedCount = 0;2537 var embeds = this._getAllNodesWithTag(node, [2538 "object",2539 "embed",2540 "iframe",2541 ]);25422543 for (var i = 0; i < embeds.length; i++) {2544 // If this embed has attribute that matches video regex, don't delete it.2545 for (var j = 0; j < embeds[i].attributes.length; j++) {2546 if (this._allowedVideoRegex.test(embeds[i].attributes[j].value)) {2547 return false;2548 }2549 }25502551 // For embed with <object> tag, check inner HTML as well.2552 if (2553 embeds[i].tagName === "object" &&2554 this._allowedVideoRegex.test(embeds[i].innerHTML)2555 ) {2556 return false;2557 }25582559 embedCount++;2560 }25612562 var innerText = this._getInnerText(node);25632564 // toss any node whose inner text contains nothing but suspicious words2565 if (2566 this.REGEXPS.adWords.test(innerText) ||2567 this.REGEXPS.loadingWords.test(innerText)2568 ) {2569 return true;2570 }25712572 var contentLength = innerText.length;2573 var linkDensity = this._getLinkDensity(node);2574 var textishTags = ["SPAN", "LI", "TD"].concat(2575 Array.from(this.DIV_TO_P_ELEMS)2576 );2577 var textDensity = this._getTextDensity(node, textishTags);2578 var isFigureChild = this._hasAncestorTag(node, "figure");25792580 // apply shadiness checks, then check for exceptions2581 const shouldRemoveNode = () => {2582 const errs = [];2583 if (!isFigureChild && img > 1 && p / img < 0.5) {2584 errs.push(`Bad p to img ratio (img=${img}, p=${p})`);2585 }2586 if (!isList && li > p) {2587 errs.push(`Too many li's outside of a list. (li=${li} > p=${p})`);2588 }2589 if (input > Math.floor(p / 3)) {2590 errs.push(`Too many inputs per p. (input=${input}, p=${p})`);2591 }2592 if (2593 !isList &&2594 !isFigureChild &&2595 headingDensity < 0.9 &&2596 contentLength < 25 &&2597 (img === 0 || img > 2) &&2598 linkDensity > 02599 ) {2600 errs.push(2601 `Suspiciously short. (headingDensity=${headingDensity}, img=${img}, linkDensity=${linkDensity})`2602 );2603 }2604 if (2605 !isList &&2606 weight < 25 &&2607 linkDensity > 0.2 + this._linkDensityModifier2608 ) {2609 errs.push(2610 `Low weight and a little linky. (linkDensity=${linkDensity})`2611 );2612 }2613 if (weight >= 25 && linkDensity > 0.5 + this._linkDensityModifier) {2614 errs.push(2615 `High weight and mostly links. (linkDensity=${linkDensity})`2616 );2617 }2618 if ((embedCount === 1 && contentLength < 75) || embedCount > 1) {2619 errs.push(2620 `Suspicious embed. (embedCount=${embedCount}, contentLength=${contentLength})`2621 );2622 }2623 if (img === 0 && textDensity === 0) {2624 errs.push(2625 `No useful content. (img=${img}, textDensity=${textDensity})`2626 );2627 }26282629 if (errs.length) {2630 this.log("Checks failed", errs);2631 return true;2632 }26332634 return false;2635 };26362637 var haveToRemove = shouldRemoveNode();26382639 // Allow simple lists of images to remain in pages2640 if (isList && haveToRemove) {2641 for (var x = 0; x < node.children.length; x++) {2642 let child = node.children[x];2643 // Don't filter in lists with li's that contain more than one child2644 if (child.children.length > 1) {2645 return haveToRemove;2646 }2647 }2648 let li_count = node.getElementsByTagName("li").length;2649 // Only allow the list to remain if every li contains an image2650 if (img == li_count) {2651 return false;2652 }2653 }2654 return haveToRemove;2655 }2656 return false;2657 });2658 },26592660 /**2661 * Clean out elements that match the specified conditions2662 *2663 * @param Element2664 * @param Function determines whether a node should be removed2665 * @return void2666 **/2667 _cleanMatchedNodes(e, filter) {2668 var endOfSearchMarkerNode = this._getNextNode(e, true);2669 var next = this._getNextNode(e);2670 while (next && next != endOfSearchMarkerNode) {2671 if (filter.call(this, next, next.className + " " + next.id)) {2672 next = this._removeAndGetNext(next);2673 } else {2674 next = this._getNextNode(next);2675 }2676 }2677 },26782679 /**2680 * Clean out spurious headers from an Element.2681 *2682 * @param Element2683 * @return void2684 **/2685 _cleanHeaders(e) {2686 let headingNodes = this._getAllNodesWithTag(e, ["h1", "h2"]);2687 this._removeNodes(headingNodes, function (node) {2688 let shouldRemove = this._getClassWeight(node) < 0;2689 if (shouldRemove) {2690 this.log("Removing header with low class weight:", node);2691 }2692 return shouldRemove;2693 });2694 },26952696 /**2697 * Check if this node is an H1 or H2 element whose content is mostly2698 * the same as the article title.2699 *2700 * @param Element the node to check.2701 * @return boolean indicating whether this is a title-like header.2702 */2703 _headerDuplicatesTitle(node) {2704 if (node.tagName != "H1" && node.tagName != "H2") {2705 return false;2706 }2707 var heading = this._getInnerText(node, false);2708 this.log("Evaluating similarity of header:", heading, this._articleTitle);2709 return this._textSimilarity(this._articleTitle, heading) > 0.75;2710 },27112712 _flagIsActive(flag) {2713 return (this._flags & flag) > 0;2714 },27152716 _removeFlag(flag) {2717 this._flags = this._flags & ~flag;2718 },27192720 _isProbablyVisible(node) {2721 // Have to null-check node.style and node.className.includes to deal with SVG and MathML nodes.2722 return (2723 (!node.style || node.style.display != "none") &&2724 (!node.style || node.style.visibility != "hidden") &&2725 !node.hasAttribute("hidden") &&2726 //check for "fallback-image" so that wikimedia math images are displayed2727 (!node.hasAttribute("aria-hidden") ||2728 node.getAttribute("aria-hidden") != "true" ||2729 (node.className &&2730 node.className.includes &&2731 node.className.includes("fallback-image")))2732 );2733 },27342735 /**2736 * Runs readability.2737 *2738 * Workflow:2739 * 1. Prep the document by removing script tags, css, etc.2740 * 2. Build readability's DOM tree.2741 * 3. Grab the article content from the current dom tree.2742 * 4. Replace the current DOM tree with the new one.2743 * 5. Read peacefully.2744 *2745 * @return void2746 **/2747 parse() {2748 // Avoid parsing too large documents, as per configuration option2749 if (this._maxElemsToParse > 0) {2750 var numTags = this._doc.getElementsByTagName("*").length;2751 if (numTags > this._maxElemsToParse) {2752 throw new Error(2753 "Aborting parsing document; " + numTags + " elements found"2754 );2755 }2756 }27572758 // Unwrap image from noscript2759 this._unwrapNoscriptImages(this._doc);27602761 // Extract JSON-LD metadata before removing scripts2762 var jsonLd = this._disableJSONLD ? {} : this._getJSONLD(this._doc);27632764 // Remove script tags from the document.2765 this._removeScripts(this._doc);27662767 this._prepDocument();27682769 var metadata = this._getArticleMetadata(jsonLd);2770 this._metadata = metadata;2771 this._articleTitle = metadata.title;27722773 var articleContent = this._grabArticle();2774 if (!articleContent) {2775 return null;2776 }27772778 this.log("Grabbed: " + articleContent.innerHTML);27792780 this._postProcessContent(articleContent);27812782 // If we haven't found an excerpt in the article's metadata, use the article's2783 // first paragraph as the excerpt. This is used for displaying a preview of2784 // the article's content.2785 if (!metadata.excerpt) {2786 var paragraphs = articleContent.getElementsByTagName("p");2787 if (paragraphs.length) {2788 metadata.excerpt = paragraphs[0].textContent.trim();2789 }2790 }27912792 var textContent = articleContent.textContent;2793 return {2794 title: this._articleTitle,2795 byline: metadata.byline || this._articleByline,2796 dir: this._articleDir,2797 lang: this._articleLang,2798 content: this._serializer(articleContent),2799 textContent,2800 length: textContent.length,2801 excerpt: metadata.excerpt,2802 siteName: metadata.siteName || this._articleSiteName,2803 publishedTime: metadata.publishedTime,2804 };2805 },2806};28072808if (typeof module === "object") {2809 /* eslint-disable-next-line no-redeclare */2810 /* global module */2811 module.exports = Readability;2812}2813