HTML 77.2%
TypeScript 10.5%
Python 9.6%
JavaScript 2.5%
1"""HTML parsing with selectolax: title, meta, canonical, Open Graph, JSON-LD, embedded JSON (Next.js/Nuxt/data-props),2headings, tables, links, cleaned text."""3from __future__ import annotations45import html as htmlmod6import json7import re8from dataclasses import dataclass, field9from typing import Any10from urllib.parse import urljoin1112from selectolax.parser import HTMLParser, Node1314_WS = re.compile(r"[ \t\r\f\v]+")15_NL = re.compile(r"\n{3,}")16_SKIP_TAGS = {"script", "style", "noscript", "svg", "template", "iframe", "canvas", "nav", "footer", "form", "button"}171819@dataclass20class HtmlDoc:21 url: str22 title: str | None = None23 canonical: str | None = None24 description: str | None = None25 lang: str | None = None26 meta: dict[str, str] = field(default_factory=dict)27 og: dict[str, str] = field(default_factory=dict)28 json_ld: list[Any] = field(default_factory=list)29 embedded_json: dict[str, Any] = field(default_factory=dict) # id/key -> parsed JSON30 headings: list[tuple[int, str]] = field(default_factory=list)31 tables: list[dict[str, Any]] = field(default_factory=list) # {"headers": [...], "rows": [[...]], "caption": str}32 links: list[tuple[str, str]] = field(default_factory=list) # (absolute href, anchor text)33 text: str = ""34 published_at: str | None = None35 modified_at: str | None = None36 tree: HTMLParser | None = field(default=None, repr=False)3738 def structured(self) -> dict[str, Any]:39 """JSON-serialisable summary stored in `snapshots.structured`."""40 return {41 "title": self.title, "canonical": self.canonical, "description": self.description, "lang": self.lang,42 "meta": {k: v for k, v in self.meta.items() if len(v) < 500}, "og": self.og,43 "json_ld": self.json_ld[:20], "embedded_json_keys": list(self.embedded_json)[:50],44 "headings": self.headings[:200], "tables": self.tables[:40], "published_at": self.published_at, "modified_at": self.modified_at,45 "link_count": len(self.links), "text_length": len(self.text),46 }4748 def css(self, selector: str) -> list[Node]:49 return self.tree.css(selector) if self.tree else []5051 def css_first(self, selector: str) -> Node | None:52 return self.tree.css_first(selector) if self.tree else None5354 def links_matching(self, pattern: str | re.Pattern[str]) -> list[tuple[str, str]]:55 rx = re.compile(pattern) if isinstance(pattern, str) else pattern56 seen: set[str] = set()57 out: list[tuple[str, str]] = []58 for href, text in self.links:59 if href not in seen and rx.search(href):60 seen.add(href)61 out.append((href, text))62 return out636465def clean_text(s: str) -> str:66 s = htmlmod.unescape(s)67 s = _WS.sub(" ", s)68 s = "\n".join(line.strip() for line in s.split("\n"))69 return _NL.sub("\n\n", s).strip()707172def node_text(node: Node, *, separator: str = " ") -> str:73 return clean_text(node.text(separator=separator, strip=True))747576def parse_html(content: str | bytes, url: str = "", *, keep_tree: bool = True, max_links: int = 5000) -> HtmlDoc:77 tree = HTMLParser(content)78 doc = HtmlDoc(url=url, tree=tree if keep_tree else None)79 if (html_node := tree.css_first("html")) is not None:80 doc.lang = html_node.attributes.get("lang")81 if (t := tree.css_first("title")) is not None:82 doc.title = clean_text(t.text()) or None8384 for m in tree.css("meta"):85 a = m.attributes86 key = a.get("property") or a.get("name") or a.get("itemprop")87 val = a.get("content")88 if not key or val is None:89 continue90 key = key.strip().lower()91 val = val.strip()92 if key.startswith(("og:", "twitter:", "article:")):93 doc.og[key] = val94 else:95 doc.meta.setdefault(key, val)96 doc.description = doc.meta.get("description") or doc.og.get("og:description")97 if not doc.title:98 doc.title = doc.og.get("og:title")99 doc.published_at = (doc.og.get("article:published_time") or doc.meta.get("date") or doc.meta.get("pubdate")100 or doc.meta.get("publish_date") or doc.meta.get("datepublished") or doc.meta.get("dc.date.issued"))101 doc.modified_at = doc.og.get("article:modified_time") or doc.meta.get("last-modified") or doc.meta.get("datemodified")102103 for link in tree.css("link[rel]"):104 rel = (link.attributes.get("rel") or "").lower()105 href = link.attributes.get("href")106 if "canonical" in rel and href:107 doc.canonical = urljoin(url, href)108109 for s in tree.css("script"):110 stype = (s.attributes.get("type") or "").lower()111 sid = s.attributes.get("id") or ""112 raw = s.text() or ""113 if not raw.strip():114 continue115 if "ld+json" in stype:116 parsed = _loads_lenient(raw)117 if parsed is not None:118 if isinstance(parsed, list):119 doc.json_ld.extend(parsed)120 else:121 doc.json_ld.append(parsed)122 elif stype in ("application/json", "text/json") or sid in ("__NEXT_DATA__", "__NUXT_DATA__", "__remixContext"):123 parsed = _loads_lenient(raw)124 if parsed is not None:125 doc.embedded_json[sid or f"json_{len(doc.embedded_json)}"] = parsed126 # Hugging Face / Svelte style `data-props` attributes and generic data-* JSON blobs127 for n in tree.css("[data-props]"):128 raw = n.attributes.get("data-props") or ""129 parsed = _loads_lenient(raw)130 if parsed is not None:131 key = f"data-props:{n.attributes.get('data-target') or len(doc.embedded_json)}"132 doc.embedded_json.setdefault(key, parsed)133134 for level in range(1, 5):135 for h in tree.css(f"h{level}"):136 txt = node_text(h)137 if txt:138 doc.headings.append((level, txt[:300]))139140 for table in tree.css("table")[:60]:141 parsed_table = _parse_table(table)142 if parsed_table["rows"]:143 doc.tables.append(parsed_table)144145 seen: set[str] = set()146 for a in tree.css("a[href]"):147 href = a.attributes.get("href") or ""148 if not href or href.startswith(("#", "javascript:", "mailto:", "tel:")):149 continue150 absolute = urljoin(url, href.strip())151 if absolute in seen:152 continue153 seen.add(absolute)154 doc.links.append((absolute, node_text(a)[:200]))155 if len(doc.links) >= max_links:156 break157158 doc.text = extract_text(HTMLParser(content)) # fresh parse: extract_text prunes nav/footer/button/noscript, doc.tree must stay intact159 return doc160161162def extract_text(tree: HTMLParser) -> str:163 body = tree.body or tree.root164 if body is None:165 return ""166 for tag in _SKIP_TAGS:167 for n in body.css(tag):168 n.decompose()169 for n in body.css("[aria-hidden='true'], .sr-only, [hidden]"):170 n.decompose()171 parts: list[str] = []172 block_tags = {"p", "div", "section", "article", "li", "h1", "h2", "h3", "h4", "h5", "h6", "tr", "br", "pre", "blockquote", "td", "th", "dd", "dt"}173 for node in body.traverse(include_text=True):174 if node.tag == "-text":175 t = node.text(deep=False)176 if t and t.strip():177 parts.append(t)178 elif node.tag in block_tags:179 parts.append("\n")180 return clean_text("".join(parts))181182183def _parse_table(table: Node) -> dict[str, Any]:184 headers: list[str] = []185 rows: list[list[str]] = []186 caption_node = table.css_first("caption")187 caption = node_text(caption_node) if caption_node else None188 for tr in table.css("tr"):189 cells = tr.css("th, td")190 if not cells:191 continue192 values = [node_text(c)[:500] for c in cells]193 if not headers and all(c.tag == "th" for c in cells):194 headers = values195 else:196 rows.append(values)197 if len(rows) > 500:198 break199 return {"caption": caption, "headers": headers, "rows": rows}200201202def _loads_lenient(raw: str) -> Any | None:203 raw = raw.strip()204 if not raw:205 return None206 try:207 return json.loads(raw)208 except json.JSONDecodeError:209 pass210 try:211 return json.loads(htmlmod.unescape(raw))212 except json.JSONDecodeError:213 return None214215216def find_in_json(obj: Any, key: str, *, max_hits: int = 50) -> list[Any]:217 """Depth-first search of every value under `key` inside a nested JSON structure."""218 hits: list[Any] = []219220 def walk(o: Any) -> None:221 if len(hits) >= max_hits:222 return223 if isinstance(o, dict):224 for k, v in o.items():225 if k == key:226 hits.append(v)227 walk(v)228 elif isinstance(o, list):229 for v in o:230 walk(v)231232 walk(obj)233 return hits234235236__all__ = ["HtmlDoc", "clean_text", "extract_text", "find_in_json", "node_text", "parse_html"]237