TypeScript 97.5%
SQL 1.4%
Python 0.8%
1/**2 * Dependency-free HTML → Markdown conversion, page metadata and link extraction.3 *4 * The converter builds a lightweight DOM from a forgiving tokenizer, optionally isolates the main5 * content (readability-style density scoring), then serialises to Markdown. It is intended for6 * LLM/RAG pipelines and crawling: stable, compact output rather than perfect fidelity.7 */8import { decodeEntities } from "./text";9import type { PageLink, PageMetadata } from "./schema";1011// ---------------------------------------------------------------------------12// Tokenizer / DOM-lite13// ---------------------------------------------------------------------------14export interface HNode {15 type: "element" | "text";16 tag: string;17 attrs: Record<string, string>;18 children: HNode[];19 text: string;20 parent: HNode | null;21}2223const VOID = new Set(["area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"]);24const RAW_TEXT = new Set(["script", "style", "noscript", "template", "svg", "math", "iframe", "canvas", "object", "textarea"]);25const BLOCK = new Set([26 "address", "article", "aside", "blockquote", "body", "center", "dd", "details", "dialog", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6",27 "header", "hr", "html", "li", "main", "nav", "ol", "p", "pre", "section", "summary", "table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul",28]);29/** Elements whose content is closed implicitly by a new block start. */30const AUTO_CLOSE: Record<string, Set<string>> = {31 p: BLOCK,32 li: new Set(["li"]),33 dt: new Set(["dt", "dd"]),34 dd: new Set(["dt", "dd"]),35 tr: new Set(["tr"]),36 td: new Set(["td", "th", "tr"]),37 th: new Set(["td", "th", "tr"]),38 option: new Set(["option"]),39};4041function makeNode(type: HNode["type"], tag: string, parent: HNode | null): HNode {42 return { type, tag, attrs: {}, children: [], text: "", parent };43}4445const ATTR_RE = /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;4647export function parseHtml(html: string): HNode {48 const root = makeNode("element", "#root", null);49 let cur = root;50 let i = 0;51 const n = html.length;52 const pushText = (t: string) => {53 if (!t) return;54 const node = makeNode("text", "#text", cur);55 node.text = t;56 cur.children.push(node);57 };58 while (i < n) {59 const lt = html.indexOf("<", i);60 if (lt === -1) {61 pushText(html.slice(i));62 break;63 }64 if (lt > i) pushText(html.slice(i, lt));65 if (html.startsWith("<!--", lt)) {66 const end = html.indexOf("-->", lt + 4);67 i = end === -1 ? n : end + 3;68 continue;69 }70 if (html.startsWith("<![CDATA[", lt)) {71 const end = html.indexOf("]]>", lt);72 i = end === -1 ? n : end + 3;73 continue;74 }75 if (html[lt + 1] === "!" || html[lt + 1] === "?") {76 const end = html.indexOf(">", lt);77 i = end === -1 ? n : end + 1;78 continue;79 }80 const gt = findTagEnd(html, lt);81 if (gt === -1) {82 pushText(html.slice(lt));83 break;84 }85 const raw = html.slice(lt + 1, gt);86 i = gt + 1;87 if (raw.startsWith("/")) {88 const tag = raw.slice(1).trim().toLowerCase().split(/\s/)[0]!;89 // close up to matching open element90 let p: HNode | null = cur;91 while (p && p !== root && p.tag !== tag) p = p.parent;92 if (p && p !== root) cur = p.parent ?? root;93 continue;94 }95 const m = raw.match(/^([a-zA-Z][a-zA-Z0-9:-]*)/);96 if (!m) {97 pushText("<" + raw + ">");98 continue;99 }100 const tag = m[1]!.toLowerCase();101 const selfClosing = raw.endsWith("/");102 const attrs: Record<string, string> = {};103 const attrStr = raw.slice(m[0].length, selfClosing ? -1 : undefined);104 for (const am of attrStr.matchAll(ATTR_RE)) {105 const k = am[1]!.toLowerCase();106 attrs[k] = decodeEntities(am[2] ?? am[3] ?? am[4] ?? "");107 }108 // implicit closes109 let p: HNode | null = cur;110 while (p && p !== root) {111 const ac = AUTO_CLOSE[p.tag];112 if (ac && ac.has(tag)) {113 cur = p.parent ?? root;114 break;115 }116 if (p.tag === "p" && BLOCK.has(tag)) {117 cur = p.parent ?? root;118 break;119 }120 p = p.parent;121 }122 const node = makeNode("element", tag, cur);123 node.attrs = attrs;124 cur.children.push(node);125 if (RAW_TEXT.has(tag) && !selfClosing) {126 const close = html.toLowerCase().indexOf(`</${tag}`, i);127 const end = close === -1 ? n : close;128 const t = makeNode("text", "#text", node);129 t.text = html.slice(i, end);130 node.children.push(t);131 i = close === -1 ? n : html.indexOf(">", close) + 1 || n;132 continue;133 }134 if (!VOID.has(tag) && !selfClosing) cur = node;135 }136 return root;137}138139function findTagEnd(html: string, from: number): number {140 let q: string | null = null;141 for (let j = from + 1; j < html.length; j++) {142 const c = html[j]!;143 if (q) {144 if (c === q) q = null;145 } else if (c === '"' || c === "'") {146 // only treat as quote when inside attribute area (after a space or =)147 const prev = html[j - 1];148 if (prev === "=" || prev === " " || prev === "\t" || prev === "\n") q = c;149 } else if (c === ">") return j;150 else if (c === "<" && j > from + 1) return -1 + 0 * j; // malformed: give up on this tag151 }152 return -1;153}154155// ---------------------------------------------------------------------------156// Queries157// ---------------------------------------------------------------------------158export function walk(node: HNode, fn: (n: HNode) => boolean | void): void {159 if (fn(node) === false) return;160 for (const c of node.children) walk(c, fn);161}162163export function findAll(root: HNode, pred: (n: HNode) => boolean): HNode[] {164 const out: HNode[] = [];165 walk(root, (n) => {166 if (n.type === "element" && pred(n)) out.push(n);167 });168 return out;169}170171export function textOf(node: HNode): string {172 let s = "";173 walk(node, (n) => {174 if (n.type === "element" && RAW_TEXT.has(n.tag)) return false;175 if (n.type === "text") s += n.text;176 });177 return decodeEntities(s).replace(/\s+/g, " ").trim();178}179180// ---------------------------------------------------------------------------181// Main content extraction (readability-lite)182// ---------------------------------------------------------------------------183const NOISE_TAGS = new Set(["nav", "header", "footer", "aside", "form", "button", "select", "input", "label", "dialog", "menu"]);184const NOISE_RE = /(^|[\s_-])(nav|menu|sidebar|side-bar|footer|header|banner|cookie|consent|gdpr|modal|popup|newsletter|subscribe|social|share|sharing|related|recommend|promo|advert|ad-|ads|sponsor|breadcrumb|comment|widget|toolbar|skip|masthead|sitemap|legal|copyright|login|signup|search)([\s_-]|$)/i;185const CONTENT_RE = /(^|[\s_-])(article|content|main|post|entry|body|story|text|blog|product|description|prose|markdown|documentation|docs)([\s_-]|$)/i;186187function classId(n: HNode): string {188 return `${n.attrs["class"] ?? ""} ${n.attrs["id"] ?? ""}`;189}190191export function isNoise(n: HNode): boolean {192 if (n.type !== "element") return false;193 if (RAW_TEXT.has(n.tag)) return true;194 if (NOISE_TAGS.has(n.tag)) return true;195 if (n.attrs["hidden"] !== undefined || /display\s*:\s*none/i.test(n.attrs["style"] ?? "")) return true;196 if (n.attrs["aria-hidden"] === "true") return true;197 const role = n.attrs["role"];198 if (role && /navigation|banner|contentinfo|complementary|dialog|search|menu/i.test(role)) return true;199 if (n.tag === "div" || n.tag === "section" || n.tag === "ul" || n.tag === "span") {200 const ci = classId(n);201 if (NOISE_RE.test(ci) && !CONTENT_RE.test(ci)) return true;202 }203 return false;204}205206/** Pick the element that most likely holds the primary content. Falls back to <body>/root. */207export function mainContent(root: HNode): HNode {208 const explicit = findAll(root, (n) => n.tag === "article" || n.tag === "main" || n.attrs["role"] === "main" || n.attrs["itemprop"] === "articleBody");209 const body = findAll(root, (n) => n.tag === "body")[0] ?? root;210 const scored: Array<{ node: HNode; score: number }> = [];211 const candidates = explicit.length ? explicit : findAll(body, (n) => ["div", "section", "td", "article", "main"].includes(n.tag));212 const bodyLen = Math.max(1, textOf(body).length);213 for (const c of candidates) {214 if (isNoise(c)) continue;215 const text = textOf(c);216 if (text.length < 200) continue;217 const paragraphs = findAll(c, (n) => n.tag === "p" || n.tag === "h1" || n.tag === "h2" || n.tag === "h3" || n.tag === "li" || n.tag === "pre").length;218 const links = findAll(c, (n) => n.tag === "a").reduce((s, a) => s + textOf(a).length, 0);219 const linkDensity = links / Math.max(1, text.length);220 let score = text.length / bodyLen + Math.min(paragraphs, 40) / 40;221 score *= 1 - Math.min(0.9, linkDensity);222 if (c.tag === "article" || c.tag === "main" || c.attrs["role"] === "main") score *= 1.6;223 if (CONTENT_RE.test(classId(c))) score *= 1.2;224 scored.push({ node: c, score });225 }226 if (!scored.length) return body;227 scored.sort((a, b) => b.score - a.score);228 const best = scored[0]!;229 // Guard against choosing a tiny fragment: require the pick to hold ≥ 25% of the body text.230 if (textOf(best.node).length < bodyLen * 0.25 && !explicit.length) return body;231 return best.node;232}233234// ---------------------------------------------------------------------------235// Markdown serialisation236// ---------------------------------------------------------------------------237export interface MarkdownOptions {238 baseUrl?: string;239 /** Isolate the main content (default true). */240 mainContent?: boolean;241 /** Keep images as  (default true). */242 images?: boolean;243 /** Keep hyperlinks as [text](href) (default true); false renders plain text. */244 links?: boolean;245 /** Drop navigation/footers/asides etc. even inside the main content (default true). */246 stripNoise?: boolean;247 /** Maximum output length in characters (default 2,000,000). */248 maxLength?: number;249}250251function absolutize(href: string | undefined, base?: string): string | null {252 if (!href) return null;253 const h = href.trim();254 if (!h || h.startsWith("javascript:") || h.startsWith("data:") || h.startsWith("mailto:") || h.startsWith("tel:") || h === "#") return h.startsWith("mailto:") || h.startsWith("tel:") ? h : null;255 try {256 return base ? new URL(h, base).toString() : new URL(h).toString();257 } catch {258 return null;259 }260}261262function esc(s: string): string {263 return s.replace(/([\\`*_{}[\]<>])/g, "\\$1");264}265266type WriterOpts = Required<Pick<MarkdownOptions, "images" | "links" | "stripNoise">> & { baseUrl?: string; /** Text length of the rendered scope, used to keep "noise" landmarks that actually hold the content (e.g. a <nav> of language links on a portal). */ scopeLen?: number };267268class MdWriter {269 out: string[] = [];270 constructor(private readonly opts: WriterOpts) {}271272 /** Noise check that spares landmark elements carrying a large share of the scope's text. */273 private noise(n: HNode): boolean {274 if (!this.opts.stripNoise || !isNoise(n)) return false;275 if (n.type === "element" && (n.tag === "nav" || n.tag === "aside" || n.tag === "header" || n.tag === "footer" || n.tag === "form") && this.opts.scopeLen) {276 const t = textOf(n).length;277 if (t >= 120 && t >= this.opts.scopeLen * 0.35) return false;278 }279 return true;280 }281282 block(s: string) {283 const t = s.replace(/\n{3,}/g, "\n\n").trim();284 if (!t) return;285 this.out.push(t);286 }287288 render(node: HNode): string {289 this.out = [];290 this.renderChildren(node, "");291 return this.out.join("\n\n").replace(/\n{3,}/g, "\n\n").trim();292 }293294 private inline(node: HNode): string {295 let s = "";296 for (const c of node.children) {297 if (c.type === "text") {298 s += decodeEntities(c.text).replace(/\s+/g, " ");299 continue;300 }301 if (this.noise(c)) continue;302 switch (c.tag) {303 case "br":304 s += " \n";305 break;306 case "strong":307 case "b": {308 const t = this.inline(c).trim();309 s += t ? `**${t}**` : "";310 break;311 }312 case "em":313 case "i": {314 const t = this.inline(c).trim();315 s += t ? `*${t}*` : "";316 break;317 }318 case "del":319 case "s":320 case "strike": {321 const t = this.inline(c).trim();322 s += t ? `~~${t}~~` : "";323 break;324 }325 case "code":326 case "kbd":327 case "samp":328 case "var": {329 const t = textOf(c);330 s += t ? `\`${t.replace(/`/g, "\\`")}\`` : "";331 break;332 }333 case "a": {334 const t = this.inline(c).trim();335 const href = absolutize(c.attrs["href"], this.opts.baseUrl);336 if (!t && !href) break;337 if (!this.opts.links || !href) s += t;338 else s += `[${t || href}](${href})`;339 break;340 }341 case "img": {342 if (!this.opts.images) break;343 const src = absolutize(c.attrs["src"] ?? c.attrs["data-src"] ?? (c.attrs["srcset"] ?? "").split(/[\s,]/)[0], this.opts.baseUrl);344 if (!src) break;345 const alt = (c.attrs["alt"] ?? "").replace(/\s+/g, " ").trim();346 s += ``;347 break;348 }349 case "sup":350 s += `^${this.inline(c).trim()}`;351 break;352 case "sub":353 s += `_${this.inline(c).trim()}`;354 break;355 case "q":356 s += `"${this.inline(c).trim()}"`;357 break;358 case "input": {359 if (c.attrs["type"] === "checkbox") s += c.attrs["checked"] !== undefined ? "[x] " : "[ ] ";360 break;361 }362 default:363 if (BLOCK.has(c.tag)) {364 // block inside inline context: flush as text with breaks365 s += "\n" + this.inlineBlock(c) + "\n";366 } else s += this.inline(c);367 }368 }369 return s;370 }371372 private inlineBlock(node: HNode): string {373 const w = new MdWriter(this.opts);374 return w.render(node);375 }376377 private renderChildren(node: HNode, prefix: string) {378 let inlineBuf = "";379 const flush = () => {380 const t = inlineBuf.replace(/[ \t]+/g, " ").replace(/ *\n */g, "\n").trim();381 if (t) this.block(prefix + t.split("\n").join("\n" + prefix));382 inlineBuf = "";383 };384 for (const c of node.children) {385 if (c.type === "text") {386 inlineBuf += decodeEntities(c.text).replace(/\s+/g, " ");387 continue;388 }389 if (this.noise(c)) continue;390 if (!BLOCK.has(c.tag) && !["table", "ul", "ol", "dl"].includes(c.tag)) {391 inlineBuf += this.inline({ ...c, children: [c] } as HNode);392 continue;393 }394 flush();395 this.renderBlock(c, prefix);396 }397 flush();398 }399400 private renderBlock(c: HNode, prefix: string) {401 switch (c.tag) {402 case "h1":403 case "h2":404 case "h3":405 case "h4":406 case "h5":407 case "h6": {408 const level = Number(c.tag[1]);409 const t = this.inline(c).replace(/\s+/g, " ").trim();410 if (t) this.block(`${prefix}${"#".repeat(level)} ${t}`);411 return;412 }413 case "p":414 case "div":415 case "section":416 case "article":417 case "main":418 case "header":419 case "footer":420 case "aside":421 case "nav":422 case "address":423 case "center":424 case "details":425 case "summary":426 case "figure":427 case "dialog":428 case "fieldset":429 case "form":430 case "body":431 case "html": {432 const hasBlocks = c.children.some((k) => k.type === "element" && (BLOCK.has(k.tag) || ["table", "ul", "ol", "dl"].includes(k.tag)));433 if (hasBlocks) this.renderChildren(c, prefix);434 else {435 const t = this.inline(c).replace(/[ \t]+/g, " ").trim();436 if (t) this.block(prefix + t.split("\n").map((l) => l.trim()).join("\n" + prefix));437 }438 return;439 }440 case "figcaption": {441 const t = this.inline(c).trim();442 if (t) this.block(`${prefix}*${t}*`);443 return;444 }445 case "blockquote": {446 const inner = new MdWriter(this.opts).render(c);447 if (inner) this.block(inner.split("\n").map((l) => `${prefix}> ${l}`).join("\n"));448 return;449 }450 case "pre": {451 const codeEl = c.children.find((k) => k.type === "element" && k.tag === "code");452 const lang = ((codeEl?.attrs["class"] ?? c.attrs["class"] ?? "").match(/(?:language|lang)-([a-z0-9+#-]+)/i) ?? [])[1] ?? "";453 let code = "";454 walk(c, (n) => {455 if (n.type === "text") code += decodeEntities(n.text);456 });457 code = code.replace(/^\n+|\n+$/g, "");458 const fence = code.includes("```") ? "````" : "```";459 this.block(`${prefix}${fence}${lang}\n${code.split("\n").map((l) => prefix + l).join("\n")}\n${prefix}${fence}`);460 return;461 }462 case "hr":463 this.block(`${prefix}---`);464 return;465 case "ul":466 case "ol": {467 const ordered = c.tag === "ol";468 let idx = Number(c.attrs["start"] ?? 1) || 1;469 const items: string[] = [];470 for (const li of c.children) {471 if (li.type !== "element" || li.tag !== "li") continue;472 const marker = ordered ? `${idx++}. ` : "- ";473 const w = new MdWriter(this.opts);474 const inner = w.render(li);475 if (!inner) continue;476 const lines = inner.split("\n");477 const pad = " ".repeat(marker.length);478 items.push(prefix + marker + lines[0] + (lines.length > 1 ? "\n" + lines.slice(1).map((l) => (l ? prefix + pad + l : "")).join("\n") : ""));479 }480 if (items.length) this.block(items.join("\n"));481 return;482 }483 case "li": {484 const w = new MdWriter(this.opts);485 const inner = w.render(c);486 if (inner) this.block(prefix + "- " + inner.split("\n").join("\n" + prefix + " "));487 return;488 }489 case "dl": {490 const lines: string[] = [];491 for (const k of c.children) {492 if (k.type !== "element") continue;493 if (k.tag === "dt") lines.push(`${prefix}**${this.inline(k).trim()}**`);494 if (k.tag === "dd") lines.push(`${prefix}: ${this.inline(k).trim()}`);495 }496 if (lines.length) this.block(lines.join("\n"));497 return;498 }499 case "table": {500 const rows: string[][] = [];501 let headerRow: string[] | null = null;502 const trs = findAll(c, (n) => n.tag === "tr").filter((tr) => {503 // exclude nested tables' rows504 let p = tr.parent;505 while (p && p !== c) {506 if (p.tag === "table") return false;507 p = p.parent;508 }509 return true;510 });511 for (const tr of trs) {512 const cells = tr.children.filter((k) => k.type === "element" && (k.tag === "td" || k.tag === "th"));513 if (!cells.length) continue;514 const vals = cells.map((cell) => this.inline(cell).replace(/\s*\n\s*/g, " ").replace(/\|/g, "\\|").trim());515 const isHeader = !headerRow && cells.every((cell) => cell.tag === "th");516 if (isHeader) headerRow = vals;517 else rows.push(vals);518 }519 if (!headerRow && !rows.length) return;520 const width = Math.max(headerRow?.length ?? 0, ...rows.map((r) => r.length));521 const pad = (r: string[]) => [...r, ...Array(Math.max(0, width - r.length)).fill("")];522 const head = headerRow ? pad(headerRow) : Array(width).fill(" ");523 const lines = [`${prefix}| ${head.join(" | ")} |`, `${prefix}| ${Array(width).fill("---").join(" | ")} |`, ...rows.map((r) => `${prefix}| ${pad(r).join(" | ")} |`)];524 this.block(lines.join("\n"));525 return;526 }527 case "tr":528 case "td":529 case "th":530 case "thead":531 case "tbody":532 case "tfoot":533 case "dd":534 case "dt": {535 const t = this.inline(c).trim();536 if (t) this.block(prefix + t);537 return;538 }539 default: {540 this.renderChildren(c, prefix);541 }542 }543 }544}545546export function htmlToMarkdown(html: string, opts: MarkdownOptions = {}): string {547 const root = parseHtml(html);548 const scope = opts.mainContent === false ? (findAll(root, (n) => n.tag === "body")[0] ?? root) : mainContent(root);549 const writer = new MdWriter({ images: opts.images ?? true, links: opts.links ?? true, stripNoise: opts.stripNoise ?? true, baseUrl: opts.baseUrl, scopeLen: textOf(scope).length });550 let md = writer.render(scope);551 // Prepend the document title when the content does not already start with a heading.552 const title = findAll(root, (n) => n.tag === "title")[0];553 const t = title ? textOf(title) : "";554 if (t && !/^#\s/.test(md) && !md.startsWith(`# ${t}`)) md = `# ${esc(t)}\n\n${md}`;555 const max = opts.maxLength ?? 2_000_000;556 return md.length > max ? md.slice(0, max) : md;557}558559/** Readable text (main content, boilerplate removed). Falls back to whole document. */560export function htmlToMainText(html: string): string {561 const root = parseHtml(html);562 const scope = mainContent(root);563 const w = new MdWriter({ images: false, links: false, stripNoise: true, scopeLen: textOf(scope).length });564 return w565 .render(scope)566 .replace(/^#+\s*/gm, "")567 .replace(/\*\*|~~|(?<!\\)\*/g, "")568 .replace(/\\([\\`*_{}[\]<>])/g, "$1")569 .trim();570}571572// ---------------------------------------------------------------------------573// Metadata & links574// ---------------------------------------------------------------------------575export function registrableHost(hostname: string): string {576 const parts = hostname.toLowerCase().replace(/^www\./, "").split(".");577 if (parts.length <= 2) return parts.join(".");578 const sld = new Set(["co", "com", "org", "net", "gov", "edu", "ac", "gc", "qc", "on", "bc"]);579 if (parts.length >= 3 && sld.has(parts[parts.length - 2]!) && parts[parts.length - 1]!.length === 2) return parts.slice(-3).join(".");580 return parts.slice(-2).join(".");581}582583export function extractPageMetadata(html: string, baseUrl: string): { page: PageMetadata; links: PageLink[] } {584 const root = parseHtml(html.length > 3_000_000 ? html.slice(0, 3_000_000) : html);585 const head = findAll(root, (n) => n.tag === "head")[0] ?? root;586 const titleEl = findAll(head, (n) => n.tag === "title")[0] ?? findAll(root, (n) => n.tag === "title")[0];587 const og: Record<string, string> = {};588 let description: string | null = null;589 let canonical: string | null = null;590 for (const m of findAll(root, (n) => n.tag === "meta")) {591 const name = (m.attrs["name"] ?? m.attrs["property"] ?? "").toLowerCase();592 const content = (m.attrs["content"] ?? "").trim();593 if (!name || !content) continue;594 if (name === "description" && !description) description = content.slice(0, 1000);595 if (name.startsWith("og:") || name.startsWith("twitter:") || name.startsWith("article:")) og[name] = content.slice(0, 1000);596 }597 for (const l of findAll(root, (n) => n.tag === "link")) {598 if ((l.attrs["rel"] ?? "").toLowerCase().split(/\s+/).includes("canonical")) {599 canonical = absolutize(l.attrs["href"], baseUrl);600 break;601 }602 }603 const htmlEl = findAll(root, (n) => n.tag === "html")[0];604 const lang = htmlEl?.attrs["lang"]?.trim().slice(0, 16) || og["og:locale"]?.slice(0, 16) || null;605606 let baseHref = baseUrl;607 const baseEl = findAll(head, (n) => n.tag === "base")[0];608 if (baseEl?.attrs["href"]) baseHref = absolutize(baseEl.attrs["href"], baseUrl) ?? baseUrl;609610 const seen = new Set<string>();611 const links: PageLink[] = [];612 let host = "";613 try {614 host = registrableHost(new URL(baseUrl).hostname);615 } catch {616 /* ignore */617 }618 for (const a of findAll(root, (n) => n.tag === "a" || n.tag === "area")) {619 const url = absolutize(a.attrs["href"], baseHref);620 if (!url || !/^https?:/i.test(url)) continue;621 const clean = url.replace(/#.*$/, "");622 if (!clean || seen.has(clean)) continue;623 seen.add(clean);624 let internal = false;625 try {626 internal = registrableHost(new URL(clean).hostname) === host;627 } catch {628 /* ignore */629 }630 const rel = (a.attrs["rel"] ?? "").toLowerCase();631 links.push({ url: clean, text: textOf(a).slice(0, 200) || (a.attrs["title"] ?? a.attrs["aria-label"] ?? "").slice(0, 200), internal, nofollow: /\bnofollow\b/.test(rel) });632 if (links.length >= 5000) break;633 }634 return {635 page: {636 title: titleEl ? textOf(titleEl).slice(0, 300) || null : og["og:title"]?.slice(0, 300) ?? null,637 description: description ?? og["og:description"] ?? null,638 canonical,639 lang,640 og,641 links_count: links.length,642 },643 links,644 };645}646647/** Glob (`*`, `**`) or `/regex/flags` pattern matcher for URL filtering. */648export function urlPatternMatcher(patterns: string[] | undefined): ((url: string) => boolean) | null {649 if (!patterns?.length) return null;650 const res: RegExp[] = [];651 for (const raw of patterns) {652 const p = raw.trim();653 if (!p) continue;654 const rx = p.match(/^\/(.+)\/([a-z]*)$/);655 if (rx) {656 try {657 res.push(new RegExp(rx[1]!, rx[2]!.replace(/[^gimsuy]/g, "")));658 continue;659 } catch {660 /* fall through to glob */661 }662 }663 const escaped = p.replace(/[.+^${}()|[\]\\?]/g, "\\$&").replace(/\*\*/g, "\u0000").replace(/\*/g, "[^/]*").replace(/\u0000/g, ".*");664 res.push(new RegExp(p.includes("://") || p.startsWith("/") ? `^${escaped}$` : escaped, "i"));665 }666 if (!res.length) return null;667 return (url: string) => res.some((r) => r.test(url));668}669670/** Normalise a URL for crawl de-duplication: strip fragment, tracking params, trailing slash, sort query. */671export function normalizeCrawlUrl(url: string): string | null {672 try {673 const u = new URL(url);674 if (!/^https?:$/.test(u.protocol)) return null;675 u.hash = "";676 u.hostname = u.hostname.toLowerCase();677 const drop = ["utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", "utm_id", "gclid", "fbclid", "mc_cid", "mc_eid", "ref", "ref_", "_ga", "yclid", "msclkid", "igshid", "spm"];678 for (const k of drop) u.searchParams.delete(k);679 u.searchParams.sort();680 if (u.pathname.length > 1 && u.pathname.endsWith("/")) u.pathname = u.pathname.slice(0, -1);681 if ((u.protocol === "http:" && u.port === "80") || (u.protocol === "https:" && u.port === "443")) u.port = "";682 return u.toString();683 } catch {684 return null;685 }686}687688const NON_HTML_EXT = /\.(jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|mp4|mp3|wav|ogg|webm|mov|avi|zip|gz|tgz|rar|7z|tar|pdf|docx?|xlsx?|pptx?|exe|dmg|apk|css|js|mjs|json|xml|rss|atom|woff2?|ttf|eot|otf)(\?.*)?$/i;689690/** Whether a URL is likely to be an HTML page worth crawling. */691export function looksLikePage(url: string): boolean {692 try {693 const u = new URL(url);694 return !NON_HTML_EXT.test(u.pathname);695 } catch {696 return false;697 }698}699