TypeScript 97.5%
SQL 1.4%
Python 0.8%
1import { describe, expect, it } from "vitest";2import { extractPageMetadata, htmlToMarkdown, htmlToMainText, normalizeCrawlUrl, parseRobots, parseSitemap, robotsAllows, urlPatternMatcher, looksLikePage } from "../src/index";34const PAGE = `<!doctype html><html lang="fr"><head><title>Titre & test</title>5<meta name="description" content="Une description."><meta property="og:title" content="OG Titre"><link rel="canonical" href="/canon">6</head><body>7<nav><a href="/a">Nav A</a><a href="/b">Nav B</a></nav>8<main><article>9<h1>Bonjour</h1>10<p>Premier <strong>paragraphe</strong> avec un <a href="/lien?utm_source=x#frag">lien</a> et une <img src="/img.png" alt="image">.</p>11<h2>Liste</h2>12<ul><li>Un</li><li>Deux <em>deux</em></li></ul>13<ol start="3"><li>Trois</li></ol>14<pre><code class="language-js">const x = 1;\nconsole.log(x);</code></pre>15<table><thead><tr><th>A</th><th>B</th></tr></thead><tbody><tr><td>1</td><td>2 | 3</td></tr></tbody></table>16<blockquote><p>Citation</p></blockquote>17</article></main>18<footer><a href="https://other.example/x" rel="nofollow">Externe</a> © 2026</footer>19<script>var a = 1;</script>20</body></html>`;2122describe("markdown", () => {23 it("converts the main content to markdown", () => {24 const md = htmlToMarkdown(PAGE, { baseUrl: "https://site.example/dir/page" });25 expect(md).toContain("# Bonjour");26 expect(md).toContain("Premier **paragraphe** avec un [lien](https://site.example/lien?utm_source=x#frag) et une .");27 expect(md).toContain("- Un\n- Deux *deux*");28 expect(md).toContain("3. Trois");29 expect(md).toContain("```js\nconst x = 1;\nconsole.log(x);\n```");30 expect(md).toContain("| A | B |\n| --- | --- |\n| 1 | 2 \\| 3 |");31 expect(md).toContain("> Citation");32 expect(md).not.toContain("Nav A");33 expect(md).not.toContain("var a = 1");34 });35 it("keeps a <nav> that carries the page's actual content (portal pages)", () => {36 const portal = `<html><head><title>Portal</title></head><body><main><h1>Portal</h1><nav class="central-featured">${Array.from({ length: 10 }, (_, i) => `<div class="lang"><a href="/l${i}"><strong>Language ${i}</strong> <small>${i}00 000+ articles</small></a></div>`).join("")}</nav></main><footer><a href="/terms">Terms</a></footer></body></html>`;37 const md = htmlToMarkdown(portal, { baseUrl: "https://p.example/" });38 expect(md).toContain("[**Language 3** 300 000+ articles](https://p.example/l3)");39 expect(md).not.toContain("Terms");40 });41 it("extracts readable main text", () => {42 const t = htmlToMainText(PAGE);43 expect(t).toContain("Bonjour");44 expect(t).toContain("Premier paragraphe");45 expect(t).not.toContain("Nav A");46 });47 it("extracts metadata and links", () => {48 const { page, links } = extractPageMetadata(PAGE, "https://site.example/dir/page");49 expect(page.title).toBe("Titre & test");50 expect(page.description).toBe("Une description.");51 expect(page.canonical).toBe("https://site.example/canon");52 expect(page.lang).toBe("fr");53 expect(page.og["og:title"]).toBe("OG Titre");54 expect(page.links_count).toBe(links.length);55 expect(links.find((l) => l.url === "https://site.example/a")?.internal).toBe(true);56 const ext = links.find((l) => l.url.startsWith("https://other.example"));57 expect(ext).toMatchObject({ internal: false, nofollow: true, text: "Externe" });58 expect(links.find((l) => l.url.includes("#"))).toBeUndefined();59 });60 it("normalises crawl URLs", () => {61 expect(normalizeCrawlUrl("HTTPS://Site.Example/path/?b=2&a=1&utm_source=x#frag")).toBe("https://site.example/path?a=1&b=2");62 expect(normalizeCrawlUrl("mailto:x@y.z")).toBeNull();63 expect(looksLikePage("https://a.b/c.pdf")).toBe(false);64 expect(looksLikePage("https://a.b/c")).toBe(true);65 });66 it("matches glob and regex patterns", () => {67 const m = urlPatternMatcher(["https://a.b/blog/*", "/\\/docs\\//"])!;68 expect(m("https://a.b/blog/post-1")).toBe(true);69 expect(m("https://a.b/blog/2024/x")).toBe(false);70 expect(m("https://a.b/docs/x")).toBe(true);71 expect(m("https://a.b/other")).toBe(false);72 });73 it("parses robots.txt and sitemaps", () => {74 const r = parseRobots("User-agent: *\nDisallow: /private/\nAllow: /private/ok\nCrawl-delay: 2\nSitemap: https://a.b/sitemap.xml\n\nUser-agent: fetchabot\nDisallow: /secret");75 expect(r.disallow).toEqual(["/secret"]);76 expect(robotsAllows(r, "https://a.b/secret/x")).toBe(false);77 expect(robotsAllows(r, "https://a.b/public")).toBe(true);78 const star = parseRobots("User-agent: *\nDisallow: /private/\nAllow: /private/ok");79 expect(robotsAllows(star, "https://a.b/private/x")).toBe(false);80 expect(robotsAllows(star, "https://a.b/private/ok")).toBe(true);81 expect(star.sitemaps).toEqual([]);82 expect(r.sitemaps).toEqual(["https://a.b/sitemap.xml"]);83 expect(parseSitemap("<urlset><url><loc>https://a.b/1</loc></url><url><loc>https://a.b/2</loc></url></urlset>").urls).toEqual(["https://a.b/1", "https://a.b/2"]);84 expect(parseSitemap("<sitemapindex><sitemap><loc>https://a.b/s1.xml</loc></sitemap></sitemapindex>").sitemaps).toEqual(["https://a.b/s1.xml"]);85 });86});87