import { describe, expect, it } from "vitest"; import { extractPageMetadata, htmlToMarkdown, htmlToMainText, normalizeCrawlUrl, parseRobots, parseSitemap, robotsAllows, urlPatternMatcher, looksLikePage } from "../src/index"; const PAGE = `Titre & test

Bonjour

Premier paragraphe avec un lien et une .

Liste

Un
Deux deux

Trois

const x = 1;\nconsole.log(x);

A	B
1	2 \| 3

Citation

`; describe("markdown", () => { it("converts the main content to markdown", () => { const md = htmlToMarkdown(PAGE, { baseUrl: "https://site.example/dir/page" }); expect(md).toContain("# Bonjour"); expect(md).toContain("Premier **paragraphe** avec un [lien](https://site.example/lien?utm_source=x#frag) et une ![image](https://site.example/img.png)."); expect(md).toContain("- Un\n- Deux *deux*"); expect(md).toContain("3. Trois"); expect(md).toContain("```js\nconst x = 1;\nconsole.log(x);\n```"); expect(md).toContain("| A | B |\n| --- | --- |\n| 1 | 2 \\| 3 |"); expect(md).toContain("> Citation"); expect(md).not.toContain("Nav A"); expect(md).not.toContain("var a = 1"); }); it("keeps a

that carries the page's actual content (portal pages)", () => { const portal = `Portal

Portal

`; const md = htmlToMarkdown(portal, { baseUrl: "https://p.example/" }); expect(md).toContain("[**Language 3** 300 000+ articles](https://p.example/l3)"); expect(md).not.toContain("Terms"); }); it("extracts readable main text", () => { const t = htmlToMainText(PAGE); expect(t).toContain("Bonjour"); expect(t).toContain("Premier paragraphe"); expect(t).not.toContain("Nav A"); }); it("extracts metadata and links", () => { const { page, links } = extractPageMetadata(PAGE, "https://site.example/dir/page"); expect(page.title).toBe("Titre & test"); expect(page.description).toBe("Une description."); expect(page.canonical).toBe("https://site.example/canon"); expect(page.lang).toBe("fr"); expect(page.og["og:title"]).toBe("OG Titre"); expect(page.links_count).toBe(links.length); expect(links.find((l) => l.url === "https://site.example/a")?.internal).toBe(true); const ext = links.find((l) => l.url.startsWith("https://other.example")); expect(ext).toMatchObject({ internal: false, nofollow: true, text: "Externe" }); expect(links.find((l) => l.url.includes("#"))).toBeUndefined(); }); it("normalises crawl URLs", () => { expect(normalizeCrawlUrl("HTTPS://Site.Example/path/?b=2&a=1&utm_source=x#frag")).toBe("https://site.example/path?a=1&b=2"); expect(normalizeCrawlUrl("mailto:x@y.z")).toBeNull(); expect(looksLikePage("https://a.b/c.pdf")).toBe(false); expect(looksLikePage("https://a.b/c")).toBe(true); }); it("matches glob and regex patterns", () => { const m = urlPatternMatcher(["https://a.b/blog/*", "/\\/docs\\//"])!; expect(m("https://a.b/blog/post-1")).toBe(true); expect(m("https://a.b/blog/2024/x")).toBe(false); expect(m("https://a.b/docs/x")).toBe(true); expect(m("https://a.b/other")).toBe(false); }); it("parses robots.txt and sitemaps", () => { const r = parseRobots("User-agent: *\nDisallow: /private/\nAllow: /private/ok\nCrawl-delay: 2\nSitemap: https://a.b/sitemap.xml\n\nUser-agent: fetchabot\nDisallow: /secret"); expect(r.disallow).toEqual(["/secret"]); expect(robotsAllows(r, "https://a.b/secret/x")).toBe(false); expect(robotsAllows(r, "https://a.b/public")).toBe(true); const star = parseRobots("User-agent: *\nDisallow: /private/\nAllow: /private/ok"); expect(robotsAllows(star, "https://a.b/private/x")).toBe(false); expect(robotsAllows(star, "https://a.b/private/ok")).toBe(true); expect(star.sitemaps).toEqual([]); expect(r.sitemaps).toEqual(["https://a.b/sitemap.xml"]); expect(parseSitemap("https://a.b/1https://a.b/2").urls).toEqual(["https://a.b/1", "https://a.b/2"]); expect(parseSitemap("https://a.b/s1.xml").sitemaps).toEqual(["https://a.b/s1.xml"]); }); });