// author: simon-pierre boucher import { describe, expect, it } from "vitest"; import { extract } from "./pipeline.js"; const article = ` How Tendrils Climb

How Tendrils Climb

${"Tendrils are specialized stems that coil around supports to hoist the plant upward. ".repeat(6)}

${"They respond to touch through a process called thigmotropism which is fascinating. ".repeat(6)}

Read the deep dive
`; describe("extract pipeline", () => { it("returns markdown, metadata, structured and links", () => { const r = extract(article, { url: "https://plants.example/guide", onlyMainContent: true }); expect(r.ok).toBe(true); if (!r.ok) return; const d = r.value; expect(d.metadata.title).toBe("How Tendrils Climb"); expect(d.metadata.author).toBe("Botanist"); expect(d.structured.jsonld.length).toBe(1); expect(d.markdown).toContain("thigmotropism"); expect(d.textLength).toBeGreaterThan(200); }); it("absolutizes links and flags internal vs external", () => { const r = extract(article, { url: "https://plants.example/guide" }); if (!r.ok) throw new Error("expected ok"); const deep = r.value.links.find((l) => l.url.endsWith("/deep-dive")); expect(deep?.url).toBe("https://plants.example/deep-dive"); expect(deep?.isInternal).toBe(true); }); it("drops chrome (nav/footer) content from markdown in onlyMainContent mode", () => { const r = extract(article, { url: "https://plants.example/guide", onlyMainContent: true }); if (!r.ok) throw new Error("expected ok"); expect(r.value.markdown).not.toContain("Privacy"); }); it("honors excludeTags", () => { const html = `

${"keep this important content here for density ".repeat(8)}

` + `
buy now discount offer
`; const r = extract(html, { url: "https://x.example/", onlyMainContent: false, excludeTags: [".promo-box"] }); if (!r.ok) throw new Error("expected ok"); expect(r.value.markdown).not.toContain("buy now"); expect(r.value.markdown).toContain("keep this"); }); it("uses density fallback on a listing page with little prose", () => { const items = Array.from({ length: 8 }, (_, i) => `
  • Product ${i} $${i}0.00 in stock
    • `).join(""); const html = `
    ` + `

    ${"Some descriptive listing text that gives the container real density weight. ".repeat(4)}

    `; const r = extract(html, { url: "https://shop.example/list", onlyMainContent: true }); if (!r.ok) throw new Error("expected ok"); expect(r.value.markdown).toContain("Product 0"); expect(r.value.markdown).toContain("Product 7"); }); });