spb/datacenterindex

1import { describe, expect, it } from "vitest";2import { load } from "cheerio";3import { applyTransform, cutTeasers, embeddedJson, evalRule, extractAddress, extractGeo, jsonLd, jsonPath, mainText, pdfText, publishedDate, walkJson } from "./extract.js";45const HTML = `<html><head><title>Page</title>6<meta property="og:title" content="OG Title"><meta name="description" content="Desc">7<meta property="article:published_time" content="2026-09-01T10:00:00Z">8<script type="application/ld+json">{"@context":"https://schema.org","@graph":[{"@type":"Place","name":"DC1","geo":{"@type":"GeoCoordinates","latitude":"39.04","longitude":-77.49},"address":{"@type":"PostalAddress","streetAddress":"1 Main St","addressLocality":"Ashburn","addressRegion":"VA","postalCode":"20147","addressCountry":"US"}}]}</script>9<script id="__NEXT_DATA__" type="application/json">{"props":{"pageProps":{"facility":{"code":"IAD1","mw":"12.5 MW","items":[{"n":1},{"n":2}]}}}}</script>10<script>window.__INITIAL_STATE__ = {"a":{"b":[{"c":"deep"}]}};</script>11</head><body><h1>DC1</h1><ul class="specs"><li>Power: 12.5 MW</li><li>Area: 10,000 sqm</li></ul><span class="status">Under Construction</span></body></html>`;1213describe("jsonLd / embeddedJson / jsonPath / walkJson", () => {14  it("flattens @graph and filters by @type case-insensitively", () => {15    expect(jsonLd(HTML)).toHaveLength(1);16    expect(jsonLd(HTML, "place")[0]?.name).toBe("DC1");17    expect(jsonLd(HTML, "Organization")).toEqual([]);18  });19  it("finds __NEXT_DATA__ and window.__INITIAL_STATE__ blobs", () => {20    const ids = embeddedJson(HTML).map((b) => b.id);21    expect(ids).toContain("__NEXT_DATA__");22    expect(ids).toContain("__INITIAL_STATE__");23  });24  it("jsonPath supports $.a.b[0].c and plain a.b", () => {25    const data = embeddedJson(HTML).find((b) => b.id === "__NEXT_DATA__")!.data;26    expect(jsonPath(data, "$.props.pageProps.facility.code")).toBe("IAD1");27    expect(jsonPath(data, "props.pageProps.facility.items[1].n")).toBe(2);28    expect(jsonPath(data, "$.props.missing.x")).toBeUndefined();29    expect(jsonPath({ a: { b: [{ c: "deep" }] } }, "a.b[0].c")).toBe("deep");30  });31  it("walkJson collects matching nodes depth-first", () => {32    const hits = walkJson({ x: [{ code: "A" }, { y: { code: "B" } }], code: "C" }, (o) => typeof o.code === "string");33    expect(hits.map((h) => h.code)).toEqual(["C", "A", "B"]);34  });35});3637describe("applyTransform", () => {38  it("normalizes MW / area / money / dates / status / type / country / numbers / text", () => {39    expect(applyTransform("0.779MW", "mw")).toBe(0.779);40    expect(applyTransform("1.2 GW", "mw")).toBe(1200);41    expect(applyTransform("10,000 sqm", "sqm")).toBe(10000);42    expect(applyTransform("$2 billion", "money")).toEqual({ amount: 2_000_000_000, currency: "USD" });43    expect(applyTransform("Q2 2027", "date")).toBe("2027-Q2");44    expect(applyTransform("Under Construction", "status")).toBe("under_construction");45    expect(applyTransform("Germany", "country")).toBe("DE");46    expect(applyTransform("US", "country")).toBe("US");47    expect(applyTransform("1,234 racks", "int")).toBe(1234);48    expect(applyTransform("PUE 1.25", "float")).toBe(1.25);49    expect(applyTransform("  a   b ", "trim")).toBe("a b");50    expect(applyTransform(" X ", "lower")).toBe("x");51    expect(applyTransform("yes", "bool")).toBe(true);52    expect(applyTransform("no", "bool")).toBe(false);53    expect(applyTransform(null, "mw")).toBeNull();54    expect(applyTransform("  raw  ", undefined)).toBe("raw");55  });56});5758describe("evalRule", () => {59  const $ = load(HTML);60  it("selector, attr, all, regex+group, meta, json-ld, embedded json, default", () => {61    expect(evalRule($, HTML, "h1")).toEqual({ value: "DC1", method: "selector:h1" });62    expect(evalRule($, HTML, { selector: ".specs li", all: true }).value).toEqual(["Power: 12.5 MW", "Area: 10,000 sqm"]);63    expect(evalRule($, HTML, { selector: ".specs li", all: true, regex: "Power: (.+)", transform: "mw" }).value).toEqual([12.5]);64    expect(evalRule($, HTML, { selector: 'meta[name="description"]', attr: "content" }).value).toBe("Desc");65    expect(evalRule($, HTML, { meta: "og:title" }).value).toBe("OG Title");66    expect(evalRule($, HTML, { jsonld: "Place", jsonPath: "address.addressLocality" })).toEqual({ value: "Ashburn", method: "json-ld:Place.address.addressLocality" });67    expect(evalRule($, HTML, { jsonPath: "$.props.pageProps.facility.mw", transform: "mw" }).value).toBe(12.5);68    expect(evalRule($, HTML, { selector: ".nope", default: "fallback" }).value).toBe("fallback");69    expect(evalRule($, HTML, { regex: "Area: ([\\d,]+) sqm", transform: "int" }).value).toBe(10000);70    expect(evalRule($, HTML, { selector: ".status", transform: "status" }).value).toBe("under_construction");71  });72  it("scopes to a container", () => {73    const scope = $(".specs");74    expect(evalRule($, HTML, { selector: "li" }, scope).value).toBe("Power: 12.5 MW");75  });76});7778describe("geo / address / date", () => {79  it("extracts JSON-LD geo and address, meta geo.position, data attributes and Google Maps links", () => {80    expect(extractGeo(HTML)).toEqual({ lat: 39.04, lng: -77.49, method: "json-ld:geo" });81    expect(extractAddress(HTML)).toEqual({ street: "1 Main St", city: "Ashburn", region: "VA", postal: "20147", country: "US" });82    expect(extractGeo('<meta name="geo.position" content="48.85;2.35">')).toMatchObject({ lat: 48.85, lng: 2.35 });83    expect(extractGeo('<div data-lat="1.5" data-lng="2.5"></div>')).toMatchObject({ lat: 1.5, lng: 2.5, method: "attr:data-lat" });84    expect(extractGeo('<a href="https://www.google.com/maps/place/x/@51.5,-0.12,17z">m</a>')).toMatchObject({ lat: 51.5, lng: -0.12 });85    expect(extractGeo("<p>nothing</p>")).toBeNull();86  });87  it("reads the publication date from meta tags", () => {88    expect(publishedDate(HTML)).toBe("2026-09-01");89    expect(publishedDate("<html></html>")).toBeNull();90  });91});9293describe("mainText", () => {94  const story = `<p>${"The developer secured planning permission for a 12MW data center in Bucharest. ".repeat(6)}</p>`;95  it("keeps the largest <article>, drops related / recommended / trending / sidebar / aside / footer blocks and cuts trailing teasers", () => {96    const html = `<html><body><nav>Home</nav><main>97<article class="card"><a>Microsoft expands plans for La Porte campus</a></article>98<article itemprop="mainEntity"><h1>S+B Gruppe to build data center in Bucharest</h1>${story}99<div class="block-auto_featured_content"><h2>More in Construction &amp; Site Selection</h2><a>Ignis to build DayOne's 300MW data center</a></div></article>100<div class="related-articles"><a>Google onboarding case study</a></div><div id="sidebar-popular"><a>Equinix opens LD14</a></div>101<section class="recommended-for-you"><a>Vantage raises $2bn</a></section><div class="trending-now"><a>Digital Realty results</a></div></main>102<aside><a>AWS buys site</a></aside><footer>Amazon Web Services</footer></body></html>`;103    const t = mainText(html);104    expect(t).toContain("S+B Gruppe to build data center in Bucharest");105    expect(t).toContain("planning permission");106    for (const leak of ["Microsoft", "Ignis", "More in", "Google", "Equinix", "Vantage", "Digital Realty", "AWS", "Amazon"]) expect(t, leak).not.toContain(leak);107  });108  it("cutTeasers only cuts after the story has started and only on a heading line", () => {109    expect(cutTeasers("Related\nshort lead")).toBe("Related\nshort lead");110    const body = "x".repeat(400);111    expect(cutTeasers(`${body}\n Tags \n Bucharest\n Comments`)).toBe(body);112    expect(cutTeasers(`${body}\nMore in the article follows here with a long sentence about the project itself.`)).toContain("More in the article");113  });114});115116describe("pdfText limits", () => {117  it("rejects oversized PDFs before parsing", async () => {118    await expect(pdfText(Buffer.alloc(16 * 1024 * 1024), 60, { maxBytes: 15 * 1024 * 1024 })).rejects.toThrow(/too large/);119  });120  it("throws on garbage input instead of hanging", async () => {121    await expect(pdfText(Buffer.from("%PDF-1.4 definitely not a pdf"), 60, { timeoutMs: 5_000 })).rejects.toThrow();122  });123});124