spb/company-atlas

1from __future__ import annotations23from conftest import fixture_text45from companyatlas.sdk import normalize as N678def test_noise_normalisation_replaces_dates_times_counters_tokens() -> None:9    s = "Updated 3 minutes ago on 2024-05-01 at 10:34 PM · March 5, 2024 · © 2024 Acme · 1,234 views · token=abcdef0123456789abcdef0123456789 ?v=123&x=1"10    n = N.normalized_text(s)11    assert "<rel>" in n and "<date>" in n and "<time>" in n and "<count>" in n and "<copyright>" in n and "<hex>" in n and "?v=<q>" in n12    assert "2024" not in n.replace("<date>", "")131415def test_text_hash_ignores_noise_but_not_prices() -> None:16    assert N.text_hash("Price $29 per month, updated 2 days ago") == N.text_hash("Price $29 per month, updated 5 hours ago")17    assert N.text_hash("Price $29 per month") != N.text_hash("Price $39 per month")181920def test_simhash_near_duplicates_and_hamming() -> None:21    a = N.simhash("The quick brown fox jumps over the lazy dog near the river bank today")22    b = N.simhash("The quick brown fox jumps over the lazy dog near the river bank tonight")23    c = N.simhash("Completely different content about pricing plans and enterprise contracts")24    assert N.hamming(a, b) < N.hamming(a, c)25    assert N.simhash("") == 0262728def test_parse_pricing_page_blocks_meta_links() -> None:29    page = N.parse(fixture_text("generic_html", "pricing.html"), url="https://www.acme-cloud.example/pricing")30    assert page.title == "Pricing — Acme Cloud"31    assert page.lang == "en"32    assert page.meta["description"].startswith("Simple")33    assert page.meta["canonical"] == "https://www.acme-cloud.example/pricing"34    assert page.feeds == ["https://www.acme-cloud.example/blog/feed.xml"]35    kinds = [b.kind for b in page.blocks]36    assert kinds.count("pricing_plan") == 337    assert "hero" in kinds and "nav" in kinds and "footer" in kinds and "faq" in kinds38    # script/style/cookie banner/hidden content dropped39    assert "dataLayer" not in page.text and "We use cookies" not in page.text and "Last updated" not in page.text40    # nav/footer are low weight, pricing plans high41    assert max(b.weight for b in page.blocks if b.kind == "pricing_plan") > max(b.weight for b in page.blocks if b.kind in ("nav", "footer"))42    regions = {ln.region for ln in page.links}43    assert {"nav", "footer", "main"} <= regions44    assert page.main_selector == "main"454647def test_block_keys_are_stable_under_reordering() -> None:48    html = fixture_text("generic_html", "pricing.html")49    a = N.parse(html, url="https://x.example/pricing")50    # swap the Starter and Pro cards51    swapped = html.replace('<h3>Starter</h3>', '<h3>TMP</h3>').replace('<h3>Pro</h3>', '<h3>Starter</h3>').replace('<h3>TMP</h3>', '<h3>Pro</h3>')52    b = N.parse(swapped, url="https://x.example/pricing")53    keys_a = {blk.key for blk in a.blocks if blk.kind == "pricing_plan"}54    keys_b = {blk.key for blk in b.blocks if blk.kind == "pricing_plan"}55    # names moved but content per card changed, so at least the Enterprise card key is identical56    assert keys_a & keys_b57    assert N.structural_hash(a.blocks) != N.structural_hash(b.blocks) or [x.key for x in a.blocks] == [x.key for x in b.blocks]585960def test_jsonld_and_microdata_extraction() -> None:61    page = N.parse(fixture_text("generic_html", "leadership.html"), url="https://www.acme-cloud.example/about/leadership")62    assert "organizations" in page.jsonld63    assert page.meta["same_as"][0].startswith("https://www.linkedin.com/")64    assert page.jsonld["persons"][0]["name"] == "Jane Doe"65    micro = N.extract_microdata(N.LexborHTMLParser('<div itemscope itemtype="https://schema.org/Person"><span itemprop="name">Ada Lovelace</span><span itemprop="jobTitle">CTO</span></div>'))66    assert micro["persons"][0]["name"] == "Ada Lovelace"676869def test_language_guess() -> None:70    assert N.language_guess("whatever", "fr-CA") == "fr"71    assert N.language_guess("Nous sommes une entreprise qui offre des services pour les clients et les partenaires dans le monde") == "fr"72    assert N.language_guess("We are a company that offers services for customers and partners in the world with our team") == "en"73    assert N.language_guess("これは日本語のテキストです") == "ja"747576def test_card_containers_are_split_into_children() -> None:77    page = N.parse(fixture_text("generic_html", "newsroom.html"), url="https://www.acme-cloud.example/news")78    assert [b.kind for b in page.blocks].count("news_item") == 479    home = N.parse(fixture_text("generic_html", "homepage.html"), url="https://www.acme-cloud.example/")80    assert [b.kind for b in home.blocks].count("product_card") == 381