spb/company-atlas

1"""Precision rules (connectors/_precision): the navigation / CTA / cookie-consent / marketing noise observed in production2(M2U64, 2026-09-13) is rejected while every real item in the fixtures is still extracted — precision must not cost recall."""3from __future__ import annotations45from datetime import UTC, datetime67import pytest8from conftest import fixture_path910from companyatlas.connectors import _precision as P11from companyatlas.fetch import file_result12from companyatlas.sdk import connector as C13from companyatlas.sdk.models import (14    ExtractedJob,15    ExtractedLocation,16    ExtractedNewsItem,17    ExtractedPerson,18    ExtractedPlan,19    ExtractedProduct,20    Extraction,21)22from companyatlas.services.pipeline import _drop_corrupt_entities2324BASE = "https://www.ex.example"25CFG = {"canonical_domain": "ex.example"}262728def _run(surface: str, name: str, path: str):  # type: ignore[no-untyped-def]29    url = BASE + path30    return C.get("generic-html-v1").extract({"url": url, "surface": surface, "config": CFG}, file_result(fixture_path("generic_html", name), url=url))313233# ------------------------------------------------------------------------------------------------------------ fixtures end to end343536def test_careers_cta_anchors_rejected_real_listings_kept() -> None:37    ex = _run("careers", "careers_cta_noise.html", "/nl/jobs")38    titles = [j.title for j in ex.jobs]39    assert titles == ["Senior Backend Software Engineer - Infrastructure", "Verpleegkundige spoedgevallen", "Data Analyst", "Accountmanager KMO", "Technicien réseau"]40    assert not {"Voor starters die willen gáán", "Early careers bij Telenet group", "Meer info", "Sales", "Alle vacatures", "Waarom werken bij ons?"} & set(titles)41    by = {j.title: j for j in ex.jobs}42    assert by["Senior Backend Software Engineer - Infrastructure"].location_text == "London, United Kingdom"43    assert by["Senior Backend Software Engineer - Infrastructure"].department == "Engineering" and by["Senior Backend Software Engineer - Infrastructure"].country == "GB"44    assert by["Data Analyst"].external_id == "12933" and by["Data Analyst"].country == "DE"          # id stripped from display, kept for identity45    assert by["Verpleegkundige spoedgevallen"].city == "Gent" and by["Verpleegkundige spoedgevallen"].country == "BE"46    assert [b.kind for b in ex.blocks].count("job_listing") == 5474849def test_careers_german_gender_markers_and_cta() -> None:50    ex = _run("careers", "careers_de.html", "/karriere")51    assert [j.title for j in ex.jobs] == ["Softwareentwickler Backend", "Projektleiter Anlagenbau", "Ausbildung zum Industriemechaniker 2027",52                                          "Werkstudent Marketing", "Pflegefachkraft Intensivstation"]53    assert [j.location_text for j in ex.jobs] == ["München", "Stuttgart", "Hamburg", "Berlin", "Köln"]   # bare known cities are locations54    assert ex.jobs[0].country is None                                                                  # …but no country is guessed555657def test_leadership_swapped_cards_and_contact_links() -> None:58    ex = _run("leadership", "leadership_swapped.html", "/leadership")59    people = {p.name: p for p in ex.people}60    assert set(people) == {"Jane Doe", "Brian Jacobson", "Priya Natarajan", "Marc van der Berg", "Samuel Adebayo"}61    assert people["Jane Doe"].title == "Chief Executive Officer" and people["Jane Doe"].role_category == "ceo" and people["Jane Doe"].is_executive62    assert people["Samuel Adebayo"].title == "Chair of the Board" and people["Samuel Adebayo"].role_category == "chair"63    assert people["Brian Jacobson"].title == "Chief Financial Officer" and people["Brian Jacobson"].role_category == "cfo"64    assert people["Priya Natarajan"].title is None and people["Priya Natarajan"].role_category == "other"   # "Contact" is not a title65    assert people["Marc van der Berg"].role_category == "coo"666768def test_leadership_japanese() -> None:69    ex = _run("leadership", "leadership_ja.html", "/company/officers")70    people = {p.name: p for p in ex.people}71    assert set(people) == {"山田 太郎", "佐藤 花子", "鈴木 一郎", "高橋 美咲"}72    assert people["山田 太郎"].role_category == "ceo" and people["山田 太郎"].is_executive73    assert people["佐藤 花子"].role_category == "cfo" and people["佐藤 花子"].is_executive74    assert people["鈴木 一郎"].role_category == "board" and not people["鈴木 一郎"].is_executive75    assert people["高橋 美咲"].role_category == "vp"767778def test_locations_cookie_categories_and_nav_cards_rejected() -> None:79    ex = _run("locations", "locations_cookie_noise.html", "/locations")80    locs = {loc.name: loc for loc in ex.locations}81    assert set(locs) == {"Hormuz Grand Hotel", "Dubai Office", "London", "Singapore", "Rotterdam Warehouse"}82    assert locs["Hormuz Grand Hotel"].country == "OM" and locs["Hormuz Grand Hotel"].city is None       # "Oman | Hormuz Grand Hotel" repaired83    assert locs["Dubai Office"].city == "Dubai" and locs["Dubai Office"].country == "AE"                 # not "Level 12"84    assert locs["Rotterdam Warehouse"].city == "Rotterdam" and locs["Rotterdam Warehouse"].kind == "warehouse" and locs["Rotterdam Warehouse"].country == "NL"85    assert locs["London"].address_text.startswith("1 Finsbury Avenue") and locs["Singapore"].city == "Singapore"86    assert all(loc.kind != "store" for loc in ex.locations)                                              # "…providers store data" is not a shop878889def test_pricing_marketing_headings_rejected_real_tiers_kept() -> None:90    ex = _run("pricing", "pricing_marketing_noise.html", "/pricing")91    plans = {p.plan_name: p for p in ex.plans}92    assert {"Free", "Starter", "Team", "Business", "Enterprise"} <= set(plans)93    assert not {"Win your market with Similar Example for businesses", "Worry-free roaming.", "Unlock the full potential of your data", "Most popular", "Plans"} & set(plans)94    assert plans["Free"].price == 0 and plans["Free"].price_text == "$0 forever"95    assert plans["Starter"].price == 125 and plans["Starter"].billing_period == "month" and plans["Starter"].price_text == "$125 per month, billed annually"96    assert plans["Team"].price == 333 and plans["Team"].currency == "EUR" and plans["Team"].price_text == "Starting at €333 / month"97    assert plans["Business"].price == 1199 and plans["Business"].unit == "user" and plans["Business"].price_text == "US$ 1,199 per user / month"98    assert plans["Enterprise"].contact_sales and plans["Enterprise"].price is None and plans["Enterprise"].price_text == "Talk to sales"99    assert all("destina" not in (p.price_text or "") for p in ex.plans)                                  # never a truncated sentence100101102def test_products_nav_words_and_slogans_rejected() -> None:103    ex = _run("products", "products_news_noise.html", "/products")104    assert [p.name for p in ex.products] == ["Atlas Metrics™", "Atlas Logs®", "Atlas Traces"]              # ® / ™ kept105106107def test_news_pagination_and_category_labels_rejected() -> None:108    ex = _run("newsroom", "products_news_noise.html", "/news")109    titles = [n.title for n in ex.news]110    assert titles == ["Acme launches Atlas AI, an assistant for cloud operations", "Q2 results", "Acme and BigCo announce strategic partnership"]111    assert ex.news[1].published_at.date().isoformat() == "2026-08-28"                                   # short title kept because a date confirms it112113114def test_existing_fixtures_recall_unchanged() -> None:115    assert {p.plan_name for p in _run("pricing", "pricing.html", "/pricing").plans} == {"Starter", "Pro", "Enterprise"}116    assert len(_run("leadership", "leadership.html", "/about/leadership").people) == 7117    assert len(_run("locations", "locations.html", "/company/locations").locations) == 8118    assert len(_run("careers", "careers.html", "/careers").jobs) == 5119    assert len(_run("newsroom", "newsroom.html", "/news").news) == 4120121122# ------------------------------------------------------------------------------------------------------------ unit rules123124125@pytest.mark.parametrize("title,url,location,ok", [126    ("Meer info", "https://x.example/nl/jobs/search?page=2", None, False),127    ("Voor starters die willen gáán", "https://x.example/nl/jobs/starters", None, False),128    ("Early careers bij Telenet group", "https://x.example/nl/jobs/early-careers", None, False),129    ("Mehr erfahren →", "https://x.example/karriere/stellen", None, False),130    ("Sales", "https://x.example/nl/jobs/12990-sales", None, False),                                   # single word131    ("Wij zoeken mensen die het verschil willen maken…", "https://x.example/jobs/13001", "Mechelen", False),132    ("Senior Backend Software Engineer - Infrastructure", None, "London, United Kingdom", True),133    ("Regional Coordinator", "https://x.example/careers/x", None, True),                               # role vocabulary134    ("Something Unusual", "https://x.example/jobs/12345-something-unusual", None, True),               # job-like URL135    ("Something Unusual", None, "Toronto, ON, Canada", True),                                          # explicit location cell136    ("Something Unusual", None, None, False),                                                          # no signal at all137    ("Infirmier(ère) de nuit", None, None, True),138    ("Ingeniero de datos", None, None, True),139    ("ソフトウェアエンジニア", None, None, True),140])141def test_job_verdict(title: str, url: str | None, location: str | None, ok: bool) -> None:142    assert P.job_verdict(title, url=url, location=location).ok is ok143144145@pytest.mark.parametrize("raw,clean,ident", [146    ("Data Analyst (m/w/d) [12933]", "Data Analyst", "12933"),147    ("Accountmanager KMO - Apply now", "Accountmanager KMO", None),148    ("Technicien réseau (h/f)", "Technicien réseau", None),149    ("Werkstudent Marketing (all genders)", "Werkstudent Marketing", None),150    ("Product Manager (Job ID: 44812) →", "Product Manager", "44812"),151    ("Senior Engineer (REQ-501)", "Senior Engineer", "REQ-501"),152])153def test_clean_job_title(raw: str, clean: str, ident: str | None) -> None:154    assert P.clean_job_title(raw) == (clean, ident)155156157@pytest.mark.parametrize("name,title,expected", [158    ("Chief Executive Officer", "Jane Doe", ("Jane Doe", "Chief Executive Officer")),                # swapped card159    ("Chair Emeritus", "Warner Bros. Discovery", None),                                               # role as name, company as title160    ("Brian Jacobson", "Contact", ("Brian Jacobson", None)),161    ("Board of Directors", "Meet the people who govern the company", None),162    ("Leadership", "Read more", None),163    ("Contact", "Media relations", None),164    ("Our team", None, None),165    ("Marc van der Berg", "Chief Operating Officer", ("Marc van der Berg", "Chief Operating Officer")),166    ("Jane Doe", "Jane leads the company since 2019 and previously ran BigCo.", ("Jane Doe", None)),   # bio sentence is not a title167    ("Dr. Aiko Tanaka", "Head of People", ("Dr. Aiko Tanaka", "Head of People")),168    ("Jane Doe 2", "CEO", None),169    ("Edgar S. Woolard, Jr.", "Key person", ("Edgar S. Woolard, Jr.", "Key person")),                  # abbreviation dot kept170])171def test_normalize_person(name: str, title: str | None, expected: tuple[str, str | None] | None) -> None:172    assert P.normalize_person(name, title) == expected173174175@pytest.mark.parametrize("name", [176    "Judy McGrath", "F. William McNabb III", "Michael G. McCaffery", "Catherine MacGregor", "Francis deSouza", "Calvin McDonald",177    "José Vicente de los Mozos", "Alexander Trotman, Baron Trotman", "Stephen Green, Baron Green of Hurstpierpoint", "Thomas John Watson, Sr.",178    "Leonardo DiCaprio", "Marc van der Berg", "María García-López", "Tom O'Neill", "山田 太郎",179])180def test_real_world_names_are_names(name: str) -> None:181    assert P.looks_like_person_name(name)182183184@pytest.mark.parametrize("name", ["Meet the team", "Chief Executive Officer", "Read more", "Warner Bros. Discovery", "Key person", "Doe, Jane", "株式会社サンプル"])185def test_non_names_are_rejected(name: str) -> None:186    assert not P.looks_like_person_name(name)187188189def test_role_category_after_swap_is_consistent() -> None:190    ex = Extraction(text="", blocks=[], people=[ExtractedPerson(name="Chief Executive Officer", title="Jane Doe", role_category="other", is_executive=False)])191    P.apply_precision(ex, html_jobs=False)192    assert ex.people[0].name == "Jane Doe" and ex.people[0].role_category == "ceo" and ex.people[0].is_executive193194195@pytest.mark.parametrize("loc,ok", [196    (ExtractedLocation(name="Performance & Analytics", kind="store", city="Allows use of behavioural data to optimise performance"), False),197    (ExtractedLocation(name="Contact Us", city="Contact Us", country="KR"), False),198    (ExtractedLocation(name="Careers", country="KR"), False),199    (ExtractedLocation(name="Sign up for our newsletter", city="Paris", country="FR"), False),200    (ExtractedLocation(name="Acme Regional Hub"), False),                                             # no evidence of a place201    (ExtractedLocation(name="Acme Regional Hub", kind="factory"), True),                              # explicit kind label202    (ExtractedLocation(name="Berlin"), True),                                                         # known city203    (ExtractedLocation(name="548 Market Street, Suite 200"), True),                                   # street address204    (ExtractedLocation(name="Oman", city="Hormuz Grand Hotel", country="OM"), True),205    (ExtractedLocation(name="We are present in twelve countries across three continents.", country="US"), False),206])207def test_location_verdict(loc: ExtractedLocation, ok: bool) -> None:208    assert P.location_verdict(loc).ok is ok209210211def test_normalize_location_repairs_country_as_name() -> None:212    fixed = P.normalize_location(ExtractedLocation(name="Oman", city="Hormuz Grand Hotel", country="OM"))213    assert fixed is not None and (fixed.name, fixed.city, fixed.country) == ("Hormuz Grand Hotel", None, "OM")214    kept = P.normalize_location(ExtractedLocation(name="Oman", city="Muscat", country="OM"))215    assert kept is not None and (kept.name, kept.city) == ("Oman", "Muscat")                          # a real city stays a city216    cleaned = P.normalize_location(ExtractedLocation(name="Dubai Office", city="Level 12", region="Emirates Towers", country="AE"))217    assert cleaned is not None and cleaned.city is None                                               # "Level 12" is not a city218219220@pytest.mark.parametrize("text,expected", [221    ("Stay connected in the U.S. ($13/day) and over 200 international destinations", "$13/day"),222    ("$99 per user / month, billed annually", "$99 per user / month, billed annually"),223    ("Starting at €333 / month", "Starting at €333 / month"),224    ("US$ 1,199 per user / month", "US$ 1,199 per user / month"),225    ("Talk to sales", "Talk to sales"),226    ("Free", "Free"),227    ("Up to 5 users and email support", None),228])229def test_price_text_from(text: str, expected: str | None) -> None:230    assert P.price_text_from(text) == expected231232233@pytest.mark.parametrize("name,ok", [234    ("Win your market with Similarweb for businesses", False), ("Worry-free roaming.", False), ("Most popular", False), ("Plans", False),235    ("Unlock the full potential of your data", False), ("Talk to sales", False), ("Pro", True), ("Business Plus", True), ("Free", True),236    ("Enterprise", True), ("Team (annual)", True),237])238def test_plan_name_ok(name: str, ok: bool) -> None:239    assert P.plan_name_ok(name) is ok240241242def test_plan_verdict_requires_price_contact_or_free() -> None:243    assert P.plan_verdict(ExtractedPlan(plan_name="Pro", price=29.0, price_text="$29 per month")).ok244    assert P.plan_verdict(ExtractedPlan(plan_name="Enterprise", contact_sales=True, price_text="Contact sales")).ok245    assert P.plan_verdict(ExtractedPlan(plan_name="Free", price=0.0, price_text="Free")).ok246    assert not P.plan_verdict(ExtractedPlan(plan_name="Pro", price=None, price_text="Everything you need")).ok247    assert not P.plan_verdict(ExtractedPlan(plan_name="Worry-free roaming.", price=13.0, price_text="n the U.S. ($13/day) and over 200 international destina")).ok248249250@pytest.mark.parametrize("name,ok", [251    ("Overview", False), ("Learn more", False), ("All products", False), ("Solutions", False), ("Discover how Atlas helps teams ship faster.", False),252    ("Atlas Metrics™", True), ("Atlas Logs®", True), ("Microsoft 365", True), ("Discover", True), ("Produits", False), ("製品一覧", False),253])254def test_product_verdict(name: str, ok: bool) -> None:255    assert P.product_verdict(name).ok is ok256257258def test_news_verdict() -> None:259    assert not P.news_verdict("Read more").ok and not P.news_verdict("Older posts »").ok and not P.news_verdict("Press releases").ok260    assert not P.news_verdict("3").ok and not P.news_verdict("Page 2").ok and not P.news_verdict("Actualités").ok261    assert not P.news_verdict("Q2 results").ok262    assert P.news_verdict("Q2 results", published_at=datetime(2026, 8, 28, tzinfo=UTC)).ok263    assert P.news_verdict("Q2 results", url="https://x.example/news/2026/08/q2").ok264    assert P.news_verdict("Acme launches Atlas AI, an assistant for cloud operations").ok265266267# ------------------------------------------------------------------------------------------------------------ pipeline last line of defence268269270def _noisy_extraction() -> Extraction:271    return Extraction(272        text="ok", blocks=[],273        jobs=[ExtractedJob(title="ML Engineer"), ExtractedJob(title="Meer info", url="https://x.example/jobs"), ExtractedJob(title="Sales")],274        people=[ExtractedPerson(name="Jane Doe", title="CEO"), ExtractedPerson(name="Chair Emeritus", title="Warner Bros. Discovery"),275                ExtractedPerson(name="Brian Jacobson", title="Contact")],276        products=[ExtractedProduct(name="Atlas Metrics"), ExtractedProduct(name="Overview")],277        plans=[ExtractedPlan(plan_name="Pro", price=29.0, price_text="$29 per month"),278               ExtractedPlan(plan_name="Win your market with Similarweb for businesses", contact_sales=True, price_text="Talk to sales")],279        locations=[ExtractedLocation(name="Berlin", country="DE"), ExtractedLocation(name="Performance & Analytics", kind="store")],280        news=[ExtractedNewsItem(title="Quarterly results published", url="https://x.example/a"), ExtractedNewsItem(title="Read more", url="https://x.example/b")],281    )282283284def test_pipeline_filter_applies_precision_for_html_connector() -> None:285    ex = _noisy_extraction()286    _drop_corrupt_entities(ex, connector_id="generic-html-v1")287    assert [j.title for j in ex.jobs] == ["ML Engineer"]288    assert [(p.name, p.title) for p in ex.people] == [("Jane Doe", "CEO"), ("Brian Jacobson", None)]289    assert [p.name for p in ex.products] == ["Atlas Metrics"]290    assert [p.plan_name for p in ex.plans] == ["Pro"]291    assert [loc.name for loc in ex.locations] == ["Berlin"]292    assert [n.title for n in ex.news] == ["Quarterly results published"]293294295def test_pipeline_filter_trusts_structured_job_boards() -> None:296    ex = _noisy_extraction()297    _drop_corrupt_entities(ex, connector_id="greenhouse-v1")298    assert [j.title for j in ex.jobs] == ["ML Engineer", "Meer info", "Sales"]                          # ATS jobs are never filtered by title rules299    assert [p.name for p in ex.products] == ["Atlas Metrics"]                                            # other rules still apply300