"""Test data factories for the intelligence layer (slug prefix `ztest-`; `cleanup()` removes everything by cascade).""" from __future__ import annotations import contextlib import uuid from datetime import UTC, datetime, timedelta from typing import Any import pytest from companyatlas.db import dispose, execute, fetch_one, jsonb, transaction from companyatlas.ids import new_id, normalize_alias, stable_hash PREFIX = "ztest-" CONNECTOR_HTML = "ztest-generic-html-v1" CONNECTOR_ATS = "ztest-greenhouse-v1" def _uid() -> str: return uuid.uuid4().hex[:8] async def ensure_reference(conn) -> None: # type: ignore[no-untyped-def] for code, name in (("CA", "Canada"), ("US", "United States"), ("JP", "Japan"), ("DE", "Germany"), ("GB", "United Kingdom")): await execute(conn, "insert into countries (code, name) values (:c, :n) on conflict (code) do nothing", c=code, n=name) for cid, cat, mode in ((CONNECTOR_HTML, "homepage", "http"), (CONNECTOR_ATS, "jobs_board", "json")): await execute(conn, "insert into connectors (id, name, version, category, fetch_mode) values (:id, :id, 'v1', :cat, :mode) on conflict (id) do nothing", id=cid, cat=cat, mode=mode) async def make_company(conn, *, name: str | None = None, country: str = "CA", industries: list[str] | None = None, # type: ignore[no-untyped-def] first_observed_days_ago: int | None = 30) -> dict[str, Any]: await ensure_reference(conn) uid = _uid() slug = f"{PREFIX}{uid}" display = name or f"ZTest Corp {uid}" cid = new_id("company") first = datetime.now(UTC) - timedelta(days=first_observed_days_ago) if first_observed_days_ago is not None else None await execute(conn, """ insert into companies (id, slug, display_name, canonical_domain, website, industries, country, onboarding_status, first_observed_at, last_observed_at) values (:id, :slug, :name, :domain, :web, cast(:ind as text[]), :country, 'active', :first, :first)""", id=cid, slug=slug, name=display, domain=f"{slug}.example", web=f"https://{slug}.example", ind=industries or [], country=country, first=first) await execute(conn, "insert into company_aliases (company_id, alias, alias_norm) values (:c, :a, :n) on conflict do nothing", c=cid, a=display, n=normalize_alias(display)) return {"id": cid, "slug": slug, "display_name": display, "country": country, "industries": industries or [], "canonical_domain": f"{slug}.example"} async def make_sensor(conn, company: dict[str, Any], surface: str, *, connector_id: str = CONNECTOR_HTML, path: str | None = None, # type: ignore[no-untyped-def] status: str = "active", created_days_ago: int = 30, interval_s: int = 86400) -> dict[str, Any]: sid = new_id("sensor") url = f"https://{company['canonical_domain']}/{path or surface}" await execute(conn, """ insert into sensors (id, company_id, surface, connector_id, url, canonical_url, domain, status, base_interval_s, current_interval_s, created_at, last_success_at) values (:id, :c, :surface, :conn, :url, :url, :domain, :status, :iv, :iv, :created, now())""", id=sid, c=company["id"], surface=surface, conn=connector_id, url=url, domain=company["canonical_domain"], status=status, iv=interval_s, created=datetime.now(UTC) - timedelta(days=created_days_ago)) return {"id": sid, "company_id": company["id"], "surface": surface, "connector_id": connector_id, "url": url} async def make_snapshot(conn, sensor: dict[str, Any], *, version_no: int = 1, title: str | None = None, fetched_at: datetime | None = None) -> str: # type: ignore[no-untyped-def] snap = new_id("snapshot") h = stable_hash(snap) await execute(conn, """ insert into snapshots (id, sensor_id, company_id, version_no, fetched_at, content_hash, normalized_hash, structural_hash, title) values (:id, :s, :c, :v, :at, :h, :h, :h, :title)""", id=snap, s=sensor["id"], c=sensor["company_id"], v=version_no, at=fetched_at or datetime.now(UTC), h=h, title=title) return snap async def make_change(conn, sensor: dict[str, Any], *, significance: float = 0.6, kind: str | None = None, structured_delta: dict[str, Any] | None = None, # type: ignore[no-untyped-def] diff: dict[str, Any] | None = None, detected_at: datetime | None = None, status: str = "pending") -> dict[str, Any]: from companyatlas.taxonomy import change_kind kind = kind or str(change_kind(significance)) before = await make_snapshot(conn, sensor, version_no=1, fetched_at=(detected_at or datetime.now(UTC)) - timedelta(days=1)) after = await make_snapshot(conn, sensor, version_no=2, fetched_at=detected_at) diff = diff or {"added": [], "removed": [], "modified": [], "moved": [], "counts": {"added": 0, "removed": 0, "modified": 0, "moved": 0}, "text_delta_ratio": 0.1, "similarity": 0.9, "reasons": []} cid = new_id("change") await execute(conn, """ insert into changes (id, sensor_id, company_id, surface, snapshot_before, snapshot_after, detected_at, significance, kind, blocks_added, blocks_removed, blocks_modified, text_delta_ratio, similarity, diff, structured_delta, status) values (:id, :s, :c, :surface, :before, :after, :at, :sig, :kind, :ba, :br, :bm, :ratio, :sim, cast(:diff as jsonb), cast(:delta as jsonb), :status)""", id=cid, s=sensor["id"], c=sensor["company_id"], surface=sensor["surface"], before=before, after=after, at=detected_at or datetime.now(UTC), sig=significance, kind=kind, ba=len(diff.get("added") or []), br=len(diff.get("removed") or []), bm=len(diff.get("modified") or []), ratio=diff.get("text_delta_ratio") or 0, sim=diff.get("similarity"), diff=jsonb(diff), delta=jsonb(structured_delta or {}), status=status) return await fetch_one(conn, "select * from changes where id = :id", id=cid) or {"id": cid} async def make_job(conn, company: dict[str, Any], *, title: str = "Software Engineer", first_seen_days_ago: float = 10, removed_days_ago: float | None = None, # type: ignore[no-untyped-def] country: str | None = "CA", is_ai: bool = False, remote: bool = False, department: str | None = "Engineering", sensor_id: str | None = None) -> str: jid = new_id("job") now = datetime.now(UTC) first = now - timedelta(days=first_seen_days_ago) removed = now - timedelta(days=removed_days_ago) if removed_days_ago is not None else None await execute(conn, """ insert into jobs (id, company_id, sensor_id, fingerprint, title, department, country, remote, first_seen_at, last_seen_at, removed_at, status, is_ai) values (:id, :c, :s, :fp, :title, :dep, :country, :remote, :first, :last, :removed, :status, :ai)""", id=jid, c=company["id"], s=sensor_id, fp=stable_hash(jid), title=title, dep=department, country=country, remote=remote, first=first, last=removed or now, removed=removed, status="no_longer_listed" if removed else "open", ai=is_ai) return jid async def make_event(conn, company: dict[str, Any], *, subtype: str, importance: float = 0.6, days_ago: float = 1, title: str | None = None, # type: ignore[no-untyped-def] tags: list[str] | None = None, surface: str | None = None, sensor_id: str | None = None) -> str: from companyatlas.taxonomy import EVENT_SUBTYPES, EventType, confidence_label eid = new_id("event") etype = str(EVENT_SUBTYPES.get(subtype, (EventType.OTHER, 0.3))[0]) await execute(conn, """ insert into events (id, company_id, sensor_id, surface, event_type, event_subtype, importance, confidence, confidence_label, title, tags, detected_at, dedupe_key) values (:id, :c, :s, :surface, :t, :st, :imp, 0.8, :label, :title, cast(:tags as text[]), :at, :dk)""", id=eid, c=company["id"], s=sensor_id, surface=surface, t=etype, st=subtype, imp=importance, label=confidence_label(0.8), title=title or f"{subtype} test event", tags=tags or [], at=datetime.now(UTC) - timedelta(days=days_ago), dk=stable_hash(eid)) return eid async def make_location(conn, company: dict[str, Any], *, name: str, country: str, city: str | None = None, first_seen_days_ago: float = 5) -> str: # type: ignore[no-untyped-def] lid = new_id("location") await execute(conn, """insert into locations (id, company_id, kind, name, name_norm, city, country, first_seen_at, last_seen_at) values (:id, :c, 'office', :name, :norm, :city, :country, :first, now())""", id=lid, c=company["id"], name=name, norm=normalize_alias(name) + _uid(), city=city, country=country, first=datetime.now(UTC) - timedelta(days=first_seen_days_ago)) return lid async def make_observation(conn, sensor: dict[str, Any], *, days_ago: float, changed: bool = False, not_modified: bool = False) -> str: # type: ignore[no-untyped-def] oid = new_id("observation") await execute(conn, """insert into observations (id, sensor_id, company_id, fetched_at, status_code, changed, not_modified) values (:id, :s, :c, :at, 200, :changed, :nm)""", id=oid, s=sensor["id"], c=sensor["company_id"], at=datetime.now(UTC) - timedelta(days=days_ago), changed=changed, nm=not_modified) return oid @pytest.fixture async def intel_db(): """DB fixture for the intelligence tests: a fresh engine bound to this test's event loop (other modules run on a session-scoped loop), skip when Postgres is unreachable, dispose afterwards. Import it into a test module to register it.""" from companyatlas.config import settings from companyatlas.db import fetch_val with contextlib.suppress(Exception): await dispose() try: async with transaction() as conn: ok = (await fetch_val(conn, "select 1")) == 1 except Exception: # noqa: BLE001 ok = False with contextlib.suppress(Exception): await dispose() if not ok: pytest.skip(f"database not reachable: {settings.database_url.split('@')[-1]}") try: yield finally: with contextlib.suppress(Exception): await dispose() async def cleanup() -> None: async with transaction() as conn: await execute(conn, "delete from companies where slug like :p", p=f"{PREFIX}%") await execute(conn, "delete from connectors where id like :p", p=f"{PREFIX}%") await execute(conn, "delete from alerts where name like :p", p=f"{PREFIX}%") await execute(conn, "delete from owners where token_hash like :p", p=f"{PREFIX}%") await execute(conn, "delete from global_daily where day < '2002-01-01'") await execute(conn, "delete from trends where day < '2002-01-01'") __all__ = ["CONNECTOR_ATS", "CONNECTOR_HTML", "PREFIX", "cleanup", "ensure_reference", "intel_db", "make_change", "make_company", "make_event", "make_job", "make_location", "make_observation", "make_sensor", "make_snapshot"]