Company Atlas kernel: config, taxonomy, ids, urls, fetch, archive, db, schema 0001, SDK models, CLI, API skeleton, docs

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 12 days ago (Sep 12, 2026)

34 changed files +3,598 −0

added .env.example +21 −0

		`@@ -0,0 +1,21 @@`
	1	+# Company Atlas — local development. Copy to .env (never commit .env).
	2	+APP_ENV=development
	3	+CA_SITE_URL=http://localhost:8370
	4	+DATABASE_URL=postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas
	5	+CA_DATA_DIR=./data
	6	+CA_API_HOST=127.0.0.1
	7	+CA_API_PORT=8371
	8	+CA_ADMIN_TOKEN=dev-admin-token
	9	+CA_LOG_JSON=0
	10	+# Crawler identity (robots.txt honoured; contact address public)
	11	+CA_USER_AGENT="CompanyAtlasBot/0.1 (+https://www.company-atlas.co/bot; contact@spboucher.ai)"
	12	+CA_FETCH_CONCURRENCY=16
	13	+# LLM enrichment (OpenAI-compatible; MacLustr llm-api.io). Optional — deterministic pipeline works without it.
	14	+CA_LLM_BASE_URL=https://www.llm-api.io/v1
	15	+CA_LLM_API_KEY=
	16	+CA_LLM_SMALL_MODEL=qwen3-4b-instruct-2507-4bit
	17	+CA_LLM_MEDIUM_MODEL=qwen3.6-35b-a3b-4bit
	18	+CA_LLM_LARGE_MODEL=qwen3.8-27b-4bit
	19	+# Web
	20	+API_URL=http://127.0.0.1:8371
	21	+NEXT_PUBLIC_SITE_URL=http://localhost:8370

added .gitignore +23 −0

		`@@ -0,0 +1,23 @@`
	1	+.venv/
	2	+__pycache__/
	3	+*.pyc
	4	+.pytest_cache/
	5	+.ruff_cache/
	6	+*.egg-info/
	7	+.env
	8	+.env.*
	9	+!.env.example
	10	+node_modules/
	11	+apps/web/.next/
	12	+apps/web/.next-*/
	13	+apps/web/next-env.d.ts
	14	+*.tsbuildinfo
	15	+/data/
	16	+/tmp/
	17	+logs/
	18	+.DS_Store
	19	+.claude/
	20	+deploy/.admin-token
	21	+deploy/.llm-key
	22	+deploy/rendered/
	23	+apps/web/qa/screens/

added CLAUDE.md +57 −0

		`@@ -0,0 +1,57 @@`
	1	+# CLAUDE.md — Company Atlas (repo guide)
	2	+
	3	+Product: Company Atlas — The Live Atlas of Global Companies. A distributed public-web sensor network attached to companies:
	4	+observations → snapshots → changes → structured events → metrics → intelligence. The accumulated history is the product.
	5	+Full product specification (200 sections, non-negotiable principles): `docs/PRODUCT-SPEC.md`. Read it before changing behaviour.
	6	+
	7	+Domains: canonical `https://www.company-atlas.co` today (DNS A → MacLustr Tunnel 51.161.112.61). `www.company-atlas.com` is the
	8	+spec's primary hostname but its nameservers are at Vercel, not GoDaddy — once its A records point to the gateway, add it as a tunnel
	9	+route/redirect and flip `CA_SITE_URL`. Apex → www 308.
	10	+
	11	+## Stack
	12	+- Python 3.12 `src/companyatlas` (FastAPI, SQLAlchemy Core + asyncpg, Alembic SQL forward-only, httpx, selectolax, feedparser, zstd),
	13	+ CLI `catlas` (commands auto-discovered from `companyatlas/commands/*.py`).
	14	+- Postgres 17 only (no Redis): queue = `queue_jobs` with `SKIP LOCKED`; caches in-process (`api/common.TTLCache`).
	15	+- Objects: content-addressed zstd store `CA_DATA_DIR/objects/ab/cd/<sha256>.zst` (`archive.py`).
	16	+- Web: `apps/web` Next 16 + React 19 + Tailwind v4 (Geist), SSR, rewrites `/api/v1/` → FastAPI loopback. Ports: prod web 8360* / api 8361;
	17	+ dev web 8370 / api 8371 (see `.env.example`).
	18	+- LLM enrichment: OpenAI-compatible `CA_LLM_BASE_URL` (MacLustr llm-api.io, key "company-atlas" in `deploy/.llm-key`, git-ignored). Optional; deterministic first.
	19	+
	20	+## Layout
	21	+```
	22	+src/companyatlas/ config.py taxonomy.py ids.py urls.py fetch.py archive.py db/ logging.py cli.py
	23	+ sdk/ (models, normalize, diff, connector) connectors/ services/ (discovery, pipeline, scheduler, events, llm, metrics…)
	24	+ api/ (main, common, routers/*) commands/ (CLI groups) registry/ (seed loader)
	25	+migrations/versions/0001_initial.py registry/ (companies ndjson, industries.yaml, countries.csv) prompts/ fixtures/ tests/
	26	+apps/web/ docs/ (PRODUCT-SPEC, ARCHITECTURE, API, DATA-MODEL, CONNECTORS, SCORING, DEPLOY, OPERATIONS) deploy/ (mld manifest, scripts)
	27	+```
	28	+Architecture, ownership map and boundaries: `docs/ARCHITECTURE.md`. API contract: `docs/API.md`.
	29	+
	30	+## Operating rules (spec §182)
	31	+1. Inspect existing architecture first; preserve working functionality; avoid rewrites; reuse the SDK.
	32	+2. Never fabricate data. No inference presented as fact: careful language (detected, no longer listed, signal), confidence labels.
	33	+3. Historical-first: never overwrite or delete history; new versions, `status` columns, forward-only migrations.
	34	+4. Raw vs interpreted stay separate (objects ↔ snapshots ↔ changes ↔ events). Reprocessing must never require re-fetching.
	35	+5. LLMs are enrichment, not the crawler: deterministic fetch → normalize → hash → diff → significance → LLM only if useful, budgeted.
	36	+6. Only `fetch.Fetcher` talks to the network: SSRF guard, robots, per-domain rate/concurrency, size and redirect caps. Never bypass
	37	+ authentication or challenges; never collect private data.
	38	+7. No magic numbers: tunables in `config.Settings` / `taxonomy.py`; bump formula/prompt/connector versions when behaviour changes.
	39	+8. Mobile is first-class; dark and light both designed; dense, readable, terminal-grade UI.
	40	+9. Tests use fixtures (`fixtures/`), never the live web (`-m live` opt-in). Run `.venv/bin/pytest -q`, `.venv/bin/ruff check src tests`,
	41	+ `pnpm typecheck` after meaningful changes.
	42	+10. Secrets only in env / git-ignored files (`deploy/.admin-token`, `deploy/.llm-key`); the rendered manifest lives on M1M32.
	43	+
	44	+## Dev quickstart
	45	+```bash
	46	+uv venv --python 3.12 .venv && uv pip install --python .venv/bin/python -e '.[dev]'
	47	+createdb -O companyatlas companyatlas # role companyatlas/companyatlas, extensions pg_trgm + uuid-ossp
	48	+cp .env.example .env && .venv/bin/catlas migrate && .venv/bin/catlas seed
	49	+.venv/bin/catlas onboard --limit 50 # discovery + sensors for the first companies
	50	+.venv/bin/catlas schedule # scheduler + workers (Ctrl-C to stop)
	51	+.venv/bin/catlas api # http://127.0.0.1:8371/api/v1/docs
	52	+pnpm install && pnpm dev:web # http://localhost:8370
	53	+```
	54	+
	55	+## Deploy (MacLustr, via mld — see docs/DEPLOY.md)
	56	+Node M2U64 (pin), dir `~/apps/company-atlas`, data `~/company-atlas-data`, PM2 `company-atlas-api` / `company-atlas-scheduler` /
	57	+`company-atlas-web`. `deploy/render-manifest.sh --push` → `mld stage . company-atlas` → `mld deploy company-atlas --node M2U64`.

added alembic.ini +31 −0

		`@@ -0,0 +1,31 @@`
	1	+[alembic]
	2	+script_location = migrations
	3	+prepend_sys_path = src
	4	+path_separator = os
	5	+
	6	+[loggers]
	7	+keys = root,alembic
	8	+
	9	+[handlers]
	10	+keys = console
	11	+
	12	+[formatters]
	13	+keys = generic
	14	+
	15	+[logger_root]
	16	+level = WARN
	17	+handlers = console
	18	+
	19	+[logger_alembic]
	20	+level = INFO
	21	+handlers =
	22	+qualname = alembic
	23	+
	24	+[handler_console]
	25	+class = StreamHandler
	26	+args = (sys.stderr,)
	27	+level = NOTSET
	28	+formatter = generic
	29	+
	30	+[formatter_generic]
	31	+format = %(levelname)-5.5s [%(name)s] %(message)s

added docs/API.md +176 −0

		`@@ -0,0 +1,176 @@`
	1	+# Company Atlas API — contract (v1)
	2	+
	3	+Base: `/api/v1` · JSON (orjson) · UTC ISO-8601 timestamps (`Z`) · immutable ids (`co_…`, `sen_…`, `evt_…`) · slugs for public URLs.
	4	+Public GET endpoints need no auth. Owner endpoints (watchlists, alerts) take `X-CA-Owner-Token` (client-generated random string ≥ 24 chars, stored hashed).
	5	+Admin endpoints take `X-CA-Admin-Token`. Errors: `{ "detail": string }` with 4xx/5xx. Pagination: `?page=1&per_page=25` (max 200) →
	6	+`{ items, page, per_page, total, pages }`. Lists accept `sort` where documented; unknown params are ignored.
	7	+
	8	+Interpretive language is careful by design (spec §167–168): events say "no longer listed", never "fired"; anything inferred carries a
	9	+`confidence` (0–1) and a `confidence_label` (`VERIFIED \| HIGH_CONFIDENCE \| LIKELY \| INFERRED \| LOW_CONFIDENCE`).
	10	+
	11	+## Shapes
	12	+
	13	+```ts
	14	+type CompanyCard = {
	15	+ id: string; slug: string; display_name: string; legal_name: string \| null; canonical_domain: string; website: string;
	16	+ description: string \| null; industries: string[]; industry_primary: string \| null; country: string \| null; hq_city: string \| null;
	17	+ hq_region: string \| null; public_company: boolean; ticker: string \| null; exchange: string \| null; founded_year: number \| null;
	18	+ employees_band: string \| null; logo_url: string \| null; status: string; onboarding_status: string; importance: number; tier: 1\|2\|3\|4;
	19	+ metrics: Partial<Record<Metric, number>>; // activity_score, hiring_momentum_30d, product_velocity, ai_adoption, corporate_change_index, open_jobs …
	20	+ counts: { sensors: number; observations: number; changes: number; events: number; jobs_open: number };
	21	+ last_event_at: string \| null; last_observed_at: string \| null;
	22	+ sparkline?: number[]; // 30 daily activity values when `?sparkline=1` (or on detail)
	23	+};
	24	+type Metric = 'activity_score'\|'hiring_momentum_7d'\|'hiring_momentum_30d'\|'hiring_momentum_90d'\|'open_jobs'\|'ai_adoption'\|'product_velocity'\|
	25	+ 'geo_expansion'\|'developer_momentum'\|'communication_activity'\|'pricing_activity'\|'leadership_activity'\|'corporate_change_index'\|'anomaly_score'\|'historical_coverage';
	26	+
	27	+type Event = {
	28	+ id: string; company: { id: string; slug: string; display_name: string; canonical_domain: string; country: string\|null; logo_url: string\|null };
	29	+ event_type: string; event_subtype: string; importance: number; confidence: number; confidence_label: string;
	30	+ title: string; summary: string \| null; old_value: string \| null; new_value: string \| null;
	31	+ payload: Record<string, unknown>; entities: Record<string, unknown>; tags: string[];
	32	+ detected_at: string; effective_at: string \| null; published_at: string \| null;
	33	+ source_url: string \| null; surface: string \| null; sensor_id: string \| null; change_id: string \| null; cluster_id: string \| null;
	34	+ origin: 'deterministic'\|'llm'\|'hybrid'\|'backfill'; model_name: string \| null; prompt_version: string \| null; status: 'active'\|'retracted'\|'duplicate'\|'review';
	35	+ sources?: { source_url: string; surface: string \| null; detected_at: string; kind: string; sensor_id: string \| null }[]; // detail only
	36	+};
	37	+
	38	+type Sensor = {
	39	+ id: string; company_id: string; surface: string; connector_id: string; url: string; canonical_url: string; domain: string;
	40	+ status: string; tier: 'A'\|'B'\|'C'\|'D'\|'E'; quality_score: number; discovery_confidence: number; discovery_method: string \| null;
	41	+ current_interval_s: number; next_run_at: string; last_run_at: string \| null; last_success_at: string \| null; last_change_at: string \| null;
	42	+ last_status: number \| null; last_failure_class: string \| null; consecutive_failures: number;
	43	+ observation_count: number; snapshot_count: number; change_count: number; meaningful_change_count: number; event_count: number; created_at: string;
	44	+};
	45	+
	46	+type Snapshot = { id: string; sensor_id: string; version_no: number; fetched_at: string; title: string \| null; language: string \| null;
	47	+ text_length: number \| null; block_count: number \| null; extracted_summary: Record<string, number>; content_hash: string; previous_snapshot_id: string \| null };
	48	+
	49	+type Change = { id: string; sensor_id: string; surface: string; company_id: string; detected_at: string; significance: number; kind: string;
	50	+ blocks_added: number; blocks_removed: number; blocks_modified: number; text_delta_ratio: number; similarity: number \| null;
	51	+ snapshot_before: string \| null; snapshot_after: string; diff?: DiffPayload; structured_delta?: Record<string, unknown> };
	52	+
	53	+type DiffPayload = { added: BlockDelta[]; removed: BlockDelta[]; modified: BlockDelta[]; moved: string[]; counts: Record<string, number>;
	54	+ text_delta_ratio: number; similarity: number; reasons: string[] };
	55	+type BlockDelta = { key: string; kind: string; path: string; before: string \| null; after: string \| null; weight: number; similarity: number \| null };
	56	+
	57	+type Job = { id: string; title: string; department: string \| null; location_text: string \| null; city: string \| null; country: string \| null;
	58	+ remote: boolean \| null; employment_type: string \| null; seniority: string \| null; url: string \| null; posted_at: string \| null;
	59	+ first_seen_at: string; last_seen_at: string; removed_at: string \| null; status: 'open'\|'no_longer_listed'; is_ai: boolean };
	60	+
	61	+type Person = { id: string; name: string; title: string \| null; role_category: string \| null; is_executive: boolean; first_seen_at: string; last_seen_at: string; removed_at: string \| null; status: string; source_url: string \| null };
	62	+type Product = { id: string; name: string; category: string \| null; description: string \| null; url: string \| null; first_seen_at: string; last_seen_at: string; removed_at: string \| null; status: string };
	63	+type Plan = { id: string; plan_name: string; price: number \| null; price_text: string \| null; currency: string \| null; billing_period: string \| null; unit: string \| null; features: string[]; contact_sales: boolean; version_no: number; valid_from: string; valid_to: string \| null; status: string; source_url: string \| null };
	64	+type Location = { id: string; kind: string; name: string \| null; city: string \| null; region: string \| null; country: string \| null; lat: number \| null; lon: number \| null; first_seen_at: string; last_seen_at: string; removed_at: string \| null; status: string; source_url: string \| null };
	65	+type NewsItem = { id: string; title: string; url: string; summary: string \| null; category: string \| null; published_at: string \| null; first_seen_at: string; language: string \| null };
	66	+type MetricPoint = { day: string; value: number; confidence: number };
	67	+type Signal = { id: string; company_id: string \| null; scope: string; scope_key: string \| null; kind: string; strength: number; confidence: number; title: string; explanation: string \| null; evidence: Record<string, unknown>; window_days: number; detected_at: string; status: string };
	68	+```
	69	+
	70	+## Endpoints
	71	+
	72	+### Platform
	73	+\| Method \| Path \| Notes \|
	74	+\|---\|---\|---\|
	75	+\| GET \| `/health`, `/ready` \| service health (also at `/api/v1/health`) \|
	76	+\| GET \| `/stats` \| `{ companies, companies_active, sensors, sensors_active, observations, snapshots, changes, meaningful_changes, events, jobs_open, countries, industries, observations_today, changes_today, events_today, dataset_started_at, dataset_age_days, oldest_history_days, last_observation_at, archive: {objects, bytes} }` (cached 60 s) \|
	77	+\| GET \| `/stats/history?days=90` \| `{ items: GlobalDaily[] }` — `{ day, companies_active, sensors_active, observations, changes, meaningful_changes, events, events_by_type, jobs_open, jobs_new, jobs_removed, activity_index }` \|
	78	+\| GET \| `/system` \| public aggregate health only: `{ sensors_online, sensors_failing, observations_today, events_today, countries_covered, queue_lag_s, scheduler_last_tick_at, fetch_per_min, success_rate_24h }` \|
	79	+\| GET \| `/pulse` \| homepage aggregate: `{ stats, live: Event[12], movers: CompanyCard[10], hiring: CompanyCard[8], launches: Event[8], pricing: Event[8], ai: CompanyCard[8], industries: IndustryRow[12], countries: CountryRow[12], trending: TrendRow[10], activity_index: {value, delta_7d, series: MetricPoint[30]}, map: MapBucket[] }` (cached 60 s) \|
	80	+\| GET \| `/live?limit=50&since=<iso>&event_type=&min_importance=` \| latest active events (no cache); `since` returns only newer than that time \|
	81	+\| GET \| `/live/stream` \| SSE: `event: event` with an `Event` JSON per message (poll-based, ~5 s), `event: heartbeat` every 20 s, `?since=` supported \|
	82	+
	83	+### Companies
	84	+\| Method \| Path \| Notes \|
	85	+\|---\|---\|---\|
	86	+\| GET \| `/companies` \| filters: `q, country, industry, tier, public (bool), status, has_events (bool), sort=activity\\|events\\|hiring\\|name\\|importance\\|recent, sparkline=1` → page of `CompanyCard` \|
	87	+\| GET \| `/companies/{slug_or_id}` \| `CompanyCard & { aliases: string[], domains: {domain, kind}[], relationships: {kind, company: {slug, display_name} \\| null, to_name, valid_from, valid_to, confidence}[], metrics_detail: {metric, value, confidence, computed_at, inputs}[], sensors_by_surface: Record<string, number>, coverage: {historical_coverage, first_observed_at, days_observed, sensor_uptime}, signals: Signal[], sparklines: {activity_30d: number[], hiring_90d: number[]} }` \|
	88	+\| GET \| `/companies/{slug}/events` \| filters `event_type, event_subtype, since, until, min_importance, surface`; `sort=recent\\|importance` → page of `Event` \|
	89	+\| GET \| `/companies/{slug}/timeline?filter=all\\|products\\|jobs\\|pricing\\|leadership\\|locations\\|legal\\|news\\|developer&limit=200` \| `{ items: (Event & {day: string})[], days: {day, count}[] }` grouped for the timeline UI \|
	90	+\| GET \| `/companies/{slug}/metrics?metric=activity_score&days=90` \| `{ current: {metric, value, confidence, computed_at, formula_version, inputs}[], series: Record<Metric, MetricPoint[]> }` \|
	91	+\| GET \| `/companies/{slug}/jobs?status=open\\|removed\\|all&q=&country=&ai=1` \| page of `Job` + `{ summary: {open, new_7d, removed_7d, ai_open, by_country: {country,n}[], by_department: {department,n}[], remote_ratio} }` as `meta` \|
	92	+\| GET \| `/companies/{slug}/people` \| `{ listed: Person[], no_longer_listed: Person[] }` \|
	93	+\| GET \| `/companies/{slug}/products` \| `{ listed: Product[], removed: Product[] }` \|
	94	+\| GET \| `/companies/{slug}/pricing` \| `{ current: Plan[], history: Plan[] }` \|
	95	+\| GET \| `/companies/{slug}/locations` \| `{ items: Location[], countries: string[] }` \|
	96	+\| GET \| `/companies/{slug}/news?limit=50` \| `{ items: NewsItem[] }` \|
	97	+\| GET \| `/companies/{slug}/sensors` \| `{ items: Sensor[] }` \|
	98	+\| GET \| `/companies/{slug}/history` \| `{ sensors: (Sensor & { versions: Snapshot[] })[] }` — historical page viewer index (max 20 versions per sensor) \|
	99	+\| GET \| `/companies/{slug}/similar?limit=8` \| `{ items: CompanyCard[] }` (same industry/country, closest importance) \|
	100	+\| GET \| `/companies/compare?companies=stripe,adyen,block` \| `{ companies: CompanyCard[], metrics: Record<Metric, Record<slug, number>>, series: Record<slug, MetricPoint[]>, events_30d: Record<slug, Record<event_type, number>>, jobs: Record<slug, {open, ai_open, new_30d}>, locations: Record<slug, number> }` (2–6 companies) \|
	101	+
	102	+### Sensors, snapshots, changes (provenance)
	103	+\| Method \| Path \| Notes \|
	104	+\|---\|---\|---\|
	105	+\| GET \| `/sensors/{id}` \| `Sensor & { company: CompanyRef, latest_snapshot: Snapshot \\| null }` \|
	106	+\| GET \| `/sensors/{id}/snapshots?limit=50` \| `{ items: Snapshot[] }` \|
	107	+\| GET \| `/sensors/{id}/changes?limit=50` \| `{ items: Change[] }` \|
	108	+\| GET \| `/snapshots/{id}` \| `Snapshot & { text: string (≤ 200 kB), blocks: Block[], extracted: object }` \|
	109	+\| GET \| `/snapshots/{id}/diff/{other_id}` \| `{ before: Snapshot, after: Snapshot, diff: DiffPayload }` (computed on demand) \|
	110	+\| GET \| `/changes/{id}` \| `Change` with `diff` and `structured_delta`, plus `events: Event[]` \|
	111	+\| GET \| `/events/{id}` \| `Event` detail with `sources`, `change` (Change summary), `company` \|
	112	+
	113	+### Events
	114	+\| Method \| Path \| Notes \|
	115	+\|---\|---\|---\|
	116	+\| GET \| `/events` \| filters `event_type, event_subtype, country, industry, since, until, min_importance, min_confidence, q, surface, origin, company`; `sort=recent\\|importance` → page of `Event` \|
	117	+\| GET \| `/events/types` \| `{ types: {event_type, subtypes: {event_subtype, count_30d}[], count_30d}[] }` \|
	118	+\| GET \| `/events/summary?days=7&group=type\\|industry\\|country` \| `{ items: {key, count, delta_pct}[] }` \|
	119	+
	120	+### Rankings, industries, countries, signals, trends
	121	+\| Method \| Path \| Notes \|
	122	+\|---\|---\|---\|
	123	+\| GET \| `/rankings?kind=most_active\\|hiring_growth\\|hiring_decline\\|product_velocity\\|ai_active\\|geo_expansion\\|developer_momentum\\|pricing_changes\\|unusual_activity&window=24h\\|7d\\|30d\\|90d\\|1y&country=&industry=&limit=50` \| `{ kind, window, items: (CompanyCard & { rank: number; value: number; delta: number \\| null })[] }` \|
	124	+\| GET \| `/industries` \| `{ items: IndustryRow[] }` — `{ slug, name, parent_slug, companies, events_7d, events_30d, hiring_momentum_30d, activity_score, ai_adoption, top_event_types: string[] }` \|
	125	+\| GET \| `/industries/{slug}` \| `IndustryRow & { description, companies: CompanyCard[24] (most active), events: Event[20], hiring: {open, new_30d, removed_30d, momentum_30d}, series: MetricPoint[90] (activity), countries: {country, companies}[], trending: TrendRow[] }` \|
	126	+\| GET \| `/countries` \| `{ items: CountryRow[] }` — `{ code, name, region, companies, events_7d, events_30d, hiring_momentum_30d, activity_score, industry_mix: {industry, companies}[], lat, lon }` \|
	127	+\| GET \| `/countries/{code}` \| `CountryRow & { companies: CompanyCard[24], events: Event[20], movers: CompanyCard[10], new_entrants: CompanyCard[10], series: MetricPoint[90], industries: IndustryRow[] }` \|
	128	+\| GET \| `/signals?kind=&scope=company\\|industry\\|country\\|global&limit=50` \| `{ items: Signal[] }` \|
	129	+\| GET \| `/trends?window=7d\\|30d\\|90d&limit=30` \| `{ items: TrendRow[] }` — `{ term, mentions, companies, momentum, series: number[] }` \|
	130	+\| GET \| `/map?metric=events_30d\\|companies\\|hiring` \| `{ buckets: MapBucket[] }` — `{ lat, lon, country, city: string \\| null, companies: number, events_30d: number, jobs_open: number, top: {slug, display_name}[] }` (clustered by city/country, ≤ 600 buckets) \|
	131	+\| GET \| `/index` \| Global Corporate Activity Index: `{ value, baseline: 100, delta_7d, delta_30d, series: MetricPoint[365], by_type: Record<string, number>, by_country: {key, value}[], by_industry: {key, value}[], formula_version }` \|
	132	+
	133	+### Search
	134	+\| Method \| Path \| Notes \|
	135	+\|---\|---\|---\|
	136	+\| GET \| `/search?q=&types=companies,events,industries,countries,people,products&limit=10` \| `{ query, companies: CompanyCard[], events: Event[], industries: IndustryRow[], countries: CountryRow[], people: (Person & {company: CompanyRef})[], products: (Product & {company: CompanyRef})[], took_ms }` \|
	137	+\| GET \| `/search/suggest?q=` \| `{ items: { kind: 'company'\\|'industry'\\|'country'\\|'event_type', label, sublabel, href }[] }` (≤ 10, < 50 ms) \|
	138	+\| GET \| `/ask?q=` \| natural-language routing (deterministic parser → structured query; LLM optional): `{ interpretation: {filters…}, answer: string, companies: CompanyCard[], events: Event[], sources: string[] }` \|
	139	+
	140	+### Watchlists & alerts (owner token)
	141	+\| Method \| Path \| Notes \|
	142	+\|---\|---\|---\|
	143	+\| GET/POST \| `/watchlist` \| GET → `{ items: CompanyCard[], events: Event[30] (for the watched companies) }`; POST `{ company: slug }` adds; DELETE `/watchlist/{slug}` removes \|
	144	+\| GET/POST \| `/alerts` \| POST `{ name, company?: slug, condition: {event_types?: string[], min_importance?: number, metrics?: {activity_score?: {gt: number}}}, channel: 'web'\\|'webhook', target?: url }`; DELETE `/alerts/{id}` \|
	145	+\| GET \| `/alerts/deliveries?limit=50` \| recent deliveries for this owner \|
	146	+
	147	+### Exports & docs
	148	+\| Method \| Path \| Notes \|
	149	+\|---\|---\|---\|
	150	+\| GET \| `/export/events.{json,ndjson,csv}?since=&event_type=&country=&limit=10000` \| streamed export \|
	151	+\| GET \| `/export/companies.{json,ndjson,csv}?country=&industry=` \| streamed export \|
	152	+\| GET \| `/export/jobs.ndjson?company=&since=` \| \|
	153	+\| GET \| `/sitemap?kind=companies\\|industries\\|countries&page=` \| `{ items: {slug, updated_at}[], pages }` — only companies with `indexed = true` \|
	154	+\| GET \| `/methodology` \| `{ metrics: {metric, formula_version, description, inputs: string[]}[], significance_bands, event_types: string[], confidence_labels }` \|
	155	+
	156	+### Admin (`X-CA-Admin-Token`)
	157	+\| Method \| Path \| Notes \|
	158	+\|---\|---\|---\|
	159	+\| GET \| `/admin/overview` \| `{ companies_by_status, sensors_by_status, sensors_by_tier, queue: {pending, running, dead, oldest_pending_s}, llm: {pending, done_today, failed_today, budget_left}, failures_24h_by_class, fetch_rate_1h, change_rate_1h, meaningful_rate_1h, storage: {objects, bytes}, workers: {name, last_seen_at, inflight}[], cost_today: {fetch, browser, llm} }` \|
	160	+\| GET \| `/admin/connectors` \| `{ items: {id, name, version, category, enabled, sensors_active, sensors_failing, success_rate_24h, avg_latency_ms, change_rate_24h, errors_24h, last_run_at}[] }` \|
	161	+\| GET \| `/admin/sensors?status=&domain=&connector=&company=&filter=healthy\\|failing\\|stale\\|blocked\\|redirected\\|low_quality\\|high_activity&page=` \| page of `Sensor & {company: CompanyRef}` \|
	162	+\| POST \| `/admin/sensors/{id}/{action}` \| action ∈ `pause, resume, retry, rediscover, retire, run_now`; body `{ interval_s?, connector_id? }` for `set_interval`, `set_connector` \|
	163	+\| GET \| `/admin/companies?onboarding_status=&page=` · POST `/admin/companies` `{ website, display_name?, country?, industries? }` (create + queue discovery) · POST `/admin/companies/{slug}/rediscover` \|
	164	+\| GET \| `/admin/failures?class=&since=&page=` · GET `/admin/queue?kind=&status=` · POST `/admin/queue/requeue-dead` \|
	165	+\| GET \| `/admin/llm?status=&page=` · GET `/admin/reviews?kind=&status=open` · POST `/admin/reviews/{id}` `{ resolution: 'accepted'\\|'rejected', note? }` \|
	166	+\| POST \| `/admin/events/{id}/retract` `{ reason }` · POST `/admin/events/{id}/restore` \|
	167	+\| GET \| `/admin/quality` \| `{ coverage: {companies_active_pct, sensors_active_pct}, freshness: {sensors_checked_24h_pct, stale}, duplicate_rate, event_confidence_avg, unknown_surfaces, failed_sensors, calibration: {correct, duplicate, noise, misclassified} }` \|
	168	+\| GET \| `/admin/costs?days=30` \| `{ items: {day, dimension, key, units, cost_estimate}[], per_1000_companies, per_million_observations, per_meaningful_event }` \|
	169	+\| POST \| `/admin/cache/clear` \| \|
	170	+
	171	+## Conventions for implementers
	172	+- Every list endpoint is bounded (`per_page ≤ 200`, `limit ≤ 500`), uses indexed predicates, and returns `Cache-Control: public, max-age=60` for public aggregates (`/pulse`, `/stats`, `/rankings`, `/industries`, `/countries`) and `no-store` for `/live*`, owner and admin routes.
	173	+- Company lookups accept slug or id. Unknown → 404 `{detail: "company not found"}`.
	174	+- Text search: Postgres FTS (`companies.search`, `events.search`) + trigram fallback for short/partial queries; never LIKE on unindexed columns.
	175	+- Numbers: metrics are 0–100 floats rounded to 1 decimal except `hiring_momentum_*` (percentage, may be negative) and `open_jobs` (int).
	176	+- Never fabricate: when a metric has no inputs, omit it (or `null`) rather than returning 0 as if measured.

added docs/ARCHITECTURE.md +61 −0

		`@@ -0,0 +1,61 @@`
	1	+# Company Atlas — architecture & module ownership
	2	+
	3	+One Python package (`src/companyatlas`, CLI `catlas`) runs every logical service of the spec as processes/commands, plus a Next 16 web app
	4	+(`apps/web`). Postgres 17 is the only stateful dependency (entities, history, queue via `SKIP LOCKED`, metrics); raw objects live in a
	5	+content-addressed zstd store on disk (`CA_DATA_DIR/objects`). No Redis. Workers are stateless and idempotent: any node with
	6	+`DATABASE_URL` + the data dir (or its own object store) can run `catlas schedule` / `catlas worker`.
	7	+
	8	+```
	9	+registry/ (seed companies, industries, countries) ──catlas seed──▶ companies
	10	+ │ onboarding queue
	11	+ ▼
	12	+ services/discovery.py (homepage → robots → sitemaps → nav → ATS/feeds → classify → sensors)
	13	+ │
	14	+ scheduler (services/scheduler.py) claims due sensors ─────┤ adaptive intervals, domain budgets, priority
	15	+ ▼
	16	+ services/pipeline.py run_sensor(): fetch (fetch.py) → observation → connector.extract → Extraction
	17	+ → normalize/fingerprint (sdk/normalize.py) → snapshot (+ objects)
	18	+ → sdk/diff.py block diff + significance → changes row (+ structured_delta)
	19	+ → entity reconciliation (jobs/people/products/plans/locations/news tables)
	20	+ │ changes.status = 'pending'
	21	+ ▼
	22	+ services/events.py process_changes(): deterministic events from structured deltas + diff → events (+ clusters, dedupe)
	23	+ → LLM enrichment jobs when useful (services/llm/*, prompts/ versioned, budgeted)
	24	+ ▼
	25	+ services/metrics.py hourly scores (activity, hiring momentum, product velocity, AI adoption, geo, developer, CCI),
	26	+ daily aggregates (company_daily, global_daily, coverage-normalised index), baselines → anomaly,
	27	+ signals, trends
	28	+ ▼
	29	+ api/ (FastAPI, docs/API.md) ◀── apps/web (Next 16, SSR + SSE live feed) ◀── Caddy (MacLustr Tunnel) ◀── www.company-atlas.co
	30	+```
	31	+
	32	+## Ownership map (who writes what)
	33	+
	34	+\| Area \| Modules \| CLI (`companyatlas/commands/*.py`, `register(app)`) \|
	35	+\|---\|---\|---\|
	36	+\| Kernel (done) \| `config.py`, `taxonomy.py`, `ids.py`, `urls.py`, `fetch.py`, `archive.py`, `db/`, `logging.py`, `sdk/models.py`, migration 0001, `api/main.py`, `api/common.py`, `cli.py` \| `migrate`, `api`, `version` \|
	37	+\| Crawl core \| `sdk/normalize.py`, `sdk/diff.py`, `sdk/connector.py` (+ registry), `connectors/*`, `services/discovery.py`, `services/pipeline.py`, `services/scheduler.py`, `services/repair.py` \| `crawl.py`: `onboard`, `discover`, `run-sensor`, `schedule`, `sensors`, `repair`, `connectors` \|
	38	+\| Intelligence \| `services/events.py`, `services/clustering.py`, `services/llm/{gateway,enrich,schemas}.py`, `prompts/*`, `services/metrics.py`, `services/signals.py`, `services/trends.py`, `services/alerts.py`, `services/digest.py` \| `intel.py`: `process-changes`, `enrich`, `metrics`, `daily`, `signals`, `alerts` \|
	39	+\| Seeds \| `registry/` data files, `companyatlas/registry/seed.py`, `scripts/seed_wikidata.py`, `scripts/seed_edgar.py` \| `seed.py`: `seed`, `import-companies` \|
	40	+\| API \| `api/routers/*.py` (auto-included; `ORDER` for precedence), `api/sse.py`, `api/ratelimit.py` \| — \|
	41	+\| Web \| `apps/web` \| — \|
	42	+\| Ops \| `deploy/.mld.json`, `deploy/render-manifest.sh`, `deploy/first-run.sh`, `scripts/backup.sh` \| `ops.py`: `stats`, `status`, `backup`, `retention` \|
	43	+
	44	+## Boundaries (contracts)
	45	+- Crawl → Intelligence: `changes` rows with `status='pending'`, `diff` (bounded JSON, `BlockDiff.to_json()`), `structured_delta`
	46	+ (shape documented in `sdk/models.py`), `significance`, `kind`. Entity tables already reconciled (first_seen/last_seen/removed_at).
	47	+- Intelligence → API: `events`, `event_clusters`, `event_sources`, `metrics_current`, `metric_series`, `company_daily`, `global_daily`,
	48	+ `baselines`, `signals`, `trends`, `alert_deliveries`.
	49	+- API → Web: `docs/API.md`. The web never touches the database.
	50	+- Seeds → Crawl: `companies` with `onboarding_status='pending'` + `queue_jobs(kind='discover')`.
	51	+
	52	+## Conventions
	53	+- IDs: `ids.new_id(kind)` (prefixed ULIDs). Slugs from `ids.slugify`. Alias keys from `ids.normalize_alias`.
	54	+- SQL: plain text via `db.fetch_all/fetch_one/execute` (SQLAlchemy Core, asyncpg). Cast ambiguous binds (`cast(:x as text)`),
	55	+ arrays as `any(cast(:ids as text[]))`, real `datetime`/`date` objects as params, `jsonb(value)` + `cast(:p as jsonb)`.
	56	+- Timestamps UTC. Never delete history; use `status` columns. Migrations forward-only, additive, in `migrations/versions/000N_*.py`.
	57	+- Politeness: only `fetch.Fetcher` talks to the network (SSRF guard, robots, per-domain governor, size caps). Never bypass challenges.
	58	+- Language: "detected", "observed", "no longer listed", "appears", "signal", "inferred". Never "fired", "laid off", "shut down".
	59	+- Config: tunables in `config.Settings` / `taxonomy.py`; formula versions bump when weights change.
	60	+- Tests: fixtures in `fixtures/`, never live network in CI (`@pytest.mark.live` for opt-in checks).
	61	+- Logging: `logging.getLogger(__name__)` with `extra={…}` (JSON in prod).

added docs/PRODUCT-SPEC.md +164 −0

		`@@ -0,0 +1,164 @@`
	1	+# Company Atlas — Product Specification (master brief, verbatim from the founder, 2026-09-12)
	2	+
	3	+## 0. PROJECT IDENTITY
	4	+
	5	+Product name: Company Atlas
	6	+Primary domain: `https://www.company-atlas.com`
	7	+Canonical hostname: `www.company-atlas.com`
	8	+Product category: Global company intelligence / continuous company monitoring / historical corporate data platform
	9	+Core concept: Build the continuously updated historical record of the world's companies.
	10	+
	11	+Company Atlas is NOT a static company directory.
	12	+
	13	+Company Atlas is a global intelligence platform composed of thousands, and eventually millions, of persistent public-web connectors/sensors attached to companies around the world.
	14	+
	15	+Each connector continuously observes a specific public surface of a company: corporate website, careers pages, job boards, newsroom, investor relations, pricing, products, services, documentation, APIs, changelogs, leadership pages, office/location pages, sustainability pages, partnerships, customer stories, legal pages, terms, privacy policies, status pages, support portals, blogs, research pages, GitHub/public development activity where appropriate, public structured feeds, sitemaps, public subdomains, other legally/publicly accessible corporate surfaces.
	16	+
	17	+The platform must continuously turn these observations into: 1. raw observations, 2. normalized snapshots, 3. structural fingerprints, 4. detected changes, 5. structured corporate events, 6. historical timelines, 7. proprietary metrics, 8. comparative intelligence, 9. search, 10. APIs, 11. alerts, 12. datasets.
	18	+
	19	+The core competitive advantage is time. Every day the platform operates, its historical dataset becomes harder to reproduce.
	20	+
	21	+> Anyone can crawl a website today. Very few can reconstruct exactly how 100,000 companies changed over the previous five years.
	22	+
	23	+Company Atlas should become that historical record.
	24	+
	25	+# 1. NORTH STAR
	26	+
	27	+> Build the most comprehensive continuously updated machine-readable history of how companies evolve.
	28	+
	29	+We are not merely collecting pages. We are measuring corporate change. Examples: Which companies are accelerating hiring? Cutting hiring? Changed pricing? Launched products? Quietly removed products? Entered new countries? Added AI-related roles? Changed executive leadership? Expanding developer ecosystems? Changing positioning? Moving toward enterprise customers? Changed their terms? Opened or closed offices? Increasing public communication activity? Which industries are changing fastest? Which geographies attract expansion? Which companies appear to be preparing launches? Which businesses suddenly have abnormal website activity?
	30	+
	31	+# 2. PRODUCT PRINCIPLES
	32	+
	33	+## 2.1 Historical-first
	34	+Never overwrite history when a new version arrives. Changes create new versions. We care about state(t0), state(t1), state(t2)… not just current_state. Current state is simply the latest historical state.
	35	+
	36	+## 2.2 Everything important should be timestamped
	37	+discovered_at, first_seen_at, last_seen_at, fetched_at, changed_at, published_at, effective_at, processed_at — depending on entity type.
	38	+
	39	+## 2.3 Raw data and interpreted data must remain separate
	40	+Never destroy source observations after interpretation. RAW SOURCE → RAW SNAPSHOT → NORMALIZED SNAPSHOT → DIFF → EVENT → LLM ENRICHMENT → METRICS. This allows future reprocessing when extraction models improve.
	41	+
	42	+## 2.4 LLMs are enrichment engines, not the crawler
	43	+Do NOT send every fetched webpage to an expensive model. fetch → normalize → hash → compare → structural diff → semantic-change candidate → LLM only if useful.
	44	+
	45	+## 2.5 Build reusable connector families
	46	+Do NOT hard-code every company individually. The system combines generic connectors + site adapters + discovered endpoints + company-specific configuration and must be capable of creating thousands of connectors automatically.
	47	+
	48	+## 2.6 Every crawl should answer a question
	49	+Did pricing change? Did the leadership team change? Did jobs increase? Was a product introduced? Were locations added? Did documentation change? Specialized sensors produce better data than storing full websites.
	50	+
	51	+# 3. SCALE TARGETS
	52	+Phase 1: 5,000 companies / 25,000+ active sensors. Phase 2: 25,000 / 150,000+. Phase 3: 100,000 / 750,000+. Phase 4: 1,000,000 / 5M–15M. An important company could have 10–100 monitored surfaces; long-tail companies 2–10. Sensors are lightweight entities.
	53	+
	54	+# 4. TERMINOLOGY
	55	+Company canonical organization (Stripe, Inc.). Domain canonical corporate domain (stripe.com). Surface logical information surface (careers, pricing, newsroom, investor_relations, products, leadership, locations). Connector a software strategy able to collect a type of source (generic_html_connector, sitemap_connector, greenhouse_connector, lever_connector, json_endpoint_connector, rss_connector). Sensor a deployed connector instance attached to a specific source (Company: Stripe · Surface: Careers · Connector: generic_careers · URL: https://stripe.com/jobs) — sensors execute repeatedly. Observation one fetch/read of a sensor. Snapshot normalized representation of an observation. Change difference between snapshots. Event a meaningful interpreted corporate change.
	56	+
	57	+# 5. HIGH-LEVEL ARCHITECTURE
	58	+Company Registry / Graph → Discovery Engine + Scheduler → Connector Workers (HTTP, Browser, Feeds) → Raw Storage → Normalization → Fingerprint / Diff → Change Detector → (insignificant → archive \| meaningful → Event Pipeline → LLM Enrichment → Event Store → Metrics Engine → API / Search / UI).
	59	+
	60	+# 6. COMPANY REGISTRY
	61	+Canonical company entity: id, slug, legal_name, display_name, canonical_domain, website, description, industry[], country, headquarters, founded_year, company_type, public_company, ticker, status, created_at, updated_at. Keep factual provenance. Do not merge entities based only on similar names.
	62	+
	63	+# 7. COMPANY GRAPH
	64	+Relationships: company → domain, brand, parent, subsidiary, executive, office, product, competitor, acquisition, investor, technology, job, industry, country, announcement, event. Every edge supports temporal validity where relevant (valid_from, valid_to, first_seen, last_seen, source).
	65	+
	66	+# 8. CONNECTOR ARCHITECTURE
	67	+Thousands of connectors but not thousands of unrelated codebases. Base interface: discover / fetch / extract / normalize / fingerprint / compare / emit_events. Metadata: connector_id, name, version, category, fetch_mode, supports_discovery, supports_incremental.
	68	+
	69	+# 9. CONNECTOR FAMILIES
	70	+9.1 Core website connectors: homepage, about, company, leadership, team, products, services, solutions, industries, customers, partners, locations, contact, pricing, documentation, developer, API, changelog, blog, news, press, research, careers, legal, terms, privacy, security, trust, sustainability, ESG, investor relations.
	71	+9.2 Discovery connectors: robots.txt, sitemap.xml, sitemap index, RSS, Atom, navigation discovery, HTML link graph, JSON-LD, schema.org, alternate language URLs, subdomain discovery, known URL patterns.
	72	+9.3 Job connectors: Greenhouse-like, Lever-like, Workday public career surfaces, SmartRecruiters-like, Ashby-like, custom career pages, JSON-backed boards, HTML listings. Avoid relying on a single vendor API; prefer public endpoints already used by public pages when lawful and appropriate.
	73	+
	74	+# 10–11. SENSOR CREATION & AUTOMATIC DISCOVERY
	75	+canonical domain → homepage fetch → robots inspection → sitemap discovery → navigation analysis → subdomain discovery → URL classification → surface identification → sensor creation. Signals: anchor text, URL paths, navigation hierarchy, page titles, schema markup, sitemap metadata, headings, CMS conventions, subdomains. Classes: CAREERS, NEWSROOM, BLOG, PRODUCTS, PRICING, ABOUT, LEADERSHIP, LOCATIONS, INVESTOR_RELATIONS, DOCUMENTATION, CHANGELOG, LEGAL, OTHER — each with a confidence.
	76	+
	77	+# 12. SENSOR QUALITY SCORE
	78	+source reliability × extraction confidence × historical stability × semantic importance × recency → quality_score 0–100. Low quality sensors may be automatically reviewed.
	79	+
	80	+# 13. FETCH MODES
	81	+A — lightweight HTTP (default: HTML, JSON, XML, RSS, sitemaps). B — browser rendering only when needed, pooled. C — public frontend network endpoint extraction where a public page loads data from public endpoints; store provenance. Never bypass authentication, access controls, CAPTCHAs; never collect non-public information.
	82	+
	83	+# 14. POLITENESS / COMPLIANCE
	84	+Domain-specific rate limiting, robots policy awareness, retry budgets, crawl-delay, backoff, concurrency caps, clear user agent, attribution, provenance, suppression mechanism, legal/compliance flags. Never collect content requiring unauthorized access; never circumvent authentication; no private customer data.
	85	+
	86	+# 15–16. SCHEDULER & ADAPTIVE CRAWLING
	87	+Tiers: A 5–15 min · B 30–60 min · C 6 h · D 24 h · E 3–7 days. Frequency adapts: significant changes → temporary burst (1/15 min) then decay. next_interval = base × stability × importance × failure × activity.
	88	+
	89	+# 17–20. CHANGE DETECTION, NORMALIZATION, BLOCK DIFFING, SIGNIFICANCE
	90	+Noise (dates, analytics IDs, rotating testimonials, randomized content, tracking params, ads, cookie banners, session ids) is normalized away. Hashes: raw_hash, normalized_hash, structural_hash, semantic_hash. Normalization: remove scripts/styles, normalize whitespace, remove dynamic attributes, normalize URLs, strip tracking/session params, canonicalize headings, extract primary content, preserve structured data → normalized DOM, plaintext, semantic blocks, structured fields. Block-level diffing over semantic blocks (header, hero, product card, pricing plan, job listing, executive bio, office location, news article, FAQ, table) with stable identities → added / removed / modified / moved. Significance: 0–0.20 noise · 0.20–0.40 minor · 0.40–0.65 meaningful · 0.65–0.85 major · 0.85–1.00 critical. Inputs: % text changed, semantic similarity, page importance, affected structured entities, novelty, cross-source confirmation, historical baseline.
	91	+
	92	+# 21–23. EVENT TAXONOMY, MODEL, DEDUPLICATION
	93	+Types: PRODUCT, PRICING, HIRING, LEADERSHIP, LOCATION, FINANCING, M&A, PARTNERSHIP, STRATEGY, TECHNOLOGY, LEGAL, MARKETING, DEVELOPER, SECURITY, OPERATIONS, SUSTAINABILITY, INVESTOR_RELATIONS, COMMUNICATION, OTHER. Subtypes: PRODUCT_LAUNCH, PRODUCT_REMOVAL, PRODUCT_RENAME, PRICE_INCREASE, PRICE_DECREASE, NEW_PRICING_TIER, JOB_COUNT_INCREASE, JOB_COUNT_DECREASE, NEW_EXECUTIVE, EXECUTIVE_REMOVED, NEW_OFFICE, OFFICE_REMOVED, COUNTRY_EXPANSION, NEW_PARTNERSHIP, ACQUISITION, DIVESTITURE, API_LAUNCH, DOCUMENTATION_CHANGE, TERMS_CHANGE, BRAND_REPOSITIONING. Event model: id, company_id, event_type, event_subtype, importance, confidence, title, summary, old_value, new_value, detected_at, effective_at, source_url, sensor_id, snapshot_before, snapshot_after, model_version. The same corporate event appearing on homepage, press release, blog, pricing page, IR → clustered into one canonical event with sources; corroboration increases confidence.
	94	+
	95	+# 24–25. LLM ENRICHMENT & MODEL ABSTRACTION
	96	+LLMs receive only relevant changed content (company metadata, source type, before blocks, after blocks, structured changes). Tasks: classification, summary, importance, entity extraction, structured event generation, industry tagging, sentiment where appropriate, strategy interpretation. Structured JSON validated against schemas. Never hard-code one vendor: LLMProvider / EmbeddingProvider / RerankerProvider; local models, OpenAI-compatible endpoints; work with local cluster inference wherever practical.
	97	+
	98	+# 26–28. STORAGE
	99	+Every relevant version retained (sensor_id, url, fetched_at, status_code, content_hash, normalized_hash, storage_pointer, content_type, size_bytes); compressed object storage; dedupe by hash. Layers: PostgreSQL (entities/metadata), object storage (raw), columnar analytics, search engine, vector index — abstracted. Internal event bus topics: company.created, sensor.created, sensor.fetch.requested/completed, snapshot.created, change.detected, event.generated, metric.updated, alert.triggered; async idempotent workers.
	100	+
	101	+# 29–37. PROPRIETARY METRICS
	102	+Company Activity Score (0–100: website changes, product changes, news frequency, job movement, leadership changes, documentation, pricing). Hiring Momentum (open jobs, new/removed, departments, locations, seniority, remote ratio, skills; 7/30/90-day, YoY). AI Adoption Score (observable public signals only: AI products, jobs, docs, marketing, partnerships, research, leadership roles — never claim internal use without evidence). Product Velocity. Geographic Expansion Score. Developer Momentum. Corporate Change Index = 0.25 hiring + 0.20 product + 0.15 geographic + 0.15 leadership + 0.10 developer + 0.10 communication + 0.05 pricing (weights to become empirical). Unusual Activity Detection from per-company baselines ("Companies behaving unusually today"). Cross-company signals (AI hiring acceleration by industry, SaaS pricing increases, US manufacturing expansion…).
	103	+
	104	+# 38–48. PRODUCT SURFACES
	105	+Industry Atlas (living index per industry: activity, companies, events, hiring, expansion, trending). Country Atlas (/country/canada …: activity, hiring, expansion, industry mix, top movers, new entrants). Company profile `/company/stripe`: header with Activity Score, Hiring Momentum, Product Velocity, AI Adoption; sections Overview, Timeline, Signals, Jobs, Products, Locations, Leadership, Technology, Sources, Historical. Company Timeline (dated entries, filters all/products/jobs/pricing/leadership/locations/legal/news/developer). Historical Page Viewer (semantic diff between versions). Global Live Feed on the homepage (cards: company, change, "2 minutes ago", incremental updates). Homepage hero The Live Atlas of Global Companies with animated counters (companies, sensors, observations, changes, structured events) and sections: Hero, Live Activity Feed, Companies Moving Fastest, Global Activity Map, Hiring Momentum, Product Launches, Pricing Changes, AI Adoption, Industries, Countries, Trending Signals, Platform Statistics, API/Data CTA. Map with clustering (HQs, expansions, offices, event density, hiring). Search over companies, industries, events, products, people, locations, keywords, technologies ("companies hiring AI engineers in Canada"). Later: "Ask Company Atlas" natural-language search always linking back to events and sources.
	106	+
	107	+# 49–50. PROVENANCE & CONFIDENCE
	108	+Every important fact traceable (Source, First seen, Last checked, Historical evidence, Confidence). Statuses: VERIFIED, HIGH CONFIDENCE, LIKELY, INFERRED, LOW CONFIDENCE. Never present inference as verified fact without labeling.
	109	+
	110	+# 51–54. API, EVENT API, STREAMING, EXPORTS
	111	+API-first `/api/v1`: companies, companies/{id}, events, metrics, jobs, history, events (filters event_type, country, since…), industries, countries, search. Streaming later (WebSocket, SSE, webhooks). Exports JSON, CSV, Parquet, NDJSON.
	112	+
	113	+# 55–56. USERS & WATCHLISTS
	114	+Public browsing without account. Accounts only for watchlists, alerts, API access, saved searches, dashboards, exports. Watchlists with alerts (product launch, pricing change, leadership change, hiring spike, location expansion, activity anomaly).
	115	+
	116	+# 57–66. INTERNAL ADMIN & OPERATIONS
	117	+/admin modules: companies, domains, sensors, connectors, crawl queue, worker health, failures, snapshots, changes, events, duplicates, LLM jobs, metrics, sources, alerts, system statistics. Connector Control Center (version, active sensors, success rate, latency, change detection rate, errors, last deployment). Sensor Control Center (filters healthy/failing/stale/blocked/redirected/low quality/high activity; actions pause/resume/retry/rediscover/change frequency/change connector). Auto-repair (retry → re-fetch sitemap → rediscover navigation → replacement URL → compare content identity → migrate sensor; flag if uncertain). Connector versioning (generic-pricing-v1 → v2; version stored with observations). Failure classes DNS, TIMEOUT, HTTP_4XX, HTTP_5XX, BOT_CHALLENGE, PARSING, SCHEMA, REDIRECT, PAGE_REMOVED, RATE_LIMIT, UNKNOWN with distinct retry policies. Domain budgets (max_concurrency, requests_per_minute, daily_budget, browser_budget, retry_budget). Priority queue (importance, watchlists, recent activity, time since last crawl, sensor value, reliability, cost). Cost accounting per company/sensor/connector/fetch/browser/LLM/GB/event. Observability (fetch/sec, success rate, changes/sec, meaningful event rate, queue lag, browser utilization, storage growth, LLM jobs, false positive rate, connector failures).
	118	+
	119	+# 67–69. INFRASTRUCTURE
	120	+Logical services: atlas-web, atlas-api, atlas-scheduler, atlas-discovery, atlas-fetch-http, atlas-fetch-browser, atlas-normalizer, atlas-diff, atlas-events, atlas-llm, atlas-metrics, atlas-search, atlas-admin, atlas-worker-manager. Stateless workers, horizontal scaling (add machine → register worker → joins queue), configurable node roles.
	121	+
	122	+# 70–82. DATA MODEL DETAILS
	123	+Tables: companies, domains, company_aliases, company_relationships, surfaces, connectors, sensors, observations, snapshots, changes, events, event_sources, people, jobs, products, locations, metrics, metric_series, crawl_runs, failures, users, watchlists, alerts. Metrics are time series (never only latest). Jobs: title, department, location, remote, employment_type, seniority, skills, salary where public, posted_at, first_seen, last_seen, removed_at. Leadership: name, title, role category, first/last seen, source — disappearance is "No longer listed on monitored leadership page", never "Fired". Products: new/changed/renamed/removed from public catalog. Pricing: plan, currency, billing period, price, features, usage units, enterprise/contact sales — preserve every version. Locations: office/store/factory/warehouse/lab/headquarters with city/region/country — never invent exact addresses. Technology signals: observed / stated / inferred. Newsroom: title, URL, published_at, category, entities, summary (first-party = high provenance). Investor relations for public companies (not a substitute for regulated filings). Legal change tracking with semantic summaries ("Section 7 was materially updated"), source versions preserved. Duplicate company resolution by multiple signals; never auto-merge on name alone. Internationalization: store original language + normalized English summaries (source_language, normalized_language, translation_model).
	124	+
	125	+# 83–88. URL DESIGN, SEO, DESIGN LANGUAGE
	126	+Clean URLs: /company/apple, /industry/artificial-intelligence, /country/canada, /events, /live, /rankings, /search, /api, /about. SEO: metadata, OpenGraph, canonical, schema markup, sitemaps; do not index thin profiles until sufficient data. Design: premium intelligence terminal blended with a modern data atlas — dense but readable, beautiful typography, live counters, micro visualizations, timelines, maps, tables, sparklines, confidence indicators, real-time status; avoid generic startup cards, excessive gradients, cartoon visuals, unused whitespace, clutter. Desktop: powerful tables, keyboard search, filters, multi-column, comparison, density. Mobile first-class: responsive typography, bottom navigation (Home, Live, Search, Rankings, Watchlist), touch controls, sticky search, compact metrics, collapsible filters, swipe-friendly timelines, fast loads. Live visual language: green live dot, "17 sec ago", new-event animation, counter increments, sparkline updates — never chaotic.
	127	+
	128	+# 89–93. RANKINGS, COMPARISON, GLOBAL INDEX, SIGNALS, PREDICTIONS
	129	+Rankings: Most Active, Fastest Hiring Growth/Decline, Highest Product Velocity, Most AI-Active, Fastest Geographic Expansion, Highest Developer Momentum, Most Pricing Changes, Most Unusual Activity — windows 24h/7d/30d/90d/1y. Comparison `/company/compare?companies=stripe,adyen,block` (activity, hiring, product velocity, AI, locations, events). Global Corporate Activity Index (baseline 100 = normalized historical activity; by industry, country, size, event type). Signals (hiring surge, hiring freeze, launch buildup, international expansion, pricing migration, developer push, enterprise repositioning, AI acceleration) labeled as signals, not facts. Predictions only as explicitly probabilistic ("Possible launch preparation signal — Confidence 63%").
	130	+
	131	+# 94–98. FLYWHEEL, MOAT, RETENTION, CAS, BACKFILL
	132	+More companies → sensors → observations → baselines → anomalies → events → metrics → users → watchlists → prioritization → dataset value. Moat = historical snapshots, normalized entities, change events, sensor reliability history, historical metrics, comparisons, graph. Long-term retention; identical snapshots share content objects; keep observation metadata. Content-addressable storage sha256 `objects/ab/cd/hash`. Backfills marked collection_method = backfill, never blurred with live.
	133	+
	134	+# 99–106. SEED, FIRST TARGET, CONNECTOR FACTORY, ONBOARDING, REGISTRY, FIXTURES, ADAPTERS, SDK
	135	+Seed diversified companies (technology, AI, finance, banking, insurance, retail, energy, manufacturing, healthcare, biotech, pharma, transportation, aerospace, telecom, media, real estate, construction, logistics, automotive, consumer) across US, Canada, Europe, UK, Japan, South Korea, India, Australia, Latin America, Middle East, Africa, Southeast Asia. First target 5,000+ companies / 25,000+ functioning sensors. Bootstrap connector factory: domain → surfaces[{type, url, connector, confidence, frequency}]. Mass onboarding pipeline fully parallelizable (import → canonicalization → domain validation → discovery → surface mapping → sensor generation → initial fetch → quality validation → scheduler activation). Machine-readable connector registry (manifest, schema, implementation, fixtures, tests, version). Fixtures for every major connector; never depend entirely on live sites in CI. Site-specific adapters inherit generic logic. Connector SDK decorator style.
	136	+
	137	+# 107–112. EXTRACTION & BANDWIDTH
	138	+Semantic extraction (DOM, ARIA, headings, lists, tables, JSON-LD, microdata, embedded JSON) over brittle selectors. Structured data first. Crawl fingerprinting (etag, last-modified, content-length, status, hash, DOM fingerprint) with conditional requests. Compression, connection pooling, streaming, size limits, selective HEAD. Do not download images/videos/fonts/binaries/tracking scripts by default. Screenshots only for high-impact changes/human review.
	139	+
	140	+# 113–131. IMPORTANCE, SECURITY, CANONICALIZATION, LOOPS, PAGE VALUE, QUALITY, REVIEW, AUDIT, VERSIONING, BACKPRESSURE, IDEMPOTENCY, LOCKS, HEALTH, PUBLIC STATS
	141	+Event importance ≠ confidence; company importance affects crawl priority only. Security: strict URL validation, SSRF protection, DNS rebinding protection, private-IP blocking, download/redirect limits, content-type checks, sandboxed parsing, secret separation; crawler must NEVER access localhost, RFC1918, metadata endpoints, cluster internal domains, Tailscale/private hosts, file://, unix sockets. URL canonicalization (scheme, host casing, ports, tracking params, fragments, duplicate slashes, session params) with the original preserved. Loop prevention (calendars, facets, random params, endless pagination, session IDs) via budgets. Page value classifier for discovered URLs. Data quality dashboard (coverage, freshness, accuracy, duplicate rate, event confidence, stale/failed sensors, unknown surfaces). Human review queues (company merge, major event, low-confidence extraction, sensor migration, legal-sensitive inference, unexpected activity) feeding evaluation data. Auditability: what changed, where, when, evidence, extraction version, model. Model + prompt versioning (prompts in code, e.g. prompts/event-classifier/v3.md). Reprocessing without recollecting. Backpressure prioritizes important/high-value/fresh/active work without losing queued work. Job idempotency (sensor_id + scheduled_window). Distributed locks. /health /ready /metrics per service. Public /system page with aggregate health only. Public statistics counters.
	142	+
	143	+# 132–148. VISUALIZATIONS, THEMES, PERFORMANCE, CACHING, RATE LIMITS, AUTH, PRIVACY, ALERTS, DIGESTS, TRENDS, COVERAGE NORMALIZATION, BASELINES, CALIBRATION, REGRESSION
	144	+Sparklines, time series, heatmaps, maps, histograms, rankings, timelines, network graphs (few pie charts). Dark/light both intentional (default follows system). Fast SSR, progressive hydration, cached public pages, optimized queries. Cache layers (request, query, entity, search, homepage aggregate) — not live feeds. API rate-limit tiers anonymous/authenticated/paid/internal. Auth (if added): passkeys, OAuth, magic links. Privacy: no consumer profiling; executive data limited to legitimate public professional context. Alerts (activity > 80, new executive, price change, job count −30 %, new country, product launch) via email/web/webhook. Weekly company digest and market digests. Trend engine with rolling windows and momentum (7/30/90 d) normalized by coverage growth. Coverage normalization is critical: long-term indices adjust for sensors, company population, crawl frequency, industry/country coverage. Baseline engine per sensor/company (changes/week, jobs, announcements, volatility). Quality calibration (correct / duplicate / noise / misclassified). Regression tests against historical fixtures before changing normalizers, diff engine, extractors, prompts.
	145	+
	146	+# 149–155. STACK, REPO, ENV, DOMAIN, DEPLOYMENT, BACKUPS
	147	+Next.js/TypeScript/React front; Python and/or TypeScript services; PostgreSQL; Redis-backed queue initially or durable broker; S3-compatible object storage; columnar analytics as scale requires; full-text search; Dockerized services; no vendor coupling. Monorepo apps/ services/ packages/ connectors/ prompts/ fixtures/ scripts/ infra/ docs/. `.env.example`, never commit secrets (DATABASE_URL, REDIS_URL, OBJECT_STORAGE_*, LLM_ENDPOINT, LLM_API_KEY, SEARCH_URL). Production hostname www.company-atlas.com with apex redirect, HTTPS, no hard-coded IPs. Rolling updates, health checks, restart policies, persistent storage separation, logs, metrics, service discovery, environment separation. Back up PostgreSQL, configuration, indexes, connector metadata; object storage redundancy; test restores; object storage is not a backup — a second independent copy is required.
	148	+
	149	+# 156–161. MVP & PHASES
	150	+MVP: 5,000 companies, 25,000 sensors, continuous monitoring, company profiles, live feed, historical timeline, change detection, event extraction, search, rankings, activity score, admin monitoring. MVP connector categories: homepage, careers, news, products, pricing, leadership, locations, blog, documentation, investor relations, sitemap. MVP events: NEW_JOB, JOB_REMOVED, JOB_COUNT_CHANGE, NEW_PRODUCT, PRODUCT_REMOVED, PRICING_CHANGE, LEADERSHIP_CHANGE, NEW_LOCATION, NEWS_RELEASE, DOC_CHANGE. Phase 2: company graph, comparisons, industry/country indices, watchlists, alerts, API, historical viewer, AI adoption, developer momentum. Phase 3: 100k+ companies, advanced graph, anomaly engine, NL search, cross-company trends, datasets, streaming API, historical analytics. Phase 4: 1M+ companies, 5M+ sensors.
	151	+
	152	+# 162–168. COPY, POSITIONING, BRAND, ERROR UX, TRANSPARENCY, ETHICS, LANGUAGE
	153	+Headline The Live Atlas of Global Companies. Supporting: "Company Atlas continuously observes the public web to track how companies evolve — products, hiring, pricing, leadership, locations, technology, strategy and more." Alt: "Thousands of companies. Millions of observations. One continuously growing historical record." Positioning: not a directory, not a news aggregator, not a web archive, not a financial terminal — a continuously updated corporate observation network. Brand: intelligent, global, precise, technical, credible, alive, data-rich, premium; not sci-fi. Error UX: "No monitored evidence available yet." / "Last successfully checked 3 days ago." — never fabricate. Every user-facing event links to its public source; if the source disappears, say so. Ethical inference: "72 monitored listings are no longer visible", never "Company laid off 72 employees". Language: detected, observed, publicly listed, no longer listed, appears, signal, inferred, confirmed by.
	154	+
	155	+# 169–181. STATES, PROVENANCE GRAPH, SOURCE UI, TIME ZONES, IDS, SOFT DELETES, CORRECTIONS, SCHEMA EVOLUTION, INDEX REPRODUCIBILITY, NO MAGIC NUMBERS, FEATURE FLAGS, EXPERIMENTATION, DOCS
	156	+Company states ACTIVE, POSSIBLY_INACTIVE, WEBSITE_UNAVAILABLE, ACQUIRED, DISSOLVED, UNKNOWN (require corroboration). Provenance graph entity field → extraction → snapshot → observation → sensor → URL. Event source UI lists each source with detection times. UTC storage, local rendering, preserve source timezone. Immutable internal ids; slugs may change; never name as key. Soft deletes via statuses. Corrections: status = retracted with audit history. Versioned schemas and careful migrations. Reproducible indices (formula version, inputs, normalization window, computed_at). No magic numbers (typed config). Feature flags for experimental systems. A/B evaluation of extractors/normalizers/classifiers/schedules (unstable metrics labeled). Docs: architecture, connector-sdk, event-taxonomy, data-model, deployment, operations, scoring.
	157	+
	158	+# 182–183. CLAUDE CODE OPERATING RULES & STYLE
	159	+1. Inspect existing architecture first. 2. Preserve working functionality. 3. Avoid unnecessary rewrites. 4. Prefer reusable abstractions. 5. Add tests with new connector behavior. 6. Keep schemas explicit. 7. Never introduce fabricated data. 8. Never hard-code secrets. 9. Do not break mobile. 10. Do not break historical reproducibility. 11. Do not delete historical data during migrations without explicit reason. 12. Run lint/typecheck/tests after meaningful changes. Favor small composable services, typed contracts, schemas, idempotent workers, deterministic processing, clear observability; avoid giant functions, silent error handling, unbounded crawling, LLMs for deterministic tasks, vendor lock-in.
	160	+
	161	+# 184–200. PHILOSOPHY & FINAL DEFINITION
	162	+Every company on Earth should feel like it has virtual sensors attached (APPLE 63 sensors, NVIDIA 48, STRIPE 41, LOCAL BUSINESS 5) living continuously. "Thousands of connectors" is not marketing: 25,000 deployed sensors initially, then 100,000, 1,000,000, 10,000,000+ — configuration-driven and auto-discoverable. Track dataset age and oldest continuous history prominently. Historical Completeness Score per company (sensor uptime, continuity, source coverage, failed periods). Show data density (active sensors, observations, historical changes, structured events). Discovery feedback loop (event → URL discovery → new sensor). Sensor retirement keeps history and seeks successors. Domain migration continuity. Acquisition continuity (ACQUIRED_BY relationship, no blind merges). Entity resolution with stored confidence. Published API docs (/api) with examples, schemas, pagination, rate limits, filters, webhooks. Monetization readiness (free public, pro research, enterprise API, bulk datasets, alerts, custom monitoring) without crippling the free product. Data licensing metadata; redistribute derived data/changes/facts/metadata rather than raw content where required. Long-term: 1M+ companies, 10M+ sensors, billions of observations, years of continuous history.
	163	+
	164	+> Company Atlas is a distributed global sensor network for companies. Each company receives persistent monitoring sensors. Each sensor produces observations. Observations become historical snapshots. Snapshots produce changes. Changes become structured events. Events become metrics. Metrics become intelligence. And every day the platform runs, the dataset becomes more valuable. Optimize for what this dataset becomes after 1, 3, 5, 10 years. That accumulated history is the product.

added migrations/env.py +45 −0

		`@@ -0,0 +1,45 @@`
	1	+"""Alembic environment (async engine). Migrations are plain SQL executed through `op.execute`."""
	2	+from __future__ import annotations
	3	+
	4	+import asyncio
	5	+from logging.config import fileConfig
	6	+
	7	+from alembic import context
	8	+from sqlalchemy.ext.asyncio import create_async_engine
	9	+
	10	+from companyatlas.config import settings
	11	+
	12	+config = context.config
	13	+if config.config_file_name is not None:
	14	+ fileConfig(config.config_file_name)
	15	+
	16	+target_metadata = None
	17	+
	18	+
	19	+def run_migrations_offline() -> None:
	20	+ context.configure(url=settings.sync_database_url, literal_binds=True, dialect_opts={"paramstyle": "named"})
	21	+ with context.begin_transaction():
	22	+ context.run_migrations()
	23	+
	24	+
	25	+def do_run_migrations(connection) -> None: # type: ignore[no-untyped-def]
	26	+ context.configure(connection=connection, target_metadata=target_metadata, transaction_per_migration=True)
	27	+ with context.begin_transaction():
	28	+ context.run_migrations()
	29	+
	30	+
	31	+async def run_async_migrations() -> None:
	32	+ engine = create_async_engine(settings.database_url, poolclass=None)
	33	+ async with engine.connect() as connection:
	34	+ await connection.run_sync(do_run_migrations)
	35	+ await engine.dispose()
	36	+
	37	+
	38	+def run_migrations_online() -> None:
	39	+ asyncio.run(run_async_migrations())
	40	+
	41	+
	42	+if context.is_offline_mode():
	43	+ run_migrations_offline()
	44	+else:
	45	+ run_migrations_online()

added migrations/script.py.mako +21 −0

		`@@ -0,0 +1,21 @@`
	1	+"""${message}
	2	+
	3	+Revision ID: ${up_revision}
	4	+Revises: ${down_revision \| comma,n}
	5	+"""
	6	+from __future__ import annotations
	7	+
	8	+from alembic import op
	9	+
	10	+revision = ${repr(up_revision)}
	11	+down_revision = ${repr(down_revision)}
	12	+branch_labels = ${repr(branch_labels)}
	13	+depends_on = ${repr(depends_on)}
	14	+
	15	+
	16	+def upgrade() -> None:
	17	+ ${upgrades if upgrades else "pass"}
	18	+
	19	+
	20	+def downgrade() -> None:
	21	+ raise RuntimeError("forward-only")

added migrations/versions/0001_initial.py +791 −0

		`@@ -0,0 +1,791 @@`
	1	+"""Company Atlas — initial schema (spec §6–7, §22, §26, §70–76).
	2	+
	3	+Historical-first: nothing here is ever overwritten by the pipeline — observations, snapshots, changes and events are append-only;
	4	+entities (jobs, people, products, plans, locations) carry first_seen / last_seen / removed_at and a status instead of being deleted.
	5	+
	6	+Revision ID: 0001
	7	+Revises:
	8	+"""
	9	+from __future__ import annotations
	10	+
	11	+from alembic import op
	12	+
	13	+revision = "0001"
	14	+down_revision = None
	15	+branch_labels = None
	16	+depends_on = None
	17	+
	18	+SQL = r"""
	19	+create extension if not exists pg_trgm;
	20	+create extension if not exists "uuid-ossp";
	21	+
	22	+-- ============================================================================================================ reference
	23	+create table if not exists industries (
	24	+ slug text primary key,
	25	+ name text not null,
	26	+ parent_slug text references industries(slug),
	27	+ description text,
	28	+ keywords text[] not null default '{}',
	29	+ sort_order int not null default 100
	30	+);
	31	+
	32	+create table if not exists countries (
	33	+ code char(2) primary key,
	34	+ name text not null,
	35	+ region text,
	36	+ subregion text,
	37	+ lat double precision,
	38	+ lon double precision
	39	+);
	40	+
	41	+-- ============================================================================================================ companies
	42	+create table if not exists companies (
	43	+ id text primary key,
	44	+ slug text not null unique,
	45	+ legal_name text,
	46	+ display_name text not null,
	47	+ canonical_domain text not null unique,
	48	+ website text not null,
	49	+ description text,
	50	+ industries text[] not null default '{}',
	51	+ industry_primary text references industries(slug),
	52	+ country char(2) references countries(code),
	53	+ hq_city text,
	54	+ hq_region text,
	55	+ founded_year int,
	56	+ company_type text,
	57	+ public_company boolean not null default false,
	58	+ ticker text,
	59	+ exchange text,
	60	+ employees_band text,
	61	+ employees int,
	62	+ wikidata_id text unique,
	63	+ lei text,
	64	+ sec_cik text,
	65	+ logo_url text,
	66	+ status text not null default 'ACTIVE',
	67	+ onboarding_status text not null default 'pending',
	68	+ onboarding_error text,
	69	+ importance real not null default 0.2,
	70	+ tier smallint not null default 4,
	71	+ indexed boolean not null default false,
	72	+ source_meta jsonb not null default '{}'::jsonb,
	73	+ stats jsonb not null default '{}'::jsonb,
	74	+ discovered_at timestamptz not null default now(),
	75	+ first_observed_at timestamptz,
	76	+ last_observed_at timestamptz,
	77	+ last_change_at timestamptz,
	78	+ last_event_at timestamptz,
	79	+ created_at timestamptz not null default now(),
	80	+ updated_at timestamptz not null default now(),
	81	+ search tsvector generated always as (
	82	+ setweight(to_tsvector('simple', coalesce(display_name, '')), 'A') \|\|
	83	+ setweight(to_tsvector('simple', coalesce(legal_name, '')), 'B') \|\|
	84	+ setweight(to_tsvector('simple', coalesce(canonical_domain, '')), 'B') \|\|
	85	+ setweight(to_tsvector('english', coalesce(description, '')), 'C')) stored
	86	+);
	87	+create index if not exists companies_search_idx on companies using gin (search);
	88	+create index if not exists companies_name_trgm_idx on companies using gin (display_name gin_trgm_ops);
	89	+create index if not exists companies_domain_trgm_idx on companies using gin (canonical_domain gin_trgm_ops);
	90	+create index if not exists companies_country_idx on companies (country);
	91	+create index if not exists companies_industry_idx on companies using gin (industries);
	92	+create index if not exists companies_importance_idx on companies (importance desc);
	93	+create index if not exists companies_onboarding_idx on companies (onboarding_status) where onboarding_status <> 'active';
	94	+create index if not exists companies_last_event_idx on companies (last_event_at desc nulls last);
	95	+
	96	+create table if not exists company_aliases (
	97	+ company_id text not null references companies(id) on delete cascade,
	98	+ alias text not null,
	99	+ alias_norm text not null,
	100	+ kind text not null default 'alias', -- alias \| legal \| brand \| former \| ticker \| native
	101	+ source text,
	102	+ primary key (company_id, alias_norm)
	103	+);
	104	+create index if not exists company_aliases_norm_idx on company_aliases (alias_norm);
	105	+
	106	+create table if not exists domains (
	107	+ id text primary key,
	108	+ company_id text not null references companies(id) on delete cascade,
	109	+ domain text not null,
	110	+ kind text not null default 'primary', -- primary \| alias \| redirect \| subdomain \| former
	111	+ first_seen_at timestamptz not null default now(),
	112	+ last_seen_at timestamptz not null default now(),
	113	+ status text not null default 'active',
	114	+ unique (domain, company_id)
	115	+);
	116	+create index if not exists domains_domain_idx on domains (domain);
	117	+
	118	+create table if not exists company_relationships (
	119	+ id text primary key,
	120	+ from_company_id text not null references companies(id) on delete cascade,
	121	+ to_company_id text references companies(id) on delete set null,
	122	+ to_name text,
	123	+ kind text not null, -- PARENT_OF \| SUBSIDIARY_OF \| ACQUIRED_BY \| ACQUIRED \| PARTNER_OF \| COMPETITOR_OF \| INVESTOR_IN \| BRAND_OF
	124	+ valid_from date,
	125	+ valid_to date,
	126	+ first_seen_at timestamptz not null default now(),
	127	+ last_seen_at timestamptz not null default now(),
	128	+ source_url text,
	129	+ confidence real not null default 0.5,
	130	+ provenance jsonb not null default '{}'::jsonb
	131	+);
	132	+create index if not exists company_relationships_from_idx on company_relationships (from_company_id, kind);
	133	+create index if not exists company_relationships_to_idx on company_relationships (to_company_id, kind);
	134	+
	135	+-- ============================================================================================================ connectors / sensors
	136	+create table if not exists connectors (
	137	+ id text primary key, -- generic-html-v1, greenhouse-v1 …
	138	+ name text not null,
	139	+ version text not null,
	140	+ category text not null, -- surface
	141	+ fetch_mode text not null default 'http',
	142	+ supports_discovery boolean not null default false,
	143	+ supports_incremental boolean not null default true,
	144	+ default_interval_s int not null default 86400,
	145	+ enabled boolean not null default true,
	146	+ created_at timestamptz not null default now(),
	147	+ updated_at timestamptz not null default now(),
	148	+ stats jsonb not null default '{}'::jsonb
	149	+);
	150	+
	151	+create table if not exists sensors (
	152	+ id text primary key,
	153	+ company_id text not null references companies(id) on delete cascade,
	154	+ surface text not null,
	155	+ connector_id text not null references connectors(id),
	156	+ url text not null,
	157	+ canonical_url text not null,
	158	+ domain text not null,
	159	+ discovery_confidence real not null default 0.5,
	160	+ discovery_method text, -- nav \| sitemap \| robots \| pattern \| ats \| feed \| manual \| event
	161	+ quality_score real not null default 50,
	162	+ status text not null default 'pending',
	163	+ tier char(1) not null default 'D',
	164	+ base_interval_s int not null default 86400,
	165	+ current_interval_s int not null default 86400,
	166	+ next_run_at timestamptz not null default now(),
	167	+ last_run_at timestamptz,
	168	+ last_success_at timestamptz,
	169	+ last_change_at timestamptz,
	170	+ last_meaningful_change_at timestamptz,
	171	+ last_status int,
	172	+ last_failure_class text,
	173	+ last_error text,
	174	+ consecutive_failures int not null default 0,
	175	+ consecutive_unchanged int not null default 0,
	176	+ etag text,
	177	+ last_modified text,
	178	+ last_content_hash text,
	179	+ last_normalized_hash text,
	180	+ last_structural_hash text,
	181	+ last_snapshot_id text,
	182	+ snapshot_count int not null default 0,
	183	+ observation_count int not null default 0,
	184	+ change_count int not null default 0,
	185	+ meaningful_change_count int not null default 0,
	186	+ event_count int not null default 0,
	187	+ config jsonb not null default '{}'::jsonb,
	188	+ priority real not null default 0.5,
	189	+ claimed_by text,
	190	+ claimed_at timestamptz,
	191	+ created_at timestamptz not null default now(),
	192	+ updated_at timestamptz not null default now(),
	193	+ retired_at timestamptz,
	194	+ unique (company_id, canonical_url)
	195	+);
	196	+create index if not exists sensors_due_idx on sensors (next_run_at) where status in ('active', 'failing', 'pending');
	197	+create index if not exists sensors_company_idx on sensors (company_id, surface);
	198	+create index if not exists sensors_domain_idx on sensors (domain);
	199	+create index if not exists sensors_status_idx on sensors (status);
	200	+create index if not exists sensors_connector_idx on sensors (connector_id);
	201	+create index if not exists sensors_claimed_idx on sensors (claimed_at) where claimed_by is not null;
	202	+
	203	+create table if not exists domain_budgets (
	204	+ domain text primary key,
	205	+ max_concurrency int not null default 2,
	206	+ requests_per_minute int not null default 20,
	207	+ daily_budget int not null default 600,
	208	+ browser_budget int not null default 20,
	209	+ used_today int not null default 0,
	210	+ browser_used_today int not null default 0,
	211	+ budget_day date not null default current_date,
	212	+ crawl_delay_s real,
	213	+ blocked_until timestamptz,
	214	+ block_reason text,
	215	+ notes text,
	216	+ updated_at timestamptz not null default now()
	217	+);
	218	+
	219	+-- ============================================================================================================ observations / snapshots
	220	+create table if not exists observations (
	221	+ id text primary key,
	222	+ sensor_id text not null references sensors(id) on delete cascade,
	223	+ company_id text not null references companies(id) on delete cascade,
	224	+ fetched_at timestamptz not null default now(),
	225	+ status_code int,
	226	+ duration_ms int,
	227	+ transport text not null default 'http',
	228	+ final_url text,
	229	+ redirects int not null default 0,
	230	+ not_modified boolean not null default false,
	231	+ changed boolean not null default false,
	232	+ failure_class text,
	233	+ error text,
	234	+ content_hash text,
	235	+ normalized_hash text,
	236	+ structural_hash text,
	237	+ object_key text,
	238	+ size_bytes int,
	239	+ content_type text,
	240	+ connector_version text,
	241	+ collection_method text not null default 'live',
	242	+ worker text
	243	+);
	244	+create index if not exists observations_sensor_idx on observations (sensor_id, fetched_at desc);
	245	+create index if not exists observations_company_idx on observations (company_id, fetched_at desc);
	246	+create index if not exists observations_time_idx on observations (fetched_at desc);
	247	+create index if not exists observations_failure_idx on observations (failure_class, fetched_at desc) where failure_class is not null;
	248	+
	249	+create table if not exists snapshots (
	250	+ id text primary key,
	251	+ sensor_id text not null references sensors(id) on delete cascade,
	252	+ company_id text not null references companies(id) on delete cascade,
	253	+ observation_id text references observations(id) on delete set null,
	254	+ previous_snapshot_id text,
	255	+ version_no int not null default 1,
	256	+ fetched_at timestamptz not null default now(),
	257	+ content_hash text not null,
	258	+ normalized_hash text not null,
	259	+ structural_hash text not null,
	260	+ object_key text, -- raw bytes
	261	+ text_key text, -- normalized text
	262	+ blocks_key text, -- JSON blocks
	263	+ extracted jsonb not null default '{}'::jsonb, -- structured fields (jobs, prices, people, locations, news, products, meta)
	264	+ extracted_summary jsonb not null default '{}'::jsonb, -- small counters for listings (job_count, plan_count …)
	265	+ title text,
	266	+ language text,
	267	+ size_bytes int,
	268	+ text_length int,
	269	+ block_count int,
	270	+ content_type text,
	271	+ connector_version text,
	272	+ collection_method text not null default 'live'
	273	+);
	274	+create index if not exists snapshots_sensor_idx on snapshots (sensor_id, fetched_at desc);
	275	+create index if not exists snapshots_company_idx on snapshots (company_id, fetched_at desc);
	276	+
	277	+create table if not exists changes (
	278	+ id text primary key,
	279	+ sensor_id text not null references sensors(id) on delete cascade,
	280	+ company_id text not null references companies(id) on delete cascade,
	281	+ surface text not null,
	282	+ snapshot_before text references snapshots(id) on delete set null,
	283	+ snapshot_after text not null references snapshots(id) on delete cascade,
	284	+ detected_at timestamptz not null default now(),
	285	+ significance real not null,
	286	+ kind text not null, -- noise \| minor \| meaningful \| major \| critical
	287	+ blocks_added int not null default 0,
	288	+ blocks_removed int not null default 0,
	289	+ blocks_modified int not null default 0,
	290	+ blocks_moved int not null default 0,
	291	+ text_delta_ratio real not null default 0,
	292	+ similarity real,
	293	+ diff jsonb not null default '{}'::jsonb, -- block-level diff (bounded)
	294	+ structured_delta jsonb not null default '{}'::jsonb, -- typed deltas (jobs added/removed, prices, people …)
	295	+ status text not null default 'pending', -- pending \| processed \| enriched \| archived
	296	+ processed_at timestamptz,
	297	+ diff_version text not null default 'diff-v1'
	298	+);
	299	+create index if not exists changes_company_idx on changes (company_id, detected_at desc);
	300	+create index if not exists changes_sensor_idx on changes (sensor_id, detected_at desc);
	301	+create index if not exists changes_kind_idx on changes (kind, detected_at desc);
	302	+create index if not exists changes_pending_idx on changes (status) where status = 'pending';
	303	+
	304	+-- ============================================================================================================ events
	305	+create table if not exists event_clusters (
	306	+ id text primary key,
	307	+ company_id text not null references companies(id) on delete cascade,
	308	+ cluster_key text not null unique,
	309	+ event_type text not null,
	310	+ event_subtype text,
	311	+ title text,
	312	+ first_detected_at timestamptz not null default now(),
	313	+ last_detected_at timestamptz not null default now(),
	314	+ source_count int not null default 1,
	315	+ surfaces text[] not null default '{}',
	316	+ confidence real not null default 0.5,
	317	+ canonical_event_id text
	318	+);
	319	+
	320	+create table if not exists events (
	321	+ id text primary key,
	322	+ company_id text not null references companies(id) on delete cascade,
	323	+ sensor_id text references sensors(id) on delete set null,
	324	+ change_id text references changes(id) on delete set null,
	325	+ cluster_id text references event_clusters(id) on delete set null,
	326	+ surface text,
	327	+ event_type text not null,
	328	+ event_subtype text not null,
	329	+ importance real not null default 0.5,
	330	+ confidence real not null default 0.7,
	331	+ confidence_label text not null default 'LIKELY',
	332	+ title text not null,
	333	+ summary text,
	334	+ old_value text,
	335	+ new_value text,
	336	+ payload jsonb not null default '{}'::jsonb,
	337	+ entities jsonb not null default '{}'::jsonb, -- {jobs:[…], people:[…], products:[…], locations:[…], amounts:[…]}
	338	+ tags text[] not null default '{}',
	339	+ detected_at timestamptz not null default now(),
	340	+ effective_at timestamptz,
	341	+ published_at timestamptz,
	342	+ source_url text,
	343	+ snapshot_before text,
	344	+ snapshot_after text,
	345	+ language text,
	346	+ origin text not null default 'deterministic', -- deterministic \| llm \| hybrid \| backfill
	347	+ model_provider text,
	348	+ model_name text,
	349	+ model_version text,
	350	+ prompt_version text,
	351	+ schema_version text not null default 'event-v1',
	352	+ status text not null default 'active',
	353	+ retracted_reason text,
	354	+ dedupe_key text unique,
	355	+ created_at timestamptz not null default now(),
	356	+ search tsvector generated always as (
	357	+ setweight(to_tsvector('english', coalesce(title, '')), 'A') \|\|
	358	+ setweight(to_tsvector('english', coalesce(summary, '')), 'B')) stored
	359	+);
	360	+create index if not exists events_company_idx on events (company_id, detected_at desc);
	361	+create index if not exists events_time_idx on events (detected_at desc) where status = 'active';
	362	+create index if not exists events_type_idx on events (event_type, detected_at desc);
	363	+create index if not exists events_subtype_idx on events (event_subtype, detected_at desc);
	364	+create index if not exists events_importance_idx on events (importance desc, detected_at desc);
	365	+create index if not exists events_search_idx on events using gin (search);
	366	+create index if not exists events_cluster_idx on events (cluster_id);
	367	+create index if not exists events_tags_idx on events using gin (tags);
	368	+
	369	+create table if not exists event_sources (
	370	+ event_id text not null references events(id) on delete cascade,
	371	+ sensor_id text references sensors(id) on delete set null,
	372	+ source_url text not null,
	373	+ snapshot_id text,
	374	+ surface text,
	375	+ detected_at timestamptz not null default now(),
	376	+ kind text not null default 'primary',
	377	+ primary key (event_id, source_url)
	378	+);
	379	+
	380	+-- ============================================================================================================ extracted entities
	381	+create table if not exists jobs (
	382	+ id text primary key,
	383	+ company_id text not null references companies(id) on delete cascade,
	384	+ sensor_id text references sensors(id) on delete set null,
	385	+ external_id text,
	386	+ fingerprint text not null,
	387	+ title text not null,
	388	+ department text,
	389	+ team text,
	390	+ location_text text,
	391	+ city text,
	392	+ region text,
	393	+ country char(2),
	394	+ remote boolean,
	395	+ employment_type text,
	396	+ seniority text,
	397	+ skills text[] not null default '{}',
	398	+ salary_min numeric,
	399	+ salary_max numeric,
	400	+ salary_currency text,
	401	+ salary_period text,
	402	+ url text,
	403	+ description_hash text,
	404	+ posted_at timestamptz,
	405	+ first_seen_at timestamptz not null default now(),
	406	+ last_seen_at timestamptz not null default now(),
	407	+ removed_at timestamptz,
	408	+ status text not null default 'open', -- open \| no_longer_listed
	409	+ is_ai boolean not null default false,
	410	+ is_engineering boolean not null default false,
	411	+ raw jsonb not null default '{}'::jsonb,
	412	+ unique (company_id, fingerprint)
	413	+);
	414	+create index if not exists jobs_company_idx on jobs (company_id, status);
	415	+create index if not exists jobs_first_seen_idx on jobs (first_seen_at desc);
	416	+create index if not exists jobs_country_idx on jobs (country) where status = 'open';
	417	+create index if not exists jobs_ai_idx on jobs (company_id) where is_ai and status = 'open';
	418	+create index if not exists jobs_title_trgm_idx on jobs using gin (title gin_trgm_ops);
	419	+
	420	+create table if not exists people (
	421	+ id text primary key,
	422	+ company_id text not null references companies(id) on delete cascade,
	423	+ sensor_id text references sensors(id) on delete set null,
	424	+ name text not null,
	425	+ name_norm text not null,
	426	+ title text,
	427	+ role_category text, -- ceo \| cfo \| cto \| coo \| founder \| president \| chair \| board \| vp \| head \| other
	428	+ is_executive boolean not null default false,
	429	+ first_seen_at timestamptz not null default now(),
	430	+ last_seen_at timestamptz not null default now(),
	431	+ removed_at timestamptz,
	432	+ status text not null default 'listed', -- listed \| no_longer_listed
	433	+ source_url text,
	434	+ unique (company_id, name_norm)
	435	+);
	436	+create index if not exists people_company_idx on people (company_id, status);
	437	+
	438	+create table if not exists products (
	439	+ id text primary key,
	440	+ company_id text not null references companies(id) on delete cascade,
	441	+ sensor_id text references sensors(id) on delete set null,
	442	+ name text not null,
	443	+ name_norm text not null,
	444	+ category text,
	445	+ description text,
	446	+ url text,
	447	+ first_seen_at timestamptz not null default now(),
	448	+ last_seen_at timestamptz not null default now(),
	449	+ removed_at timestamptz,
	450	+ status text not null default 'listed',
	451	+ unique (company_id, name_norm)
	452	+);
	453	+create index if not exists products_company_idx on products (company_id, status);
	454	+
	455	+create table if not exists pricing_plans (
	456	+ id text primary key,
	457	+ company_id text not null references companies(id) on delete cascade,
	458	+ sensor_id text references sensors(id) on delete set null,
	459	+ plan_name text not null,
	460	+ plan_norm text not null,
	461	+ currency text,
	462	+ billing_period text, -- month \| year \| one_time \| usage \| contact
	463	+ price numeric,
	464	+ price_text text,
	465	+ unit text,
	466	+ features jsonb not null default '[]'::jsonb,
	467	+ contact_sales boolean not null default false,
	468	+ version_no int not null default 1,
	469	+ valid_from timestamptz not null default now(),
	470	+ valid_to timestamptz,
	471	+ first_seen_at timestamptz not null default now(),
	472	+ last_seen_at timestamptz not null default now(),
	473	+ status text not null default 'current', -- current \| superseded \| removed
	474	+ source_url text
	475	+);
	476	+create index if not exists pricing_plans_company_idx on pricing_plans (company_id, status);
	477	+create index if not exists pricing_plans_current_idx on pricing_plans (company_id, plan_norm) where status = 'current';
	478	+
	479	+create table if not exists locations (
	480	+ id text primary key,
	481	+ company_id text not null references companies(id) on delete cascade,
	482	+ sensor_id text references sensors(id) on delete set null,
	483	+ kind text not null default 'office', -- headquarters \| office \| store \| factory \| warehouse \| lab \| data_center \| other
	484	+ name text,
	485	+ name_norm text not null,
	486	+ city text,
	487	+ region text,
	488	+ country char(2),
	489	+ lat double precision,
	490	+ lon double precision,
	491	+ first_seen_at timestamptz not null default now(),
	492	+ last_seen_at timestamptz not null default now(),
	493	+ removed_at timestamptz,
	494	+ status text not null default 'listed',
	495	+ source_url text,
	496	+ unique (company_id, name_norm)
	497	+);
	498	+create index if not exists locations_company_idx on locations (company_id, status);
	499	+create index if not exists locations_country_idx on locations (country) where status = 'listed';
	500	+
	501	+create table if not exists news_items (
	502	+ id text primary key,
	503	+ company_id text not null references companies(id) on delete cascade,
	504	+ sensor_id text references sensors(id) on delete set null,
	505	+ url text not null,
	506	+ canonical_url text not null,
	507	+ title text not null,
	508	+ summary text,
	509	+ category text, -- press \| blog \| changelog \| research \| ir \| other
	510	+ published_at timestamptz,
	511	+ first_seen_at timestamptz not null default now(),
	512	+ language text,
	513	+ entities jsonb not null default '{}'::jsonb,
	514	+ unique (company_id, canonical_url)
	515	+);
	516	+create index if not exists news_items_company_idx on news_items (company_id, coalesce(published_at, first_seen_at) desc);
	517	+create index if not exists news_items_time_idx on news_items (first_seen_at desc);
	518	+
	519	+-- ============================================================================================================ metrics
	520	+create table if not exists metrics_current (
	521	+ company_id text not null references companies(id) on delete cascade,
	522	+ metric text not null,
	523	+ value double precision not null,
	524	+ confidence real not null default 0.5,
	525	+ inputs jsonb not null default '{}'::jsonb,
	526	+ formula_version text not null,
	527	+ computed_at timestamptz not null default now(),
	528	+ primary key (company_id, metric)
	529	+);
	530	+create index if not exists metrics_current_metric_idx on metrics_current (metric, value desc);
	531	+
	532	+create table if not exists metric_series (
	533	+ company_id text not null references companies(id) on delete cascade,
	534	+ metric text not null,
	535	+ day date not null,
	536	+ value double precision not null,
	537	+ confidence real not null default 0.5,
	538	+ formula_version text not null,
	539	+ primary key (company_id, metric, day)
	540	+);
	541	+create index if not exists metric_series_metric_day_idx on metric_series (metric, day desc);
	542	+
	543	+create table if not exists company_daily (
	544	+ company_id text not null references companies(id) on delete cascade,
	545	+ day date not null,
	546	+ observations int not null default 0,
	547	+ changes int not null default 0,
	548	+ meaningful_changes int not null default 0,
	549	+ events int not null default 0,
	550	+ events_by_type jsonb not null default '{}'::jsonb,
	551	+ jobs_open int,
	552	+ jobs_new int not null default 0,
	553	+ jobs_removed int not null default 0,
	554	+ jobs_ai_open int,
	555	+ news_items int not null default 0,
	556	+ sensors_active int,
	557	+ primary key (company_id, day)
	558	+);
	559	+create index if not exists company_daily_day_idx on company_daily (day desc);
	560	+
	561	+create table if not exists baselines (
	562	+ company_id text not null references companies(id) on delete cascade,
	563	+ metric text not null,
	564	+ mean double precision not null,
	565	+ stddev double precision not null,
	566	+ samples int not null,
	567	+ window_days int not null,
	568	+ computed_at timestamptz not null default now(),
	569	+ primary key (company_id, metric)
	570	+);
	571	+
	572	+create table if not exists global_daily (
	573	+ day date primary key,
	574	+ companies_active int not null default 0,
	575	+ sensors_active int not null default 0,
	576	+ observations int not null default 0,
	577	+ changes int not null default 0,
	578	+ meaningful_changes int not null default 0,
	579	+ events int not null default 0,
	580	+ events_by_type jsonb not null default '{}'::jsonb,
	581	+ jobs_open int,
	582	+ jobs_new int not null default 0,
	583	+ jobs_removed int not null default 0,
	584	+ activity_index double precision, -- coverage-normalised Global Corporate Activity Index (100 = baseline)
	585	+ by_country jsonb not null default '{}'::jsonb,
	586	+ by_industry jsonb not null default '{}'::jsonb,
	587	+ computed_at timestamptz not null default now()
	588	+);
	589	+
	590	+create table if not exists signals (
	591	+ id text primary key,
	592	+ company_id text references companies(id) on delete cascade,
	593	+ scope text not null default 'company', -- company \| industry \| country \| global
	594	+ scope_key text,
	595	+ kind text not null, -- hiring_surge \| hiring_freeze \| launch_buildup \| expansion \| pricing_migration \| developer_push \| enterprise_repositioning \| ai_acceleration \| abnormal_activity
	596	+ strength real not null,
	597	+ confidence real not null,
	598	+ title text not null,
	599	+ explanation text,
	600	+ evidence jsonb not null default '{}'::jsonb,
	601	+ window_days int not null default 30,
	602	+ detected_at timestamptz not null default now(),
	603	+ expires_at timestamptz,
	604	+ status text not null default 'active'
	605	+);
	606	+create index if not exists signals_company_idx on signals (company_id, detected_at desc);
	607	+create index if not exists signals_scope_idx on signals (scope, scope_key, detected_at desc);
	608	+
	609	+create table if not exists trends (
	610	+ term text not null,
	611	+ day date not null,
	612	+ mentions int not null default 0,
	613	+ companies int not null default 0,
	614	+ primary key (term, day)
	615	+);
	616	+
	617	+-- ============================================================================================================ operations
	618	+create table if not exists queue_jobs (
	619	+ id text primary key,
	620	+ kind text not null, -- discover \| run_sensor \| enrich_change \| recompute_metrics \| digest \| repair_sensor
	621	+ key text not null unique, -- idempotency (sensor_id + window …)
	622	+ payload jsonb not null default '{}'::jsonb,
	623	+ priority real not null default 0.5,
	624	+ run_at timestamptz not null default now(),
	625	+ locked_at timestamptz,
	626	+ locked_by text,
	627	+ attempts int not null default 0,
	628	+ max_attempts int not null default 3,
	629	+ status text not null default 'pending', -- pending \| running \| done \| failed \| dead
	630	+ last_error text,
	631	+ created_at timestamptz not null default now(),
	632	+ finished_at timestamptz
	633	+);
	634	+create index if not exists queue_jobs_due_idx on queue_jobs (kind, priority desc, run_at) where status = 'pending';
	635	+create index if not exists queue_jobs_status_idx on queue_jobs (status, kind);
	636	+
	637	+create table if not exists llm_jobs (
	638	+ id text primary key,
	639	+ kind text not null, -- classify_change \| summarize_event \| extract_entities \| classify_industry
	640	+ ref_id text not null,
	641	+ company_id text references companies(id) on delete cascade,
	642	+ model text,
	643	+ prompt_version text,
	644	+ status text not null default 'pending',
	645	+ attempts int not null default 0,
	646	+ request_tokens int,
	647	+ response_tokens int,
	648	+ latency_ms int,
	649	+ result jsonb,
	650	+ error text,
	651	+ created_at timestamptz not null default now(),
	652	+ started_at timestamptz,
	653	+ finished_at timestamptz
	654	+);
	655	+create index if not exists llm_jobs_status_idx on llm_jobs (status, created_at);
	656	+create index if not exists llm_jobs_ref_idx on llm_jobs (ref_id);
	657	+
	658	+create table if not exists failures (
	659	+ id text primary key,
	660	+ sensor_id text references sensors(id) on delete cascade,
	661	+ company_id text references companies(id) on delete cascade,
	662	+ at timestamptz not null default now(),
	663	+ failure_class text not null,
	664	+ status_code int,
	665	+ message text,
	666	+ url text
	667	+);
	668	+create index if not exists failures_time_idx on failures (at desc);
	669	+create index if not exists failures_sensor_idx on failures (sensor_id, at desc);
	670	+
	671	+create table if not exists crawl_runs (
	672	+ id text primary key,
	673	+ kind text not null, -- scheduler_tick \| onboarding \| metrics \| daily \| backup \| repair
	674	+ worker text,
	675	+ started_at timestamptz not null default now(),
	676	+ finished_at timestamptz,
	677	+ stats jsonb not null default '{}'::jsonb,
	678	+ error text
	679	+);
	680	+create index if not exists crawl_runs_kind_idx on crawl_runs (kind, started_at desc);
	681	+
	682	+create table if not exists review_queue (
	683	+ id text primary key,
	684	+ kind text not null, -- company_merge \| major_event \| low_confidence \| sensor_migration \| legal_sensitive \| unexpected_activity \| blocked_source
	685	+ ref_id text,
	686	+ company_id text references companies(id) on delete cascade,
	687	+ payload jsonb not null default '{}'::jsonb,
	688	+ status text not null default 'open', -- open \| accepted \| rejected \| resolved
	689	+ resolution text,
	690	+ created_at timestamptz not null default now(),
	691	+ resolved_at timestamptz
	692	+);
	693	+create index if not exists review_queue_open_idx on review_queue (kind, created_at) where status = 'open';
	694	+
	695	+create table if not exists cost_ledger (
	696	+ day date not null,
	697	+ dimension text not null, -- fetch \| browser \| llm \| storage_gb \| event
	698	+ key text not null default '',
	699	+ units double precision not null default 0,
	700	+ cost_estimate double precision not null default 0,
	701	+ primary key (day, dimension, key)
	702	+);
	703	+
	704	+-- ============================================================================================================ users-light (no accounts required)
	705	+create table if not exists owners (
	706	+ token_hash text primary key,
	707	+ created_at timestamptz not null default now(),
	708	+ last_seen_at timestamptz not null default now(),
	709	+ email text,
	710	+ plan text not null default 'free'
	711	+);
	712	+
	713	+create table if not exists watchlists (
	714	+ id text primary key,
	715	+ owner_hash text not null references owners(token_hash) on delete cascade,
	716	+ name text not null default 'My Watchlist',
	717	+ created_at timestamptz not null default now()
	718	+);
	719	+create table if not exists watchlist_items (
	720	+ watchlist_id text not null references watchlists(id) on delete cascade,
	721	+ company_id text not null references companies(id) on delete cascade,
	722	+ added_at timestamptz not null default now(),
	723	+ primary key (watchlist_id, company_id)
	724	+);
	725	+
	726	+create table if not exists alerts (
	727	+ id text primary key,
	728	+ owner_hash text not null references owners(token_hash) on delete cascade,
	729	+ company_id text references companies(id) on delete cascade,
	730	+ name text not null,
	731	+ condition jsonb not null, -- {event_types:[…], min_importance, metrics:{activity_score:{gt:80}}, industries, countries}
	732	+ channel text not null default 'web', -- web \| email \| webhook
	733	+ target text,
	734	+ enabled boolean not null default true,
	735	+ created_at timestamptz not null default now(),
	736	+ last_fired_at timestamptz
	737	+);
	738	+create table if not exists alert_deliveries (
	739	+ id text primary key,
	740	+ alert_id text not null references alerts(id) on delete cascade,
	741	+ event_id text references events(id) on delete cascade,
	742	+ delivered_at timestamptz not null default now(),
	743	+ channel text not null,
	744	+ status text not null default 'queued',
	745	+ detail text
	746	+);
	747	+create index if not exists alert_deliveries_alert_idx on alert_deliveries (alert_id, delivered_at desc);
	748	+
	749	+create table if not exists api_keys (
	750	+ id text primary key,
	751	+ key_hash text not null unique,
	752	+ prefix text not null,
	753	+ name text not null,
	754	+ tier text not null default 'authenticated', -- anonymous \| authenticated \| paid \| internal
	755	+ owner_hash text references owners(token_hash) on delete set null,
	756	+ created_at timestamptz not null default now(),
	757	+ last_used_at timestamptz,
	758	+ request_count bigint not null default 0,
	759	+ revoked_at timestamptz
	760	+);
	761	+
	762	+create table if not exists settings_kv (
	763	+ key text primary key,
	764	+ value jsonb not null,
	765	+ updated_at timestamptz not null default now()
	766	+);
	767	+insert into settings_kv (key, value) values ('dataset_started_at', to_jsonb(now())) on conflict (key) do nothing;
	768	+"""
	769	+
	770	+
	771	+def upgrade() -> None:
	772	+ for stmt in _split(SQL):
	773	+ op.execute(stmt)
	774	+
	775	+
	776	+def downgrade() -> None:
	777	+ raise RuntimeError("forward-only migrations: historical data is never dropped")
	778	+
	779	+
	780	+def _split(sql: str) -> list[str]:
	781	+ out: list[str] = []
	782	+ buf: list[str] = []
	783	+ for line in sql.splitlines():
	784	+ buf.append(line)
	785	+ if line.rstrip().endswith(";"):
	786	+ stmt = "\n".join(buf).strip()
	787	+ body = "\n".join(x for x in stmt.splitlines() if not x.strip().startswith("--")).strip()
	788	+ if body and body != ";":
	789	+ out.append(stmt)
	790	+ buf = []
	791	+ return out

added package.json +14 −0

		`@@ -0,0 +1,14 @@`
	1	+{
	2	+ "name": "company-atlas",
	3	+ "version": "0.1.0",
	4	+ "private": true,
	5	+ "description": "Company Atlas — the live atlas of global companies. Web workspace (backend is Python, see pyproject.toml).",
	6	+ "packageManager": "pnpm@11.1.2",
	7	+ "engines": { "node": ">=22" },
	8	+ "scripts": {
	9	+ "dev:web": "pnpm --filter @company-atlas/web run dev",
	10	+ "build": "pnpm --filter @company-atlas/web run build",
	11	+ "start:web": "pnpm --filter @company-atlas/web run start",
	12	+ "typecheck": "pnpm -r run typecheck"
	13	+ }
	14	+}

added pnpm-workspace.yaml +2 −0

		`@@ -0,0 +1,2 @@`
	1	+packages:
	2	+ - apps/*

added pyproject.toml +53 −0

		`@@ -0,0 +1,53 @@`
	1	+[project]
	2	+name = "companyatlas"
	3	+version = "0.1.0"
	4	+description = "Company Atlas — the live atlas of global companies: a distributed public-web sensor network that turns corporate change into a continuously updated historical record"
	5	+requires-python = ">=3.12"
	6	+dependencies = [
	7	+ "fastapi>=0.115",
	8	+ "uvicorn[standard]>=0.30",
	9	+ "pydantic>=2.8",
	10	+ "pydantic-settings>=2.4",
	11	+ "sqlalchemy[asyncio]>=2.0.35",
	12	+ "asyncpg>=0.30",
	13	+ "alembic>=1.13",
	14	+ "httpx[http2]>=0.27",
	15	+ "orjson>=3.10",
	16	+ "typer>=0.12",
	17	+ "rich>=13",
	18	+ "apscheduler>=3.10,<4",
	19	+ "python-ulid>=2.7",
	20	+ "pyyaml>=6",
	21	+ "python-dateutil>=2.9",
	22	+ "tenacity>=9",
	23	+ "python-slugify>=8",
	24	+ "selectolax>=0.3.21",
	25	+ "feedparser>=6.0.11",
	26	+ "rapidfuzz>=3.9",
	27	+ "zstandard>=0.23",
	28	+ "tldextract>=5.1",
	29	+ "sse-starlette>=2.1",
	30	+]
	31	+
	32	+[project.optional-dependencies]
	33	+dev = ["pytest>=8", "pytest-asyncio>=0.24", "ruff>=0.6", "respx>=0.21"]
	34	+browser = ["playwright>=1.47"]
	35	+
	36	+[project.scripts]
	37	+catlas = "companyatlas.cli:app"
	38	+
	39	+[build-system]
	40	+requires = ["hatchling"]
	41	+build-backend = "hatchling.build"
	42	+
	43	+[tool.hatch.build.targets.wheel]
	44	+packages = ["src/companyatlas"]
	45	+
	46	+[tool.ruff]
	47	+line-length = 130
	48	+target-version = "py312"
	49	+
	50	+[tool.pytest.ini_options]
	51	+testpaths = ["tests"]
	52	+asyncio_mode = "auto"
	53	+markers = ["live: hits real external endpoints (skipped unless -m live)"]

added src/companyatlas/__init__.py +2 −0

		`@@ -0,0 +1,2 @@`
	1	+"""Company Atlas — a distributed global sensor network for companies."""
	2	+__version__ = "0.1.0"

added src/companyatlas/api/common.py +118 −0

		`@@ -0,0 +1,118 @@`
	1	+"""Shared API helpers: JSON response class (orjson), pagination, admin/owner auth dependencies, tiny in-process TTL cache."""
	2	+from __future__ import annotations
	3	+
	4	+import hashlib
	5	+import hmac
	6	+import time
	7	+from collections import OrderedDict
	8	+from typing import Any
	9	+
	10	+import orjson
	11	+from fastapi import Header, HTTPException, Query, Request
	12	+from fastapi.responses import JSONResponse
	13	+
	14	+from companyatlas.config import settings
	15	+
	16	+
	17	+class AtlasJSONResponse(JSONResponse):
	18	+ media_type = "application/json"
	19	+
	20	+ def render(self, content: Any) -> bytes:
	21	+ return orjson.dumps(content, option=orjson.OPT_NON_STR_KEYS \| orjson.OPT_UTC_Z \| orjson.OPT_SERIALIZE_NUMPY, default=_default)
	22	+
	23	+
	24	+def _default(obj: Any) -> Any:
	25	+ if hasattr(obj, "isoformat"):
	26	+ return obj.isoformat()
	27	+ if hasattr(obj, "__float__"):
	28	+ return float(obj)
	29	+ if isinstance(obj, set \| frozenset):
	30	+ return sorted(obj)
	31	+ raise TypeError(f"not serialisable: {type(obj)!r}")
	32	+
	33	+
	34	+# ------------------------------------------------------------------------------------------------ pagination
	35	+
	36	+
	37	+class PageParams:
	38	+ def __init__(self, page: int = Query(1, ge=1), per_page: int = Query(25, ge=1, le=200)):
	39	+ self.page = page
	40	+ self.per_page = per_page
	41	+
	42	+ @property
	43	+ def offset(self) -> int:
	44	+ return (self.page - 1) * self.per_page
	45	+
	46	+
	47	+def page_payload(items: list[Any], total: int, p: PageParams) -> dict[str, Any]:
	48	+ return {"items": items, "page": p.page, "per_page": p.per_page, "total": total, "pages": max(1, -(-total // p.per_page))}
	49	+
	50	+
	51	+# ------------------------------------------------------------------------------------------------ auth
	52	+
	53	+
	54	+def require_admin(x_ca_admin_token: str \| None = Header(None, alias="X-CA-Admin-Token")) -> None:
	55	+ if not settings.admin_token or not x_ca_admin_token or not hmac.compare_digest(x_ca_admin_token, settings.admin_token):
	56	+ raise HTTPException(status_code=401, detail="admin token required")
	57	+
	58	+
	59	+def owner_hash(x_ca_owner_token: str \| None = Header(None, alias="X-CA-Owner-Token")) -> str:
	60	+ """Anonymous owner identity for watchlists/alerts: a client-generated random token (≥ 24 chars), stored hashed."""
	61	+ if not x_ca_owner_token or len(x_ca_owner_token) < 24 or len(x_ca_owner_token) > 200:
	62	+ raise HTTPException(status_code=401, detail="owner token required (X-CA-Owner-Token, ≥ 24 characters)")
	63	+ return hashlib.sha256(x_ca_owner_token.encode()).hexdigest()
	64	+
	65	+
	66	+def client_ip(request: Request) -> str:
	67	+ fwd = request.headers.get("x-forwarded-for")
	68	+ if fwd:
	69	+ return fwd.split(",")[0].strip()
	70	+ return request.client.host if request.client else "0.0.0.0"
	71	+
	72	+
	73	+# ------------------------------------------------------------------------------------------------ cache (per process)
	74	+
	75	+
	76	+class TTLCache:
	77	+ def __init__(self, max_items: int = 4096):
	78	+ self._data: OrderedDict[str, tuple[float, Any]] = OrderedDict()
	79	+ self._max = max_items
	80	+
	81	+ def get(self, key: str) -> Any \| None:
	82	+ item = self._data.get(key)
	83	+ if item is None:
	84	+ return None
	85	+ exp, value = item
	86	+ if exp < time.monotonic():
	87	+ self._data.pop(key, None)
	88	+ return None
	89	+ self._data.move_to_end(key)
	90	+ return value
	91	+
	92	+ def set(self, key: str, value: Any, ttl_s: float) -> None:
	93	+ self._data[key] = (time.monotonic() + ttl_s, value)
	94	+ self._data.move_to_end(key)
	95	+ while len(self._data) > self._max:
	96	+ self._data.popitem(last=False)
	97	+
	98	+ def clear(self, prefix: str \| None = None) -> None:
	99	+ if prefix is None:
	100	+ self._data.clear()
	101	+ else:
	102	+ for k in [k for k in self._data if k.startswith(prefix)]:
	103	+ self._data.pop(k, None)
	104	+
	105	+
	106	+cache = TTLCache()
	107	+
	108	+
	109	+async def cached(key: str, ttl_s: float, producer): # type: ignore[no-untyped-def]
	110	+ hit = cache.get(key)
	111	+ if hit is not None:
	112	+ return hit
	113	+ value = await producer()
	114	+ cache.set(key, value, ttl_s)
	115	+ return value
	116	+
	117	+
	118	+__all__ = ["AtlasJSONResponse", "PageParams", "TTLCache", "cache", "cached", "client_ip", "owner_hash", "page_payload", "require_admin"]

added src/companyatlas/api/main.py +131 −0

		`@@ -0,0 +1,131 @@`
	1	+"""FastAPI application — `/api/v1`. Loopback-only in production (Next.js proxies `/api/v1/*`); docs at `/api/v1/docs`.
	2	+
	3	+Routers live in `companyatlas.api.routers.<name>` and expose `router`; they are auto-included in alphabetical order, except that
	4	+modules listing `ORDER = n` are sorted by that first (literal paths must be registered before `/{slug}` catch-alls).
	5	+"""
	6	+from __future__ import annotations
	7	+
	8	+import asyncio
	9	+import importlib
	10	+import logging
	11	+import pkgutil
	12	+import time
	13	+from contextlib import asynccontextmanager
	14	+from datetime import UTC, datetime
	15	+from typing import Any
	16	+
	17	+from fastapi import FastAPI, Request
	18	+from fastapi.exceptions import RequestValidationError
	19	+from fastapi.middleware.cors import CORSMiddleware
	20	+from fastapi.middleware.gzip import GZipMiddleware
	21	+from starlette.exceptions import HTTPException as StarletteHTTPException
	22	+
	23	+import companyatlas
	24	+from companyatlas.api.common import AtlasJSONResponse
	25	+from companyatlas.config import settings
	26	+from companyatlas.db import connection, dispose, fetch_val
	27	+from companyatlas.logging import setup_logging
	28	+
	29	+log = logging.getLogger("companyatlas.api")
	30	+API_VERSION = "1.0"
	31	+_STARTED = time.time()
	32	+
	33	+
	34	+@asynccontextmanager
	35	+async def lifespan(app: FastAPI): # type: ignore[no-untyped-def]
	36	+ setup_logging(service="ca-api")
	37	+ settings.ensure_dirs()
	38	+ log.info("api started", extra={"version": companyatlas.__version__, "api": API_VERSION, "env": settings.app_env, "port": settings.api_port})
	39	+ yield
	40	+ await dispose()
	41	+
	42	+
	43	+app = FastAPI(title="Company Atlas API", version=companyatlas.__version__, lifespan=lifespan, default_response_class=AtlasJSONResponse,
	44	+ docs_url="/api/v1/docs", redoc_url=None, openapi_url="/api/v1/openapi.json",
	45	+ description="The live atlas of global companies: companies, sensors, observations, changes, structured events, metrics, rankings, "
	46	+ "industries and countries — with provenance and history on every fact. Contract: docs/API.md.")
	47	+
	48	+app.add_middleware(GZipMiddleware, minimum_size=1024)
	49	+_origins = {settings.site_url, "https://www.company-atlas.co", "https://company-atlas.co", "https://www.company-atlas.com", "https://company-atlas.com",
	50	+ "http://localhost:8370", "http://127.0.0.1:8370", "http://localhost:8360", "http://127.0.0.1:8360"}
	51	+app.add_middleware(CORSMiddleware, allow_origins=sorted(o for o in _origins if o), allow_methods=["GET", "POST", "PATCH", "DELETE", "OPTIONS"],
	52	+ allow_headers=["*"], expose_headers=["etag", "cache-control", "x-api-version", "x-ratelimit-remaining"], max_age=600)
	53	+
	54	+
	55	+@app.middleware("http")
	56	+async def security_headers(request: Request, call_next): # type: ignore[no-untyped-def]
	57	+ try:
	58	+ response = await call_next(request)
	59	+ except Exception:
	60	+ log.exception("unhandled error", extra={"route": request.url.path})
	61	+ return AtlasJSONResponse({"detail": "internal server error"}, status_code=500)
	62	+ response.headers["x-content-type-options"] = "nosniff"
	63	+ response.headers["referrer-policy"] = "strict-origin-when-cross-origin"
	64	+ response.headers["x-api-version"] = API_VERSION
	65	+ return response
	66	+
	67	+
	68	+@app.exception_handler(StarletteHTTPException)
	69	+async def http_error(request: Request, exc: StarletteHTTPException): # type: ignore[no-untyped-def]
	70	+ detail = exc.detail if isinstance(exc.detail, str) else str(exc.detail)
	71	+ return AtlasJSONResponse({"detail": detail}, status_code=exc.status_code, headers=dict(exc.headers or {}))
	72	+
	73	+
	74	+@app.exception_handler(RequestValidationError)
	75	+async def validation_error(request: Request, exc: RequestValidationError): # type: ignore[no-untyped-def]
	76	+ errs = exc.errors()[:5]
	77	+ msg = "; ".join(f"{'.'.join(str(x) for x in e.get('loc', []) if x not in ('query', 'body', 'path'))}: {e.get('msg')}" for e in errs) or "invalid request"
	78	+ return AtlasJSONResponse({"detail": msg, "errors": [{"loc": e.get("loc"), "msg": e.get("msg"), "type": e.get("type")} for e in errs]}, status_code=422)
	79	+
	80	+
	81	+@app.exception_handler(Exception)
	82	+async def unhandled(request: Request, exc: Exception): # type: ignore[no-untyped-def]
	83	+ log.exception("unhandled error", extra={"route": request.url.path})
	84	+ return AtlasJSONResponse({"detail": "internal server error"}, status_code=500)
	85	+
	86	+
	87	+async def _health() -> dict[str, Any]:
	88	+ db_ok = False
	89	+ try:
	90	+ async with connection() as conn:
	91	+ db_ok = (await asyncio.wait_for(fetch_val(conn, "select 1"), timeout=3)) == 1
	92	+ except Exception: # noqa: BLE001
	93	+ db_ok = False
	94	+ return {"status": "ok" if db_ok else "degraded", "version": companyatlas.__version__, "api_version": API_VERSION, "db": db_ok,
	95	+ "llm": {"configured": settings.llm_configured}, "uptime_s": int(time.time() - _STARTED), "time": datetime.now(UTC)}
	96	+
	97	+
	98	+@app.get("/health", tags=["health"])
	99	+async def health_root() -> dict[str, Any]:
	100	+ return await _health()
	101	+
	102	+
	103	+@app.get("/api/v1/health", tags=["health"])
	104	+async def health_v1() -> dict[str, Any]:
	105	+ return await _health()
	106	+
	107	+
	108	+@app.get("/ready", tags=["health"])
	109	+async def ready() -> dict[str, Any]:
	110	+ h = await _health()
	111	+ return {"ready": h["db"]}
	112	+
	113	+
	114	+def _include_routers() -> None:
	115	+ import companyatlas.api.routers as pkg
	116	+
	117	+ mods = []
	118	+ for m in pkgutil.iter_modules(pkg.__path__):
	119	+ if m.name.startswith("_"):
	120	+ continue
	121	+ module = importlib.import_module(f"companyatlas.api.routers.{m.name}")
	122	+ router = getattr(module, "router", None)
	123	+ if router is not None:
	124	+ mods.append((getattr(module, "ORDER", 50), m.name, router))
	125	+ for _order, _name, router in sorted(mods, key=lambda x: (x[0], x[1])):
	126	+ app.include_router(router)
	127	+
	128	+
	129	+_include_routers()
	130	+
	131	+__all__ = ["API_VERSION", "app"]

added src/companyatlas/api/routers/__init__.py +1 −0

		`@@ -0,0 +1 @@`
	1	+"""Package."""

added src/companyatlas/archive.py +77 −0

		`@@ -0,0 +1,77 @@`
	1	+"""Content-addressable object store for raw observations, normalized text and block structures (spec §26, §96, §97).
	2	+
	3	+ objects/<sha256[0:2]>/<sha256[2:4]>/<sha256>.zst
	4	+
	5	+Objects are written once (dedupe by hash) and never mutated; many observations may point at the same key. zstd level 9 gives
	6	+~8× on HTML. `key` = sha256 hex of the uncompressed bytes; the relative path is derived, so the store can move as a whole.
	7	+"""
	8	+from __future__ import annotations
	9	+
	10	+import hashlib
	11	+import os
	12	+from pathlib import Path
	13	+
	14	+import zstandard as zstd
	15	+
	16	+from companyatlas.config import settings
	17	+
	18	+_cctx = zstd.ZstdCompressor(level=9)
	19	+_dctx = zstd.ZstdDecompressor()
	20	+
	21	+
	22	+def object_path(key: str, root: Path \| None = None) -> Path:
	23	+ root = root or settings.objects_dir
	24	+ return root / key[:2] / key[2:4] / f"{key}.zst"
	25	+
	26	+
	27	+def sha256_hex(data: bytes) -> str:
	28	+ return hashlib.sha256(data).hexdigest()
	29	+
	30	+
	31	+def put_bytes(data: bytes, *, key: str \| None = None) -> tuple[str, int, bool]:
	32	+ """Store bytes; return (key, stored_size_bytes, created). Idempotent."""
	33	+ key = key or sha256_hex(data)
	34	+ path = object_path(key)
	35	+ if path.exists():
	36	+ return key, path.stat().st_size, False
	37	+ path.parent.mkdir(parents=True, exist_ok=True)
	38	+ tmp = path.with_suffix(f".zst.tmp{os.getpid()}")
	39	+ with open(tmp, "wb") as fh:
	40	+ fh.write(_cctx.compress(data))
	41	+ tmp.replace(path)
	42	+ return key, path.stat().st_size, True
	43	+
	44	+
	45	+def put_text(text: str) -> tuple[str, int, bool]:
	46	+ return put_bytes(text.encode("utf-8"))
	47	+
	48	+
	49	+def get_bytes(key: str) -> bytes:
	50	+ with open(object_path(key), "rb") as fh:
	51	+ return _dctx.decompress(fh.read(), max_output_size=64 * 1024 * 1024)
	52	+
	53	+
	54	+def get_text(key: str) -> str:
	55	+ return get_bytes(key).decode("utf-8", errors="replace")
	56	+
	57	+
	58	+def exists(key: str) -> bool:
	59	+ return object_path(key).exists()
	60	+
	61	+
	62	+def store_stats(root: Path \| None = None) -> dict[str, int]:
	63	+ root = root or settings.objects_dir
	64	+ total = count = 0
	65	+ if root.exists():
	66	+ for dirpath, _dirs, files in os.walk(root):
	67	+ for f in files:
	68	+ if f.endswith(".zst"):
	69	+ try:
	70	+ total += os.stat(os.path.join(dirpath, f)).st_size
	71	+ count += 1
	72	+ except OSError:
	73	+ pass
	74	+ return {"objects": count, "bytes": total}
	75	+
	76	+
	77	+__all__ = ["exists", "get_bytes", "get_text", "object_path", "put_bytes", "put_text", "sha256_hex", "store_stats"]

added src/companyatlas/cli.py +90 −0

		`@@ -0,0 +1,90 @@`
	1	+"""`catlas` — Company Atlas operations CLI.
	2	+
	3	+Command groups live in `companyatlas.commands.<module>`; each module exposes `register(app: typer.Typer) -> None` and is
	4	+auto-discovered here, so crawl, intelligence, API and seed commands can evolve independently.
	5	+"""
	6	+from __future__ import annotations
	7	+
	8	+import asyncio
	9	+import importlib
	10	+import logging
	11	+import pkgutil
	12	+from pathlib import Path
	13	+from typing import Annotated
	14	+
	15	+import typer
	16	+from rich.console import Console
	17	+
	18	+from companyatlas.config import settings
	19	+from companyatlas.logging import setup_logging
	20	+
	21	+app = typer.Typer(name="catlas", help="Company Atlas — the live atlas of global companies.", no_args_is_help=True, add_completion=False)
	22	+console = Console(stderr=True)
	23	+out = Console()
	24	+
	25	+
	26	+@app.callback()
	27	+def _main(verbose: Annotated[bool, typer.Option("--verbose", "-v")] = False) -> None:
	28	+ setup_logging(level=logging.DEBUG if verbose else logging.INFO, service="catlas")
	29	+ settings.ensure_dirs()
	30	+
	31	+
	32	+def run_async(coro): # type: ignore[no-untyped-def]
	33	+ """Run a coroutine and dispose the shared DB engine afterwards (for CLI commands)."""
	34	+ from companyatlas.db import dispose
	35	+
	36	+ async def wrapper(): # type: ignore[no-untyped-def]
	37	+ try:
	38	+ return await coro
	39	+ finally:
	40	+ await dispose()
	41	+
	42	+ return asyncio.run(wrapper())
	43	+
	44	+
	45	+@app.command()
	46	+def migrate(revision: str = "head") -> None:
	47	+ """Apply database migrations (forward-only)."""
	48	+ from alembic import command
	49	+ from alembic.config import Config
	50	+
	51	+ root = Path(__file__).resolve().parents[2]
	52	+ cfg = Config(str(root / "alembic.ini"))
	53	+ cfg.set_main_option("script_location", str(root / "migrations"))
	54	+ command.upgrade(cfg, revision)
	55	+ out.print("[green]migrations applied[/]")
	56	+
	57	+
	58	+@app.command()
	59	+def api(host: str \| None = None, port: int \| None = None, workers: int = 1, reload: bool = False) -> None:
	60	+ """Serve the FastAPI application (development)."""
	61	+ import uvicorn
	62	+
	63	+ uvicorn.run("companyatlas.api.main:app", host=host or settings.api_host, port=port or settings.api_port, workers=workers, reload=reload,
	64	+ proxy_headers=True, access_log=False)
	65	+
	66	+
	67	+@app.command()
	68	+def version() -> None:
	69	+ import companyatlas
	70	+
	71	+ out.print(companyatlas.__version__)
	72	+
	73	+
	74	+def _discover_commands() -> None:
	75	+ try:
	76	+ import companyatlas.commands as pkg
	77	+ except ImportError:
	78	+ return
	79	+ for mod in pkgutil.iter_modules(pkg.__path__):
	80	+ if mod.name.startswith("_"):
	81	+ continue
	82	+ module = importlib.import_module(f"companyatlas.commands.{mod.name}")
	83	+ register = getattr(module, "register", None)
	84	+ if callable(register):
	85	+ register(app)
	86	+
	87	+
	88	+_discover_commands()
	89	+
	90	+__all__ = ["app", "console", "out", "run_async"]

added src/companyatlas/commands/__init__.py +1 −0

		`@@ -0,0 +1 @@`
	1	+"""Package."""

added src/companyatlas/config.py +130 −0

		`@@ -0,0 +1,130 @@`
	1	+"""Runtime settings (environment variables, `CA_` prefix). Never log `settings.model_dump()` — it contains secrets.
	2	+
	3	+Every tunable that the product spec calls a "magic number" (crawl intervals, thresholds, weights, retry counts) lives here or in
	4	+`companyatlas.taxonomy` as typed configuration — never scattered as literals through the pipeline.
	5	+"""
	6	+from __future__ import annotations
	7	+
	8	+from functools import lru_cache
	9	+from pathlib import Path
	10	+
	11	+from pydantic import Field
	12	+from pydantic_settings import BaseSettings, SettingsConfigDict
	13	+
	14	+
	15	+class Settings(BaseSettings):
	16	+ model_config = SettingsConfigDict(env_file=(".env",), env_file_encoding="utf-8", extra="ignore")
	17	+
	18	+ app_env: str = Field("development", alias="APP_ENV")
	19	+ site_url: str = Field("https://www.company-atlas.co", alias="CA_SITE_URL")
	20	+ canonical_hosts: str = Field("www.company-atlas.co,www.company-atlas.com", alias="CA_CANONICAL_HOSTS")
	21	+
	22	+ database_url: str = Field("postgresql+asyncpg://companyatlas:companyatlas@127.0.0.1:5432/companyatlas", alias="DATABASE_URL")
	23	+ db_pool_size: int = Field(8, alias="CA_DB_POOL_SIZE")
	24	+ db_max_overflow: int = Field(8, alias="CA_DB_MAX_OVERFLOW")
	25	+
	26	+ data_dir: Path = Field(Path("./data"), alias="CA_DATA_DIR")
	27	+ api_host: str = Field("127.0.0.1", alias="CA_API_HOST")
	28	+ api_port: int = Field(8371, alias="CA_API_PORT")
	29	+ admin_token: str = Field("", alias="CA_ADMIN_TOKEN")
	30	+ log_json: bool = Field(True, alias="CA_LOG_JSON")
	31	+ tz: str = Field("America/Toronto", alias="CA_TZ")
	32	+
	33	+ # ---------------------------------------------------------------- crawler / politeness
	34	+ user_agent: str = Field("CompanyAtlasBot/0.1 (+https://www.company-atlas.co/bot; contact@spboucher.ai)", alias="CA_USER_AGENT")
	35	+ http_timeout_s: float = Field(30.0, alias="CA_HTTP_TIMEOUT")
	36	+ max_body_bytes: int = Field(8 * 1024 * 1024, alias="CA_MAX_BODY_BYTES")
	37	+ default_rate_per_min: int = Field(20, alias="CA_DEFAULT_RATE_PER_MIN")
	38	+ domain_max_concurrency: int = Field(2, alias="CA_DOMAIN_MAX_CONCURRENCY")
	39	+ domain_daily_budget: int = Field(600, alias="CA_DOMAIN_DAILY_BUDGET")
	40	+ respect_robots: bool = Field(True, alias="CA_RESPECT_ROBOTS")
	41	+ fetch_concurrency: int = Field(16, alias="CA_FETCH_CONCURRENCY")
	42	+ browser_enabled: bool = Field(False, alias="CA_BROWSER_ENABLED")
	43	+ browser_concurrency: int = Field(2, alias="CA_BROWSER_CONCURRENCY")
	44	+ max_redirects: int = Field(5, alias="CA_MAX_REDIRECTS")
	45	+ fetch_retries: int = Field(2, alias="CA_FETCH_RETRIES")
	46	+
	47	+ # ---------------------------------------------------------------- discovery
	48	+ discovery_max_pages: int = Field(12, alias="CA_DISCOVERY_MAX_PAGES")
	49	+ discovery_max_sitemap_urls: int = Field(5000, alias="CA_DISCOVERY_MAX_SITEMAP_URLS")
	50	+ discovery_min_confidence: float = Field(0.55, alias="CA_DISCOVERY_MIN_CONFIDENCE")
	51	+ discovery_max_sensors_per_company: int = Field(40, alias="CA_DISCOVERY_MAX_SENSORS")
	52	+ onboarding_concurrency: int = Field(12, alias="CA_ONBOARDING_CONCURRENCY")
	53	+
	54	+ # ---------------------------------------------------------------- scheduling (seconds)
	55	+ scheduler_tick_s: int = Field(15, alias="CA_SCHEDULER_TICK_S")
	56	+ scheduler_claim_batch: int = Field(200, alias="CA_SCHEDULER_CLAIM_BATCH")
	57	+ min_interval_s: int = Field(15 * 60, alias="CA_MIN_INTERVAL_S")
	58	+ max_interval_s: int = Field(7 * 86400, alias="CA_MAX_INTERVAL_S")
	59	+ burst_interval_s: int = Field(15 * 60, alias="CA_BURST_INTERVAL_S")
	60	+ burst_decay: float = Field(1.6, alias="CA_BURST_DECAY")
	61	+ stability_growth: float = Field(1.25, alias="CA_STABILITY_GROWTH")
	62	+ failure_growth: float = Field(2.0, alias="CA_FAILURE_GROWTH")
	63	+ stale_after_failures: int = Field(6, alias="CA_STALE_AFTER_FAILURES")
	64	+ retire_after_failures: int = Field(30, alias="CA_RETIRE_AFTER_FAILURES")
	65	+
	66	+ # ---------------------------------------------------------------- change detection
	67	+ noise_threshold: float = Field(0.20, alias="CA_NOISE_THRESHOLD")
	68	+ meaningful_threshold: float = Field(0.40, alias="CA_MEANINGFUL_THRESHOLD")
	69	+ major_threshold: float = Field(0.65, alias="CA_MAJOR_THRESHOLD")
	70	+ critical_threshold: float = Field(0.85, alias="CA_CRITICAL_THRESHOLD")
	71	+ keep_noise_snapshots: bool = Field(False, alias="CA_KEEP_NOISE_SNAPSHOTS")
	72	+
	73	+ # ---------------------------------------------------------------- LLM enrichment (optional)
	74	+ llm_base_url: str = Field("", alias="CA_LLM_BASE_URL")
	75	+ llm_api_key: str = Field("", alias="CA_LLM_API_KEY")
	76	+ llm_small_model: str = Field("qwen3-4b-instruct-2507-4bit", alias="CA_LLM_SMALL_MODEL")
	77	+ llm_medium_model: str = Field("qwen3.6-35b-a3b-4bit", alias="CA_LLM_MEDIUM_MODEL")
	78	+ llm_large_model: str = Field("qwen3.8-27b-4bit", alias="CA_LLM_LARGE_MODEL")
	79	+ llm_timeout_s: float = Field(300.0, alias="CA_LLM_TIMEOUT")
	80	+ llm_enabled: bool = Field(True, alias="CA_LLM_ENABLED")
	81	+ llm_daily_budget: int = Field(1500, alias="CA_LLM_DAILY_BUDGET")
	82	+ llm_min_significance: float = Field(0.40, alias="CA_LLM_MIN_SIGNIFICANCE")
	83	+ worker_concurrency: int = Field(1, alias="CA_WORKER_CONCURRENCY")
	84	+
	85	+ # ---------------------------------------------------------------- metrics / retention
	86	+ metrics_cron: str = Field("7 * * * *", alias="CA_METRICS_CRON")
	87	+ daily_cron: str = Field("20 0 * * *", alias="CA_DAILY_CRON")
	88	+ backup_cron: str = Field("35 4 * * *", alias="CA_BACKUP_CRON")
	89	+ baseline_window_days: int = Field(56, alias="CA_BASELINE_WINDOW_DAYS")
	90	+ anomaly_z: float = Field(2.5, alias="CA_ANOMALY_Z")
	91	+ seo_min_events: int = Field(1, alias="CA_SEO_MIN_EVENTS")
	92	+ seo_min_sensors: int = Field(3, alias="CA_SEO_MIN_SENSORS")
	93	+
	94	+ @property
	95	+ def objects_dir(self) -> Path:
	96	+ return self.data_dir / "objects"
	97	+
	98	+ @property
	99	+ def logs_dir(self) -> Path:
	100	+ return self.data_dir / "logs"
	101	+
	102	+ @property
	103	+ def backups_dir(self) -> Path:
	104	+ return self.data_dir / "backups"
	105	+
	106	+ @property
	107	+ def cache_dir(self) -> Path:
	108	+ return self.data_dir / "cache"
	109	+
	110	+ @property
	111	+ def sync_database_url(self) -> str:
	112	+ return self.database_url.replace("+asyncpg", "")
	113	+
	114	+ @property
	115	+ def llm_configured(self) -> bool:
	116	+ return bool(self.llm_enabled and self.llm_base_url)
	117	+
	118	+ def ensure_dirs(self) -> None:
	119	+ for d in (self.objects_dir, self.logs_dir, self.backups_dir, self.cache_dir):
	120	+ d.mkdir(parents=True, exist_ok=True)
	121	+
	122	+
	123	+@lru_cache
	124	+def get_settings() -> Settings:
	125	+ return Settings()
	126	+
	127	+
	128	+settings = get_settings()
	129	+
	130	+__all__ = ["Settings", "get_settings", "settings"]

added src/companyatlas/connectors/__init__.py +1 −0

		`@@ -0,0 +1 @@`
	1	+"""Package."""

added src/companyatlas/db/__init__.py +75 −0

		`@@ -0,0 +1,75 @@`
	1	+"""Database access: SQLAlchemy Core (async, asyncpg) with plain SQL. One engine per process."""
	2	+from __future__ import annotations
	3	+
	4	+import json
	5	+from collections.abc import AsyncIterator, Mapping, Sequence
	6	+from contextlib import asynccontextmanager
	7	+from typing import Any
	8	+
	9	+from sqlalchemy import text
	10	+from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, create_async_engine
	11	+
	12	+from companyatlas.config import settings
	13	+
	14	+_engine: AsyncEngine \| None = None
	15	+
	16	+
	17	+def engine() -> AsyncEngine:
	18	+ global _engine
	19	+ if _engine is None:
	20	+ _engine = create_async_engine(settings.database_url, pool_size=8, max_overflow=8, pool_pre_ping=True, pool_recycle=1800,
	21	+ connect_args={"server_settings": {"application_name": "companyatlas", "jit": "off"}})
	22	+ return _engine
	23	+
	24	+
	25	+async def dispose() -> None:
	26	+ global _engine
	27	+ if _engine is not None:
	28	+ await _engine.dispose()
	29	+ _engine = None
	30	+
	31	+
	32	+@asynccontextmanager
	33	+async def connection() -> AsyncIterator[AsyncConnection]:
	34	+ async with engine().connect() as conn:
	35	+ yield conn
	36	+
	37	+
	38	+@asynccontextmanager
	39	+async def transaction() -> AsyncIterator[AsyncConnection]:
	40	+ async with engine().begin() as conn:
	41	+ yield conn
	42	+
	43	+
	44	+def jsonb(value: Any) -> str:
	45	+ """Serialise a Python value for a `cast(:x as jsonb)` parameter."""
	46	+ return json.dumps(value, default=str, ensure_ascii=False)
	47	+
	48	+
	49	+async def execute(conn: AsyncConnection, sql: str, /, **params: Any) -> None:
	50	+ await conn.execute(text(sql), params)
	51	+
	52	+
	53	+async def fetch_all(conn: AsyncConnection, sql: str, /, **params: Any) -> list[dict[str, Any]]:
	54	+ result = await conn.execute(text(sql), params)
	55	+ return [dict(r._mapping) for r in result]
	56	+
	57	+
	58	+async def fetch_one(conn: AsyncConnection, sql: str, /, **params: Any) -> dict[str, Any] \| None:
	59	+ result = await conn.execute(text(sql), params)
	60	+ row = result.first()
	61	+ return dict(row._mapping) if row is not None else None
	62	+
	63	+
	64	+async def fetch_val(conn: AsyncConnection, sql: str, /, **params: Any) -> Any:
	65	+ result = await conn.execute(text(sql), params)
	66	+ row = result.first()
	67	+ return row[0] if row is not None else None
	68	+
	69	+
	70	+async def execute_many(conn: AsyncConnection, sql: str, rows: Sequence[Mapping[str, Any]]) -> None:
	71	+ if rows:
	72	+ await conn.execute(text(sql), list(rows))
	73	+
	74	+
	75	+__all__ = ["connection", "dispose", "engine", "execute", "execute_many", "fetch_all", "fetch_one", "fetch_val", "jsonb", "transaction"]

added src/companyatlas/fetch.py +472 −0

		`@@ -0,0 +1,472 @@`
	1	+"""Fetch transport (spec §13–14, §109–118): httpx direct mode with conditional requests, per-domain rate limiting and concurrency
	2	+caps, robots.txt awareness, bounded redirects (each hop SSRF-checked), size limits, failure classification and an optional
	3	+headless-browser mode that is only used when a sensor's connector asks for it.
	4	+
	5	+Crawler inputs are untrusted: every destination and every redirect hop is validated before a socket is opened.
	6	+"""
	7	+from __future__ import annotations
	8	+
	9	+import asyncio
	10	+import hashlib
	11	+import ipaddress
	12	+import logging
	13	+import socket
	14	+import time
	15	+from dataclasses import dataclass, field
	16	+from datetime import UTC, datetime
	17	+from urllib.parse import urljoin, urlparse
	18	+from urllib.robotparser import RobotFileParser
	19	+
	20	+import httpx
	21	+
	22	+from companyatlas.config import settings
	23	+from companyatlas.taxonomy import FailureClass
	24	+from companyatlas.urls import registrable_domain
	25	+
	26	+log = logging.getLogger(__name__)
	27	+
	28	+TRANSIENT_STATUS = {408, 425, 429, 500, 502, 503, 504}
	29	+BLOCK_STATUS = {401, 403, 451, 999}
	30	+REDIRECT_STATUS = {301, 302, 303, 307, 308}
	31	+
	32	+# ---------------------------------------------------------------------------------------------- SSRF guard (spec §115–116)
	33	+_BLOCKED_SUFFIXES = (".local", ".internal", ".localhost", ".localdomain", ".lan", ".home", ".corp", ".intranet", ".maclustr.io", ".ts.net")
	34	+_BLOCKED_HOSTS = {"localhost", "metadata.google.internal", "metadata", "instance-data", "kubernetes.default", "kubernetes.default.svc"}
	35	+_BLOCKED_NETWORKS = [ipaddress.ip_network(n) for n in (
	36	+ "0.0.0.0/8", "10.0.0.0/8", "100.64.0.0/10", "127.0.0.0/8", "169.254.0.0/16", "172.16.0.0/12", "192.0.0.0/24", "192.168.0.0/16",
	37	+ "198.18.0.0/15", "240.0.0.0/4", "::/128", "::1/128", "fc00::/7", "fe80::/10", "::ffff:0:0/96", "64:ff9b::/96",
	38	+)]
	39	+
	40	+
	41	+class BlockedDestination(ValueError):
	42	+ """The URL points at a private, local or otherwise non-public destination."""
	43	+
	44	+
	45	+def _ip_blocked(ip: ipaddress.IPv4Address \| ipaddress.IPv6Address) -> bool:
	46	+ if isinstance(ip, ipaddress.IPv6Address) and ip.ipv4_mapped is not None:
	47	+ ip = ip.ipv4_mapped
	48	+ if isinstance(ip, ipaddress.IPv6Address) and ip in ipaddress.ip_network("64:ff9b::/96"):
	49	+ ip = ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF) # NAT64: judge the embedded IPv4
	50	+ if ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_unspecified or ip.is_reserved or ip.is_multicast:
	51	+ return True
	52	+ return any(ip in net for net in _BLOCKED_NETWORKS)
	53	+
	54	+
	55	+def _parse_ip(host: str) -> ipaddress.IPv4Address \| ipaddress.IPv6Address \| None:
	56	+ try:
	57	+ return ipaddress.ip_address(host.strip("[]").split("%")[0])
	58	+ except ValueError:
	59	+ return None
	60	+
	61	+
	62	+def _resolve(host: str, port: int) -> list[str]:
	63	+ try:
	64	+ infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
	65	+ except socket.gaierror:
	66	+ return []
	67	+ return sorted({info[4][0] for info in infos})
	68	+
	69	+
	70	+def validate_destination(url: str, *, resolved_ips: list[str] \| None = None) -> None:
	71	+ p = urlparse(url.strip())
	72	+ if p.scheme.lower() not in ("http", "https"):
	73	+ raise BlockedDestination(f"scheme {p.scheme!r}")
	74	+ host = (p.hostname or "").strip().lower().rstrip(".")
	75	+ if not host:
	76	+ raise BlockedDestination("no host")
	77	+ if p.username or p.password:
	78	+ raise BlockedDestination("credentials in URL")
	79	+ literal = _parse_ip(host)
	80	+ if literal is None and (host in _BLOCKED_HOSTS or host.endswith(_BLOCKED_SUFFIXES) or "." not in host):
	81	+ raise BlockedDestination(f"local host {host!r}")
	82	+ if literal is not None:
	83	+ if _ip_blocked(literal):
	84	+ raise BlockedDestination(f"non-public address {host}")
	85	+ return
	86	+ ips = resolved_ips if resolved_ips is not None else _resolve(host, p.port or (443 if p.scheme.lower() == "https" else 80))
	87	+ for raw in ips:
	88	+ ip = _parse_ip(raw)
	89	+ if ip is not None and _ip_blocked(ip):
	90	+ raise BlockedDestination(f"{host} resolves to non-public address {raw}")
	91	+
	92	+
	93	+async def validate_destination_async(url: str) -> None:
	94	+ p = urlparse(url.strip())
	95	+ host = (p.hostname or "").strip().lower()
	96	+ validate_destination(url, resolved_ips=[])
	97	+ if host and _parse_ip(host) is None:
	98	+ ips = await asyncio.to_thread(_resolve, host, p.port or (443 if p.scheme.lower() == "https" else 80))
	99	+ validate_destination(url, resolved_ips=ips)
	100	+
	101	+
	102	+# ---------------------------------------------------------------------------------------------- results / errors
	103	+
	104	+
	105	+class FetchError(Exception):
	106	+ def __init__(self, message: str, *, status: int \| None = None, url: str = "", failure: FailureClass = FailureClass.UNKNOWN):
	107	+ super().__init__(message)
	108	+ self.status = status
	109	+ self.url = url
	110	+ self.failure = failure
	111	+
	112	+
	113	+class BlockedError(FetchError):
	114	+ """Access denied by the origin or by robots — never bypassed."""
	115	+
	116	+
	117	+class NotModified(Exception):
	118	+ """HTTP 304 — content unchanged since our stored validators."""
	119	+
	120	+ def __init__(self, duration_ms: int = 0):
	121	+ super().__init__("not modified")
	122	+ self.duration_ms = duration_ms
	123	+
	124	+
	125	+@dataclass
	126	+class FetchResult:
	127	+ url: str
	128	+ final_url: str
	129	+ status: int
	130	+ headers: dict[str, str]
	131	+ content: bytes
	132	+ content_type: str
	133	+ fetched_at: datetime
	134	+ duration_ms: int
	135	+ transport: str = "http"
	136	+ redirects: int = 0
	137	+ sha256: str = field(init=False)
	138	+
	139	+ def __post_init__(self) -> None:
	140	+ self.sha256 = hashlib.sha256(self.content).hexdigest()
	141	+
	142	+ @property
	143	+ def etag(self) -> str \| None:
	144	+ return self.headers.get("etag")
	145	+
	146	+ @property
	147	+ def last_modified(self) -> str \| None:
	148	+ return self.headers.get("last-modified")
	149	+
	150	+ @property
	151	+ def text(self) -> str:
	152	+ enc = "utf-8"
	153	+ ct = self.content_type.lower()
	154	+ if "charset=" in ct:
	155	+ enc = ct.split("charset=", 1)[1].split(";")[0].strip().strip('"') or "utf-8"
	156	+ try:
	157	+ return self.content.decode(enc, errors="replace")
	158	+ except LookupError:
	159	+ return self.content.decode("utf-8", errors="replace")
	160	+
	161	+ @property
	162	+ def is_html(self) -> bool:
	163	+ head = self.content[:512].lstrip().lower()
	164	+ return "html" in self.content_type or head.startswith((b"<!doctype html", b"<html")) or b"<html" in head
	165	+
	166	+ @property
	167	+ def is_json(self) -> bool:
	168	+ return "json" in self.content_type or self.content[:1] in (b"{", b"[")
	169	+
	170	+ @property
	171	+ def is_xml(self) -> bool:
	172	+ return "xml" in self.content_type or self.content[:5] == b"<?xml" or self.content[:200].lstrip().startswith((b"<rss", b"<feed", b"<urlset", b"<sitemapindex"))
	173	+
	174	+ def json(self): # type: ignore[no-untyped-def]
	175	+ import orjson
	176	+
	177	+ return orjson.loads(self.content)
	178	+
	179	+
	180	+def classify_exception(exc: BaseException) -> FailureClass:
	181	+ if isinstance(exc, FetchError):
	182	+ return exc.failure
	183	+ if isinstance(exc, httpx.ConnectTimeout \| httpx.ReadTimeout \| httpx.WriteTimeout \| httpx.PoolTimeout \| asyncio.TimeoutError \| TimeoutError):
	184	+ return FailureClass.TIMEOUT
	185	+ if isinstance(exc, httpx.ConnectError):
	186	+ msg = str(exc).lower()
	187	+ if "nodename" in msg or "name or service" in msg or "getaddrinfo" in msg or "temporary failure in name" in msg or "no address" in msg:
	188	+ return FailureClass.DNS
	189	+ return FailureClass.TIMEOUT
	190	+ return FailureClass.UNKNOWN
	191	+
	192	+
	193	+def _status_failure(status: int) -> FailureClass:
	194	+ if status == 429:
	195	+ return FailureClass.RATE_LIMIT
	196	+ if status in (404, 410):
	197	+ return FailureClass.PAGE_REMOVED
	198	+ if status in BLOCK_STATUS:
	199	+ return FailureClass.BOT_CHALLENGE if status in (403, 999) else FailureClass.HTTP_4XX
	200	+ if 400 <= status < 500:
	201	+ return FailureClass.HTTP_4XX
	202	+ return FailureClass.HTTP_5XX
	203	+
	204	+
	205	+# ---------------------------------------------------------------------------------------------- politeness primitives
	206	+
	207	+
	208	+class DomainGovernor:
	209	+ """Per-domain minimum spacing + concurrency cap (in-process). Cluster-wide fairness comes from Postgres domain budgets."""
	210	+
	211	+ def __init__(self) -> None:
	212	+ self._next: dict[str, float] = {}
	213	+ self._sem: dict[str, asyncio.Semaphore] = {}
	214	+ self._delay: dict[str, float] = {}
	215	+ self._lock = asyncio.Lock()
	216	+
	217	+ def set_crawl_delay(self, domain: str, seconds: float) -> None:
	218	+ self._delay[domain] = min(120.0, max(0.0, seconds))
	219	+
	220	+ async def acquire(self, domain: str, per_min: int) -> asyncio.Semaphore:
	221	+ sem = self._sem.get(domain)
	222	+ if sem is None:
	223	+ sem = self._sem[domain] = asyncio.Semaphore(settings.domain_max_concurrency)
	224	+ await sem.acquire()
	225	+ async with self._lock:
	226	+ spacing = max(60.0 / max(1, per_min), self._delay.get(domain, 0.0))
	227	+ now = time.monotonic()
	228	+ ready = self._next.get(domain, 0.0)
	229	+ wait = max(0.0, ready - now)
	230	+ self._next[domain] = max(now, ready) + spacing
	231	+ if wait > 0:
	232	+ await asyncio.sleep(wait)
	233	+ return sem
	234	+
	235	+
	236	+class RobotsCache:
	237	+ def __init__(self) -> None:
	238	+ self._cache: dict[str, tuple[float, RobotFileParser \| None, float \| None]] = {}
	239	+
	240	+ async def policy(self, client: httpx.AsyncClient, url: str) -> tuple[bool, float \| None]:
	241	+ """(allowed, crawl_delay_seconds)."""
	242	+ if not settings.respect_robots:
	243	+ return True, None
	244	+ p = urlparse(url)
	245	+ key = f"{p.scheme}://{p.netloc}"
	246	+ cached = self._cache.get(key)
	247	+ if cached is None or cached[0] < time.monotonic():
	248	+ rp: RobotFileParser \| None = RobotFileParser()
	249	+ delay: float \| None = None
	250	+ try:
	251	+ r = await client.get(f"{key}/robots.txt", timeout=15, follow_redirects=False)
	252	+ if r.status_code == 200 and len(r.content) < 512 * 1024 and b"<html" not in r.content[:512].lower():
	253	+ rp.parse(r.text.splitlines()) # type: ignore[union-attr]
	254	+ agent = settings.user_agent.split("/")[0]
	255	+ try:
	256	+ d = rp.crawl_delay(agent) or rp.crawl_delay("*") # type: ignore[union-attr]
	257	+ delay = float(d) if d else None
	258	+ except Exception: # noqa: BLE001
	259	+ delay = None
	260	+ else:
	261	+ rp = None
	262	+ except Exception: # noqa: BLE001
	263	+ rp = None
	264	+ self._cache[key] = (time.monotonic() + 12 * 3600, rp, delay)
	265	+ cached = self._cache[key]
	266	+ rp, delay = cached[1], cached[2]
	267	+ if rp is None:
	268	+ return True, None
	269	+ agent = settings.user_agent.split("/")[0]
	270	+ try:
	271	+ return (rp.can_fetch(agent, url) or rp.can_fetch("*", url)), delay
	272	+ except Exception: # noqa: BLE001
	273	+ return True, delay
	274	+
	275	+
	276	+governor = DomainGovernor()
	277	+robots = RobotsCache()
	278	+
	279	+
	280	+# ---------------------------------------------------------------------------------------------- fetcher
	281	+
	282	+
	283	+class Fetcher:
	284	+ """Shared httpx client (one per worker process). `get()` raises NotModified / FetchError / BlockedError."""
	285	+
	286	+ def __init__(self, *, timeout_s: float \| None = None, headers: dict[str, str] \| None = None, http2: bool = True,
	287	+ max_connections: int \| None = None):
	288	+ self.timeout_s = timeout_s or settings.http_timeout_s
	289	+ self.headers = {"User-Agent": settings.user_agent,
	290	+ "Accept": "text/html,application/xhtml+xml,application/xml,application/json,application/rss+xml,text/;q=0.9,/*;q=0.7",
	291	+ "Accept-Language": "en-US,en;q=0.9,fr;q=0.6,de;q=0.4,*;q=0.2", "Accept-Encoding": "gzip, deflate, br",
	292	+ **(headers or {})}
	293	+ self._client: httpx.AsyncClient \| None = None
	294	+ self._http2 = http2
	295	+ self._max_connections = max_connections or max(16, settings.fetch_concurrency * 2)
	296	+
	297	+ async def __aenter__(self) -> Fetcher:
	298	+ await self.open()
	299	+ return self
	300	+
	301	+ async def __aexit__(self, *exc: object) -> None:
	302	+ await self.close()
	303	+
	304	+ async def open(self) -> None:
	305	+ if self._client is None:
	306	+ self._client = httpx.AsyncClient(headers=self.headers, timeout=httpx.Timeout(self.timeout_s, connect=15), follow_redirects=False,
	307	+ http2=self._http2, limits=httpx.Limits(max_connections=self._max_connections,
	308	+ max_keepalive_connections=self._max_connections // 2))
	309	+
	310	+ async def close(self) -> None:
	311	+ if self._client is not None:
	312	+ await self._client.aclose()
	313	+ self._client = None
	314	+
	315	+ @property
	316	+ def client(self) -> httpx.AsyncClient:
	317	+ if self._client is None:
	318	+ raise RuntimeError("Fetcher not opened — use `async with Fetcher() as f` or `await f.open()`")
	319	+ return self._client
	320	+
	321	+ async def get(self, url: str, *, etag: str \| None = None, last_modified: str \| None = None, min_bytes: int = 1,
	322	+ retries: int \| None = None, accept: str \| None = None, rate_per_min: int \| None = None, respect_robots: bool = True,
	323	+ max_bytes: int \| None = None) -> FetchResult:
	324	+ client = self.client
	325	+ retries = settings.fetch_retries if retries is None else retries
	326	+ try:
	327	+ await validate_destination_async(url)
	328	+ except BlockedDestination as exc:
	329	+ raise FetchError(f"blocked destination: {exc}", url=url, failure=FailureClass.BLOCKED_DESTINATION) from exc
	330	+ domain = registrable_domain(url)
	331	+ if respect_robots:
	332	+ allowed, delay = await robots.policy(client, url)
	333	+ if delay:
	334	+ governor.set_crawl_delay(domain, delay)
	335	+ if not allowed:
	336	+ raise BlockedError(f"robots.txt disallows {url}", url=url, failure=FailureClass.ROBOTS)
	337	+ headers: dict[str, str] = {}
	338	+ if etag:
	339	+ headers["If-None-Match"] = etag
	340	+ if last_modified:
	341	+ headers["If-Modified-Since"] = last_modified
	342	+ if accept:
	343	+ headers["Accept"] = accept
	344	+ current = url
	345	+ hops = 0
	346	+ attempt = 0
	347	+ while True:
	348	+ sem = await governor.acquire(registrable_domain(current), rate_per_min or settings.default_rate_per_min)
	349	+ t0 = time.perf_counter()
	350	+ try:
	351	+ async with client.stream("GET", current, headers=headers) as r:
	352	+ if r.status_code == 304:
	353	+ raise NotModified(int((time.perf_counter() - t0) * 1000))
	354	+ if r.status_code in REDIRECT_STATUS:
	355	+ location = r.headers.get("location")
	356	+ if not location:
	357	+ raise FetchError(f"http {r.status_code} without Location", status=r.status_code, url=url, failure=FailureClass.REDIRECT)
	358	+ hops += 1
	359	+ if hops > settings.max_redirects:
	360	+ raise FetchError(f"too many redirects (> {settings.max_redirects})", status=r.status_code, url=url, failure=FailureClass.REDIRECT)
	361	+ nxt = urljoin(current, location)
	362	+ try:
	363	+ await validate_destination_async(nxt)
	364	+ except BlockedDestination as exc:
	365	+ raise FetchError(f"blocked redirect {current} → {nxt}: {exc}", status=r.status_code, url=url,
	366	+ failure=FailureClass.BLOCKED_DESTINATION) from exc
	367	+ if registrable_domain(nxt) != registrable_domain(current):
	368	+ headers.pop("If-None-Match", None)
	369	+ headers.pop("If-Modified-Since", None)
	370	+ current = nxt
	371	+ continue
	372	+ if r.status_code in TRANSIENT_STATUS and attempt < retries:
	373	+ retry_after = r.headers.get("retry-after")
	374	+ delay = min(60.0, float(retry_after)) if retry_after and retry_after.isdigit() else 2.0 * (2 ** attempt)
	375	+ attempt += 1
	376	+ await asyncio.sleep(delay)
	377	+ continue
	378	+ if r.status_code in BLOCK_STATUS:
	379	+ raise BlockedError(f"http {r.status_code} for {current}", status=r.status_code, url=url, failure=_status_failure(r.status_code))
	380	+ if r.status_code >= 400:
	381	+ raise FetchError(f"http {r.status_code} for {current}", status=r.status_code, url=url, failure=_status_failure(r.status_code))
	382	+ return await self._read(r, url=url, final_url=current, t0=t0, min_bytes=min_bytes, hops=hops, max_bytes=max_bytes)
	383	+ except (NotModified, FetchError):
	384	+ raise
	385	+ except (httpx.TimeoutException, httpx.TransportError) as exc:
	386	+ if attempt < retries:
	387	+ attempt += 1
	388	+ await asyncio.sleep(1.5 * (2 ** attempt))
	389	+ continue
	390	+ raise FetchError(f"{exc.__class__.__name__}: {exc}", url=url, failure=classify_exception(exc)) from exc
	391	+ finally:
	392	+ sem.release()
	393	+
	394	+ async def _read(self, r: httpx.Response, *, url: str, final_url: str, t0: float, min_bytes: int, hops: int, max_bytes: int \| None) -> FetchResult:
	395	+ limit = max_bytes or settings.max_body_bytes
	396	+ declared = r.headers.get("content-length")
	397	+ if declared and declared.isdigit() and int(declared) > limit:
	398	+ raise FetchError(f"declared size {declared} exceeds {limit}", status=r.status_code, url=url, failure=FailureClass.TOO_LARGE)
	399	+ chunks: list[bytes] = []
	400	+ size = 0
	401	+ async for chunk in r.aiter_bytes():
	402	+ size += len(chunk)
	403	+ if size > limit:
	404	+ raise FetchError(f"body exceeds {limit} bytes", status=r.status_code, url=url, failure=FailureClass.TOO_LARGE)
	405	+ chunks.append(chunk)
	406	+ content = b"".join(chunks)
	407	+ if len(content) < min_bytes:
	408	+ raise FetchError(f"short body ({len(content)} bytes)", status=r.status_code, url=url, failure=FailureClass.PARSING)
	409	+ res = FetchResult(url=url, final_url=final_url, status=r.status_code, headers={k.lower(): v for k, v in r.headers.items()},
	410	+ content=content, content_type=r.headers.get("content-type", "").lower(), fetched_at=datetime.now(UTC),
	411	+ duration_ms=int((time.perf_counter() - t0) * 1000), transport="http", redirects=hops)
	412	+ if res.is_html and looks_like_challenge(content):
	413	+ raise BlockedError(f"anti-bot challenge page at {final_url}", status=r.status_code, url=url, failure=FailureClass.BOT_CHALLENGE)
	414	+ return res
	415	+
	416	+ # ------------------------------------------------------------------------------------------ optional browser mode (spec §13 B)
	417	+ async def get_rendered(self, url: str, *, wait_ms: int = 1500) -> FetchResult:
	418	+ """Headless Chromium render — pooled by `settings.browser_concurrency`; only when a connector declares fetch_mode=browser."""
	419	+ if not settings.browser_enabled:
	420	+ raise FetchError("browser mode disabled", url=url, failure=FailureClass.UNKNOWN)
	421	+ await validate_destination_async(url)
	422	+ async with _browser_slots():
	423	+ from playwright.async_api import async_playwright # optional dependency
	424	+
	425	+ t0 = time.perf_counter()
	426	+ async with async_playwright() as p:
	427	+ browser = await p.chromium.launch(headless=True)
	428	+ try:
	429	+ ctx = await browser.new_context(user_agent=settings.user_agent, java_script_enabled=True)
	430	+ await ctx.route("*/", lambda route: route.abort() if route.request.resource_type in ("image", "media", "font") else route.continue_())
	431	+ page = await ctx.new_page()
	432	+ resp = await page.goto(url, wait_until="domcontentloaded", timeout=int(self.timeout_s * 1000))
	433	+ await page.wait_for_timeout(wait_ms)
	434	+ html = await page.content()
	435	+ status = resp.status if resp else 200
	436	+ final = page.url
	437	+ finally:
	438	+ await browser.close()
	439	+ return FetchResult(url=url, final_url=final, status=status, headers={}, content=html.encode(), content_type="text/html; charset=utf-8",
	440	+ fetched_at=datetime.now(UTC), duration_ms=int((time.perf_counter() - t0) * 1000), transport="browser")
	441	+
	442	+
	443	+_browser_sem: asyncio.Semaphore \| None = None
	444	+
	445	+
	446	+def _browser_slots() -> asyncio.Semaphore:
	447	+ global _browser_sem
	448	+ if _browser_sem is None:
	449	+ _browser_sem = asyncio.Semaphore(settings.browser_concurrency)
	450	+ return _browser_sem
	451	+
	452	+
	453	+def looks_like_challenge(content: bytes) -> bool:
	454	+ if len(content) > 80_000:
	455	+ return False
	456	+ head = content[:20000].lower()
	457	+ markers = (b"just a moment", b"cf-chl-", b"challenge-platform", b"attention required", b"verify you are human", b"access denied",
	458	+ b"captcha", b"perimeterx", b"_pxappid", b"datadome", b"enable javascript and cookies to continue", b"request unsuccessful. incapsula",
	459	+ b"bot detection", b"are you a robot")
	460	+ return sum(m in head for m in markers) >= 2
	461	+
	462	+
	463	+def file_result(path: str, *, url: str, content_type: str = "text/html") -> FetchResult:
	464	+ """Wrap a fixture file as a FetchResult (tests, `catlas run-sensor --file`)."""
	465	+ with open(path, "rb") as fh:
	466	+ content = fh.read()
	467	+ return FetchResult(url=url, final_url=url, status=200, headers={}, content=content, content_type=content_type,
	468	+ fetched_at=datetime.now(UTC), duration_ms=0, transport="file")
	469	+
	470	+
	471	+__all__ = ["BlockedDestination", "BlockedError", "FetchError", "FetchResult", "Fetcher", "NotModified", "classify_exception", "file_result",
	472	+ "governor", "looks_like_challenge", "robots", "validate_destination", "validate_destination_async"]

added src/companyatlas/ids.py +81 −0

		`@@ -0,0 +1,81 @@`
	1	+"""Stable internal identifiers: prefixed ULIDs (`co_01J…`). Slugs, names and URLs change; ids never do (spec §173)."""
	2	+from __future__ import annotations
	3	+
	4	+import hashlib
	5	+import re
	6	+
	7	+from slugify import slugify as _slugify
	8	+from ulid import ULID
	9	+
	10	+PREFIXES: dict[str, str] = {
	11	+ "company": "co",
	12	+ "domain": "dom",
	13	+ "sensor": "sen",
	14	+ "observation": "obs",
	15	+ "snapshot": "snap",
	16	+ "change": "chg",
	17	+ "event": "evt",
	18	+ "cluster": "cl",
	19	+ "job": "job",
	20	+ "person": "person",
	21	+ "product": "prod",
	22	+ "plan": "plan",
	23	+ "location": "loc",
	24	+ "news": "news",
	25	+ "relationship": "rel",
	26	+ "queue_job": "qj",
	27	+ "llm_job": "llm",
	28	+ "review": "rev",
	29	+ "failure": "fail",
	30	+ "crawl_run": "run",
	31	+ "watchlist": "wl",
	32	+ "alert": "alert",
	33	+ "api_key": "key",
	34	+ "signal": "sig",
	35	+}
	36	+
	37	+
	38	+def new_id(kind: str) -> str:
	39	+ prefix = PREFIXES.get(kind)
	40	+ if prefix is None:
	41	+ raise ValueError(f"unknown id kind {kind!r}")
	42	+ return f"{prefix}_{ULID()}"
	43	+
	44	+
	45	+def kind_of(entity_id: str) -> str \| None:
	46	+ prefix = entity_id.split("_", 1)[0]
	47	+ for kind, p in PREFIXES.items():
	48	+ if p == prefix:
	49	+ return kind
	50	+ return None
	51	+
	52	+
	53	+_slug_clean = re.compile(r"[^a-z0-9.-]+")
	54	+
	55	+
	56	+def slugify(text: str, *, max_length: int = 80) -> str:
	57	+ """URL slug (`stripe`, `jpmorgan-chase`, `l-oreal`)."""
	58	+ s = _slugify(text, lowercase=True, regex_pattern=r"[^a-z0-9-]+", replacements=[("&", " and "), ("/", "-"), ("_", "-"), ("@", "-at-")])
	59	+ s = _slug_clean.sub("-", s).strip("-.")
	60	+ return s[:max_length].rstrip("-.") or "company"
	61	+
	62	+
	63	+def normalize_alias(text: str) -> str:
	64	+ """Deterministic alias key: lowercase ASCII, punctuation collapsed, legal suffixes dropped. Resolution only — never display."""
	65	+ s = _slugify(text, lowercase=True, regex_pattern=r"[^a-z0-9]+")
	66	+ s = s.replace("-", "")
	67	+ for suffix in ("incorporated", "corporation", "limited", "company", "holdings", "group", "inc", "corp", "ltd", "plc", "llc",
	68	+ "gmbh", "ag", "sa", "nv", "bv", "se", "ab", "oyj", "asa", "spa", "srl", "kk", "co"):
	69	+ if s.endswith(suffix) and len(s) > len(suffix) + 2:
	70	+ s = s[: -len(suffix)]
	71	+ break
	72	+ return s
	73	+
	74	+
	75	+def stable_hash(*parts: str, length: int = 24) -> str:
	76	+ """Deterministic key for idempotent rows (queue jobs `sensor_id + window`, job fingerprints, cluster keys)."""
	77	+ h = hashlib.sha256("\x1f".join(parts).encode("utf-8")).hexdigest()
	78	+ return h[:length]
	79	+
	80	+
	81	+__all__ = ["PREFIXES", "kind_of", "new_id", "normalize_alias", "slugify", "stable_hash"]

added src/companyatlas/logging.py +56 −0

		`@@ -0,0 +1,56 @@`
	1	+"""Structured logging (JSON in production, human-readable in development)."""
	2	+from __future__ import annotations
	3	+
	4	+import json
	5	+import logging
	6	+import sys
	7	+from datetime import UTC, datetime
	8	+
	9	+from companyatlas.config import settings
	10	+
	11	+_RESERVED = {"name", "msg", "args", "levelname", "levelno", "pathname", "filename", "module", "exc_info", "exc_text", "stack_info",
	12	+ "lineno", "funcName", "created", "msecs", "relativeCreated", "thread", "threadName", "processName", "process",
	13	+ "message", "taskName"}
	14	+
	15	+
	16	+class JsonFormatter(logging.Formatter):
	17	+ def __init__(self, service: str):
	18	+ super().__init__()
	19	+ self.service = service
	20	+
	21	+ def format(self, record: logging.LogRecord) -> str:
	22	+ payload = {
	23	+ "ts": datetime.now(UTC).isoformat(timespec="milliseconds"),
	24	+ "level": record.levelname,
	25	+ "logger": record.name,
	26	+ "service": self.service,
	27	+ "msg": record.getMessage(),
	28	+ }
	29	+ for key, value in record.__dict__.items():
	30	+ if key not in _RESERVED and not key.startswith("_"):
	31	+ payload[key] = value
	32	+ if record.exc_info:
	33	+ payload["exc"] = self.formatException(record.exc_info)
	34	+ return json.dumps(payload, default=str)
	35	+
	36	+
	37	+class PlainFormatter(logging.Formatter):
	38	+ def format(self, record: logging.LogRecord) -> str:
	39	+ extras = {k: v for k, v in record.__dict__.items() if k not in _RESERVED and not k.startswith("_")}
	40	+ base = f"{datetime.now().strftime('%H:%M:%S')} {record.levelname:<7} {record.name}: {record.getMessage()}"
	41	+ if extras:
	42	+ base += " " + " ".join(f"{k}={v}" for k, v in extras.items())
	43	+ if record.exc_info:
	44	+ base += "\n" + self.formatException(record.exc_info)
	45	+ return base
	46	+
	47	+
	48	+def setup_logging(level: int = logging.INFO, service: str = "companyatlas") -> None:
	49	+ root = logging.getLogger()
	50	+ root.handlers.clear()
	51	+ handler = logging.StreamHandler(sys.stderr)
	52	+ handler.setFormatter(JsonFormatter(service) if settings.log_json else PlainFormatter())
	53	+ root.addHandler(handler)
	54	+ root.setLevel(level)
	55	+ for noisy in ("httpx", "httpcore", "apscheduler", "asyncio", "urllib3"):
	56	+ logging.getLogger(noisy).setLevel(logging.WARNING)

added src/companyatlas/registry/__init__.py +1 −0

		`@@ -0,0 +1 @@`
	1	+"""Package."""

added src/companyatlas/schemas/__init__.py +1 −0

		`@@ -0,0 +1 @@`
	1	+"""Package."""

added src/companyatlas/sdk/__init__.py +1 −0

		`@@ -0,0 +1 @@`
	1	+"""Package."""

added src/companyatlas/sdk/models.py +208 −0

		`@@ -0,0 +1,208 @@`
	1	+"""Typed contracts shared by connectors, the pipeline and the intelligence layer.
	2	+
	3	+ FetchResult (fetch.py) → Extraction (this module) → snapshot row + blocks object → BlockDiff / StructuredDelta → changes row
	4	+ ↓
	5	+ events service reads changes.structured_delta
	6	+
	7	+Everything is plain dataclasses / pydantic so it serialises to JSON for the `snapshots.extracted` and `changes.structured_delta`
	8	+columns. Never put raw HTML in these — raw bytes live in the object store, referenced by hash.
	9	+"""
	10	+from __future__ import annotations
	11	+
	12	+from dataclasses import asdict, dataclass, field
	13	+from datetime import datetime
	14	+from typing import Any, Literal
	15	+
	16	+from companyatlas.taxonomy import Surface
	17	+
	18	+BlockKind = Literal["header", "hero", "nav", "section", "heading", "paragraph", "list", "table", "product_card", "pricing_plan", "job_listing",
	19	+ "person", "location", "news_item", "faq", "footer", "code", "quote", "other"]
	20	+
	21	+
	22	+@dataclass(slots=True)
	23	+class Block:
	24	+ """A semantic block of a page (spec §19). `key` is the stable identity used for added/removed/modified/moved diffing:
	25	+ it is derived from the block kind + heading path + a fuzzy content fingerprint, never from DOM position alone."""
	26	+ key: str
	27	+ kind: str
	28	+ text: str
	29	+ path: str = "" # heading breadcrumb, e.g. "Pricing > Pro"
	30	+ hash: str = "" # sha256 of normalized text (exact)
	31	+ simhash: int = 0 # 64-bit near-duplicate fingerprint
	32	+ weight: float = 1.0 # importance weight (hero/pricing/job > footer/nav)
	33	+ order: int = 0
	34	+ attrs: dict[str, Any] = field(default_factory=dict)
	35	+
	36	+ def to_json(self) -> dict[str, Any]:
	37	+ return asdict(self)
	38	+
	39	+
	40	+@dataclass(slots=True)
	41	+class ExtractedJob:
	42	+ title: str
	43	+ url: str \| None = None
	44	+ external_id: str \| None = None
	45	+ department: str \| None = None
	46	+ team: str \| None = None
	47	+ location_text: str \| None = None
	48	+ city: str \| None = None
	49	+ region: str \| None = None
	50	+ country: str \| None = None # ISO-2 when confidently derivable, else None (never guess)
	51	+ remote: bool \| None = None
	52	+ employment_type: str \| None = None
	53	+ seniority: str \| None = None
	54	+ skills: list[str] = field(default_factory=list)
	55	+ salary_min: float \| None = None
	56	+ salary_max: float \| None = None
	57	+ salary_currency: str \| None = None
	58	+ salary_period: str \| None = None
	59	+ posted_at: datetime \| None = None
	60	+ description_hash: str \| None = None
	61	+ raw: dict[str, Any] = field(default_factory=dict)
	62	+
	63	+
	64	+@dataclass(slots=True)
	65	+class ExtractedPerson:
	66	+ name: str
	67	+ title: str \| None = None
	68	+ role_category: str \| None = None # ceo \| cfo \| cto \| coo \| founder \| president \| chair \| board \| vp \| head \| other
	69	+ is_executive: bool = False
	70	+ url: str \| None = None
	71	+
	72	+
	73	+@dataclass(slots=True)
	74	+class ExtractedProduct:
	75	+ name: str
	76	+ url: str \| None = None
	77	+ category: str \| None = None
	78	+ description: str \| None = None
	79	+
	80	+
	81	+@dataclass(slots=True)
	82	+class ExtractedPlan:
	83	+ plan_name: str
	84	+ price: float \| None = None
	85	+ price_text: str \| None = None
	86	+ currency: str \| None = None
	87	+ billing_period: str \| None = None # month \| year \| one_time \| usage \| contact
	88	+ unit: str \| None = None
	89	+ features: list[str] = field(default_factory=list)
	90	+ contact_sales: bool = False
	91	+
	92	+
	93	+@dataclass(slots=True)
	94	+class ExtractedLocation:
	95	+ name: str
	96	+ kind: str = "office" # headquarters \| office \| store \| factory \| warehouse \| lab \| data_center \| other
	97	+ city: str \| None = None
	98	+ region: str \| None = None
	99	+ country: str \| None = None
	100	+ address_text: str \| None = None # only if the page states it; never invented
	101	+
	102	+
	103	+@dataclass(slots=True)
	104	+class ExtractedNewsItem:
	105	+ title: str
	106	+ url: str
	107	+ published_at: datetime \| None = None
	108	+ summary: str \| None = None
	109	+ category: str \| None = None # press \| blog \| changelog \| research \| ir \| other
	110	+ language: str \| None = None
	111	+
	112	+
	113	+@dataclass(slots=True)
	114	+class DiscoveredUrl:
	115	+ url: str
	116	+ surface: Surface
	117	+ confidence: float
	118	+ anchor: str \| None = None
	119	+ method: str = "nav" # nav \| sitemap \| robots \| pattern \| ats \| feed \| link \| jsonld
	120	+
	121	+
	122	+@dataclass(slots=True)
	123	+class Extraction:
	124	+ """Output of `Connector.extract`. `text` is the normalized main-content text (what gets diffed at the text level),
	125	+ `blocks` the semantic blocks (block-level diff), the typed lists feed entity tables and structured deltas."""
	126	+ text: str
	127	+ blocks: list[Block]
	128	+ title: str \| None = None
	129	+ language: str \| None = None
	130	+ meta: dict[str, Any] = field(default_factory=dict) # description, og tags, canonical, generator, jsonld types…
	131	+ jobs: list[ExtractedJob] = field(default_factory=list)
	132	+ people: list[ExtractedPerson] = field(default_factory=list)
	133	+ products: list[ExtractedProduct] = field(default_factory=list)
	134	+ plans: list[ExtractedPlan] = field(default_factory=list)
	135	+ locations: list[ExtractedLocation] = field(default_factory=list)
	136	+ news: list[ExtractedNewsItem] = field(default_factory=list)
	137	+ discovered: list[DiscoveredUrl] = field(default_factory=list)
	138	+ structured_hash: str = "" # hash of the structured payload (typed lists) — set by the pipeline if empty
	139	+ normalized_hash: str = "" # hash of `text` — set by the pipeline if empty
	140	+
	141	+ def summary(self) -> dict[str, Any]:
	142	+ return {"job_count": len(self.jobs), "people_count": len(self.people), "product_count": len(self.products), "plan_count": len(self.plans),
	143	+ "location_count": len(self.locations), "news_count": len(self.news), "block_count": len(self.blocks), "text_length": len(self.text),
	144	+ "discovered_count": len(self.discovered)}
	145	+
	146	+ def structured_payload(self) -> dict[str, Any]:
	147	+ return {"jobs": [asdict(j) for j in self.jobs], "people": [asdict(p) for p in self.people], "products": [asdict(p) for p in self.products],
	148	+ "plans": [asdict(p) for p in self.plans], "locations": [asdict(loc) for loc in self.locations], "news": [asdict(n) for n in self.news],
	149	+ "meta": self.meta}
	150	+
	151	+
	152	+@dataclass(slots=True)
	153	+class BlockDelta:
	154	+ key: str
	155	+ kind: str
	156	+ path: str
	157	+ before: str \| None
	158	+ after: str \| None
	159	+ weight: float
	160	+ similarity: float \| None = None # for modified blocks
	161	+
	162	+
	163	+@dataclass(slots=True)
	164	+class BlockDiff:
	165	+ """Result of comparing two block lists (spec §19–20). `significance` ∈ [0,1] is computed by the diff engine from the weighted
	166	+ share of changed content, page importance and the typed deltas; the pipeline maps it to a ChangeKind."""
	167	+ added: list[BlockDelta] = field(default_factory=list)
	168	+ removed: list[BlockDelta] = field(default_factory=list)
	169	+ modified: list[BlockDelta] = field(default_factory=list)
	170	+ moved: list[str] = field(default_factory=list)
	171	+ text_delta_ratio: float = 0.0
	172	+ similarity: float = 1.0
	173	+ significance: float = 0.0
	174	+ reasons: list[str] = field(default_factory=list)
	175	+
	176	+ @property
	177	+ def is_empty(self) -> bool:
	178	+ return not (self.added or self.removed or self.modified)
	179	+
	180	+ def to_json(self, *, limit: int = 40) -> dict[str, Any]:
	181	+ def _cut(items: list[BlockDelta]) -> list[dict[str, Any]]:
	182	+ out = []
	183	+ for d in items[:limit]:
	184	+ out.append({"key": d.key, "kind": d.kind, "path": d.path, "before": (d.before or "")[:600] or None, "after": (d.after or "")[:600] or None,
	185	+ "weight": d.weight, "similarity": d.similarity})
	186	+ return out
	187	+ return {"added": _cut(self.added), "removed": _cut(self.removed), "modified": _cut(self.modified), "moved": self.moved[:limit],
	188	+ "counts": {"added": len(self.added), "removed": len(self.removed), "modified": len(self.modified), "moved": len(self.moved)},
	189	+ "text_delta_ratio": round(self.text_delta_ratio, 4), "similarity": round(self.similarity, 4), "reasons": self.reasons}
	190	+
	191	+
	192	+# StructuredDelta — JSON stored in `changes.structured_delta`, produced by the pipeline when it reconciles typed extractions
	193	+# with the entity tables. Shape (all keys optional, lists bounded to 200 items):
	194	+# {
	195	+# "jobs": {"added": [ {title, url, location_text, country, remote, department, is_ai} ], "removed": [...], "open_before": n, "open_after": n},
	196	+# "people": {"added": [ {name, title, role_category} ], "removed": [...], "title_changed": [ {name, before, after} ]},
	197	+# "products": {"added": [ {name, url} ], "removed": [...]},
	198	+# "plans": {"added": [ {plan_name, price, currency, billing_period} ], "removed": [...],
	199	+# "price_changed": [ {plan_name, before, after, currency, billing_period, pct} ]},
	200	+# "locations": {"added": [ {name, city, country, kind} ], "removed": [...], "new_countries": ["JP"]},
	201	+# "news": {"added": [ {title, url, published_at, category} ]},
	202	+# "meta": {"title_changed": {before, after}, "description_changed": bool, "language": "en"}
	203	+# }
	204	+StructuredDelta = dict[str, Any]
	205	+
	206	+
	207	+__all__ = ["Block", "BlockDelta", "BlockDiff", "BlockKind", "DiscoveredUrl", "ExtractedJob", "ExtractedLocation", "ExtractedNewsItem", "ExtractedPerson",
	208	+ "ExtractedPlan", "ExtractedProduct", "Extraction", "StructuredDelta"]

added src/companyatlas/services/__init__.py +1 −0

		`@@ -0,0 +1 @@`
	1	+"""Package."""

added src/companyatlas/services/periodic.py +114 −0

		`@@ -0,0 +1,114 @@`
	1	+"""Periodic task registry shared by the scheduler process.
	2	+
	3	+The crawl scheduler (`catlas schedule`, services/scheduler.py) is the single long-running production process besides the API. Any
	4	+area (intelligence, alerts, metrics, retention, backups…) registers background work here instead of shipping its own daemon:
	5	+
	6	+ from companyatlas.services.periodic import periodic
	7	+
	8	+ @periodic("process-changes", every_s=30)
	9	+ async def process_changes_task(): ...
	10	+
	11	+ @periodic("metrics-hourly", cron="7 * * * *")
	12	+ async def metrics_task(): ...
	13	+
	14	+`every_s` tasks run on a fixed cadence (first run `initial_delay_s` after start); `cron` tasks use APScheduler cron syntax in
	15	+`settings.tz`. Tasks must be idempotent and bounded (do a batch, return). Exceptions are logged, never fatal. The scheduler
	16	+imports `companyatlas.services.registry_loader` (below) which imports every module that registers tasks — add yours there.
	17	+"""
	18	+from __future__ import annotations
	19	+
	20	+import asyncio
	21	+import importlib
	22	+import logging
	23	+import time
	24	+from collections.abc import Awaitable, Callable
	25	+from dataclasses import dataclass, field
	26	+
	27	+log = logging.getLogger(__name__)
	28	+
	29	+TaskFn = Callable[[], Awaitable[object]]
	30	+
	31	+
	32	+@dataclass
	33	+class PeriodicTask:
	34	+ name: str
	35	+ fn: TaskFn
	36	+ every_s: float \| None = None
	37	+ cron: str \| None = None
	38	+ initial_delay_s: float = 5.0
	39	+ exclusive: bool = True # never overlap with itself
	40	+ last_started_at: float \| None = None
	41	+ last_finished_at: float \| None = None
	42	+ last_error: str \| None = None
	43	+ runs: int = 0
	44	+ failures: int = 0
	45	+ _lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False)
	46	+
	47	+ async def run(self) -> None:
	48	+ if self.exclusive and self._lock.locked():
	49	+ return
	50	+ async with self._lock:
	51	+ self.last_started_at = time.time()
	52	+ try:
	53	+ await self.fn()
	54	+ self.runs += 1
	55	+ self.last_error = None
	56	+ except Exception as exc: # noqa: BLE001
	57	+ self.failures += 1
	58	+ self.last_error = f"{exc.__class__.__name__}: {exc}"[:500]
	59	+ log.exception("periodic task failed", extra={"task": self.name})
	60	+ finally:
	61	+ self.last_finished_at = time.time()
	62	+
	63	+
	64	+_TASKS: dict[str, PeriodicTask] = {}
	65	+
	66	+# Modules that register periodic tasks at import time. Areas append their module path here (one line each).
	67	+TASK_MODULES: list[str] = [
	68	+ "companyatlas.services.events",
	69	+ "companyatlas.services.metrics",
	70	+ "companyatlas.services.alerts",
	71	+ "companyatlas.services.retention",
	72	+ "companyatlas.services.llm.enrich",
	73	+ "companyatlas.services.signals",
	74	+ "companyatlas.services.trends",
	75	+]
	76	+
	77	+
	78	+def periodic(name: str, *, every_s: float \| None = None, cron: str \| None = None, initial_delay_s: float = 5.0, exclusive: bool = True): # type: ignore[no-untyped-def]
	79	+ if not every_s and not cron:
	80	+ raise ValueError("periodic task needs every_s or cron")
	81	+
	82	+ def deco(fn: TaskFn) -> TaskFn:
	83	+ _TASKS[name] = PeriodicTask(name=name, fn=fn, every_s=every_s, cron=cron, initial_delay_s=initial_delay_s, exclusive=exclusive)
	84	+ return fn
	85	+
	86	+ return deco
	87	+
	88	+
	89	+def load_task_modules() -> list[str]:
	90	+ """Import every registered task module (missing modules are skipped so areas can land independently)."""
	91	+ loaded: list[str] = []
	92	+ for mod in TASK_MODULES:
	93	+ try:
	94	+ importlib.import_module(mod)
	95	+ loaded.append(mod)
	96	+ except ModuleNotFoundError as exc:
	97	+ if exc.name and (mod == exc.name or mod.startswith(exc.name + ".")):
	98	+ continue
	99	+ log.exception("task module failed to import", extra={"module": mod})
	100	+ except Exception: # noqa: BLE001
	101	+ log.exception("task module failed to import", extra={"module": mod})
	102	+ return loaded
	103	+
	104	+
	105	+def tasks() -> dict[str, PeriodicTask]:
	106	+ return _TASKS
	107	+
	108	+
	109	+def snapshot() -> list[dict[str, object]]:
	110	+ return [{"name": t.name, "every_s": t.every_s, "cron": t.cron, "runs": t.runs, "failures": t.failures, "last_started_at": t.last_started_at,
	111	+ "last_finished_at": t.last_finished_at, "last_error": t.last_error} for t in _TASKS.values()]
	112	+
	113	+
	114	+__all__ = ["TASK_MODULES", "PeriodicTask", "load_task_modules", "periodic", "snapshot", "tasks"]

added src/companyatlas/taxonomy.py +369 −0

		`@@ -0,0 +1,369 @@`
	1	+"""Shared vocabulary and typed configuration: surfaces, connector categories, schedule tiers, failure classes, event taxonomy,
	2	+significance bands, metric names and index weights. Everything here is versioned with the code (spec §21, §35, §61, §178)."""
	3	+from __future__ import annotations
	4	+
	5	+from enum import StrEnum
	6	+
	7	+# ------------------------------------------------------------------------------------------------------------ surfaces
	8	+
	9	+
	10	+class Surface(StrEnum):
	11	+ HOMEPAGE = "homepage"
	12	+ ABOUT = "about"
	13	+ LEADERSHIP = "leadership"
	14	+ PRODUCTS = "products"
	15	+ SERVICES = "services"
	16	+ SOLUTIONS = "solutions"
	17	+ INDUSTRIES = "industries"
	18	+ PRICING = "pricing"
	19	+ CAREERS = "careers"
	20	+ JOBS_BOARD = "jobs_board" # structured ATS board (Greenhouse, Lever, Ashby, SmartRecruiters, Workday, JSON)
	21	+ NEWSROOM = "newsroom"
	22	+ BLOG = "blog"
	23	+ FEED = "feed" # RSS / Atom
	24	+ DOCS = "docs"
	25	+ DEVELOPER = "developer"
	26	+ API = "api"
	27	+ CHANGELOG = "changelog"
	28	+ INVESTOR_RELATIONS = "investor_relations"
	29	+ LOCATIONS = "locations"
	30	+ CONTACT = "contact"
	31	+ CUSTOMERS = "customers"
	32	+ PARTNERS = "partners"
	33	+ LEGAL_TERMS = "legal_terms"
	34	+ LEGAL_PRIVACY = "legal_privacy"
	35	+ SECURITY = "security"
	36	+ SUSTAINABILITY = "sustainability"
	37	+ RESEARCH = "research"
	38	+ STATUS = "status"
	39	+ SUPPORT = "support"
	40	+ SITEMAP = "sitemap"
	41	+ OTHER = "other"
	42	+
	43	+
	44	+# Default base interval per surface (seconds) — the scheduler adapts around it (spec §15–16). Tier letters are derived.
	45	+SURFACE_BASE_INTERVAL_S: dict[str, int] = {
	46	+ Surface.HOMEPAGE: 6 * 3600,
	47	+ Surface.NEWSROOM: 3 * 3600,
	48	+ Surface.FEED: 2 * 3600,
	49	+ Surface.BLOG: 6 * 3600,
	50	+ Surface.CAREERS: 12 * 3600,
	51	+ Surface.JOBS_BOARD: 6 * 3600,
	52	+ Surface.PRICING: 12 * 3600,
	53	+ Surface.PRODUCTS: 24 * 3600,
	54	+ Surface.SERVICES: 48 * 3600,
	55	+ Surface.SOLUTIONS: 48 * 3600,
	56	+ Surface.INDUSTRIES: 72 * 3600,
	57	+ Surface.LEADERSHIP: 24 * 3600,
	58	+ Surface.ABOUT: 72 * 3600,
	59	+ Surface.LOCATIONS: 48 * 3600,
	60	+ Surface.CONTACT: 7 * 86400,
	61	+ Surface.DOCS: 24 * 3600,
	62	+ Surface.DEVELOPER: 24 * 3600,
	63	+ Surface.API: 24 * 3600,
	64	+ Surface.CHANGELOG: 6 * 3600,
	65	+ Surface.INVESTOR_RELATIONS: 12 * 3600,
	66	+ Surface.CUSTOMERS: 72 * 3600,
	67	+ Surface.PARTNERS: 72 * 3600,
	68	+ Surface.LEGAL_TERMS: 3 * 86400,
	69	+ Surface.LEGAL_PRIVACY: 3 * 86400,
	70	+ Surface.SECURITY: 3 * 86400,
	71	+ Surface.SUSTAINABILITY: 7 * 86400,
	72	+ Surface.RESEARCH: 48 * 3600,
	73	+ Surface.STATUS: 6 * 3600,
	74	+ Surface.SUPPORT: 7 * 86400,
	75	+ Surface.SITEMAP: 24 * 3600,
	76	+ Surface.OTHER: 3 * 86400,
	77	+}
	78	+
	79	+# Semantic importance of a surface (0–1): weights change significance and sensor quality (spec §12, §20).
	80	+SURFACE_IMPORTANCE: dict[str, float] = {
	81	+ Surface.PRICING: 1.0, Surface.JOBS_BOARD: 0.95, Surface.CAREERS: 0.85, Surface.LEADERSHIP: 0.9, Surface.NEWSROOM: 0.9,
	82	+ Surface.PRODUCTS: 0.85, Surface.INVESTOR_RELATIONS: 0.85, Surface.CHANGELOG: 0.8, Surface.LOCATIONS: 0.8, Surface.FEED: 0.75,
	83	+ Surface.HOMEPAGE: 0.7, Surface.DOCS: 0.65, Surface.API: 0.65, Surface.DEVELOPER: 0.65, Surface.LEGAL_TERMS: 0.7,
	84	+ Surface.LEGAL_PRIVACY: 0.6, Surface.SECURITY: 0.6, Surface.PARTNERS: 0.6, Surface.CUSTOMERS: 0.55, Surface.BLOG: 0.55,
	85	+ Surface.RESEARCH: 0.55, Surface.SERVICES: 0.55, Surface.SOLUTIONS: 0.5, Surface.INDUSTRIES: 0.45, Surface.ABOUT: 0.5,
	86	+ Surface.SUSTAINABILITY: 0.45, Surface.STATUS: 0.5, Surface.CONTACT: 0.3, Surface.SUPPORT: 0.3, Surface.SITEMAP: 0.4, Surface.OTHER: 0.3,
	87	+}
	88	+
	89	+
	90	+def tier_for_interval(seconds: int) -> str:
	91	+ """Schedule tier letter (spec §15): A ≤ 15 min · B ≤ 1 h · C ≤ 6 h · D ≤ 24 h · E > 24 h."""
	92	+ if seconds <= 15 * 60:
	93	+ return "A"
	94	+ if seconds <= 3600:
	95	+ return "B"
	96	+ if seconds <= 6 * 3600:
	97	+ return "C"
	98	+ if seconds <= 86400:
	99	+ return "D"
	100	+ return "E"
	101	+
	102	+
	103	+# ------------------------------------------------------------------------------------------------------------ fetch / failures
	104	+
	105	+
	106	+class FetchMode(StrEnum):
	107	+ HTTP = "http"
	108	+ BROWSER = "browser"
	109	+ FEED = "feed"
	110	+ JSON = "json"
	111	+ SITEMAP = "sitemap"
	112	+
	113	+
	114	+class FailureClass(StrEnum):
	115	+ DNS = "DNS"
	116	+ TIMEOUT = "TIMEOUT"
	117	+ HTTP_4XX = "HTTP_4XX"
	118	+ HTTP_5XX = "HTTP_5XX"
	119	+ BOT_CHALLENGE = "BOT_CHALLENGE"
	120	+ PARSING = "PARSING"
	121	+ SCHEMA = "SCHEMA"
	122	+ REDIRECT = "REDIRECT"
	123	+ PAGE_REMOVED = "PAGE_REMOVED"
	124	+ RATE_LIMIT = "RATE_LIMIT"
	125	+ ROBOTS = "ROBOTS"
	126	+ BLOCKED_DESTINATION = "BLOCKED_DESTINATION"
	127	+ TOO_LARGE = "TOO_LARGE"
	128	+ UNKNOWN = "UNKNOWN"
	129	+
	130	+
	131	+# Retry policy per failure class: (backoff multiplier on the sensor interval, failures before the sensor is marked failing)
	132	+FAILURE_POLICY: dict[str, tuple[float, int]] = {
	133	+ FailureClass.DNS: (3.0, 3),
	134	+ FailureClass.TIMEOUT: (1.5, 4),
	135	+ FailureClass.HTTP_4XX: (2.0, 3),
	136	+ FailureClass.HTTP_5XX: (1.5, 5),
	137	+ FailureClass.BOT_CHALLENGE: (4.0, 2),
	138	+ FailureClass.PARSING: (2.0, 3),
	139	+ FailureClass.SCHEMA: (2.0, 3),
	140	+ FailureClass.REDIRECT: (2.0, 2),
	141	+ FailureClass.PAGE_REMOVED: (4.0, 2),
	142	+ FailureClass.RATE_LIMIT: (3.0, 4),
	143	+ FailureClass.ROBOTS: (8.0, 1),
	144	+ FailureClass.BLOCKED_DESTINATION: (8.0, 1),
	145	+ FailureClass.TOO_LARGE: (4.0, 2),
	146	+ FailureClass.UNKNOWN: (2.0, 3),
	147	+}
	148	+
	149	+
	150	+class SensorStatus(StrEnum):
	151	+ PENDING = "pending"
	152	+ ACTIVE = "active"
	153	+ PAUSED = "paused"
	154	+ FAILING = "failing"
	155	+ STALE = "stale"
	156	+ BLOCKED = "blocked"
	157	+ REDIRECTED = "redirected"
	158	+ RETIRED = "retired"
	159	+
	160	+
	161	+class CompanyStatus(StrEnum):
	162	+ ACTIVE = "ACTIVE"
	163	+ POSSIBLY_INACTIVE = "POSSIBLY_INACTIVE"
	164	+ WEBSITE_UNAVAILABLE = "WEBSITE_UNAVAILABLE"
	165	+ ACQUIRED = "ACQUIRED"
	166	+ DISSOLVED = "DISSOLVED"
	167	+ UNKNOWN = "UNKNOWN"
	168	+
	169	+
	170	+class OnboardingStatus(StrEnum):
	171	+ PENDING = "pending"
	172	+ DISCOVERING = "discovering"
	173	+ ACTIVE = "active"
	174	+ FAILED = "failed"
	175	+ NO_WEBSITE = "no_website"
	176	+
	177	+
	178	+# ------------------------------------------------------------------------------------------------------------ changes / events
	179	+
	180	+
	181	+class ChangeKind(StrEnum):
	182	+ NOISE = "noise"
	183	+ MINOR = "minor"
	184	+ MEANINGFUL = "meaningful"
	185	+ MAJOR = "major"
	186	+ CRITICAL = "critical"
	187	+
	188	+
	189	+def change_kind(significance: float, *, noise: float = 0.20, meaningful: float = 0.40, major: float = 0.65, critical: float = 0.85) -> ChangeKind:
	190	+ if significance < noise:
	191	+ return ChangeKind.NOISE
	192	+ if significance < meaningful:
	193	+ return ChangeKind.MINOR
	194	+ if significance < major:
	195	+ return ChangeKind.MEANINGFUL
	196	+ if significance < critical:
	197	+ return ChangeKind.MAJOR
	198	+ return ChangeKind.CRITICAL
	199	+
	200	+
	201	+class EventType(StrEnum):
	202	+ PRODUCT = "PRODUCT"
	203	+ PRICING = "PRICING"
	204	+ HIRING = "HIRING"
	205	+ LEADERSHIP = "LEADERSHIP"
	206	+ LOCATION = "LOCATION"
	207	+ FINANCING = "FINANCING"
	208	+ MA = "M&A"
	209	+ PARTNERSHIP = "PARTNERSHIP"
	210	+ STRATEGY = "STRATEGY"
	211	+ TECHNOLOGY = "TECHNOLOGY"
	212	+ LEGAL = "LEGAL"
	213	+ MARKETING = "MARKETING"
	214	+ DEVELOPER = "DEVELOPER"
	215	+ SECURITY = "SECURITY"
	216	+ OPERATIONS = "OPERATIONS"
	217	+ SUSTAINABILITY = "SUSTAINABILITY"
	218	+ INVESTOR_RELATIONS = "INVESTOR_RELATIONS"
	219	+ COMMUNICATION = "COMMUNICATION"
	220	+ OTHER = "OTHER"
	221	+
	222	+
	223	+# subtype → (type, default importance). Deterministic extractors emit these; the LLM classifier may only pick from this list.
	224	+EVENT_SUBTYPES: dict[str, tuple[EventType, float]] = {
	225	+ "PRODUCT_LAUNCH": (EventType.PRODUCT, 0.75),
	226	+ "NEW_PRODUCT": (EventType.PRODUCT, 0.7),
	227	+ "PRODUCT_REMOVED": (EventType.PRODUCT, 0.6),
	228	+ "PRODUCT_RENAME": (EventType.PRODUCT, 0.5),
	229	+ "PRODUCT_UPDATE": (EventType.PRODUCT, 0.45),
	230	+ "FEATURE_LAUNCH": (EventType.PRODUCT, 0.5),
	231	+ "PRICE_INCREASE": (EventType.PRICING, 0.8),
	232	+ "PRICE_DECREASE": (EventType.PRICING, 0.75),
	233	+ "NEW_PRICING_TIER": (EventType.PRICING, 0.7),
	234	+ "PRICING_TIER_REMOVED": (EventType.PRICING, 0.65),
	235	+ "PRICING_CHANGE": (EventType.PRICING, 0.65),
	236	+ "NEW_JOB": (EventType.HIRING, 0.3),
	237	+ "JOB_REMOVED": (EventType.HIRING, 0.25),
	238	+ "JOB_COUNT_INCREASE": (EventType.HIRING, 0.55),
	239	+ "JOB_COUNT_DECREASE": (EventType.HIRING, 0.55),
	240	+ "HIRING_SURGE": (EventType.HIRING, 0.75),
	241	+ "HIRING_FREEZE_SIGNAL": (EventType.HIRING, 0.7),
	242	+ "AI_HIRING": (EventType.HIRING, 0.5),
	243	+ "NEW_EXECUTIVE": (EventType.LEADERSHIP, 0.8),
	244	+ "EXECUTIVE_NO_LONGER_LISTED": (EventType.LEADERSHIP, 0.75),
	245	+ "EXECUTIVE_TITLE_CHANGE": (EventType.LEADERSHIP, 0.6),
	246	+ "LEADERSHIP_CHANGE": (EventType.LEADERSHIP, 0.7),
	247	+ "NEW_OFFICE": (EventType.LOCATION, 0.65),
	248	+ "NEW_LOCATION": (EventType.LOCATION, 0.6),
	249	+ "OFFICE_REMOVED": (EventType.LOCATION, 0.6),
	250	+ "COUNTRY_EXPANSION": (EventType.LOCATION, 0.8),
	251	+ "FUNDING_ROUND": (EventType.FINANCING, 0.85),
	252	+ "IPO": (EventType.FINANCING, 0.95),
	253	+ "ACQUISITION": (EventType.MA, 0.9),
	254	+ "DIVESTITURE": (EventType.MA, 0.8),
	255	+ "MERGER": (EventType.MA, 0.9),
	256	+ "NEW_PARTNERSHIP": (EventType.PARTNERSHIP, 0.6),
	257	+ "PARTNERSHIP_ENDED": (EventType.PARTNERSHIP, 0.5),
	258	+ "BRAND_REPOSITIONING": (EventType.STRATEGY, 0.6),
	259	+ "STRATEGY_UPDATE": (EventType.STRATEGY, 0.5),
	260	+ "ENTERPRISE_REPOSITIONING": (EventType.STRATEGY, 0.55),
	261	+ "TECHNOLOGY_ADOPTION": (EventType.TECHNOLOGY, 0.5),
	262	+ "AI_LAUNCH": (EventType.TECHNOLOGY, 0.7),
	263	+ "TERMS_CHANGE": (EventType.LEGAL, 0.6),
	264	+ "PRIVACY_POLICY_CHANGE": (EventType.LEGAL, 0.55),
	265	+ "LEGAL_UPDATE": (EventType.LEGAL, 0.5),
	266	+ "REGULATORY": (EventType.LEGAL, 0.6),
	267	+ "CAMPAIGN_LAUNCH": (EventType.MARKETING, 0.35),
	268	+ "MESSAGING_CHANGE": (EventType.MARKETING, 0.4),
	269	+ "API_LAUNCH": (EventType.DEVELOPER, 0.7),
	270	+ "API_CHANGE": (EventType.DEVELOPER, 0.5),
	271	+ "SDK_RELEASE": (EventType.DEVELOPER, 0.5),
	272	+ "DOCUMENTATION_CHANGE": (EventType.DEVELOPER, 0.35),
	273	+ "CHANGELOG_ENTRY": (EventType.DEVELOPER, 0.4),
	274	+ "SECURITY_INCIDENT": (EventType.SECURITY, 0.85),
	275	+ "SECURITY_UPDATE": (EventType.SECURITY, 0.5),
	276	+ "OUTAGE": (EventType.OPERATIONS, 0.6),
	277	+ "OPERATIONS_UPDATE": (EventType.OPERATIONS, 0.4),
	278	+ "SUSTAINABILITY_UPDATE": (EventType.SUSTAINABILITY, 0.4),
	279	+ "EARNINGS_RELEASE": (EventType.INVESTOR_RELATIONS, 0.7),
	280	+ "INVESTOR_UPDATE": (EventType.INVESTOR_RELATIONS, 0.55),
	281	+ "NEWS_RELEASE": (EventType.COMMUNICATION, 0.45),
	282	+ "BLOG_POST": (EventType.COMMUNICATION, 0.3),
	283	+ "WEBSITE_CHANGE": (EventType.COMMUNICATION, 0.3),
	284	+ "HOMEPAGE_REDESIGN": (EventType.MARKETING, 0.45),
	285	+ "DOC_CHANGE": (EventType.DEVELOPER, 0.35),
	286	+ "OTHER": (EventType.OTHER, 0.3),
	287	+}
	288	+
	289	+# MVP high-confidence events emitted deterministically (spec §158).
	290	+MVP_EVENT_SUBTYPES = ("NEW_JOB", "JOB_REMOVED", "JOB_COUNT_INCREASE", "JOB_COUNT_DECREASE", "NEW_PRODUCT", "PRODUCT_REMOVED",
	291	+ "PRICING_CHANGE", "PRICE_INCREASE", "PRICE_DECREASE", "NEW_PRICING_TIER", "LEADERSHIP_CHANGE", "NEW_EXECUTIVE",
	292	+ "EXECUTIVE_NO_LONGER_LISTED", "NEW_LOCATION", "NEWS_RELEASE", "DOC_CHANGE", "CHANGELOG_ENTRY", "BLOG_POST")
	293	+
	294	+
	295	+class EventStatus(StrEnum):
	296	+ ACTIVE = "active"
	297	+ RETRACTED = "retracted"
	298	+ DUPLICATE = "duplicate"
	299	+ REVIEW = "review"
	300	+
	301	+
	302	+def confidence_label(confidence: float) -> str:
	303	+ """Spec §50: VERIFIED · HIGH CONFIDENCE · LIKELY · INFERRED · LOW CONFIDENCE."""
	304	+ if confidence >= 0.95:
	305	+ return "VERIFIED"
	306	+ if confidence >= 0.85:
	307	+ return "HIGH_CONFIDENCE"
	308	+ if confidence >= 0.7:
	309	+ return "LIKELY"
	310	+ if confidence >= 0.5:
	311	+ return "INFERRED"
	312	+ return "LOW_CONFIDENCE"
	313	+
	314	+
	315	+# ------------------------------------------------------------------------------------------------------------ metrics
	316	+
	317	+
	318	+class Metric(StrEnum):
	319	+ ACTIVITY_SCORE = "activity_score"
	320	+ HIRING_MOMENTUM_7D = "hiring_momentum_7d"
	321	+ HIRING_MOMENTUM_30D = "hiring_momentum_30d"
	322	+ HIRING_MOMENTUM_90D = "hiring_momentum_90d"
	323	+ OPEN_JOBS = "open_jobs"
	324	+ AI_ADOPTION = "ai_adoption"
	325	+ PRODUCT_VELOCITY = "product_velocity"
	326	+ GEO_EXPANSION = "geo_expansion"
	327	+ DEVELOPER_MOMENTUM = "developer_momentum"
	328	+ COMMUNICATION_ACTIVITY = "communication_activity"
	329	+ PRICING_ACTIVITY = "pricing_activity"
	330	+ LEADERSHIP_ACTIVITY = "leadership_activity"
	331	+ CORPORATE_CHANGE_INDEX = "corporate_change_index"
	332	+ ANOMALY_SCORE = "anomaly_score"
	333	+ HISTORICAL_COVERAGE = "historical_coverage"
	334	+
	335	+
	336	+# Corporate Change Index weights (spec §35) — formula version bumps when weights change.
	337	+CCI_FORMULA_VERSION = "cci-v1"
	338	+CCI_WEIGHTS: dict[str, float] = {
	339	+ Metric.HIRING_MOMENTUM_30D: 0.25,
	340	+ Metric.PRODUCT_VELOCITY: 0.20,
	341	+ Metric.GEO_EXPANSION: 0.15,
	342	+ Metric.LEADERSHIP_ACTIVITY: 0.15,
	343	+ Metric.DEVELOPER_MOMENTUM: 0.10,
	344	+ Metric.COMMUNICATION_ACTIVITY: 0.10,
	345	+ Metric.PRICING_ACTIVITY: 0.05,
	346	+}
	347	+METRICS_FORMULA_VERSION = "metrics-v1"
	348	+
	349	+# Observable AI-adoption signals (spec §31): keywords matched in job titles, product names, docs headings. Case-insensitive.
	350	+AI_KEYWORDS = ("machine learning", "artificial intelligence", "deep learning", " ai ", "ai/ml", "ml engineer", "llm", "large language",
	351	+ "generative", "genai", "gen ai", "nlp", "computer vision", "data scientist", "mlops", "agentic", "copilot",
	352	+ "foundation model", "neural", "inference", "rag ", "retrieval-augmented", "prompt engineer", "ai engineer", "ai product")
	353	+
	354	+# ------------------------------------------------------------------------------------------------------------ companies
	355	+
	356	+COMPANY_IMPORTANCE_TIERS = {1: "global", 2: "major", 3: "notable", 4: "long_tail"}
	357	+
	358	+
	359	+class CollectionMethod(StrEnum):
	360	+ LIVE = "live"
	361	+ BACKFILL = "backfill"
	362	+
	363	+
	364	+__all__ = [
	365	+ "AI_KEYWORDS", "CCI_FORMULA_VERSION", "CCI_WEIGHTS", "COMPANY_IMPORTANCE_TIERS", "EVENT_SUBTYPES", "FAILURE_POLICY",
	366	+ "METRICS_FORMULA_VERSION", "MVP_EVENT_SUBTYPES", "SURFACE_BASE_INTERVAL_S", "SURFACE_IMPORTANCE", "ChangeKind", "CollectionMethod",
	367	+ "CompanyStatus", "EventStatus", "EventType", "FailureClass", "FetchMode", "Metric", "OnboardingStatus", "SensorStatus", "Surface",
	368	+ "change_kind", "confidence_label", "tier_for_interval",
	369	+]

added src/companyatlas/urls.py +209 −0

		`@@ -0,0 +1,209 @@`
	1	+"""URL canonicalisation, domain identity and surface classification heuristics (spec §11, §117, §118).
	2	+
	3	+`canonicalize_url` produces the stable key stored in `sensors.canonical_url`; the original URL is always kept separately.
	4	+`classify_url` maps a URL (+ optional anchor text / title) to a Surface with a confidence in 0–1 — deterministic, no LLM.
	5	+"""
	6	+from __future__ import annotations
	7	+
	8	+import re
	9	+from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
	10	+
	11	+import tldextract
	12	+
	13	+from companyatlas.taxonomy import Surface
	14	+
	15	+_extract = tldextract.TLDExtract(suffix_list_urls=(), fallback_to_snapshot=True) # offline PSL snapshot: no network at import
	16	+
	17	+TRACKING_PREFIXES = ("utm_", "ref", "fbclid", "gclid", "dclid", "msclkid", "mc_cid", "mc_eid", "_hs", "hsa_", "igshid", "yclid", "_ga",
	18	+ "_gl", "source", "campaign", "mkt_tok", "trk", "cmpid", "s_kwcid", "ef_id", "sessionid", "session_id", "phpsessid",
	19	+ "jsessionid", "sid", "cid", "icid", "ncid", "spm", "srsltid")
	20	+SESSION_PATH_RE = re.compile(r";jsessionid=[^/?#]+", re.I)
	21	+MULTI_SLASH_RE = re.compile(r"/{2,}")
	22	+STATIC_EXT = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico", ".css", ".js", ".mjs", ".woff", ".woff2", ".ttf", ".eot", ".mp4",
	23	+ ".mp3", ".webm", ".mov", ".zip", ".gz", ".tar", ".dmg", ".exe", ".pkg", ".apk", ".ics", ".xlsx", ".pptx", ".docx")
	24	+DOCUMENT_EXT = (".pdf",)
	25	+
	26	+
	27	+def canonicalize_url(url: str) -> str:
	28	+ """Stable key: lowercase scheme/host, default ports dropped, tracking/session params removed, params sorted, fragment dropped,
	29	+ duplicate slashes collapsed, trailing slash removed (except root). Preserves path case (many servers are case-sensitive)."""
	30	+ p = urlparse(url.strip())
	31	+ scheme = (p.scheme or "https").lower()
	32	+ host = (p.hostname or "").lower().rstrip(".")
	33	+ port = p.port
	34	+ if port and not ((scheme == "https" and port == 443) or (scheme == "http" and port == 80)):
	35	+ host = f"{host}:{port}"
	36	+ path = SESSION_PATH_RE.sub("", p.path or "/")
	37	+ path = MULTI_SLASH_RE.sub("/", path)
	38	+ if len(path) > 1:
	39	+ path = path.rstrip("/") or "/"
	40	+ kept = [(k, v) for k, v in parse_qsl(p.query, keep_blank_values=False) if not k.lower().startswith(TRACKING_PREFIXES)]
	41	+ query = urlencode(sorted(kept), doseq=True)
	42	+ return urlunparse((scheme, host, path, "", query, ""))
	43	+
	44	+
	45	+def registrable_domain(url_or_host: str) -> str:
	46	+ """`https://jobs.eu.stripe.com/x` → `stripe.com` (public-suffix aware, offline)."""
	47	+ host = url_or_host if "://" not in url_or_host else (urlparse(url_or_host).hostname or "")
	48	+ host = host.lower().strip().rstrip(".")
	49	+ ext = _extract(host)
	50	+ if ext.domain and ext.suffix:
	51	+ return f"{ext.domain}.{ext.suffix}"
	52	+ return host.removeprefix("www.")
	53	+
	54	+
	55	+def host_of(url: str) -> str:
	56	+ return (urlparse(url).hostname or "").lower()
	57	+
	58	+
	59	+def same_company_host(url: str, canonical_domain: str) -> bool:
	60	+ """Is this URL on the company's registrable domain (any subdomain)?"""
	61	+ return registrable_domain(url) == registrable_domain(canonical_domain)
	62	+
	63	+
	64	+def is_static_asset(url: str) -> bool:
	65	+ path = urlparse(url).path.lower()
	66	+ return path.endswith(STATIC_EXT)
	67	+
	68	+
	69	+def is_document(url: str) -> bool:
	70	+ return urlparse(url).path.lower().endswith(DOCUMENT_EXT)
	71	+
	72	+
	73	+def absolutize(base: str, href: str) -> str \| None:
	74	+ href = (href or "").strip()
	75	+ if not href or href.startswith(("#", "mailto:", "tel:", "javascript:", "data:", "sms:", "whatsapp:")):
	76	+ return None
	77	+ try:
	78	+ out = urljoin(base, href)
	79	+ except ValueError:
	80	+ return None
	81	+ if not out.startswith(("http://", "https://")):
	82	+ return None
	83	+ return out
	84	+
	85	+
	86	+# ------------------------------------------------------------------------------------------------------ crawl-trap heuristics
	87	+
	88	+TRAP_PARAM_RE = re.compile(r"(^\|[?&])(page\|p\|offset\|start\|sort\|order\|filter\|facet\|color\|size\|price\|min\|max\|year\|month\|day\|date\|q\|s\|search)=", re.I)
	89	+CALENDAR_RE = re.compile(r"/(19\|20)\d{2}/(0?[1-9]\|1[0-2])(/\|$)")
	90	+
	91	+
	92	+def looks_like_trap(url: str) -> bool:
	93	+ p = urlparse(url)
	94	+ if p.query.count("&") >= 4:
	95	+ return True
	96	+ if TRAP_PARAM_RE.search("?" + p.query) and p.query.count("=") >= 2:
	97	+ return True
	98	+ if CALENDAR_RE.search(p.path) and p.path.count("/") > 4:
	99	+ return True
	100	+ return len(url) > 400
	101	+
	102	+
	103	+# ------------------------------------------------------------------------------------------------------ surface classification
	104	+# Each rule: (surface, path regex, anchor/title regex, base confidence). Path evidence weighs more than anchor evidence; when both
	105	+# match, confidence is boosted. Order matters only for tie-breaks (first rule wins on equal confidence).
	106	+
	107	+_R = re.compile
	108	+RULES: list[tuple[Surface, re.Pattern[str] \| None, re.Pattern[str] \| None, float]] = [
	109	+ (Surface.PRICING, _R(r"/(pricing\|plans\|plans-and-pricing\|pricing-plans\|tarifs\|preise\|precios\|prezzi\|价格)(/\|$)", re.I), _R(r"^(pricing\|plans( & pricing\| and pricing)?\|see pricing\|view pricing\|tarifs\|preise\|precios)$", re.I), 0.95),
	110	+ (Surface.JOBS_BOARD, _R(r"(boards\.greenhouse\.io\|job-boards\.greenhouse\.io\|jobs\.lever\.co\|jobs\.ashbyhq\.com\|jobs\.smartrecruiters\.com\|careers\.smartrecruiters\.com\|myworkdayjobs\.com\|apply\.workable\.com\|jobs\.jobvite\.com\|recruiting\.paylocity\.com\|bamboohr\.com/careers\|breezy\.hr\|recruitee\.com\|personio\.(de\|com)\|teamtailor\.com\|icims\.com\|taleo\.net\|successfactors\.com\|eightfold\.ai\|phenom\.com\|wd\d\.myworkdaysite\.com\|careers-page\.com\|homerun\.co\|pinpointhq\.com\|rippling-ats\.com\|jobs\.gem\.com)", re.I), None, 0.97),
	111	+ (Surface.CAREERS, _R(r"/(careers?\|jobs?\|join(-us\|us)?\|work-with-us\|work-for-us\|working-at\|open-positions\|opportunities\|vacancies\|recruit(ing\|ment)?\|employment\|karriere\|stellen\|emplois?\|carri[eè]res?\|empleo\|trabaja-con-nosotros\|lavora-con-noi\|saiyou\|採用)(/\|$)", re.I), _R(r"^(careers?\|jobs?\|join (us\|the team\|our team)\|work (with\|for\|at) us\|open (roles\|positions)\|we'?re hiring\|hiring\|opportunities\|vacancies\|karriere\|emplois?\|carri[eè]res?\|empleo)$", re.I), 0.93),
	112	+ (Surface.NEWSROOM, _R(r"/(news(room)?\|press(-releases?\|room\|-center\|-centre)?\|media(-center\|-centre\|-room)?\|announcements\|releases\|actualit[eé]s\|presse\|noticias\|prensa\|ニュース)(/\|$)", re.I), _R(r"^(news(room)?\|press( releases?\| room\| center)?\|media( center\| room)?\|announcements\|in the news\|actualit[eé]s\|presse\|noticias)$", re.I), 0.9),
	113	+ (Surface.INVESTOR_RELATIONS, _R(r"/(investors?\|investor-relations\|ir\|shareholders?\|financials?\|earnings\|sec-filings\|annual-reports?\|investisseurs\|investoren\|inversores)(/\|$)\|^https?://(ir\|investors?\|investor)\.", re.I), _R(r"^(investors?\|investor relations\|shareholders?\|financials?\|ir\|investisseurs\|investoren)$", re.I), 0.92),
	114	+ (Surface.LEADERSHIP, _R(r"/(leadership\|management(-team)?\|executive(s\|-team\|-leadership)?\|our-team\|the-team\|team\|board(-of-directors)?\|directors\|founders\|people\|who-we-are\|direction\|equipe\|équipe\|equipo\|vorstand\|management-board\|governance/(board\|leadership))(/\|$)", re.I), _R(r"^(leadership( team)?\|management( team)?\|executive (team\|leadership)\|our (team\|leadership\|people)\|meet the team\|board of directors\|founders\|team\|direction\|équipe)$", re.I), 0.85),
	115	+ (Surface.LOCATIONS, _R(r"/(locations?\|offices?\|our-offices\|where-we-are\|global-presence\|worldwide\|stores?\|store-locator\|find-a-store\|branches\|dealers?\|showrooms?\|sites\|standorte\|bureaux\|ubicaciones\|拠点)(/\|$)", re.I), _R(r"^(locations?\|our (locations\|offices)\|offices?\|where we are\|global presence\|find a store\|store locator\|branches\|standorte\|bureaux)$", re.I), 0.88),
	116	+ (Surface.CHANGELOG, _R(r"/(changelog\|change-log\|changes\|release-notes\|releases\|whats-new\|what's-new\|updates\|product-updates)(/\|$)\|^https?://(changelog\|releases\|updates)\.", re.I), _R(r"^(changelog\|release notes\|what'?s new\|product updates\|updates\|releases)$", re.I), 0.9),
	117	+ (Surface.API, _R(r"/(api\|apis\|api-reference\|api-docs\|reference)(/\|$)\|^https?://api-?docs?\.", re.I), _R(r"^(api( reference\| docs\| documentation)?\|apis\|rest api\|graphql)$", re.I), 0.82),
	118	+ (Surface.DEVELOPER, _R(r"/(developers?\|dev\|devs\|developer-portal\|platform\|sdks?\|integrations?\|build)(/\|$)\|^https?://(developers?\|dev\|build)\.", re.I), _R(r"^(developers?\|developer (portal\|center\|hub)\|for developers\|sdks?\|integrations?\|build)$", re.I), 0.8),
	119	+ (Surface.DOCS, _R(r"/(docs\|documentation\|help-center\|help\|guides?\|manuals?\|knowledge-base\|kb\|learn\|tutorials?)(/\|$)\|^https?://(docs\|documentation\|help\|support\|kb\|learn\|guides?)\.", re.I), _R(r"^(docs\|documentation\|guides?\|help center\|knowledge base\|manuals?\|tutorials?\|learn)$", re.I), 0.8),
	120	+ (Surface.STATUS, _R(r"^https?://(status\|health\|uptime\|trust)\.\|/(status\|system-status\|service-status)(/\|$)", re.I), _R(r"^(status\|system status\|service status\|status page)$", re.I), 0.85),
	121	+ (Surface.BLOG, _R(r"/(blog\|blogs\|insights\|stories\|articles\|journal\|magazine\|perspectives\|thinking\|ideas\|posts\|editorial\|le-blog)(/\|$)\|^https?://(blog\|insights\|stories\|medium)\.", re.I), _R(r"^(blog\|insights\|stories\|articles\|journal\|perspectives\|ideas\|our blog)$", re.I), 0.82),
	122	+ (Surface.RESEARCH, _R(r"/(research\|labs?\|science\|publications\|papers\|whitepapers?\|reports\|studies)(/\|$)\|^https?://(research\|labs?\|science)\.", re.I), _R(r"^(research\|labs?\|publications\|whitepapers?\|reports\|science)$", re.I), 0.78),
	123	+ (Surface.PRODUCTS, _R(r"/(products?\|product-catalog\|catalog(ue)?\|shop\|store\|collections\|portfolio\|offerings\|our-products\|produits\|produkte\|productos\|prodotti\|製品)(/\|$)\|^https?://(shop\|store\|products?)\.", re.I), _R(r"^(products?\|our products\|product catalog\|catalog(ue)?\|shop\|store\|portfolio\|offerings\|produits\|produkte)$", re.I), 0.82),
	124	+ (Surface.SERVICES, _R(r"/(services?\|what-we-do\|capabilities\|expertise\|offerings\|our-services\|prestations\|leistungen\|servicios\|servizi)(/\|$)", re.I), _R(r"^(services?\|our services\|what we do\|capabilities\|expertise\|leistungen\|prestations)$", re.I), 0.78),
	125	+ (Surface.SOLUTIONS, _R(r"/(solutions?\|use-cases\|platform\|technology\|technologies\|features)(/\|$)", re.I), _R(r"^(solutions?\|use cases\|platform\|features)$", re.I), 0.7),
	126	+ (Surface.INDUSTRIES, _R(r"/(industries\|industry\|sectors?\|markets?\|verticals?\|who-we-serve)(/\|$)", re.I), _R(r"^(industries\|sectors?\|markets?\|who we serve\|verticals?)$", re.I), 0.7),
	127	+ (Surface.CUSTOMERS, _R(r"/(customers?\|customer-stories\|case-studies\|success-stories\|clients?\|references\|testimonials\|showcase\|wall-of-love)(/\|$)", re.I), _R(r"^(customers?\|customer stories\|case studies\|success stories\|clients?\|our customers\|testimonials)$", re.I), 0.78),
	128	+ (Surface.PARTNERS, _R(r"/(partners?\|partnerships?\|partner-program\|alliances\|ecosystem\|marketplace\|resellers?\|channel)(/\|$)\|^https?://(partners?\|marketplace)\.", re.I), _R(r"^(partners?\|partnerships?\|partner program\|alliances\|ecosystem\|become a partner\|marketplace)$", re.I), 0.78),
	129	+ (Surface.SECURITY, _R(r"/(security\|trust(-center\|-portal)?\|compliance\|privacy-and-security)(/\|$)\|^https?://(security\|trust)\.", re.I), _R(r"^(security\|trust( center)?\|compliance\|trust & safety)$", re.I), 0.8),
	130	+ (Surface.SUSTAINABILITY, _R(r"/(sustainability\|esg\|responsibility\|corporate-responsibility\|csr\|impact\|environment\|climate\|social-impact\|citizenship\|purpose\|d[eé]veloppement-durable\|nachhaltigkeit\|sostenibilidad)(/\|$)\|^https?://(sustainability\|esg\|impact)\.", re.I), _R(r"^(sustainability\|esg\|(corporate \|social )?responsibility\|csr\|impact\|environment\|climate\|our impact\|purpose)$", re.I), 0.82),
	131	+ (Surface.LEGAL_PRIVACY, _R(r"/(privacy(-policy\|-notice\|-statement)?\|privacypolicy\|datenschutz\|confidentialit[eé]\|privacidad\|cookie-policy\|cookies)(/\|$)", re.I), _R(r"^(privacy( policy\| notice\| statement)?\|datenschutz(erkl[aä]rung)?\|politique de confidentialit[eé]\|cookie policy)$", re.I), 0.9),
	132	+ (Surface.LEGAL_TERMS, _R(r"/(terms(-of-(service\|use\|sale\|business))?\|tos\|legal\|legal-notice\|terms-and-conditions\|conditions\|eula\|agb\|mentions-l[eé]gales\|cgu\|cgv\|aviso-legal\|impressum\|acceptable-use(-policy)?\|aup)(/\|$)", re.I), _R(r"^(terms( of (service\|use\|sale))?\|terms (and\|&) conditions\|legal( notice)?\|agb\|mentions l[eé]gales\|impressum\|eula\|acceptable use policy)$", re.I), 0.88),
	133	+ (Surface.SUPPORT, _R(r"/(support\|customer-support\|customer-service\|contact-support\|faq\|faqs\|community\|forum)(/\|$)\|^https?://(community\|forum\|faq)\.", re.I), _R(r"^(support\|customer (support\|service\|care)\|faqs?\|community\|forum\|help & support)$", re.I), 0.72),
	134	+ (Surface.CONTACT, _R(r"/(contact(-us\|us\|-sales)?\|get-in-touch\|reach-us\|kontakt\|contactez-nous\|contacto\|お問い合わせ)(/\|$)", re.I), _R(r"^(contact( us\| sales)?\|get in touch\|talk to (us\|sales)\|kontakt\|contactez-nous\|contacto)$", re.I), 0.85),
	135	+ (Surface.ABOUT, _R(r"/(about(-us\|us\|-company\|-[a-z0-9-]+)?\|company\|our-company\|our-story\|who-we-are\|mission\|history\|overview\|corporate\|a-propos\|qui-sommes-nous\|[uü]ber-uns\|unternehmen\|sobre-nosotros\|chi-siamo\|会社概要\|企業情報)(/\|$)", re.I), _R(r"^(about( us\| the company)?\|company\|our (company\|story\|mission\|history)\|who we are\|mission\|history\|overview\|a propos\|qui sommes-nous\|[uü]ber uns\|unternehmen)$", re.I), 0.85),
	136	+ (Surface.FEED, _R(r"/(feed\|rss\|atom\|feeds)(\.xml\|/\|$)\|\.(rss\|atom)$\|/rss\.xml\|/feed\.xml\|/atom\.xml", re.I), _R(r"^(rss\|atom\|feed\|subscribe via rss)$", re.I), 0.9),
	137	+ (Surface.SITEMAP, _R(r"/sitemap[^/]*\.xml(\.gz)?$\|/sitemap_index\.xml$\|/sitemaps?/", re.I), None, 0.95),
	138	+]
	139	+
	140	+# Anchors that are almost always navigation noise, never surfaces.
	141	+_NOISE_ANCHOR = re.compile(r"^(home\|back\|next\|previous\|prev\|more\|read more\|learn more\|skip to content\|menu\|close\|login\|log in\|sign in\|sign up\|"
	142	+ r"register\|search\|cart\|account\|download\|share\|print\|top\|en\|fr\|de\|es\|it\|ja\|zh\|português\|english\|français\|deutsch)$", re.I)
	143	+
	144	+
	145	+def classify_url(url: str, *, anchor: str \| None = None, title: str \| None = None, canonical_domain: str \| None = None) -> tuple[Surface, float]:
	146	+ """Return (surface, confidence). Homepage detection needs `canonical_domain`. Unknown → (OTHER, 0.1)."""
	147	+ p = urlparse(url)
	148	+ path = p.path or "/"
	149	+ text = " ".join(x.strip() for x in (anchor, title) if x).strip()
	150	+ text_norm = re.sub(r"\s+", " ", text).strip(" -–\|·»›>").lower()
	151	+ if canonical_domain and path in ("/", "") and not p.query and registrable_domain(url) == registrable_domain(canonical_domain) \
	152	+ and host_of(url) in (canonical_domain.lower(), "www." + canonical_domain.lower(), canonical_domain.lower().removeprefix("www.")):
	153	+ return Surface.HOMEPAGE, 0.99
	154	+ if is_static_asset(url):
	155	+ return Surface.OTHER, 0.0
	156	+ best: tuple[Surface, float] = (Surface.OTHER, 0.1)
	157	+ full = f"{p.scheme}://{p.netloc}{path}"
	158	+ depth = max(0, path.strip("/").count("/"))
	159	+ for surface, path_re, anchor_re, base in RULES:
	160	+ score = 0.0
	161	+ path_hit = bool(path_re and path_re.search(full))
	162	+ anchor_hit = bool(anchor_re and text_norm and anchor_re.search(text_norm))
	163	+ if path_hit:
	164	+ score = base
	165	+ if depth >= 2:
	166	+ score -= 0.12 * (depth - 1) # /news/2024/foo is an article, not the newsroom
	167	+ if anchor_hit:
	168	+ score = max(score, base - 0.2) + (0.05 if path_hit else 0.0)
	169	+ if is_document(url):
	170	+ score -= 0.3
	171	+ if p.query and surface not in (Surface.JOBS_BOARD, Surface.SITEMAP):
	172	+ score -= 0.1
	173	+ if score > best[1]:
	174	+ best = (surface, round(min(0.99, max(0.0, score)), 3))
	175	+ if best[0] is Surface.OTHER and text_norm and _NOISE_ANCHOR.match(text_norm):
	176	+ return Surface.OTHER, 0.0
	177	+ return best
	178	+
	179	+
	180	+ATS_PATTERNS: list[tuple[str, re.Pattern[str]]] = [
	181	+ ("greenhouse", re.compile(r"https?://(?:boards\|job-boards)\.greenhouse\.io/([a-z0-9_-]+)", re.I)),
	182	+ ("greenhouse", re.compile(r"https?://boards-api\.greenhouse\.io/v1/boards/([a-z0-9_-]+)", re.I)),
	183	+ ("lever", re.compile(r"https?://jobs\.(?:eu\.)?lever\.co/([a-z0-9_-]+)", re.I)),
	184	+ ("ashby", re.compile(r"https?://jobs\.ashbyhq\.com/([a-z0-9_.-]+)", re.I)),
	185	+ ("smartrecruiters", re.compile(r"https?://(?:careers\|jobs)\.smartrecruiters\.com/([a-z0-9_-]+)", re.I)),
	186	+ ("workable", re.compile(r"https?://apply\.workable\.com/([a-z0-9_-]+)", re.I)),
	187	+ ("workday", re.compile(r"https?://([a-z0-9-]+)\.(wd\d+)\.myworkdayjobs\.com/(?:[a-z]{2}-[A-Z]{2}/)?([A-Za-z0-9_-]+)", re.I)),
	188	+ ("recruitee", re.compile(r"https?://([a-z0-9-]+)\.recruitee\.com", re.I)),
	189	+ ("personio", re.compile(r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de\|com)", re.I)),
	190	+ ("teamtailor", re.compile(r"https?://([a-z0-9-]+)\.teamtailor\.com", re.I)),
	191	+ ("bamboohr", re.compile(r"https?://([a-z0-9-]+)\.bamboohr\.com/careers", re.I)),
	192	+ ("breezy", re.compile(r"https?://([a-z0-9-]+)\.breezy\.hr", re.I)),
	193	+ ("jobvite", re.compile(r"https?://jobs\.jobvite\.com/([a-z0-9_-]+)", re.I)),
	194	+ ("pinpoint", re.compile(r"https?://([a-z0-9-]+)\.pinpointhq\.com", re.I)),
	195	+ ("rippling", re.compile(r"https?://ats\.rippling\.com/([a-z0-9_-]+)", re.I)),
	196	+]
	197	+
	198	+
	199	+def detect_ats(url: str) -> tuple[str, str] \| None:
	200	+ """(vendor, board token) when the URL is a public ATS board we have a structured connector for."""
	201	+ for vendor, pat in ATS_PATTERNS:
	202	+ m = pat.search(url)
	203	+ if m:
	204	+ return vendor, m.group(1)
	205	+ return None
	206	+
	207	+
	208	+__all__ = ["ATS_PATTERNS", "RULES", "absolutize", "canonicalize_url", "classify_url", "detect_ats", "host_of", "is_document",
	209	+ "is_static_asset", "looks_like_trap", "registrable_domain", "same_company_host"]
	210