kernel: ruff fixes, README
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
4 changed files +51 −7
added
README.md
+44 −0
@@ -0,0 +1,44 @@ | |||||
| 1 | +# Company Atlas — The Live Atlas of Global Companies | ||||
| 2 | + | ||||
| 3 | +**https://www.company-atlas.co** · a continuously updated corporate observation network. | ||||
| 4 | + | ||||
| 5 | +Company Atlas attaches persistent public-web sensors to companies (homepage, careers and job boards, newsroom, pricing, products, | ||||
| 6 | +leadership, locations, docs, changelog, investor relations, legal pages, feeds, sitemaps…). Every sensor produces observations; | ||||
| 7 | +observations become historical snapshots; snapshots produce block-level changes; meaningful changes become structured, carefully | ||||
| 8 | +worded events; events become metrics (Activity Score, Hiring Momentum, Product Velocity, AI Adoption, Corporate Change Index, Global | ||||
| 9 | +Corporate Activity Index) and signals. Nothing historical is ever overwritten. **The accumulated history is the product.** | ||||
| 10 | + | ||||
| 11 | +| | | | ||||
| 12 | +|---|---| | ||||
| 13 | +| Spec | `docs/PRODUCT-SPEC.md` (200 sections) · repo guide `CLAUDE.md` | | ||||
| 14 | +| Architecture | `docs/ARCHITECTURE.md` · crawl `docs/CRAWL.md` · connectors `docs/CONNECTORS.md` · events `docs/EVENT-TAXONOMY.md` · scoring `docs/SCORING.md` · LLM `docs/LLM.md` · seeds `docs/SEEDS.md` | | ||||
| 15 | +| API | `docs/API.md` (`/api/v1`, SSE `/api/v1/live/stream`, exports, admin) — live docs at `/api/v1/docs` | | ||||
| 16 | +| Web | `apps/web` (Next 16) — `docs/FRONTEND.md` | | ||||
| 17 | +| Deploy / ops | `docs/DEPLOY.md` · `docs/OPERATIONS.md` | | ||||
| 18 | + | ||||
| 19 | +## Stack | ||||
| 20 | +Python 3.12 (`src/companyatlas`, CLI `catlas`: FastAPI · SQLAlchemy Core + asyncpg · Alembic · httpx · selectolax · feedparser · zstd) · | ||||
| 21 | +PostgreSQL 17 (the only stateful dependency: entities, history, `SKIP LOCKED` queue, metrics) · content-addressed object store on disk · | ||||
| 22 | +Next 16 + React 19 + Tailwind v4 · optional OpenAI-compatible LLM endpoint for enrichment (MacLustr llm-api.io). | ||||
| 23 | + | ||||
| 24 | +## Quickstart | ||||
| 25 | +```bash | ||||
| 26 | +uv venv --python 3.12 .venv && uv pip install --python .venv/bin/python -e '.[dev]' | ||||
| 27 | +createdb -O companyatlas companyatlas # role companyatlas/companyatlas; extensions pg_trgm + uuid-ossp | ||||
| 28 | +cp .env.example .env | ||||
| 29 | +.venv/bin/catlas migrate && .venv/bin/catlas seed # schema + industries/countries/companies (Wikidata registry) | ||||
| 30 | +.venv/bin/catlas discover https://stripe.com --dry-run | ||||
| 31 | +.venv/bin/catlas onboard --limit 100 # discovery → sensors | ||||
| 32 | +.venv/bin/catlas schedule # crawl + intelligence loop | ||||
| 33 | +.venv/bin/catlas api # http://127.0.0.1:8371/api/v1/docs | ||||
| 34 | +pnpm install && pnpm dev:web # http://localhost:8370 | ||||
| 35 | +.venv/bin/pytest -q && .venv/bin/ruff check src tests && pnpm -r typecheck | ||||
| 36 | +``` | ||||
| 37 | + | ||||
| 38 | +## Principles (short form) | ||||
| 39 | +Historical-first · raw and interpreted data kept separate · deterministic pipeline first, LLMs only as budgeted enrichment · reusable | ||||
| 40 | +connector families, automatic discovery · polite and lawful crawling (robots, per-domain limits, SSRF guard, never bypass challenges, | ||||
| 41 | +no private data) · provenance and confidence on every fact · careful language ("no longer listed", never "fired") · no magic numbers · | ||||
| 42 | +mobile first-class · never fabricate. | ||||
| 43 | + | ||||
| 44 | +Contact: contact@spboucher.ai · crawler identity `CompanyAtlasBot` (see `/bot`). Hosted on MacLustr. | ||||
modified
src/companyatlas/commands/ops.py
+2 −2
@@ -49,7 +49,7 @@ def stats() -> None: | |||||
| 49 | started = row.get("dataset_started_at") | 49 | started = row.get("dataset_started_at") | ||
| 50 | if started: | 50 | if started: | ||
| 51 | try: | 51 | try: | ||
| 52 | − dt = datetime.fromisoformat(str(started).strip('"').replace("Z", "+00:00")) | 52 | + dt = datetime.fromisoformat(str(started).strip('"')) | ||
| 53 | row["dataset_age_days"] = (datetime.now(UTC) - dt).days | 53 | row["dataset_age_days"] = (datetime.now(UTC) - dt).days | ||
| 54 | except ValueError: | 54 | except ValueError: | ||
| 55 | pass | 55 | pass | ||
@@ -132,5 +132,5 @@ try: # register the nightly backup as a periodic task when the scheduler import | |||||
| 132 | import asyncio | 132 | import asyncio | ||
| 133 | 133 | ||||
| 134 | await asyncio.to_thread(backup, 14) | 134 | await asyncio.to_thread(backup, 14) | ||
| 135 | −except Exception: # noqa: BLE001 | 135 | +except Exception: # noqa: BLE001, S110 — optional: scheduler not present in this process | ||
| 136 | pass | 136 | pass | ||
modified
src/companyatlas/logging.py
+1 −1
@@ -37,7 +37,7 @@ class JsonFormatter(logging.Formatter): | |||||
| 37 | class PlainFormatter(logging.Formatter): | 37 | class PlainFormatter(logging.Formatter): | ||
| 38 | def format(self, record: logging.LogRecord) -> str: | 38 | def format(self, record: logging.LogRecord) -> str: | ||
| 39 | extras = {k: v for k, v in record.__dict__.items() if k not in _RESERVED and not k.startswith("_")} | 39 | extras = {k: v for k, v in record.__dict__.items() if k not in _RESERVED and not k.startswith("_")} | ||
| 40 | − base = f"{datetime.now().strftime('%H:%M:%S')} {record.levelname:<7} {record.name}: {record.getMessage()}" | 40 | + base = f"{datetime.now(UTC).astimezone().strftime('%H:%M:%S')} {record.levelname:<7} {record.name}: {record.getMessage()}" | ||
| 41 | if extras: | 41 | if extras: | ||
| 42 | base += " " + " ".join(f"{k}={v}" for k, v in extras.items()) | 42 | base += " " + " ".join(f"{k}={v}" for k, v in extras.items()) | ||
| 43 | if record.exc_info: | 43 | if record.exc_info: | ||
modified
src/companyatlas/services/periodic.py
+4 −4
@@ -53,7 +53,7 @@ class PeriodicTask: | |||||
| 53 | await self.fn() | 53 | await self.fn() | ||
| 54 | self.runs += 1 | 54 | self.runs += 1 | ||
| 55 | self.last_error = None | 55 | self.last_error = None | ||
| 56 | − except Exception as exc: # noqa: BLE001 | 56 | + except Exception as exc: | ||
| 57 | self.failures += 1 | 57 | self.failures += 1 | ||
| 58 | self.last_error = f"{exc.__class__.__name__}: {exc}"[:500] | 58 | self.last_error = f"{exc.__class__.__name__}: {exc}"[:500] | ||
| 59 | log.exception("periodic task failed", extra={"task": self.name}) | 59 | log.exception("periodic task failed", extra={"task": self.name}) | ||
@@ -98,9 +98,9 @@ def load_task_modules() -> list[str]: | |||||
| 98 | except ModuleNotFoundError as exc: | 98 | except ModuleNotFoundError as exc: | ||
| 99 | if exc.name and (mod == exc.name or mod.startswith(exc.name + ".")): | 99 | if exc.name and (mod == exc.name or mod.startswith(exc.name + ".")): | ||
| 100 | continue | 100 | continue | ||
| 101 | − log.exception("task module failed to import", extra={"module": mod}) | 101 | + log.exception("task module failed to import", extra={"task_module": mod}) | ||
| 102 | − except Exception: # noqa: BLE001 | 102 | + except Exception: | ||
| 103 | − log.exception("task module failed to import", extra={"module": mod}) | 103 | + log.exception("task module failed to import", extra={"task_module": mod}) | ||
| 104 | return loaded | 104 | return loaded | ||
| 105 | 105 | ||||
| 106 | 106 | ||||
| 107 | 107 | ||||