SPB Git forge
spboucher.ai ↗

spb/ai-atlas

Public
41commits 1branches 0releases
4.6 MBsize
maindefault branch
12 days agolast push
HTML 77.2% TypeScript 10.5% Python 9.6% JavaScript 2.5%

Lint fixes

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Simon-Pierre Boucher committed 13 days ago (Sep 11, 2026) parent 9f1db17

20 changed files +113 −52

modified src/aiatlas/db/__init__.py +1 −1
@@ -72,4 +72,4 @@ async def execute_many(conn: AsyncConnection, sql: str, rows: Sequence[Mapping[s
72 72 await conn.execute(text(sql), list(rows))
73 73
74 74
75 −__all__ = ["engine", "dispose", "connection", "transaction", "execute", "fetch_all", "fetch_one", "fetch_val", "execute_many", "jsonb"]
75 +__all__ = ["connection", "dispose", "engine", "execute", "execute_many", "fetch_all", "fetch_one", "fetch_val", "jsonb", "transaction"]
modified src/aiatlas/registry/__init__.py +2 −2
@@ -58,7 +58,7 @@ def org_by_github(gh_org: str) -> dict[str, Any] | None:
58 58 @lru_cache
59 59 def org_by_domain(domain: str) -> dict[str, Any] | None:
60 60 d = domain.lower()
61 − d = d[4:] if d.startswith("www.") else d
61 + d = d.removeprefix("www.")
62 62 for o in organizations().values():
63 63 for od in o.get("domains", []):
64 64 if d == od or d.endswith("." + od):
@@ -102,4 +102,4 @@ def provider_by_openrouter(slug: str) -> str | None:
102 102 return None
103 103
104 104
105 −__all__ = ["load", "organizations", "providers", "org_by_hf", "org_by_github", "org_by_domain", "org_ref", "provider_ref", "provider_by_openrouter", "REGISTRY_DIR"]
105 +__all__ = ["REGISTRY_DIR", "load", "org_by_domain", "org_by_github", "org_by_hf", "org_ref", "organizations", "provider_by_openrouter", "provider_ref", "providers"]
modified src/aiatlas/sdk/__init__.py +19 −3
@@ -1,7 +1,23 @@
1 1 """Connector SDK — every connector shares this infrastructure: fetch transport, raw archive, extraction, facts, change detection."""
2 2 from aiatlas.sdk.connector import BaseConnector, ConnectorError, RunContext, Target
3 3 from aiatlas.sdk.facts import Claim, EntityRef, Event, Facts, PriceObs, Relation, ResultObs
4 −from aiatlas.sdk.fetch import BlockedError, FetchError, FetchResult, Fetcher, NotModified
4 +from aiatlas.sdk.fetch import BlockedError, Fetcher, FetchError, FetchResult, NotModified
5 5
6 −__all__ = ["BaseConnector", "ConnectorError", "RunContext", "Target", "Claim", "EntityRef", "Event", "Facts", "PriceObs",
7 − "Relation", "ResultObs", "BlockedError", "FetchError", "FetchResult", "Fetcher", "NotModified"]
6 +__all__ = [
7 + "BaseConnector",
8 + "BlockedError",
9 + "Claim",
10 + "ConnectorError",
11 + "EntityRef",
12 + "Event",
13 + "Facts",
14 + "FetchError",
15 + "FetchResult",
16 + "Fetcher",
17 + "NotModified",
18 + "PriceObs",
19 + "Relation",
20 + "ResultObs",
21 + "RunContext",
22 + "Target",
23 +]
modified src/aiatlas/sdk/archive.py +1 −1
@@ -71,4 +71,4 @@ def archive_size() -> dict[str, int]:
71 71 return out
72 72
73 73
74 −__all__ = ["store_raw", "store_text", "load_raw", "load_text", "archive_size"]
74 +__all__ = ["archive_size", "load_raw", "load_text", "store_raw", "store_text"]
modified src/aiatlas/sdk/connector.py +4 −5
@@ -16,7 +16,6 @@ from dataclasses import dataclass, field
16 16 from datetime import UTC, datetime, timedelta
17 17 from typing import Any
18 18
19 −from aiatlas.config import settings
20 19 from aiatlas.db import execute, fetch_one, jsonb, transaction
21 20 from aiatlas.ids import new_id
22 21 from aiatlas.sdk import archive
@@ -24,7 +23,7 @@ from aiatlas.sdk.extract.feeds import FeedItem, parse_feed
24 23 from aiatlas.sdk.extract.html import HtmlDoc, parse_html
25 24 from aiatlas.sdk.extract.markdown import MarkdownDoc, parse_markdown
26 25 from aiatlas.sdk.facts import Facts, Target
27 −from aiatlas.sdk.fetch import BlockedError, FetchError, Fetcher, FetchResult, NotModified, canonicalize_url, file_result
26 +from aiatlas.sdk.fetch import BlockedError, Fetcher, FetchError, FetchResult, NotModified, canonicalize_url, file_result
28 27 from aiatlas.sdk.writer import FactWriter
29 28
30 29 log = logging.getLogger(__name__)
@@ -208,7 +207,7 @@ class BaseConnector:
208 207 d=f"parser_breakage:{self.name}:{started.date()}")
209 208 ctx.log.info("run finished", extra={"status": status, **{k: v for k, v in ctx.stats.__dict__.items() if k != "meta"},
210 209 "ms": int((time.perf_counter() - t0) * 1000)})
211 − except Exception as exc: # noqa: BLE001
210 + except Exception as exc:
212 211 ctx.log.exception("run failed", extra={"error": str(exc)})
213 212 async with transaction() as conn:
214 213 await self._record(conn, ctx, "failed", error=f"{exc.__class__.__name__}: {exc}"[:2000])
@@ -385,7 +384,7 @@ class BaseConnector:
385 384 parsed = parsed or self.parse(target, res)
386 385 try:
387 386 facts = await self.extract(ctx, target, res, parsed)
388 − except Exception as exc: # noqa: BLE001
387 + except Exception as exc:
389 388 ctx.log.exception("extract failed", extra={"url": res.url})
390 389 async with transaction() as conn:
391 390 await execute(conn, "insert into connector_errors (connector_name, run_id, url, error_type, message) values (:n, :r, :u, :t, :m)",
@@ -480,4 +479,4 @@ def _jsonable(obj: Any) -> Any:
480 479 return json.loads(json.dumps(obj, default=str))
481 480
482 481
483 −__all__ = ["BaseConnector", "ConnectorError", "RunContext", "RunStats", "Parsed", "Target"]
482 +__all__ = ["BaseConnector", "ConnectorError", "Parsed", "RunContext", "RunStats", "Target"]
modified src/aiatlas/sdk/extract/__init__.py +24 −4
@@ -4,9 +4,29 @@ from aiatlas.sdk.extract.dates import parse_date, parse_datetime
4 4 from aiatlas.sdk.extract.feeds import FeedItem, parse_feed
5 5 from aiatlas.sdk.extract.html import HtmlDoc, parse_html
6 6 from aiatlas.sdk.extract.markdown import MarkdownDoc, parse_front_matter, parse_markdown
7 −from aiatlas.sdk.extract.numbers import parse_context_length, parse_money_per_mtok, parse_param_count, parse_percent, parse_tokens
7 +from aiatlas.sdk.extract.numbers import (
8 + parse_context_length,
9 + parse_money_per_mtok,
10 + parse_param_count,
11 + parse_percent,
12 + parse_tokens,
13 +)
8 14 from aiatlas.sdk.extract.sitemap import parse_sitemap
9 15
10 −__all__ = ["parse_date", "parse_datetime", "FeedItem", "parse_feed", "HtmlDoc", "parse_html", "MarkdownDoc", "parse_front_matter",
11 − "parse_markdown", "parse_context_length", "parse_money_per_mtok", "parse_param_count", "parse_percent", "parse_tokens",
12 − "parse_sitemap"]
16 +__all__ = [
17 + "FeedItem",
18 + "HtmlDoc",
19 + "MarkdownDoc",
20 + "parse_context_length",
21 + "parse_date",
22 + "parse_datetime",
23 + "parse_feed",
24 + "parse_front_matter",
25 + "parse_html",
26 + "parse_markdown",
27 + "parse_money_per_mtok",
28 + "parse_param_count",
29 + "parse_percent",
30 + "parse_sitemap",
31 + "parse_tokens",
32 +]
modified src/aiatlas/sdk/extract/dates.py +2 −2
@@ -7,7 +7,7 @@ from datetime import UTC, date, datetime
7 7 from dateutil import parser as duparser
8 8
9 9 _MONTHS = "january|february|march|april|may|june|july|august|september|october|november|december|jan|feb|mar|apr|jun|jul|aug|sep|sept|oct|nov|dec"
10 −_MONTH_YEAR = re.compile(rf"\b({_MONTHS})\.?\s+(\d{{4}})\b", re.I)
10 +_MONTH_YEAR = re.compile(rf"\b({_MONTHS})\.?\s+(\d{{4}})\b", re.IGNORECASE)
11 11 _YEAR = re.compile(r"\b(19[89]\d|20[0-4]\d)\b")
12 12
13 13
@@ -68,4 +68,4 @@ def iso(dt: datetime | date | None) -> str | None:
68 68 return dt.isoformat()
69 69
70 70
71 −__all__ = ["parse_datetime", "parse_date", "iso"]
71 +__all__ = ["iso", "parse_date", "parse_datetime"]
modified src/aiatlas/sdk/extract/html.py +1 −1
@@ -233,4 +233,4 @@ def find_in_json(obj: Any, key: str, *, max_hits: int = 50) -> list[Any]:
233 233 return hits
234 234
235 235
236 −__all__ = ["HtmlDoc", "parse_html", "clean_text", "node_text", "extract_text", "find_in_json"]
236 +__all__ = ["HtmlDoc", "clean_text", "extract_text", "find_in_json", "node_text", "parse_html"]
modified src/aiatlas/sdk/extract/markdown.py +7 −9
@@ -7,8 +7,8 @@ from typing import Any
7 7
8 8 import yaml
9 9
10 −_FM = re.compile(r"\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*\r?\n", re.S)
11 −_HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$", re.M)
10 +_FM = re.compile(r"\A---[ \t]*\r?\n(.*?)\r?\n---[ \t]*\r?\n", re.DOTALL)
11 +_HEADING = re.compile(r"^(#{1,6})\s+(.*?)\s*#*\s*$", re.MULTILINE)
12 12 _LINK = re.compile(r"\[([^\]]*)\]\((https?://[^)\s]+)\)")
13 13 _TABLE_ROW = re.compile(r"^\s*\|(.+)\|\s*$")
14 14 _SEP_ROW = re.compile(r"^\s*\|?\s*:?-{2,}:?\s*(\|\s*:?-{2,}:?\s*)*\|?\s*$")
@@ -25,7 +25,7 @@ class MarkdownDoc:
25 25
26 26 def section(self, title_pattern: str) -> str:
27 27 """Body text of the first heading matching `title_pattern` (case-insensitive) up to the next heading of same/higher level."""
28 − rx = re.compile(title_pattern, re.I)
28 + rx = re.compile(title_pattern, re.IGNORECASE)
29 29 lines = self.body.split("\n")
30 30 start = None
31 31 level = 0
@@ -83,19 +83,17 @@ def _tables(body: str) -> list[dict[str, Any]]:
83 83
84 84 def _cells(line: str) -> list[str]:
85 85 inner = line.strip()
86 − if inner.startswith("|"):
87 − inner = inner[1:]
88 − if inner.endswith("|"):
89 − inner = inner[:-1]
86 + inner = inner.removeprefix("|")
87 + inner = inner.removesuffix("|")
90 88 return [re.sub(r"\*\*|`", "", c).strip() for c in inner.split("|")]
91 89
92 90
93 91 def _to_text(body: str) -> str:
94 − s = re.sub(r"```.*?```", " ", body, flags=re.S)
92 + s = re.sub(r"```.*?```", " ", body, flags=re.DOTALL)
95 93 s = re.sub(r"!\[[^\]]*\]\([^)]*\)", " ", s)
96 94 s = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", s)
97 95 s = re.sub(r"<[^>]+>", " ", s)
98 − s = re.sub(r"^[#>*\-|\s]+", "", s, flags=re.M)
96 + s = re.sub(r"^[#>*\-|\s]+", "", s, flags=re.MULTILINE)
99 97 s = re.sub(r"[ \t]+", " ", s)
100 98 return re.sub(r"\n{3,}", "\n\n", s).strip()
101 99
modified src/aiatlas/sdk/extract/numbers.py +15 −7
@@ -5,10 +5,10 @@ import re
5 5
6 6 _MULT = {"k": 1e3, "m": 1e6, "b": 1e9, "t": 1e12, "g": 1e9}
7 7
8 −_PARAMS = re.compile(r"(?<![\w.])(\d+(?:[.,]\d+)?)\s*([kKmMbBtT])\b(?:\s*(?:params?|parameters?))?", re.I)
9 −_PARAMS_WORD = re.compile(r"(\d+(?:[.,]\d+)?)\s*(billion|million|trillion)\s*(?:params?|parameters?)", re.I)
10 −_CONTEXT = re.compile(r"(\d+(?:[.,]\d+)?)\s*([kKmM])?\s*(?:tokens?|token|ctx|context)?", re.I)
11 −_MONEY = re.compile(r"(?:US)?\$\s*(\d+(?:[.,]\d+)?)\s*(?:/|per)\s*(?:1\s*)?([mMkK])\s*(?:tok(?:ens?)?)?", re.I)
8 +_PARAMS = re.compile(r"(?<![\w.])(\d+(?:[.,]\d+)?)\s*([kKmMbBtT])\b(?:\s*(?:params?|parameters?))?", re.IGNORECASE)
9 +_PARAMS_WORD = re.compile(r"(\d+(?:[.,]\d+)?)\s*(billion|million|trillion)\s*(?:params?|parameters?)", re.IGNORECASE)
10 +_CONTEXT = re.compile(r"(\d+(?:[.,]\d+)?)\s*([kKmM])?\s*(?:tokens?|token|ctx|context)?", re.IGNORECASE)
11 +_MONEY = re.compile(r"(?:US)?\$\s*(\d+(?:[.,]\d+)?)\s*(?:/|per)\s*(?:1\s*)?([mMkK])\s*(?:tok(?:ens?)?)?", re.IGNORECASE)
12 12 _MONEY_SIMPLE = re.compile(r"(?:US)?\$\s*(\d+(?:\.\d+)?)")
13 13 _PCT = re.compile(r"(-?\d+(?:\.\d+)?)\s*%")
14 14
@@ -34,7 +34,7 @@ def parse_active_params(text: str) -> int | None:
34 34 """'235B-A22B' / '30B-A3B' → active parameters of an MoE model."""
35 35 m = re.search(r"-A(\d+(?:\.\d+)?)([bBmM])\b", text)
36 36 if not m:
37 − m = re.search(r"(\d+(?:\.\d+)?)\s*([bB])\s*active", text, re.I)
37 + m = re.search(r"(\d+(?:\.\d+)?)\s*([bB])\s*active", text, re.IGNORECASE)
38 38 if not m:
39 39 return None
40 40 return int(_num(m.group(1)) * _MULT[m.group(2).lower()])
@@ -102,5 +102,13 @@ def parse_int(text: str) -> int | None:
102 102 return None
103 103
104 104
105 −__all__ = ["parse_param_count", "parse_active_params", "parse_context_length", "parse_tokens", "parse_money_per_mtok", "parse_money",
106 − "parse_percent", "parse_int"]
105 +__all__ = [
106 + "parse_active_params",
107 + "parse_context_length",
108 + "parse_int",
109 + "parse_money",
110 + "parse_money_per_mtok",
111 + "parse_param_count",
112 + "parse_percent",
113 + "parse_tokens",
114 +]
modified src/aiatlas/sdk/facts.py +14 −2
@@ -218,5 +218,17 @@ EVENT_CATEGORY_BY_TYPE: dict[str, str] = {
218 218 "mcp_server": "tool", "researcher": "company", "license": "model", "product": "tool", "robot": "hardware",
219 219 }
220 220
221 −__all__ = ["EntityRef", "Claim", "Relation", "Event", "PriceObs", "ResultObs", "Target", "Facts", "MATERIAL_PROPERTIES",
222 − "NOISY_PREFIXES", "EVENT_CATEGORY_BY_TYPE", "CONFIDENCE"]
221 +__all__ = [
222 + "CONFIDENCE",
223 + "EVENT_CATEGORY_BY_TYPE",
224 + "MATERIAL_PROPERTIES",
225 + "NOISY_PREFIXES",
226 + "Claim",
227 + "EntityRef",
228 + "Event",
229 + "Facts",
230 + "PriceObs",
231 + "Relation",
232 + "ResultObs",
233 + "Target",
234 +]
modified src/aiatlas/sdk/fetch.py +2 −2
@@ -102,7 +102,7 @@ def canonicalize_url(url: str) -> str:
102 102
103 103 def domain_of(url: str) -> str:
104 104 host = urlparse(url).netloc.lower()
105 − return host[4:] if host.startswith("www.") else host
105 + return host.removeprefix("www.")
106 106
107 107
108 108 class _RateLimiter:
@@ -326,4 +326,4 @@ def file_result(path: str, *, url: str, content_type: str = "application/octet-s
326 326 _limiter = _RateLimiter()
327 327 _robots = _Robots()
328 328
329 −__all__ = ["Fetcher", "FetchResult", "FetchError", "BlockedError", "NotModified", "canonicalize_url", "domain_of", "file_result"]
329 +__all__ = ["BlockedError", "FetchError", "FetchResult", "Fetcher", "NotModified", "canonicalize_url", "domain_of", "file_result"]
modified src/aiatlas/sdk/writer.py +9 −1
@@ -19,7 +19,15 @@ from sqlalchemy.ext.asyncio import AsyncConnection
19 19
20 20 from aiatlas.db import execute, fetch_one, jsonb
21 21 from aiatlas.ids import new_id, normalize_alias
22 −from aiatlas.sdk.facts import EVENT_CATEGORY_BY_TYPE, MATERIAL_PROPERTIES, NOISY_PREFIXES, EntityRef, Facts, PriceObs, ResultObs
22 +from aiatlas.sdk.facts import (
23 + EVENT_CATEGORY_BY_TYPE,
24 + MATERIAL_PROPERTIES,
25 + NOISY_PREFIXES,
26 + EntityRef,
27 + Facts,
28 + PriceObs,
29 + ResultObs,
30 +)
23 31 from aiatlas.sdk.resolution import Resolver
24 32
25 33 log = logging.getLogger(__name__)
modified src/aiatlas/services/cache.py +1 −1
@@ -96,4 +96,4 @@ async def close() -> None:
96 96 await asyncio.sleep(0)
97 97
98 98
99 −__all__ = ["redis", "cache_get", "cache_set", "cache_invalidate", "lock", "heartbeat", "heartbeats", "close"]
99 +__all__ = ["cache_get", "cache_invalidate", "cache_set", "close", "heartbeat", "heartbeats", "lock", "redis"]
modified src/aiatlas/services/handlers.py +2 −2
@@ -6,10 +6,10 @@ from datetime import UTC
6 6 from typing import Any
7 7
8 8 from aiatlas.db import execute, fetch_one, transaction
9 +from aiatlas.schemas import schema_for
9 10 from aiatlas.sdk import archive
10 11 from aiatlas.sdk.facts import EntityRef, Facts
11 12 from aiatlas.sdk.writer import FactWriter
12 −from aiatlas.schemas import schema_for
13 13 from aiatlas.services.jobs import handler
14 14 from aiatlas.services.llm import LLMUnavailable, gateway
15 15
@@ -253,4 +253,4 @@ async def recompute_quality(payload: dict[str, Any], job: dict[str, Any]) -> dic
253 253 return await recompute(entity_ids=payload.get("entity_ids"))
254 254
255 255
256 −__all__ = ["llm_extract", "facts_from_llm", "embed_entity", "reprocess_snapshot", "recompute_quality"]
256 +__all__ = ["embed_entity", "facts_from_llm", "llm_extract", "recompute_quality", "reprocess_snapshot"]
modified src/aiatlas/services/jobs.py +1 −1
@@ -125,4 +125,4 @@ async def requeue_stale(conn: AsyncConnection, *, older_than_minutes: int = 120)
125 125 return int(row["n"]) if row else 0
126 126
127 127
128 −__all__ = ["enqueue", "claim_next", "complete", "fail", "queue_depth", "run_worker", "handler", "requeue_stale"]
128 +__all__ = ["claim_next", "complete", "enqueue", "fail", "handler", "queue_depth", "requeue_stale", "run_worker"]
modified src/aiatlas/services/llm/gateway.py +3 −3
@@ -207,7 +207,7 @@ class LLMGateway:
207 207
208 208 def _parse_json(raw: str) -> dict[str, Any] | None:
209 209 s = raw.strip()
210 − s = re.sub(r"<think>.*?</think>", "", s, flags=re.S).strip()
210 + s = re.sub(r"<think>.*?</think>", "", s, flags=re.DOTALL).strip()
211 211 if s.startswith("```"):
212 212 s = re.sub(r"^```(?:json)?\s*", "", s)
213 213 s = re.sub(r"\s*```$", "", s)
@@ -216,7 +216,7 @@ def _parse_json(raw: str) -> dict[str, Any] | None:
216 216 return v if isinstance(v, dict) else None
217 217 except json.JSONDecodeError:
218 218 pass
219 − m = re.search(r"\{.*\}", s, flags=re.S)
219 + m = re.search(r"\{.*\}", s, flags=re.DOTALL)
220 220 if m:
221 221 try:
222 222 v = json.loads(m.group(0))
@@ -228,4 +228,4 @@ def _parse_json(raw: str) -> dict[str, Any] | None:
228 228
229 229 gateway = LLMGateway()
230 230
231 −__all__ = ["LLMGateway", "LLMResult", "LLMUnavailable", "Engine", "OpenAICompatEngine", "gateway"]
231 +__all__ = ["Engine", "LLMGateway", "LLMResult", "LLMUnavailable", "OpenAICompatEngine", "gateway"]
modified src/aiatlas/services/quality.py +1 −1
@@ -69,4 +69,4 @@ async def recompute(*, entity_ids: list[str] | None = None, limit: int = 20000)
69 69 return {"updated": updated}
70 70
71 71
72 −__all__ = ["recompute", "QUALITY_VERSION", "EXPECTED_FIELDS"]
72 +__all__ = ["EXPECTED_FIELDS", "QUALITY_VERSION", "recompute"]
modified src/aiatlas/services/search.py +3 −3
@@ -80,12 +80,12 @@ def compile_query(q: str) -> Query:
80 80 if m:
81 81 out.organization = m.group(1)
82 82 # residual free-text (remove the structured bits)
83 − residual = re.sub(r"\b(more than|over|above|less than|under|below|at least|at most|released|launched|published|since|after|before|until|with|and|in|from|by|the|a|an|context|window|params?|parameters)\b", " ", s, flags=re.I)
83 + residual = re.sub(r"\b(more than|over|above|less than|under|below|at least|at most|released|launched|published|since|after|before|until|with|and|in|from|by|the|a|an|context|window|params?|parameters)\b", " ", s, flags=re.IGNORECASE)
84 84 residual = re.sub(r"\d+(?:\.\d+)?\s*[bBmMkKtT]\b|\b20\d\d\b|[<>≤≥]", " ", residual)
85 85 for words in TYPE_WORDS.values():
86 86 for w in words:
87 − residual = re.sub(rf"\b{re.escape(w)}\b", " ", residual, flags=re.I)
88 − residual = re.sub(r"\b(open[- ]?(weight|weights|source)|proprietary|closed)\b", " ", residual, flags=re.I)
87 + residual = re.sub(rf"\b{re.escape(w)}\b", " ", residual, flags=re.IGNORECASE)
88 + residual = re.sub(r"\b(open[- ]?(weight|weights|source)|proprietary|closed)\b", " ", residual, flags=re.IGNORECASE)
89 89 out.filters["residual"] = " ".join(residual.split())
90 90 return out
91 91
modified src/aiatlas/services/stats.py +1 −1
@@ -50,4 +50,4 @@ async def history(conn: AsyncConnection, days: int = 90) -> list[dict[str, Any]]
50 50 from stats_snapshots where computed_at > now() - make_interval(days => :d) order by 1 desc, computed_at desc""", d=days)
51 51
52 52
53 −__all__ = ["live_counts", "compute_stats", "history"]
53 +__all__ = ["compute_stats", "history", "live_counts"]
54 54