scheduler: nightly canonicalize --apply (03:30 local, Redis lock 2 h) + light pass every 6 h with one-line summary; full API cache flush after each connector run documented; taxonomy: Apache Software License/Apache License → Apache-2.0, BSD License → BSD-3-Clause, framework kind 'model' and org_kind 'organization' no longer reported as unmapped
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 changed files +36 −3
modified
src/aiatlas/ontology/licenses.py
+3 −2
@@ -59,12 +59,13 @@ def _add(info: LicenseInfo) -> None: | ||
| 59 | 59 | |
| 60 | 60 | # ---------------------------------------------------------------------------------------------- permissive / copyleft (OSI) |
| 61 | 61 | _add(_permissive("Apache-2.0", "Apache License 2.0", "Apache-2.0", "https://www.apache.org/licenses/LICENSE-2.0", |
| 62 | − aliases=("apache 2.0", "apache-2", "apache2", "apache license 2.0", "apache license, version 2.0", "apache"))) | |
| 62 | + aliases=("apache 2.0", "apache-2", "apache2", "apache license 2.0", "apache license, version 2.0", "apache", "apache license", | |
| 63 | + "apache software license", "apache software license 2.0", "asl 2.0", "asl-2.0"))) | |
| 63 | 64 | _add(_permissive("MIT", "MIT License", "MIT", "https://opensource.org/license/mit", aliases=("mit license", "the mit license"))) |
| 64 | 65 | _add(LicenseInfo(key="MIT-Modified", label="Modified MIT License", category="permissive", spdx=None, commercial_use=True, redistribution=True, |
| 65 | 66 | derivatives=True, hosting_restrictions=None, attribution=True, acceptable_use=None, aliases=("modified mit", "mit-modified"))) |
| 66 | 67 | _add(_permissive("BSD-3-Clause", "BSD 3-Clause License", "BSD-3-Clause", "https://opensource.org/license/bsd-3-clause", |
| 67 | − aliases=("bsd-3", "bsd 3-clause", "bsd3", "new bsd", "modified bsd", "bsd"))) | |
| 68 | + aliases=("bsd-3", "bsd 3-clause", "bsd3", "new bsd", "modified bsd", "bsd", "bsd license", "bsd-3-clause license"))) | |
| 68 | 69 | _add(_permissive("BSD-2-Clause", "BSD 2-Clause License", "BSD-2-Clause", "https://opensource.org/license/bsd-2-clause", aliases=("bsd-2", "simplified bsd"))) |
| 69 | 70 | _add(_permissive("ISC", "ISC License", "ISC", "https://opensource.org/license/isc-license-txt")) |
| 70 | 71 | _add(_permissive("Unlicense", "The Unlicense", "Unlicense", "https://unlicense.org", attribution=False, aliases=("unlicense",))) |
modified
src/aiatlas/ontology/taxonomy.py
+6 −0
@@ -212,12 +212,18 @@ def normalize_property(entity_type: str | None, prop: str, value: Any) -> tuple[ | ||
| 212 | 212 | return value, None, [] |
| 213 | 213 | |
| 214 | 214 | |
| 215 | +# raw values that legitimately have no canonical counterpart (kept as-is, never reported as an unmapped violation) | |
| 216 | +NOT_A_VIOLATION = {("framework_kind", "model"), ("org_kind", "organization")} | |
| 217 | + | |
| 218 | + | |
| 215 | 219 | def _scalar(domain: str, value: Any, normaliser) -> tuple[Any, str | None, list[tuple[str, str, str | None]]]: |
| 216 | 220 | if not isinstance(value, str): |
| 217 | 221 | return value, None, [] |
| 218 | 222 | raw = value.strip() |
| 219 | 223 | canon = normaliser(raw) |
| 220 | 224 | if canon is None: |
| 225 | + if (domain, raw.lower()) in NOT_A_VIOLATION: | |
| 226 | + return raw, None, [] | |
| 221 | 227 | return raw, None, [(domain, raw, None)] |
| 222 | 228 | return canon, (raw if raw != canon else None), [(domain, raw, canon)] |
| 223 | 229 | |
modified
src/aiatlas/services/scheduler.py
+27 −1
@@ -34,11 +34,35 @@ async def run_connector(name: str, *, force: bool = False) -> None: | ||
| 34 | 34 | await get(name).run(force=force) |
| 35 | 35 | except Exception as exc: # noqa: BLE001 |
| 36 | 36 | log.warning("connector run failed", extra={"connector": name, "error": str(exc)}) |
| 37 | + # a connector run can touch every read model (stats, changes, benchmarks, prices, models, frontier, pulse, families…): | |
| 38 | + # flush the whole `aia:api:` namespace so new routes are covered without maintaining a prefix list | |
| 37 | 39 | await cache.cache_invalidate() |
| 38 | 40 | finally: |
| 39 | 41 | _running.discard(name) |
| 40 | 42 | |
| 41 | 43 | |
| 44 | +LIGHT_CANONICALIZE_STEPS = ["variants", "artifacts", "families", "events", "anomalies"] | |
| 45 | + | |
| 46 | + | |
| 47 | +async def canonicalize_job(*, light: bool = False) -> None: | |
| 48 | + """Nightly full `canonicalize --apply` (03:30 local) and a light pass every 6 h; one line of summary in the log.""" | |
| 49 | + from aiatlas.services.canonical import canonicalize | |
| 50 | + | |
| 51 | + async with cache.lock("canonicalize", ttl_s=2 * 3600) as ok: | |
| 52 | + if not ok: | |
| 53 | + log.info("canonicalize locked elsewhere, skipping", extra={"light": light}) | |
| 54 | + return | |
| 55 | + try: | |
| 56 | + report = await canonicalize(apply=True, steps=LIGHT_CANONICALIZE_STEPS if light else None) | |
| 57 | + summary = ", ".join(f"{s.name}={s.changes}" for s in report.steps if s.changes) or "no changes" | |
| 58 | + log.info("canonicalize %s pass: %d change(s) — %s", "light" if light else "full", report.changes, summary, | |
| 59 | + extra={"light": light, "changes": report.changes, "steps": {s.name: s.changes for s in report.steps}}) | |
| 60 | + if report.changes: | |
| 61 | + await cache.cache_invalidate() | |
| 62 | + except Exception as exc: # noqa: BLE001 | |
| 63 | + log.warning("canonicalize pass failed", extra={"light": light, "error": str(exc)}) | |
| 64 | + | |
| 65 | + | |
| 42 | 66 | async def _pop_run_now() -> list[str]: |
| 43 | 67 | """Admin `POST /admin/connectors/{name}/run` sets `aia:run-now:<name>`; consume (delete) every such key.""" |
| 44 | 68 | names: list[str] = [] |
@@ -125,6 +149,8 @@ async def main(*, with_worker: bool = True) -> None: | ||
| 125 | 149 | scheduler.add_job(hourly, "interval", minutes=60, max_instances=1, coalesce=True, id="hourly", next_run_time=datetime.now(UTC)) |
| 126 | 150 | minute, hour, *_ = settings.backup_cron.split() |
| 127 | 151 | scheduler.add_job(nightly_backup, CronTrigger(minute=minute, hour=hour, timezone=settings.tz), id="backup") |
| 152 | + scheduler.add_job(canonicalize_job, CronTrigger(minute=30, hour=3, timezone=settings.tz), id="canonicalize-full", max_instances=1, coalesce=True) | |
| 153 | + scheduler.add_job(canonicalize_job, "interval", hours=6, kwargs={"light": True}, id="canonicalize-light", max_instances=1, coalesce=True) | |
| 128 | 154 | scheduler.start() |
| 129 | 155 | log.info("scheduler started", extra={"tick_s": settings.scheduler_tick_s, "connectors": len(registry()), "worker": with_worker}) |
| 130 | 156 | async with transaction() as conn: |
@@ -137,4 +163,4 @@ async def main(*, with_worker: bool = True) -> None: | ||
| 137 | 163 | await cache.close() |
| 138 | 164 | |
| 139 | 165 | |
| 140 | −__all__ = ["hourly", "main", "run_connector", "tick"] | |
| 166 | +__all__ = ["LIGHT_CANONICALIZE_STEPS", "canonicalize_job", "hourly", "main", "run_connector", "tick"] | |
| 141 | 167 | |