scheduler: nightly canonicalize --apply (03:30 local, Redis lock 2 h) + light pass every 6 h with one-line summary; full API cache flush after each connector run documented; taxonomy: Apache Software License/Apache License → Apache-2.0, BSD License → BSD-3-Clause, framework kind 'model' and org_kind 'organization' no longer reported as unmapped

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

Simon-Pierre Boucher committed 1 mo ago (Sep 12, 2026) parent b78b6ac Browse files

3 changed files +36 −3

modified src/aiatlas/ontology/licenses.py +3 −2

				`@@ -59,12 +59,13 @@ def _add(info: LicenseInfo) -> None:`
59		59
60	# ---------------------------------------------------------------------------------------------- permissive / copyleft (OSI)	60	# ---------------------------------------------------------------------------------------------- permissive / copyleft (OSI)
61	_add(_permissive("Apache-2.0", "Apache License 2.0", "Apache-2.0", "https://www.apache.org/licenses/LICENSE-2.0",	61	_add(_permissive("Apache-2.0", "Apache License 2.0", "Apache-2.0", "https://www.apache.org/licenses/LICENSE-2.0",
62	− aliases=("apache 2.0", "apache-2", "apache2", "apache license 2.0", "apache license, version 2.0", "apache")))	62	+ aliases=("apache 2.0", "apache-2", "apache2", "apache license 2.0", "apache license, version 2.0", "apache", "apache license",
		63	+ "apache software license", "apache software license 2.0", "asl 2.0", "asl-2.0")))
63	_add(_permissive("MIT", "MIT License", "MIT", "https://opensource.org/license/mit", aliases=("mit license", "the mit license")))	64	_add(_permissive("MIT", "MIT License", "MIT", "https://opensource.org/license/mit", aliases=("mit license", "the mit license")))
64	_add(LicenseInfo(key="MIT-Modified", label="Modified MIT License", category="permissive", spdx=None, commercial_use=True, redistribution=True,	65	_add(LicenseInfo(key="MIT-Modified", label="Modified MIT License", category="permissive", spdx=None, commercial_use=True, redistribution=True,
65	derivatives=True, hosting_restrictions=None, attribution=True, acceptable_use=None, aliases=("modified mit", "mit-modified")))	66	derivatives=True, hosting_restrictions=None, attribution=True, acceptable_use=None, aliases=("modified mit", "mit-modified")))
66	_add(_permissive("BSD-3-Clause", "BSD 3-Clause License", "BSD-3-Clause", "https://opensource.org/license/bsd-3-clause",	67	_add(_permissive("BSD-3-Clause", "BSD 3-Clause License", "BSD-3-Clause", "https://opensource.org/license/bsd-3-clause",
67	− aliases=("bsd-3", "bsd 3-clause", "bsd3", "new bsd", "modified bsd", "bsd")))	68	+ aliases=("bsd-3", "bsd 3-clause", "bsd3", "new bsd", "modified bsd", "bsd", "bsd license", "bsd-3-clause license")))
68	_add(_permissive("BSD-2-Clause", "BSD 2-Clause License", "BSD-2-Clause", "https://opensource.org/license/bsd-2-clause", aliases=("bsd-2", "simplified bsd")))	69	_add(_permissive("BSD-2-Clause", "BSD 2-Clause License", "BSD-2-Clause", "https://opensource.org/license/bsd-2-clause", aliases=("bsd-2", "simplified bsd")))
69	_add(_permissive("ISC", "ISC License", "ISC", "https://opensource.org/license/isc-license-txt"))	70	_add(_permissive("ISC", "ISC License", "ISC", "https://opensource.org/license/isc-license-txt"))
70	_add(_permissive("Unlicense", "The Unlicense", "Unlicense", "https://unlicense.org", attribution=False, aliases=("unlicense",)))	71	_add(_permissive("Unlicense", "The Unlicense", "Unlicense", "https://unlicense.org", attribution=False, aliases=("unlicense",)))

modified src/aiatlas/ontology/taxonomy.py +6 −0

				`@@ -212,12 +212,18 @@ def normalize_property(entity_type: str \| None, prop: str, value: Any) -> tuple[`
212	return value, None, []	212	return value, None, []
213		213
214		214
		215	+# raw values that legitimately have no canonical counterpart (kept as-is, never reported as an unmapped violation)
		216	+NOT_A_VIOLATION = {("framework_kind", "model"), ("org_kind", "organization")}
		217	+
		218	+
215	def _scalar(domain: str, value: Any, normaliser) -> tuple[Any, str \| None, list[tuple[str, str, str \| None]]]:	219	def _scalar(domain: str, value: Any, normaliser) -> tuple[Any, str \| None, list[tuple[str, str, str \| None]]]:
216	if not isinstance(value, str):	220	if not isinstance(value, str):
217	return value, None, []	221	return value, None, []
218	raw = value.strip()	222	raw = value.strip()
219	canon = normaliser(raw)	223	canon = normaliser(raw)
220	if canon is None:	224	if canon is None:
		225	+ if (domain, raw.lower()) in NOT_A_VIOLATION:
		226	+ return raw, None, []
221	return raw, None, [(domain, raw, None)]	227	return raw, None, [(domain, raw, None)]
222	return canon, (raw if raw != canon else None), [(domain, raw, canon)]	228	return canon, (raw if raw != canon else None), [(domain, raw, canon)]
223		229

modified src/aiatlas/services/scheduler.py +27 −1

				`@@ -34,11 +34,35 @@ async def run_connector(name: str, *, force: bool = False) -> None:`
34	await get(name).run(force=force)	34	await get(name).run(force=force)
35	except Exception as exc: # noqa: BLE001	35	except Exception as exc: # noqa: BLE001
36	log.warning("connector run failed", extra={"connector": name, "error": str(exc)})	36	log.warning("connector run failed", extra={"connector": name, "error": str(exc)})
		37	+ # a connector run can touch every read model (stats, changes, benchmarks, prices, models, frontier, pulse, families…):
		38	+ # flush the whole `aia:api:` namespace so new routes are covered without maintaining a prefix list
37	await cache.cache_invalidate()	39	await cache.cache_invalidate()
38	finally:	40	finally:
39	_running.discard(name)	41	_running.discard(name)
40		42
41		43
		44	+LIGHT_CANONICALIZE_STEPS = ["variants", "artifacts", "families", "events", "anomalies"]
		45	+
		46	+
		47	+async def canonicalize_job(*, light: bool = False) -> None:
		48	+ """Nightly full `canonicalize --apply` (03:30 local) and a light pass every 6 h; one line of summary in the log."""
		49	+ from aiatlas.services.canonical import canonicalize
		50	+
		51	+ async with cache.lock("canonicalize", ttl_s=2 * 3600) as ok:
		52	+ if not ok:
		53	+ log.info("canonicalize locked elsewhere, skipping", extra={"light": light})
		54	+ return
		55	+ try:
		56	+ report = await canonicalize(apply=True, steps=LIGHT_CANONICALIZE_STEPS if light else None)
		57	+ summary = ", ".join(f"{s.name}={s.changes}" for s in report.steps if s.changes) or "no changes"
		58	+ log.info("canonicalize %s pass: %d change(s) — %s", "light" if light else "full", report.changes, summary,
		59	+ extra={"light": light, "changes": report.changes, "steps": {s.name: s.changes for s in report.steps}})
		60	+ if report.changes:
		61	+ await cache.cache_invalidate()
		62	+ except Exception as exc: # noqa: BLE001
		63	+ log.warning("canonicalize pass failed", extra={"light": light, "error": str(exc)})
		64	+
		65	+
42	async def _pop_run_now() -> list[str]:	66	async def _pop_run_now() -> list[str]:
43	"""Admin `POST /admin/connectors/{name}/run` sets `aia:run-now:<name>`; consume (delete) every such key."""	67	"""Admin `POST /admin/connectors/{name}/run` sets `aia:run-now:<name>`; consume (delete) every such key."""
44	names: list[str] = []	68	names: list[str] = []
				`@@ -125,6 +149,8 @@ async def main(*, with_worker: bool = True) -> None:`
125	scheduler.add_job(hourly, "interval", minutes=60, max_instances=1, coalesce=True, id="hourly", next_run_time=datetime.now(UTC))	149	scheduler.add_job(hourly, "interval", minutes=60, max_instances=1, coalesce=True, id="hourly", next_run_time=datetime.now(UTC))
126	minute, hour, *_ = settings.backup_cron.split()	150	minute, hour, *_ = settings.backup_cron.split()
127	scheduler.add_job(nightly_backup, CronTrigger(minute=minute, hour=hour, timezone=settings.tz), id="backup")	151	scheduler.add_job(nightly_backup, CronTrigger(minute=minute, hour=hour, timezone=settings.tz), id="backup")
		152	+ scheduler.add_job(canonicalize_job, CronTrigger(minute=30, hour=3, timezone=settings.tz), id="canonicalize-full", max_instances=1, coalesce=True)
		153	+ scheduler.add_job(canonicalize_job, "interval", hours=6, kwargs={"light": True}, id="canonicalize-light", max_instances=1, coalesce=True)
128	scheduler.start()	154	scheduler.start()
129	log.info("scheduler started", extra={"tick_s": settings.scheduler_tick_s, "connectors": len(registry()), "worker": with_worker})	155	log.info("scheduler started", extra={"tick_s": settings.scheduler_tick_s, "connectors": len(registry()), "worker": with_worker})
130	async with transaction() as conn:	156	async with transaction() as conn:
				`@@ -137,4 +163,4 @@ async def main(*, with_worker: bool = True) -> None:`
137	await cache.close()	163	await cache.close()
138		164
139		165
140	−__all__ = ["hourly", "main", "run_connector", "tick"]	166	+__all__ = ["LIGHT_CANONICALIZE_STEPS", "canonicalize_job", "hourly", "main", "run_connector", "tick"]
141		167

spb/ai-atlas

Theme