1"""Admin API (`x-aia-admin-token`): overview, connectors, runs/errors, documents & snapshots (cleaned text only), queues, LLM2accounting, review queue with entity merging, duplicates, curation, infrastructure, cache. Never returns raw archive paths."""3from __future__ import annotations45import platform6import socket7import time8from datetime import UTC, datetime9from typing import Any1011from fastapi import APIRouter, Depends, Query12from pydantic import BaseModel, Field1314from aiatlas.api.common import PAGINATION, ApiError, Pagination, page, rate_limit, require_admin15from aiatlas.config import settings16from aiatlas.connectors import registry as connector_registry17from aiatlas.db import connection, execute, fetch_all, fetch_one, fetch_val, jsonb, transaction18from aiatlas.ids import new_id19from aiatlas.sdk import archive20from aiatlas.services import cache21from aiatlas.services.jobs import enqueue, queue_depth22from aiatlas.services.llm import gateway23from aiatlas.services.merge import merge_entities24from aiatlas.services.stats import live_counts2526router = APIRouter(prefix="/api/v1/admin", tags=["admin"], dependencies=[Depends(require_admin), Depends(rate_limit("admin"))])27_STARTED = time.time()28SNAPSHOT_TEXT_LIMIT = 20 * 102429HIDDEN = ("raw_path", "text_path")303132def _public(row: dict[str, Any]) -> dict[str, Any]:33 return {k: v for k, v in row.items() if k not in HIDDEN}343536# ------------------------------------------------------------------------------------------------------------------ overview373839@router.get("/overview")40async def overview() -> dict[str, Any]:41 async with connection() as conn:42 stats = await live_counts(conn)43 queue = await queue_depth(conn)44 health = await fetch_all(conn, "select health, count(*) as n from connectors group by 1")45 errors = await fetch_val(conn, "select count(*) from connector_errors where created_at > now() - interval '24 hours'")46 llm = await fetch_one(conn, """select count(*) as jobs_24h, coalesce(sum(input_tokens), 0) + coalesce(sum(output_tokens), 0) as tokens_24h,47 count(*) filter (where status <> 'ok') as failed_24h from llm_jobs where created_at > now() - interval '24 hours'""")48 by_stage = await fetch_all(conn, "select stage, status, count(*) as n from llm_jobs where created_at > now() - interval '24 hours' group by 1, 2 order by 1, 2")49 recent_runs = await fetch_all(conn, "select * from connector_runs order by started_at desc limit 12")50 review_kinds = await fetch_all(conn, "select kind, count(*) as n from review_queue where status = 'pending' group by 1")51 counts = {h["health"]: int(h["n"]) for h in health}52 return {"stats": stats, "queue": queue, "heartbeats": await cache.heartbeats(),53 "connectors": {k: counts.get(k, 0) for k in ("ok", "degraded", "failing", "disabled", "unknown")}, "review_pending": stats.get("review_pending", 0),54 "review_by_kind": {r["kind"]: int(r["n"]) for r in review_kinds}, "recent_errors": int(errors or 0),55 "llm": {"available": gateway.available, "jobs_24h": int(llm["jobs_24h"] or 0), "tokens_24h": int(llm["tokens_24h"] or 0), "failed_24h": int(llm["failed_24h"] or 0),56 "by_stage": [{**r, "n": int(r["n"])} for r in by_stage]},57 "recent_runs": recent_runs, "archive": archive.archive_size(), "computed_at": datetime.now(UTC)}585960# ------------------------------------------------------------------------------------------------------------------ connectors616263@router.get("/connectors")64async def connectors() -> dict[str, Any]:65 async with connection() as conn:66 rows = await fetch_all(conn, """67 select c.*, s.key as source_key, s.name as source_name, s.tier as source_tier, s.domain as source_domain,68 (select row_to_json(r) from connector_runs r where r.connector_name = c.name order by r.started_at desc limit 1) as last_run,69 (select count(*) from documents d where d.connector_name = c.name) as documents,70 (select count(*) from snapshots x join documents d on d.id = x.document_id where d.connector_name = c.name) as snapshots,71 (select count(*) from connector_errors e where e.connector_name = c.name and e.created_at > now() - interval '7 days') as errors_7d72 from connectors c left join sources s on s.id = c.source_id order by c.priority, c.name""")73 known = connector_registry()74 return {"items": [{**r, "documents": int(r["documents"]), "snapshots": int(r["snapshots"]), "errors_7d": int(r["errors_7d"]), "in_code": r["name"] in known,75 "run_now_pending": await _run_now_pending(r["name"])} for r in rows],76 "unregistered_in_db": sorted(set(known) - {r["name"] for r in rows})}777879async def _run_now_pending(name: str) -> bool:80 try:81 return bool(await cache.redis().exists(f"aia:run-now:{name}"))82 except Exception: # noqa: BLE00183 return False848586class RunBody(BaseModel):87 force: bool = False888990@router.post("/connectors/{name}/run")91async def run_connector(name: str, body: RunBody | None = None) -> dict[str, Any]:92 async with connection() as conn:93 exists = await fetch_one(conn, "select name from connectors where name = :n", n=name)94 if not exists and name not in connector_registry():95 raise ApiError(404, f"unknown connector {name!r}")96 try:97 await cache.redis().set(f"aia:run-now:{name}", b"force" if (body and body.force) else b"1", ex=6 * 3600)98 except Exception as exc:99 raise ApiError(503, f"redis unavailable: {exc.__class__.__name__}") from exc100 return {"queued": True, "connector": name, "force": bool(body and body.force), "note": "consumed by the scheduler tick (aia:run-now:<name>)"}101102103class ConnectorPatch(BaseModel):104 enabled: bool | None = None105 interval_seconds: int | None = Field(None, ge=60, le=30 * 86400)106 priority: int | None = Field(None, ge=0, le=9)107108109@router.patch("/connectors/{name}")110async def patch_connector(name: str, body: ConnectorPatch) -> dict[str, Any]:111 sets, params = [], {"n": name}112 if body.enabled is not None:113 sets.append("enabled = :enabled")114 sets.append("health = case when :enabled then (case when health = 'disabled' then 'unknown' else health end) else 'disabled' end")115 sets.append("circuit_open_until = case when :enabled then null else circuit_open_until end")116 params["enabled"] = body.enabled117 if body.interval_seconds is not None:118 sets.append("interval_seconds = :iv")119 sets.append("next_run_at = least(coalesce(next_run_at, now()), coalesce(last_success_at, now()) + make_interval(secs => :iv))")120 params["iv"] = body.interval_seconds121 if body.priority is not None:122 sets.append("priority = :pr")123 params["pr"] = body.priority124 if not sets:125 raise ApiError(400, "nothing to update")126 async with transaction() as conn:127 row = await fetch_one(conn, f"update connectors set {', '.join(sets)}, updated_at = now() where name = :n returning *", **params)128 if not row:129 raise ApiError(404, f"unknown connector {name!r}")130 return row131132133@router.get("/runs")134async def runs(connector: str | None = None, status: str | None = None, limit: int = Query(50, ge=1, le=500), offset: int = Query(0, ge=0)) -> dict[str, Any]:135 where = ["true"]136 params: dict[str, Any] = {"lim": limit, "off": offset}137 if connector:138 where.append("connector_name = :c")139 params["c"] = connector140 if status:141 where.append("status = :s")142 params["s"] = status143 async with connection() as conn:144 rows = await fetch_all(conn, f"select * from connector_runs where {' and '.join(where)} order by started_at desc limit :lim offset :off", **params)145 total = await fetch_val(conn, f"select count(*) from connector_runs where {' and '.join(where)}", **{k: v for k, v in params.items() if k not in ('lim', 'off')})146 return {"items": rows, "total": int(total or 0), "limit": limit, "offset": offset}147148149@router.get("/errors")150async def errors(connector: str | None = None, limit: int = Query(100, ge=1, le=1000), offset: int = Query(0, ge=0)) -> dict[str, Any]:151 where = "connector_name = :c" if connector else "true"152 async with connection() as conn:153 rows = await fetch_all(conn, f"select * from connector_errors where {where} order by created_at desc limit :lim offset :off", c=connector, lim=limit, off=offset)154 total = await fetch_val(conn, f"select count(*) from connector_errors where {where}", c=connector)155 return {"items": rows, "total": int(total or 0), "limit": limit, "offset": offset}156157158# ------------------------------------------------------------------------------------------------------------------ documents & snapshots159160161@router.get("/documents")162async def documents(connector: str | None = None, status: str | None = None, q: str | None = Query(None, max_length=300), entity: str | None = None,163 needs_llm: int | None = Query(None, ge=0, le=1), p: Pagination = PAGINATION) -> dict[str, Any]:164 where = ["true"]165 params: dict[str, Any] = {}166 if connector:167 where.append("d.connector_name = :c")168 params["c"] = connector169 if status:170 where.append("d.status = :s")171 params["s"] = status172 if q:173 where.append("(d.url ilike :q or d.title ilike :q)")174 params["q"] = f"%{q}%"175 if entity:176 where.append("(d.entity_id = :e or e.slug = :e)")177 params["e"] = entity178 if needs_llm is not None:179 where.append("d.needs_llm = :nl")180 params["nl"] = bool(needs_llm)181 where_sql = " and ".join(where)182 async with connection() as conn:183 rows = await fetch_all(conn, f"""select d.*, e.slug as entity_slug, e.canonical_name as entity_name, e.entity_type,184 (select count(*) from snapshots x where x.document_id = d.id) as snapshots,185 (select x.processing_status from snapshots x where x.document_id = d.id order by x.observed_at desc limit 1) as last_processing_status186 from documents d left join entities e on e.id = d.entity_id where {where_sql}187 order by d.last_fetched_at desc nulls last, d.first_seen_at desc limit :lim offset :off""", lim=p.limit, off=p.offset, **params)188 total = await fetch_val(conn, f"select count(*) from documents d left join entities e on e.id = d.entity_id where {where_sql}", **params)189 return page([{**r, "snapshots": int(r["snapshots"])} for r in rows], int(total or 0), p)190191192@router.get("/documents/{doc_id}")193async def document(doc_id: str) -> dict[str, Any]:194 async with connection() as conn:195 doc = await fetch_one(conn, """select d.*, e.slug as entity_slug, e.canonical_name as entity_name, e.entity_type, s.name as source_name, s.tier as source_tier196 from documents d left join entities e on e.id = d.entity_id left join sources s on s.id = d.source_id where d.id = :id or d.url = :id limit 1""", id=doc_id)197 if not doc:198 raise ApiError(404, "document not found")199 snaps = await fetch_all(conn, """select id, run_id, url, final_url, observed_at, http_status, content_type, content_hash, byte_size, text_hash, parser_version, connector_version,200 transport, changed, processing_status, created_at, structured is not null as has_structured, diff is not null as has_diff, text_path is not null as has_text201 from snapshots where document_id = :id order by observed_at desc limit 200""", id=doc["id"])202 claims = await fetch_val(conn, "select count(*) from claims c join snapshots s on s.id = c.snapshot_id where s.document_id = :id", id=doc["id"])203 return {**doc, "snapshots": snaps, "claims_from_document": int(claims or 0)}204205206@router.get("/snapshots/{snap_id}")207async def snapshot(snap_id: str, text_limit: int = Query(SNAPSHOT_TEXT_LIMIT, ge=0, le=SNAPSHOT_TEXT_LIMIT)) -> dict[str, Any]:208 async with connection() as conn:209 snap = await fetch_one(conn, """select s.*, d.url as document_url, d.doc_type, d.connector_name, d.entity_id, e.slug as entity_slug, e.canonical_name as entity_name210 from snapshots s join documents d on d.id = s.document_id left join entities e on e.id = d.entity_id where s.id = :id""", id=snap_id)211 if not snap:212 raise ApiError(404, "snapshot not found")213 claims = await fetch_all(conn, """select c.id, c.entity_id, e.slug as entity_slug, c.property, c.value, c.unit, c.status, c.confidence, c.extractor, c.observed_at214 from claims c left join entities e on e.id = c.entity_id where c.snapshot_id = :id order by e.slug, c.property limit 500""", id=snap_id)215 events = await fetch_all(conn, "select id, entity_id, event_type, category, property, summary, importance, observed_at from change_events where snapshot_id = :id order by observed_at desc limit 200", id=snap_id)216 llm = await fetch_all(conn, "select id, task_type, stage, model, status, input_tokens, output_tokens, duration_ms, error, created_at from llm_jobs where snapshot_id = :id order by created_at desc limit 50", id=snap_id)217 text, text_error, text_len = None, None, 0218 if snap.get("text_path") and text_limit:219 try:220 full = archive.load_text(snap["text_path"])221 text_len = len(full)222 text = full[:text_limit]223 except OSError as exc:224 text_error = f"cleaned text unavailable ({exc.__class__.__name__})"225 out = _public(snap)226 out.update({"text": text, "text_chars": text_len, "text_truncated": text_len > text_limit, "text_error": text_error, "has_raw": bool(snap.get("raw_path")),227 "claims": claims, "events": events, "llm_jobs": llm})228 return out229230231# ------------------------------------------------------------------------------------------------------------------ queues232233234@router.get("/jobs")235async def jobs(status: str | None = None, kind: str | None = None, limit: int = Query(100, ge=1, le=1000), offset: int = Query(0, ge=0)) -> dict[str, Any]:236 where = ["true"]237 params: dict[str, Any] = {}238 if status:239 where.append("status = any(cast(:s as text[]))")240 params["s"] = [x.strip() for x in status.split(",") if x.strip()]241 if kind:242 where.append("kind = :k")243 params["k"] = kind244 where_sql = " and ".join(where)245 async with connection() as conn:246 rows = await fetch_all(conn, f"select * from jobs where {where_sql} order by case status when 'running' then 0 when 'queued' then 1 when 'failed' then 2 when 'dead' then 3 else 4 end, "247 f"priority, coalesce(finished_at, run_after) desc limit :lim offset :off", lim=limit, off=offset, **params)248 total = await fetch_val(conn, f"select count(*) from jobs where {where_sql}", **params)249 depth = await queue_depth(conn)250 return {"items": rows, "total": int(total or 0), "limit": limit, "offset": offset, "depth": depth}251252253@router.post("/jobs/{job_id}/retry")254async def retry_job(job_id: str) -> dict[str, Any]:255 async with transaction() as conn:256 row = await fetch_one(conn, """update jobs set status = 'queued', attempts = 0, error = null, locked_by = null, locked_at = null, finished_at = null, run_after = now()257 where id = :id and status in ('failed', 'dead', 'done') returning id, kind, status""", id=job_id)258 if not row:259 raise ApiError(404, "job not found or not retryable (must be failed, dead or done)")260 return {"ok": True, **row}261262263@router.post("/jobs/requeue-dead")264async def requeue_dead(kind: str | None = None) -> dict[str, Any]:265 where = "j.status = 'dead'" + (" and j.kind = :k" if kind else "")266 async with transaction() as conn:267 # respect the partial unique index on dedupe_key (queued|running): skip dead jobs already superseded by a live one,268 # and requeue only one dead job per dedupe_key (the most recent); the other duplicates are closed as superseded.269 n = await fetch_val(conn, f"""with pick as (270 select distinct on (coalesce(j.dedupe_key, j.id)) j.id from jobs j271 where {where} and (j.dedupe_key is null or not exists (select 1 from jobs q where q.dedupe_key = j.dedupe_key and q.status in ('queued','running')))272 order by coalesce(j.dedupe_key, j.id), j.created_at desc),273 u as (update jobs set status = 'queued', attempts = 0, error = null, locked_by = null, locked_at = null, finished_at = null, run_after = now()274 where id in (select id from pick) returning 1)275 select count(*) from u""", k=kind)276 await execute(conn, f"update jobs j set status = 'done', error = coalesce(error, '') || ' [superseded]' where {where}", k=kind)277 return {"requeued": int(n or 0)}278279280@router.get("/llm-jobs")281async def llm_jobs(limit: int = Query(100, ge=1, le=1000), offset: int = Query(0, ge=0), status: str | None = None, task: str | None = None) -> dict[str, Any]:282 where = ["true"]283 params: dict[str, Any] = {}284 if status:285 where.append("status = :s")286 params["s"] = status287 if task:288 where.append("task_type = :t")289 params["t"] = task290 where_sql = " and ".join(where)291 async with connection() as conn:292 rows = await fetch_all(conn, f"select id, job_id, task_type, stage, engine, model, node, schema_name, snapshot_id, entity_id, input_tokens, output_tokens, duration_ms, status, error, created_at "293 f"from llm_jobs where {where_sql} order by created_at desc limit :lim offset :off", lim=limit, off=offset, **params)294 total = await fetch_val(conn, f"select count(*) from llm_jobs where {where_sql}", **params)295 totals = await fetch_all(conn, """select stage, model, status, count(*) as n, coalesce(sum(input_tokens), 0) as input_tokens, coalesce(sum(output_tokens), 0) as output_tokens,296 coalesce(avg(duration_ms), 0)::int as avg_ms from llm_jobs group by 1, 2, 3 order by 1, 2, 3""")297 return {"items": rows, "total": int(total or 0), "limit": limit, "offset": offset, "totals": [{**t, "n": int(t["n"]), "input_tokens": int(t["input_tokens"]), "output_tokens": int(t["output_tokens"])} for t in totals]}298299300@router.get("/llm/health")301async def llm_health() -> dict[str, Any]:302 h = await gateway.health()303 h["base_url_configured"] = bool(settings.llm_base_url)304 return h305306307class LLMEnqueue(BaseModel):308 limit: int = Field(200, ge=1, le=5000)309 task: str | None = None310 connector: str | None = None311312313@router.post("/llm/enqueue")314async def llm_enqueue(body: LLMEnqueue | None = None) -> dict[str, Any]:315 body = body or LLMEnqueue()316 where = "s.processing_status in ('stored', 'extracted', 'llm_pending') and s.text_path is not null and (d.needs_llm or s.processing_status = 'llm_pending')"317 params: dict[str, Any] = {"lim": body.limit}318 if body.connector:319 where += " and d.connector_name = :c"320 params["c"] = body.connector321 queued = 0322 async with transaction() as conn:323 snaps = await fetch_all(conn, f"select s.id from snapshots s join documents d on d.id = s.document_id where {where} order by d.priority, s.observed_at desc limit :lim", **params)324 for s in snaps:325 payload = {"snapshot_id": s["id"], **({"task": body.task} if body.task else {})}326 if await enqueue(conn, "llm_extract", payload, priority=6, dedupe_key=f"llm:{s['id']}"):327 queued += 1328 if snaps:329 await execute(conn, "update snapshots set processing_status = 'llm_pending' where id = any(cast(:ids as text[])) and processing_status <> 'llm_pending'", ids=[s["id"] for s in snaps])330 return {"queued": queued, "candidates": len(snaps), "llm_available": gateway.available}331332333class ReprocessBody(BaseModel):334 connector: str335 url: str | None = None336337338@router.post("/reprocess")339async def reprocess(body: ReprocessBody) -> dict[str, Any]:340 if body.connector not in connector_registry():341 raise ApiError(404, f"unknown connector {body.connector!r}")342 async with transaction() as conn:343 jid = await enqueue(conn, "reprocess_snapshot", {"connector": body.connector, **({"url": body.url} if body.url else {})}, priority=3,344 dedupe_key=f"reprocess:{body.connector}:{body.url or '*'}")345 return {"queued": jid is not None, "job_id": jid}346347348# ------------------------------------------------------------------------------------------------------------------ review queue & curation349350351@router.get("/review")352async def review(status: str = "pending", kind: str | None = None, limit: int = Query(100, ge=1, le=1000), offset: int = Query(0, ge=0)) -> dict[str, Any]:353 where = ["r.status = :s"] if status != "all" else ["true"]354 params: dict[str, Any] = {"s": status}355 if kind:356 where.append("r.kind = :k")357 params["k"] = kind358 where_sql = " and ".join(where)359 async with connection() as conn:360 rows = await fetch_all(conn, f"""select r.*, (select jsonb_agg(jsonb_build_object('id', e.id, 'slug', e.slug, 'name', e.canonical_name, 'entity_type', e.entity_type, 'status', e.status) order by e.slug)361 from entities e where e.id = any(r.entity_ids)) as entities362 from review_queue r where {where_sql} order by r.created_at desc limit :lim offset :off""", lim=limit, off=offset, **params)363 total = await fetch_val(conn, f"select count(*) from review_queue r where {where_sql}", **params)364 kinds = await fetch_all(conn, "select kind, status, count(*) as n from review_queue group by 1, 2 order by 1, 2")365 return {"items": rows, "total": int(total or 0), "limit": limit, "offset": offset, "by_kind": [{**k, "n": int(k["n"])} for k in kinds]}366367368class ReviewAction(BaseModel):369 action: str = Field(..., pattern="^(approve|reject|edit)$")370 resolution: dict[str, Any] | None = None371372373@router.post("/review/{review_id}")374async def review_action(review_id: str, body: ReviewAction) -> dict[str, Any]:375 async with transaction() as conn:376 item = await fetch_one(conn, "select * from review_queue where id = :id for update", id=review_id)377 if not item:378 raise ApiError(404, "review item not found")379 if item["status"] != "pending":380 raise ApiError(409, f"review item already {item['status']}")381 resolution: dict[str, Any] = dict(body.resolution or {})382 effect: dict[str, Any] | None = None383 if body.action == "approve" and item["kind"] == "merge_candidate":384 payload = item["payload"] or {}385 ids = list(item["entity_ids"] or [])386 source = resolution.get("source_id") or payload.get("source_id") or (ids[0] if len(ids) > 1 else None)387 target = resolution.get("target_id") or payload.get("target_id") or (ids[1] if len(ids) > 1 else None)388 if not source or not target:389 raise ApiError(400, "merge_candidate needs source_id and target_id (payload or resolution)")390 try:391 effect = await merge_entities(conn, source, target)392 except (ValueError, LookupError) as exc:393 raise ApiError(400, str(exc)) from exc394 resolution.update({"merged": effect})395 elif body.action == "approve" and item["kind"] == "conflict":396 keep = resolution.get("keep_claim_id")397 if keep:398 effect = await _accept_claim(conn, keep)399 resolution["accepted"] = effect400 status = {"approve": "approved", "reject": "rejected", "edit": "edited"}[body.action]401 await execute(conn, "update review_queue set status = :st, resolution = cast(:r as jsonb), resolved_at = now() where id = :id", st=status, r=jsonb(resolution), id=review_id)402 await cache.cache_invalidate()403 return {"ok": True, "id": review_id, "status": status, "effect": effect}404405406async def _accept_claim(conn: Any, claim_id: str) -> dict[str, Any]:407 """Conflict resolution: promote one claim to current, supersede the others for the same property."""408 c = await fetch_one(conn, "select * from claims where id = :id", id=claim_id)409 if not c:410 raise ApiError(404, "claim not found")411 await execute(conn, "update claims set status = 'superseded', valid_to = now() where entity_id = :e and property = :p and id <> :id and status in ('current', 'conflicting')",412 e=c["entity_id"], p=c["property"], id=claim_id)413 await execute(conn, "update claims set status = 'current', confidence = case when confidence = 'conflicted' then 'high' else confidence end, valid_to = null where id = :id", id=claim_id)414 prov = {"source_id": c["source_id"], "snapshot_id": c["snapshot_id"], "url": c["source_url"], "tier": c["tier"], "confidence": "high", "extractor": c["extractor"],415 "observed_at": c["observed_at"].isoformat() if c["observed_at"] else None, "resolved_by": "review"}416 await execute(conn, "update entities set attributes = attributes || jsonb_build_object(:p, cast(:v as jsonb)), provenance = provenance || jsonb_build_object(:p, cast(:pv as jsonb)), updated_at = now() where id = :e",417 p=c["property"], v=jsonb(c["value"]), pv=jsonb(prov), e=c["entity_id"])418 return {"claim_id": claim_id, "entity_id": c["entity_id"], "property": c["property"]}419420421@router.get("/entities/duplicates")422async def duplicates(type: str | None = Query(None, alias="type"), limit: int = Query(100, ge=1, le=500), threshold: float = Query(0.8, ge=0.3, le=1.0)) -> dict[str, Any]:423 where = "a.entity_type = :t" if type else "true"424 async with connection() as conn:425 await execute(conn, "select set_limit(:th)", th=threshold)426 rows = await fetch_all(conn, f"""427 select a.id as a_id, a.slug as a_slug, a.canonical_name as a_name, a.entity_type, a.first_seen_at as a_first_seen_at, oa.canonical_name as a_org,428 b.id as b_id, b.slug as b_slug, b.canonical_name as b_name, b.first_seen_at as b_first_seen_at, ob.canonical_name as b_org,429 similarity(a.canonical_name, b.canonical_name) as similarity,430 (select count(*) from claims c where c.entity_id = a.id and c.status = 'current') as a_claims,431 (select count(*) from claims c where c.entity_id = b.id and c.status = 'current') as b_claims432 from entities a join entities b on b.entity_type = a.entity_type and b.id > a.id and a.canonical_name % b.canonical_name433 left join entities oa on oa.id = a.organization_id left join entities ob on ob.id = b.organization_id434 where {where} and a.merged_into is null and b.merged_into is null and similarity(a.canonical_name, b.canonical_name) > :th435 order by similarity desc, a.canonical_name limit :lim""", t=type, th=threshold, lim=limit)436 return {"threshold": threshold, "items": [{"entity_type": r["entity_type"], "similarity": round(float(r["similarity"]), 3),437 "a": {"id": r["a_id"], "slug": r["a_slug"], "name": r["a_name"], "organization": r["a_org"], "first_seen_at": r["a_first_seen_at"], "claims": int(r["a_claims"])},438 "b": {"id": r["b_id"], "slug": r["b_slug"], "name": r["b_name"], "organization": r["b_org"], "first_seen_at": r["b_first_seen_at"], "claims": int(r["b_claims"])}}439 for r in rows]}440441442class MergeBody(BaseModel):443 source_id: str444 target_id: str445446447@router.post("/entities/merge")448async def merge(body: MergeBody) -> dict[str, Any]:449 async with transaction() as conn:450 src = await fetch_one(conn, "select id from entities where id = :k or slug = :k limit 1", k=body.source_id)451 dst = await fetch_one(conn, "select id from entities where id = :k or slug = :k limit 1", k=body.target_id)452 if not src or not dst:453 raise ApiError(404, "source or target entity not found")454 try:455 result = await merge_entities(conn, src["id"], dst["id"])456 except (ValueError, LookupError) as exc:457 raise ApiError(400, str(exc)) from exc458 await cache.cache_invalidate()459 return result460461462@router.post("/entities/{entity_id}/claims/{claim_id}/retract")463async def retract_claim(entity_id: str, claim_id: str) -> dict[str, Any]:464 async with transaction() as conn:465 ent = await fetch_one(conn, "select id from entities where id = :k or slug = :k limit 1", k=entity_id)466 if not ent:467 raise ApiError(404, "entity not found")468 c = await fetch_one(conn, "select * from claims where id = :c and entity_id = :e", c=claim_id, e=ent["id"])469 if not c:470 raise ApiError(404, "claim not found for this entity")471 was_current = c["status"] == "current"472 await execute(conn, "update claims set status = 'retracted', valid_to = coalesce(valid_to, now()) where id = :c", c=claim_id)473 restored = None474 if was_current:475 prev = await fetch_one(conn, """select * from claims where entity_id = :e and property = :p and id <> :c and status in ('superseded', 'conflicting')476 order by tier, valid_from desc limit 1""", e=ent["id"], p=c["property"], c=claim_id)477 if prev:478 await execute(conn, "update claims set status = 'current', valid_to = null where id = :id", id=prev["id"])479 prov = {"source_id": prev["source_id"], "snapshot_id": prev["snapshot_id"], "url": prev["source_url"], "tier": prev["tier"], "confidence": prev["confidence"],480 "extractor": prev["extractor"], "observed_at": prev["observed_at"].isoformat() if prev["observed_at"] else None, "restored_by": "retraction"}481 await execute(conn, "update entities set attributes = attributes || jsonb_build_object(:p, cast(:v as jsonb)), provenance = provenance || jsonb_build_object(:p, cast(:pv as jsonb)), updated_at = now() where id = :e",482 p=c["property"], v=jsonb(prev["value"]), pv=jsonb(prov), e=ent["id"])483 restored = prev["id"]484 else:485 await execute(conn, "update entities set attributes = attributes - :p, provenance = provenance - :p, updated_at = now() where id = :e", p=c["property"], e=ent["id"])486 await execute(conn, """insert into change_events (id, entity_id, event_type, category, property, old_value, summary, importance, connector_name, dedupe_key)487 values (:id, :e, 'CLAIM_RETRACTED', 'source', :p, cast(:v as jsonb), :s, 1, 'curation', :dk) on conflict (dedupe_key) do nothing""",488 id=new_id("change_event"), e=ent["id"], p=c["property"], v=jsonb(c["value"]),489 s=f"Retracted claim {c['property']}", dk=f"retract:{claim_id}")490 await cache.cache_invalidate()491 return {"ok": True, "claim_id": claim_id, "property": c["property"], "was_current": was_current, "restored_claim_id": restored}492493494# ------------------------------------------------------------------------------------------------------------------ infrastructure & maintenance495496497@router.get("/infrastructure")498async def infrastructure() -> dict[str, Any]:499 async with connection() as conn:500 db = await fetch_one(conn, """select pg_database_size(current_database()) as db_bytes, current_database() as database, version() as pg_version,501 (select count(*) from pg_stat_activity where datname = current_database()) as connections,502 (select setting from pg_settings where name = 'max_connections') as max_connections""")503 tables = await fetch_all(conn, """select c.relname as table, pg_total_relation_size(c.oid) as total_bytes, pg_relation_size(c.oid) as data_bytes,504 coalesce(s.n_live_tup, 0) as rows_estimate from pg_class c join pg_namespace n on n.oid = c.relnamespace505 left join pg_stat_user_tables s on s.relid = c.oid where n.nspname = 'public' and c.relkind = 'r'506 order by pg_total_relation_size(c.oid) desc limit 20""")507 extensions = await fetch_all(conn, "select extname, extversion from pg_extension order by 1")508 redis_info: dict[str, Any] = {"ok": False}509 try:510 info = await cache.redis().info("memory")511 redis_info = {"ok": True, "used_memory": info.get("used_memory"), "used_memory_human": info.get("used_memory_human"),512 "api_cache_keys": sum([1 async for _ in cache.redis().scan_iter(match="aia:api:*", count=1000)])}513 except Exception as exc: # noqa: BLE001514 redis_info = {"ok": False, "error": exc.__class__.__name__}515 return {"hostname": socket.gethostname(), "python": platform.python_version(), "platform": platform.platform(), "api_uptime_s": int(time.time() - _STARTED),516 "env": settings.app_env, "heartbeats": await cache.heartbeats(), "archive": archive.archive_size(), "data_dir_exists": settings.data_dir.exists(),517 "database": {**(db or {}), "db_bytes": int((db or {}).get("db_bytes") or 0), "tables": tables, "extensions": extensions}, "redis": redis_info,518 "llm": {"available": gateway.available, "engine": gateway.engine.name}, "scheduler_tick_s": settings.scheduler_tick_s}519520521@router.post("/cache/flush")522async def cache_flush(prefix: str = "") -> dict[str, Any]:523 return {"flushed": await cache.cache_invalidate(prefix)}524525526@router.post("/stats/recompute")527async def stats_recompute() -> dict[str, Any]:528 from aiatlas.services.stats import compute_stats529530 counts = await compute_stats()531 await cache.cache_invalidate("/api/v1/stats")532 return {"ok": True, "entities_total": counts.get("entities_total")}533534535class QualityBody(BaseModel):536 entity_ids: list[str] | None = None537 limit: int = Field(20000, ge=1, le=200000)538539540@router.post("/quality/recompute")541async def quality_recompute(body: QualityBody | None = None) -> dict[str, Any]:542 from aiatlas.services.quality import recompute543544 body = body or QualityBody()545 res = await recompute(entity_ids=body.entity_ids, limit=body.limit)546 await cache.cache_invalidate()547 return {"ok": True, **res}548