Python 88.3%
TypeScript 7.6%
Shell 4.1%
1#!/usr/bin/env python32"""Merge generated/fragments/**/*.json into the canonical machine-readable outputs.34 python3 scripts/build_generated.py # build everything5 python3 scripts/build_generated.py --check # validate fragments only (no write)67Inputs : generated/fragments/<kind>/*.json — each file is a JSON array of records (or {"records":[...]})8Outputs : generated/models.json|csv, endpoints.json|csv, parameters.json|csv, tools.json|csv,9 streaming-events.json, errors.json|csv, headers.json, pricing.json|csv, rate-limits.json,10 deprecations.json, objects.json, sdks.json, webhook-events.json, audit-log-events.json,11 status-lifecycles.json, beta-headers.json, examples-manifest.json,12 compatibility/model-capability-matrix.{json,csv}, compatibility/model-endpoint-matrix.{json,csv},13 compatibility/model-tool-matrix.{json,csv}, compatibility/tool-model-matrix.csv,14 compatibility/feature-platform-matrix.json (anthropic clouds), index.json (counts + provenance)1516Fragment kinds are inferred from the directory name (generated/fragments/<kind>/). Unknown kinds are merged17into generated/<kind>.json verbatim so agents can add new record types without touching this script.18Records are de-duplicated on a per-kind natural key; on conflict the later file (alphabetical) wins but the19conflict is reported in generated/build-report.json.20"""21from __future__ import annotations2223import argparse24import csv25import json26import sys27from collections import defaultdict28from datetime import datetime, timezone29from pathlib import Path3031ROOT = Path(__file__).resolve().parent.parent32FRAG = ROOT / "generated" / "fragments"33OUT = ROOT / "generated"34STATUSES = {"DOCUMENTED", "LIVE_DISCOVERED", "LIVE_VERIFIED", "BETA", "PREVIEW", "LEGACY", "DEPRECATED", "RETIRED",35 "ACCOUNT_RESTRICTED", "UNVERIFIED", "FAILED_VERIFICATION", "DOCUMENTATION_INCOMPLETE", "GA"}3637# directory-name aliases (agents sometimes pluralise differently)38KIND_ALIASES = {"prices": "pricing", "price": "pricing", "model": "models", "endpoint": "endpoints",39 "parameter": "parameters", "tool": "tools", "streaming_events": "streaming-events",40 "events": "streaming-events", "error": "errors", "header": "headers", "rate_limits": "rate-limits",41 "ratelimits": "rate-limits", "deprecation": "deprecations", "object": "objects", "sdk": "sdks",42 "webhook_events": "webhook-events", "webhooks": "webhook-events", "audit_log_events": "audit-log-events",43 "lifecycles": "status-lifecycles", "status_lifecycles": "status-lifecycles", "beta_headers": "beta-headers",44 "compat": "compatibility"}4546# natural keys per kind (tuple of field names); None → no dedup47KEYS: dict[str, tuple[str, ...] | None] = {48 "models": ("provider", "id"),49 "endpoints": ("provider", "method", "path"),50 "parameters": ("provider", "endpoint", "parameter", "location", "variant", "type"),51 "tools": ("provider", "type", "name"),52 "streaming-events": ("provider", "api", "event", "direction"),53 "errors": ("provider", "http_status", "type", "code"),54 "headers": ("provider", "name", "direction"),55 "pricing": ("provider", "model_or_service", "dimension", "tier", "unit"),56 "rate-limits": None,57 "deprecations": ("provider", "subject", "shutdown_date"),58 "objects": ("provider", "name"),59 "sdks": ("provider", "language", "package"),60 "webhook-events": ("provider", "event"),61 "audit-log-events": ("provider", "event"),62 "status-lifecycles": ("provider", "resource"),63 "beta-headers": ("provider", "value"),64 "compatibility": None,65}6667CSV_COLUMNS: dict[str, list[str]] = {68 "models": ["provider", "id", "display_name", "family", "status", "release_date", "knowledge_cutoff",69 "context_window", "max_output", "input_modalities", "output_modalities", "reasoning", "streaming",70 "structured_outputs", "function_calling", "prompt_caching", "batch", "fine_tuning", "vision",71 "audio_in", "audio_out", "image_out", "video_out", "realtime", "computer_use", "web_search",72 "code_execution", "file_search", "mcp", "price_input", "price_cached_input", "price_output",73 "endpoints", "tools", "last_verified"],74 "endpoints": ["provider", "api_family", "method", "path", "name", "status", "auth", "beta_header",75 "streaming", "sdk_python", "sdk_node", "verification_result", "source"],76 "parameters": ["provider", "endpoint", "parameter", "location", "type", "required", "default", "minimum",77 "maximum", "enum", "compatible_models", "beta_header", "status", "description", "source"],78 "tools": ["provider", "type", "name", "category", "status", "compatible_models", "compatible_endpoints",79 "beta_header", "billing", "verification_result", "source"],80 "errors": ["provider", "http_status", "type", "code", "retryable", "recommended_action", "message_semantics",81 "source"],82 "pricing": ["provider", "model_or_service", "dimension", "tier", "price", "currency", "unit", "effective_notes",83 "source", "retrieved_at"],84}858687def now() -> str:88 return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")899091def load_fragment(path: Path) -> list[dict]:92 data = json.loads(path.read_text())93 if isinstance(data, dict):94 for k in ("records", "items", "data"):95 if k in data and isinstance(data[k], list):96 data = data[k]97 break98 else:99 data = [data]100 if not isinstance(data, list):101 raise ValueError(f"{path}: expected a JSON array")102 out = []103 for rec in data:104 if not isinstance(rec, dict):105 raise ValueError(f"{path}: record is not an object")106 rec.setdefault("_fragment", str(path.relative_to(ROOT)))107 out.append(rec)108 return out109110111API_CANON = [ # (substring test on lowercased `api`, canonical value)112 ("lyria", "lyria-realtime"), ("music", "lyria-realtime"), ("interaction", "interactions"),113 ("bidigeneratecontent", "live"), ("generatecontent", "generate-content"), ("voice", "voice"),114 ("deferred", "chat_completions"),115 ("websocket", "responses-websocket"), ("responses", "responses"), ("chat", "chat_completions"),116 ("/v1/completions", "completions"), ("completions", "completions"), ("translation", "realtime-translation"),117 ("realtime", "realtime"), ("live", "live"), ("transcription", "audio-transcriptions"), ("speech", "audio-speech"),118 ("images", "images"), ("webhook", "agents-webhooks"), ("agents/sessions", "agents"), ("agents", "agents"),119 ("managed", "managed-agents"), ("/v1/complete", "complete-legacy"), ("messages", "messages"),120]121122123def canon_api(rec: dict) -> None:124 """Normalize streaming_event.api to a small canonical vocabulary; keep the original in api_raw."""125 if rec.get("provider") == "gemini" and "live" in str(rec.get("api", "")).lower() and "lyria" not in str(rec.get("api", "")).lower():126 rec["api_raw"], rec["api"] = rec.get("api"), "live"127 return128 if rec.get("provider") == "anthropic" and "managed" in str(rec.get("api", "")).lower():129 rec["api_raw"], rec["api"] = rec.get("api"), "managed-agents"130 return131 a = str(rec.get("api") or "").lower()132 for needle, canon in API_CANON:133 if needle in a:134 if rec.get("api") != canon:135 rec["api_raw"] = rec.get("api")136 rec["api"] = canon137 return138139140def key_of(kind: str, rec: dict) -> tuple | None:141 fields = KEYS.get(kind)142 if fields is None:143 return None144 if kind == "objects":145 name = next((rec.get(k) for k in ("name", "object", "type", "title", "id") if rec.get(k)), None)146 if name is None:147 return None # cannot identify → keep as is148 return (json.dumps(rec.get("provider")), json.dumps(name, default=str))149 return tuple(json.dumps(rec.get(f), sort_keys=True, default=str) for f in fields)150151152def flatten(v) -> str:153 if v is None:154 return ""155 if isinstance(v, bool):156 return "true" if v else "false"157 if isinstance(v, (list, tuple)):158 return "|".join(flatten(x) for x in v)159 if isinstance(v, dict):160 return json.dumps(v, ensure_ascii=False, sort_keys=True)161 return str(v)162163164def model_row(m: dict) -> dict:165 caps = m.get("capabilities") or {}166 mod = m.get("modalities") or {}167 pr = m.get("pricing") if isinstance(m.get("pricing"), dict) else {}168169 def cap(*names):170 for n in names:171 if n in caps:172 return caps[n]173 return "unknown"174175 def price(*names):176 for n in names:177 v = pr.get(n)178 if isinstance(v, dict):179 v = v.get("price", v.get("value"))180 if v is not None:181 return v182 return ""183184 return {185 "provider": m.get("provider"), "id": m.get("id"), "display_name": m.get("display_name"),186 "family": m.get("family"), "status": m.get("status"), "release_date": m.get("release_date"),187 "knowledge_cutoff": m.get("knowledge_cutoff"), "context_window": m.get("context_window"),188 "max_output": m.get("max_output"), "input_modalities": mod.get("input"), "output_modalities": mod.get("output"),189 "reasoning": cap("reasoning", "thinking", "extended_thinking"), "streaming": cap("streaming"),190 "structured_outputs": cap("structured_outputs"), "function_calling": cap("function_calling", "tool_use"),191 "prompt_caching": cap("prompt_caching"), "batch": cap("batch"), "fine_tuning": cap("fine_tuning"),192 "vision": cap("image_in", "image_input", "vision"), "audio_in": cap("audio_in", "audio_input"),193 "audio_out": cap("audio_out", "audio_output"), "image_out": cap("image_out", "image_output", "image_generation"),194 "video_out": cap("video_out", "video_output", "video_generation"), "realtime": cap("realtime"),195 "computer_use": cap("computer_use"), "web_search": cap("web_search"),196 "code_execution": cap("code_execution", "code_interpreter"), "file_search": cap("file_search"),197 "mcp": cap("mcp"), "price_input": price("input"), "price_cached_input": price("cached_input", "cache_read"),198 "price_output": price("output"), "endpoints": sorted(names(m.get("endpoints"), "route", "path", "name")),199 "tools": sorted(names(m.get("tools"), "type", "name")),200 "last_verified": m.get("last_verified"),201 }202203204def generic_row(kind: str, rec: dict) -> dict:205 cols = CSV_COLUMNS[kind]206 row = {}207 for c in cols:208 if c == "verification_result":209 row[c] = (rec.get("verification") or {}).get("result")210 elif c == "sdk_python":211 row[c] = (rec.get("sdk") or {}).get("python")212 elif c == "sdk_node":213 row[c] = (rec.get("sdk") or {}).get("node")214 elif c == "streaming" and isinstance(rec.get("streaming"), dict):215 row[c] = rec["streaming"].get("supported")216 elif c == "source":217 src = rec.get("source") or rec.get("sources")218 if isinstance(src, list):219 src = "|".join((s.get("url") if isinstance(s, dict) else str(s)) for s in src)220 row[c] = src221 else:222 row[c] = rec.get(c)223 return row224225226def write_csv(path: Path, rows: list[dict], columns: list[str]) -> None:227 with path.open("w", newline="") as f:228 w = csv.DictWriter(f, fieldnames=columns, extrasaction="ignore")229 w.writeheader()230 for r in rows:231 w.writerow({k: flatten(r.get(k)) for k in columns})232233234def names(items, *keys: str) -> set[str]:235 """Normalize a list of strings/objects into a set of identifier strings."""236 out: set[str] = set()237 for x in items or []:238 if isinstance(x, str):239 out.add(x)240 elif isinstance(x, dict):241 for k in keys:242 v = x.get(k)243 if isinstance(v, str):244 out.add(v)245 break246 return out247248249def build_matrices(models: list[dict], tools: list[dict], endpoints: list[dict]) -> dict[str, list[dict]]:250 cap_names: list[str] = []251 seen = set()252 for m in models:253 for c in (m.get("capabilities") or {}):254 if c not in seen:255 seen.add(c)256 cap_names.append(c)257 cap_matrix = []258 for m in models:259 row = {"provider": m.get("provider"), "model": m.get("id")}260 caps = m.get("capabilities") or {}261 for c in cap_names:262 row[c] = caps.get(c, "unknown")263 cap_matrix.append(row)264265 ep_names = sorted({e for m in models for e in names(m.get("endpoints"), "route", "path", "name")})266 ep_matrix = []267 for m in models:268 row = {"provider": m.get("provider"), "model": m.get("id")}269 mine = names(m.get("endpoints"), "route", "path", "name")270 for e in ep_names:271 row[e] = e in mine272 ep_matrix.append(row)273274 tool_types = sorted({t.get("type") for t in tools if t.get("type")})275 tool_matrix = []276 for m in models:277 row = {"provider": m.get("provider"), "model": m.get("id")}278 mine = names(m.get("tools"), "type", "name")279 for t in tool_types:280 # tool record may list compatible models explicitly; union both directions281 row[t] = (t in mine) or any(282 tr.get("type") == t and tr.get("provider") == m.get("provider")283 and m.get("id") in (tr.get("compatible_models") or []) for tr in tools)284 tool_matrix.append(row)285286 tool_model = []287 for t in tools:288 tool_model.append({"provider": t.get("provider"), "tool_type": t.get("type"), "name": t.get("name"),289 "category": t.get("category"), "status": t.get("status"),290 "compatible_models": t.get("compatible_models"),291 "compatible_endpoints": t.get("compatible_endpoints"), "beta_header": t.get("beta_header")})292 return {"model-capability-matrix": cap_matrix, "model-endpoint-matrix": ep_matrix,293 "model-tool-matrix": tool_matrix, "tool-model-matrix": tool_model}294295296def main() -> int:297 ap = argparse.ArgumentParser()298 ap.add_argument("--check", action="store_true")299 args = ap.parse_args()300301 if not FRAG.exists():302 print("no fragments directory", file=sys.stderr)303 return 1304305 merged: dict[str, list[dict]] = defaultdict(list)306 report = {"built_at": now(), "fragments": [], "conflicts": [], "warnings": [], "counts": {}}307 indexes: dict[str, dict[tuple, int]] = defaultdict(dict)308 for kind_dir in sorted(p for p in FRAG.iterdir() if p.is_dir()):309 kind = KIND_ALIASES.get(kind_dir.name, kind_dir.name)310 index = indexes[kind]311 for f in sorted(kind_dir.glob("*.json")):312 try:313 recs = load_fragment(f)314 except Exception as e: # noqa: BLE001315 report["warnings"].append(f"{f}: {e}")316 continue317 report["fragments"].append({"kind": kind, "file": str(f.relative_to(ROOT)), "records": len(recs)})318 for r in recs:319 st = r.get("status")320 if isinstance(st, str):321 r["status"] = [st]322 if kind == "streaming-events":323 canon_api(r)324 for s in (r.get("status") or []):325 if s not in STATUSES:326 report["warnings"].append(f"{f}: unknown status {s!r}")327 k = key_of(kind, r)328 if k is None:329 merged[kind].append(r)330 continue331 if k in index:332 prev = merged[kind][index[k]]333 report["conflicts"].append({"kind": kind, "key": k, "kept": r["_fragment"], "dropped": prev["_fragment"]})334 merged[kind][index[k]] = r335 else:336 index[k] = len(merged[kind])337 merged[kind].append(r)338339 for kind, recs in merged.items():340 report["counts"][kind] = len(recs)341342 if args.check:343 print(json.dumps(report, indent=1, default=str))344 return 0345346 OUT.mkdir(exist_ok=True)347 (OUT / "compatibility").mkdir(exist_ok=True)348 for kind, recs in merged.items():349 if kind == "compatibility":350 continue351 (OUT / f"{kind}.json").write_text(json.dumps(recs, indent=1, ensure_ascii=False, default=str))352 if kind in CSV_COLUMNS:353 rows = [model_row(r) for r in recs] if kind == "models" else [generic_row(kind, r) for r in recs]354 write_csv(OUT / f"{kind}.csv", rows, CSV_COLUMNS[kind])355356 # compatibility fragments are copied through by file name357 for f in sorted((FRAG / "compatibility").glob("*.json")) if (FRAG / "compatibility").exists() else []:358 (OUT / "compatibility" / f.name).write_text(f.read_text())359360 matrices = build_matrices(merged.get("models", []), merged.get("tools", []), merged.get("endpoints", []))361 for name, rows in matrices.items():362 (OUT / "compatibility" / f"{name}.json").write_text(json.dumps(rows, indent=1, ensure_ascii=False, default=str))363 if rows:364 cols = list(rows[0].keys())365 for r in rows[1:]:366 for c in r:367 if c not in cols:368 cols.append(c)369 write_csv(OUT / "compatibility" / f"{name}.csv", rows, cols)370371 # examples manifest: merge examples/manifest-*.json372 ex = []373 for f in sorted((ROOT / "examples").glob("manifest-*.json")):374 try:375 data = json.loads(f.read_text())376 items = data.get("examples", data.get("files", data.get("items"))) if isinstance(data, dict) else data377 if isinstance(items, dict):378 items = [dict(v, file=k) if isinstance(v, dict) else {"file": k, "status": v} for k, v in items.items()]379 if items is None:380 items = [v for v in data.values() if isinstance(v, dict)] if isinstance(data, dict) else []381 norm = []382 for it in items:383 if isinstance(it, str):384 it = {"file": it}385 elif not isinstance(it, dict):386 continue387 it.setdefault("_manifest", f.name)388 norm.append(it)389 ex.extend(norm)390 except Exception as e: # noqa: BLE001391 report["warnings"].append(f"{f}: {e}")392 (OUT / "examples-manifest.json").write_text(json.dumps(ex, indent=1, ensure_ascii=False, default=str))393 report["counts"]["examples"] = len(ex)394395 # live request log summary396 log = ROOT / "reports" / "live-requests.jsonl"397 if log.exists():398 calls = [json.loads(l) for l in log.read_text().splitlines() if l.strip()]399 summary = defaultdict(lambda: {"calls": 0, "est_cost_usd": 0.0, "statuses": defaultdict(int)})400 for c in calls:401 s = summary[c.get("provider", "?")]402 s["calls"] += 1403 s["est_cost_usd"] += float(c.get("est_cost_usd") or 0)404 s["statuses"][str(c.get("status"))] += 1405 report["live_requests"] = {k: {"calls": v["calls"], "est_cost_usd": round(v["est_cost_usd"], 4),406 "statuses": dict(v["statuses"])} for k, v in summary.items()}407408 (OUT / "build-report.json").write_text(json.dumps(report, indent=1, default=str))409 index = {"built_at": report["built_at"], "counts": report["counts"],410 "files": sorted(str(p.relative_to(ROOT)) for p in OUT.rglob("*") if p.is_file() and "fragments" not in p.parts)}411 (OUT / "index.json").write_text(json.dumps(index, indent=1))412 print(json.dumps({"counts": report["counts"], "conflicts": len(report["conflicts"]),413 "warnings": len(report["warnings"])}, indent=1))414 return 0415416417if __name__ == "__main__":418 sys.exit(main())419