Python 88.3%
TypeScript 7.6%
Shell 4.1%
1#!/usr/bin/env python32"""Generate generated/fragments/endpoints/openai-files-vectorstores-batch-finetuning-evals.json3from the OpenAPI index (tmp/platform-spec/_index.json), the SDK map and the observed live results (2026-09-18)."""4from __future__ import annotations5import json, re6from pathlib import Path7ROOT = Path(__file__).resolve().parent.parent8IDX = json.load(open(ROOT / "tmp/platform-spec/_index.json"))9SDK = json.load(open(ROOT / "tmp/platform-spec/_sdk_map.json"))10RET = "2026-09-18"11REF = "https://developers.openai.com/api/reference/resources/"12GUIDE = "https://developers.openai.com/api/docs/guides/"13DEPREC = "https://developers.openai.com/api/docs/deprecations"1415FAM = [("/fine_tuning/alpha/graders", "graders"), ("/fine_tuning", "fine_tuning"), ("/files", "files"), ("/uploads", "uploads"),16 ("/vector_stores", "vector_stores"), ("/batches", "batches"), ("/evals", "evals")]17DEPRECATED = {"fine_tuning": "Self-serve fine-tuning is winding down: no new orgs since 2026-05-07, job creation ends 2027-01-06 (inference on existing ft: models continues until base-model deprecation).",18 "graders": "Graders are deprecated together with the Evals platform (API shutdown 2026-11-30) and the fine-tuning platform (2027-01-06).",19 "evals": "Evals platform deprecated 2026-06-03: read-only 2026-10-31, dashboard + API shut down 2026-11-30."}20SRC_PAGE = {"files": "files", "uploads": "uploads", "vector_stores": "vector_stores", "batches": "batches", "fine_tuning": "fine_tuning", "graders": "graders", "evals": "evals"}21GUIDES = {"files": [GUIDE + "file-inputs"], "uploads": [], "vector_stores": [GUIDE + "retrieval", GUIDE + "tools-file-search"], "batches": [GUIDE + "batch"],22 "fine_tuning": [GUIDE + "supervised-fine-tuning", GUIDE + "direct-preference-optimization", GUIDE + "reinforcement-fine-tuning", GUIDE + "model-optimization"],23 "graders": [GUIDE + "graders"], "evals": [GUIDE + "evals"]}2425# observed live results, keyed by "METHOD /path" (spec path form)26V = lambda result, http, note, method="live_api": {"method": method, "verified_at": RET, "result": result, "http_status": http, "request_note": note} # noqa: E73127LIVE = {28 "POST /files": V("success", 200, "multipart purposes user_data (+expires_after 3600s -> expires_at set), batch (default expires_at = +30d), evals, assistants; 200-byte txt and 1-line jsonl"),29 "GET /files": V("success", 200, "?purpose=user_data&limit=5 -> list envelope {object,data,first_id,last_id,has_more}"),30 "GET /files/{file_id}": V("success", 200, "200 for own file; 404 'No such File object' after delete"),31 "DELETE /files/{file_id}": V("success", 200, "{object:'file', id, deleted:true} for all 7 created files"),32 "GET /files/{file_id}/content": V("success", 200, "200 (raw jsonl) for purpose batch_output; 400 'Not allowed to download files of purpose: user_data' for user_data"),33 "POST /uploads": V("success", 200, "purpose user_data accepted although the OpenAPI enum lists only assistants|batch|fine-tune|vision; status pending; expires_at = created_at + 3600"),34 "POST /uploads/{upload_id}/parts": V("success", 200, "multipart field 'data' -> {object:'upload.part'}; 400 after cancel: 'Upload status is already in a cancelled state'"),35 "POST /uploads/{upload_id}/complete": V("success", 200, "part_ids -> status completed, file.purpose user_data, file.bytes == declared bytes"),36 "POST /uploads/{upload_id}/cancel": V("success", 200, "status cancelled"),37 "POST /vector_stores": V("success", 200, "name+description+metadata+expires_after(last_active_at,1d) -> status completed, usage_bytes 0"),38 "GET /vector_stores": V("success", 200, "limit=5"),39 "GET /vector_stores/{vector_store_id}": V("success", 200, "usage_bytes 1203 after a 200-byte file (index overhead); 404 for bogus id"),40 "POST /vector_stores/{vector_store_id}": V("success", 200, "rename + metadata replace"),41 "DELETE /vector_stores/{vector_store_id}": V("success", 200, "{object:'vector_store.deleted', deleted:true}"),42 "POST /vector_stores/{vector_store_id}/files": V("success", 200, "static chunking 100/20 + attributes {string,number,boolean} -> in_progress, completed after ~7.6s"),43 "GET /vector_stores/{vector_store_id}/files": V("success", 200, "?filter=completed"),44 "GET /vector_stores/{vector_store_id}/files/{file_id}": V("success", 200, "polled in_progress -> completed (7.6s)"),45 "POST /vector_stores/{vector_store_id}/files/{file_id}": V("success", 200, "attributes replaced wholesale (not merged)"),46 "DELETE /vector_stores/{vector_store_id}/files/{file_id}": V("success", 200, "{object:'vector_store.file.deleted', deleted:true}"),47 "GET /vector_stores/{vector_store_id}/files/{file_id}/content": V("success", 200, "{object:'vector_store.file_content.page', data:[{type:'text',text}], has_more, next_page}"),48 "POST /vector_stores/{vector_store_id}/search": V("success", 200, "string query + and/eq/gte filters + rewrite_query (search_query rewritten) + ranking_options; array query + ranker none; non-matching filter -> empty data"),49 "POST /vector_stores/{vector_store_id}/file_batches": V("success", 200, "files[] form with per-file attributes + chunking_strategy auto -> object 'vector_store.file_batch' (spec says vector_store.files_batch), in_progress -> completed ~4s"),50 "GET /vector_stores/{vector_store_id}/file_batches/{batch_id}": V("success", 200, "polled to completed"),51 "GET /vector_stores/{vector_store_id}/file_batches/{batch_id}/files": V("success", 200, "returned chunking_strategy.type 'static' for a file added with type auto (auto resolves to static 800/400)"),52 "POST /vector_stores/{vector_store_id}/file_batches/{batch_id}/cancel": V("failure", 500, "cancel on an already-completed batch returned HTTP 500 'The server had an error processing your request' (not tested on an in-progress batch)"),53 "POST /batches": V("success", 200, "1-request /v1/responses gpt-5.4-nano; status validating -> in_progress (2s) -> finalizing -> completed at 30s; usage populated; output file purpose batch_output with expires_at honoring output_expires_after. NOTE: a bogus input_file_id is accepted with 200 (fails later during validation)"),54 "GET /batches": V("success", 200, "limit=3"),55 "GET /batches/{batch_id}": V("success", 200, "404 for bogus id"),56 "POST /batches/{batch_id}/cancel": V("failure", 409, "cancel on a batch that had just transitioned validating -> failed returned 409 Conflict (behaviour not documented); cancel on an in-progress batch not exercised because the 1-request test batch completed in 30s"),57 "GET /fine_tuning/jobs": V("success", 200, "empty list {object,data,has_more}; metadata[owner]=x filter accepted"),58 "POST /fine_tuning/jobs": V("restricted", 403, "validation-only call with bogus training_file -> 403 code training_not_available: 'OpenAI is winding down the fine-tuning platform and your organization is no longer able to create new fine-tuning training jobs'"),59 "GET /fine_tuning/jobs/{fine_tuning_job_id}": V("success", 404, "bogus id -> 404 code fine_tune_not_found (endpoint reachable)"),60 "POST /fine_tuning/jobs/{fine_tuning_job_id}/cancel": V("success", None, "not called (no job)", method="docs_only"),61 "POST /fine_tuning/jobs/{fine_tuning_job_id}/pause": V("success", 404, "bogus id -> 404 fine_tune_not_found"),62 "POST /fine_tuning/jobs/{fine_tuning_job_id}/resume": V("success", None, "not called (no job)", method="docs_only"),63 "GET /fine_tuning/jobs/{fine_tuning_job_id}/events": V("success", 404, "bogus id -> 404 fine_tune_not_found"),64 "GET /fine_tuning/jobs/{fine_tuning_job_id}/checkpoints": V("success", 404, "bogus id -> 404 fine_tune_not_found"),65 "GET /fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions": V("restricted", 401, "401 'Missing scopes: api.fine_tuning.checkpoints.read' (requires org Owner role / admin scope)"),66 "POST /fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions": V("success", None, "not called (admin-level write)", method="docs_only"),67 "DELETE /fine_tuning/checkpoints/{fine_tuned_model_checkpoint}/permissions/{permission_id}": V("success", None, "not called (admin-level write)", method="docs_only"),68 "POST /fine_tuning/alpha/graders/validate": V("success", 200, "string_check + python graders -> {grader}; invalid operation 'contains' -> 400 type invalid_value param operation"),69 "POST /fine_tuning/alpha/graders/run": V("success", 200, "string_check reward 1.0 (0.4ms), text_similarity fuzzy_match 0.756, python 1.0 (2.3s sandbox), multi with calculate_output -> sub_rewards; no model grader run (cost)"),70 "POST /evals": V("success", 201, "custom data_source_config (item_schema + include_sample_schema) with string_check/text_similarity/python criteria -> 201; logs config + label_model criterion -> 201; returned schema wraps {item, sample} with sample.required=[model, choices]"),71 "GET /evals": V("success", 200, "list is eventually consistent: empty right after create, present 3s later"),72 "GET /evals/{eval_id}": V("success", 200, "404 after delete"),73 "POST /evals/{eval_id}": V("success", 200, "rename + metadata"),74 "DELETE /evals/{eval_id}": V("success", 200, "{object:'eval.deleted', deleted:true, eval_id}"),75 "POST /evals/{eval_id}/runs": V("success", 201, "jsonl file_content and jsonl file_id (purpose evals) with pre-filled sample {model, choices, output_text} -> 201 queued; sample without model/choices -> 400 'model is a required property'; responses source without model/input_messages -> 400"),76 "GET /evals/{eval_id}/runs": V("success", 200, "?status=completed returned 0 items right after completion (index lag)"),77 "GET /evals/{eval_id}/runs/{run_id}": V("success", 200, "queued -> in_progress -> completed in ~4s for 3 items, no sampling; result_counts + per_testing_criteria_results (with testing_criteria_id) populated"),78 "POST /evals/{eval_id}/runs/{run_id}": V("success", 200, "cancel on a completed run -> 200, status stays completed"),79 "DELETE /evals/{eval_id}/runs/{run_id}": V("success", 200, "{object:'eval.run.deleted', deleted:true, run_id}"),80 "GET /evals/{eval_id}/runs/{run_id}/output_items": V("success", 200, "3 items, status pass|fail, ?status=fail filter works; results[] {name(id), type:null, score, passed}; extra fields _datasource_item_content_hash, available_includes"),81 "GET /evals/{eval_id}/runs/{run_id}/output_items/{output_item_id}": V("success", 200, "single item"),82}83STATUS_FOR_RESULT = {"success": "LIVE_VERIFIED", "restricted": "ACCOUNT_RESTRICTED", "failure": "FAILED_VERIFICATION"}848586def family(path):87 for p, f in FAM:88 if path.startswith(p):89 return f909192def clean_sdk(s):93 return re.sub(r"<a href=\"[^\"]+\">([^<]+)</a>", r"\1", s) if s else None949596records = []97for key, op in IDX.items():98 method, path = key.split(" ", 1)99 fam = family(path)100 ver = LIVE.get(key, V("success", None, "not called", method="docs_only"))101 status = ["DOCUMENTED"]102 if ver["method"] == "live_api":103 status.append(STATUS_FOR_RESULT[ver["result"]])104 if fam in DEPRECATED:105 status.append("DEPRECATED")106 if fam == "graders":107 status.append("BETA") # alpha path108 if key == "POST /uploads":109 status.append("LIVE_DISCOVERED") # user_data purpose not in spec enum110 if key == "POST /batches/{batch_id}/cancel":111 status = ["DOCUMENTED", "LIVE_DISCOVERED"] # 409 on terminal batch: reachable, undocumented semantics112 paged = any(p in ("after", "limit") for p in op["params"]) and method == "GET" and not path.endswith(("/content",))113 resp = op["responses"]114 ok = next(iter(resp.values()), {})115 sdk = SDK.get(f"{method} /v1{path}", {})116 rec = {117 "provider": "openai", "api_family": fam, "method": method, "path": f"/v1{path}", "name": op["operationId"],118 "description": op["summary"] or "", "status": status, "auth": "Bearer API key (project or user key)",119 "beta_header": None,120 "request": {"content_type": op["body_ct"], "body_ref": op.get("body_ref")},121 "response": {"content_type": ok.get("content_type"), "body_ref": ok.get("ref"), "success_codes": sorted(int(c) for c in resp if c.isdigit() and int(c) < 400)},122 "streaming": {"supported": False, "events_ref": None},123 "pagination": ({"style": "cursor", "params": [p for p in op["params"] if p in ("after", "before", "limit", "order", "order_by", "filter", "status", "purpose", "metadata", "project_id")]} if paged else None),124 "idempotency": "safe" if method == "GET" else ("idempotent" if method == "DELETE" else "not idempotent"),125 "sdk": {"python": clean_sdk(sdk.get("python")), "node": clean_sdk(sdk.get("node"))},126 "deprecation": DEPRECATED.get(fam),127 "verification": ver,128 "sources": [{"url": REF + SRC_PAGE[fam], "retrieved_at": RET}] + [{"url": u, "retrieved_at": RET} for u in GUIDES[fam]] + ([{"url": DEPREC, "retrieved_at": RET}] if fam in DEPRECATED else []),129 }130 records.append(rec)131records.sort(key=lambda r: (r["api_family"], r["path"], r["method"]))132out = ROOT / "generated/fragments/endpoints/openai-files-vectorstores-batch-finetuning-evals.json"133out.parent.mkdir(parents=True, exist_ok=True)134out.write_text(json.dumps(records, indent=1, ensure_ascii=False) + "\n")135from collections import Counter136print(len(records), Counter(s for r in records for s in r["status"]))137