Python 88.3%
TypeScript 7.6%
Shell 4.1%
1#!/usr/bin/env python32"""Live verification of Files / Uploads / Vector stores / Batch / Fine-tuning / Graders / Evals.3Budget: < $0.05. Everything created is prefixed atlas-platform-agent and deleted at the end.4Raw sanitized responses -> tmp-live/platform/*.json ; summary -> stdout (JSON lines).5"""6from __future__ import annotations7import json, sys, time, uuid8from pathlib import Path9ROOT = Path(__file__).resolve().parent.parent10sys.path.insert(0, str(ROOT))11from scripts.live import openai_request, save_sanitized # noqa: E4021213OUT = ROOT / "tmp-live" / "platform"14OUT.mkdir(parents=True, exist_ok=True)15TAG = "atlas-platform-agent"16RUN = uuid.uuid4().hex[:6]17SUMMARY: list[dict] = []18CREATED = {"files": [], "vector_stores": [], "evals": [], "uploads": []}192021def rec(step, method, path, st, body, note="", **extra):22 save_sanitized({"status": st, "body": body if not isinstance(body, bytes) else body.decode("utf-8", "replace")},23 OUT / f"{len(SUMMARY):02d}-{step}.json")24 row = {"step": step, "method": method, "path": path, "status": st, "note": note, **extra}25 SUMMARY.append(row)26 print(json.dumps(row, default=str), flush=True)27 return body282930def multipart(fields: dict[str, str], files: dict[str, tuple[str, bytes, str]]):31 b = f"----atlas{uuid.uuid4().hex}"32 out = bytearray()33 for k, v in fields.items():34 out += f"--{b}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n".encode()35 for k, (fn, data, ct) in files.items():36 out += f"--{b}\r\nContent-Disposition: form-data; name=\"{k}\"; filename=\"{fn}\"\r\nContent-Type: {ct}\r\n\r\n".encode()37 out += data + b"\r\n"38 out += f"--{b}--\r\n".encode()39 return bytes(out), f"multipart/form-data; boundary={b}"404142def oai(method, path, json_body=None, *, data=None, content_type="application/json", note="", cost=0.0):43 st, body, hdrs = openai_request(method, path, json_body, data=data, content_type=content_type, note=note, est_cost_usd=cost)44 return st, body454647def upload_file(name: str, content: bytes, purpose: str, mime="text/plain", extra_fields=None):48 fields = {"purpose": purpose, **(extra_fields or {})}49 data, ct = multipart(fields, {"file": (name, content, mime)})50 st, body = oai("POST", "/v1/files", data=data, content_type=ct, note=f"{TAG} upload {purpose}")51 if st == 200:52 CREATED["files"].append(body["id"])53 return st, body545556def delete_file(fid):57 st, body = oai("DELETE", f"/v1/files/{fid}", note=f"{TAG} cleanup")58 rec("files.delete", "DELETE", f"/v1/files/{fid}", st, body, deleted=body.get("deleted") if isinstance(body, dict) else None)59 if st == 200 and fid in CREATED["files"]:60 CREATED["files"].remove(fid)61 return st626364# ---------------------------------------------------------------- (a) Files65txt = (f"{TAG} {RUN} - The Atlas woodchuck policy: each passenger may carry at most two woodchucks. "66 "Woodchucks must be contained during transport. Return window is 30 days.").encode()67txt = (txt + b" " * 200)[:200]68st, f_txt = upload_file(f"{TAG}-{RUN}.txt", txt, "user_data",69 extra_fields={"expires_after[anchor]": "created_at", "expires_after[seconds]": "3600"})70rec("files.create(user_data,expires_after)", "POST", "/v1/files", st, f_txt,71 id=f_txt.get("id"), bytes=f_txt.get("bytes"), status_field=f_txt.get("status"), expires_at=f_txt.get("expires_at"))72TXT_ID = f_txt.get("id")7374st, b = oai("GET", "/v1/files?purpose=user_data&limit=5", note=TAG)75rec("files.list(purpose=user_data)", "GET", "/v1/files", st, b, n=len(b.get("data", [])), has_more=b.get("has_more"))76st, b = oai("GET", f"/v1/files/{TXT_ID}", note=TAG)77rec("files.retrieve", "GET", f"/v1/files/{TXT_ID}", st, b, purpose=b.get("purpose"))78st, b = oai("GET", f"/v1/files/{TXT_ID}/content", content_type=None, note=TAG)79rec("files.content", "GET", f"/v1/files/{TXT_ID}/content", st, b, bytes=len(b) if isinstance(b, bytes) else None,80 roundtrip_ok=(b == txt))8182# batch input jsonl83line = {"custom_id": f"{TAG}-req-1", "method": "POST", "url": "/v1/responses",84 "body": {"model": "gpt-5.4-nano", "input": "Reply with OK.", "max_output_tokens": 16}}85jsonl = (json.dumps(line) + "\n").encode()86st, f_batch = upload_file(f"{TAG}-{RUN}-batch.jsonl", jsonl, "batch", mime="application/jsonl")87rec("files.create(batch)", "POST", "/v1/files", st, f_batch, id=f_batch.get("id"), expires_at=f_batch.get("expires_at"),88 purpose=f_batch.get("purpose"))89BATCH_FILE_ID = f_batch.get("id")9091# evals purpose file (format check only)92ev_jsonl = (json.dumps({"item": {"q": "ping", "expected": "OK"}}) + "\n").encode()93st, f_ev = upload_file(f"{TAG}-{RUN}-evals.jsonl", ev_jsonl, "evals", mime="application/jsonl")94rec("files.create(evals)", "POST", "/v1/files", st, f_ev, id=f_ev.get("id"), purpose=f_ev.get("purpose"))95EV_FILE_ID = f_ev.get("id")9697# ---------------------------------------------------------------- (b) Batch98t0 = time.time()99st, batch = oai("POST", "/v1/batches", {"input_file_id": BATCH_FILE_ID, "endpoint": "/v1/responses",100 "completion_window": "24h", "metadata": {"owner": TAG, "run": RUN},101 "output_expires_after": {"anchor": "created_at", "seconds": 3600}},102 note=f"{TAG} 1 request gpt-5.4-nano", cost=0.0001)103rec("batches.create", "POST", "/v1/batches", st, batch, id=batch.get("id"), status_field=batch.get("status"),104 request_counts=batch.get("request_counts"), expires_at=batch.get("expires_at"))105BATCH_ID = batch.get("id")106st, b = oai("GET", "/v1/batches?limit=3", note=TAG)107rec("batches.list", "GET", "/v1/batches", st, b, n=len(b.get("data", [])), has_more=b.get("has_more"))108109timeline = []110final = None111if BATCH_ID:112 deadline = time.time() + 190113 while time.time() < deadline:114 st, b = oai("GET", f"/v1/batches/{BATCH_ID}", note=f"{TAG} poll")115 timeline.append({"t": round(time.time() - t0, 1), "status": b.get("status"), "counts": b.get("request_counts")})116 if b.get("status") in ("completed", "failed", "expired", "cancelled"):117 final = b118 break119 time.sleep(20)120 rec("batches.retrieve(poll)", "GET", f"/v1/batches/{BATCH_ID}", st, b, timeline=timeline, final_status=b.get("status"),121 output_file_id=b.get("output_file_id"), error_file_id=b.get("error_file_id"), usage=b.get("usage"))122 if final is None:123 st, c = oai("POST", f"/v1/batches/{BATCH_ID}/cancel", note=TAG)124 rec("batches.cancel", "POST", f"/v1/batches/{BATCH_ID}/cancel", st, c, status_field=c.get("status"),125 cancelling_at=c.get("cancelling_at"))126 for i in range(4):127 time.sleep(10)128 st, c = oai("GET", f"/v1/batches/{BATCH_ID}", note=f"{TAG} poll after cancel")129 timeline.append({"t": round(time.time() - t0, 1), "status": c.get("status")})130 if c.get("status") == "cancelled":131 break132 rec("batches.retrieve(after cancel)", "GET", f"/v1/batches/{BATCH_ID}", st, c, status_field=c.get("status"),133 timeline=timeline[-5:], output_file_id=c.get("output_file_id"), error_file_id=c.get("error_file_id"))134 final = c135 for key in ("output_file_id", "error_file_id"):136 fid = (final or {}).get(key)137 if fid:138 st, content = oai("GET", f"/v1/files/{fid}/content", content_type=None, note=f"{TAG} batch {key}")139 text = content.decode() if isinstance(content, bytes) else json.dumps(content)140 shape = None141 try:142 first = json.loads(text.splitlines()[0])143 shape = {k: (type(v).__name__ if k != "response" else {kk: type(vv).__name__ for kk, vv in (v or {}).items()})144 for k, v in first.items()}145 except Exception as e: # noqa: BLE001146 shape = f"unparsed: {e}"147 rec(f"files.content({key})", "GET", f"/v1/files/{fid}/content", st, text, line_shape=shape)148 st, meta = oai("GET", f"/v1/files/{fid}", note=TAG)149 rec(f"files.retrieve({key})", "GET", f"/v1/files/{fid}", st, meta, purpose=meta.get("purpose"),150 expires_at=meta.get("expires_at"))151 CREATED["files"].append(fid)152153# ---------------------------------------------------------------- (c) Uploads154small = (f"{TAG} {RUN} upload part content\n" * 4).encode()155for purpose in ("user_data", "assistants"):156 st, up = oai("POST", "/v1/uploads", {"filename": f"{TAG}-{RUN}-upload.txt", "purpose": purpose,157 "bytes": len(small), "mime_type": "text/plain"}, note=f"{TAG} purpose={purpose}")158 rec(f"uploads.create(purpose={purpose})", "POST", "/v1/uploads", st, up, id=up.get("id"), status_field=up.get("status"),159 expires_at=up.get("expires_at"), created_at=up.get("created_at"))160 if st == 200:161 break162UP_ID = up.get("id") if st == 200 else None163if UP_ID:164 CREATED["uploads"].append(UP_ID)165 data, ct = multipart({}, {"data": ("part", small, "application/octet-stream")})166 st, part = oai("POST", f"/v1/uploads/{UP_ID}/parts", data=data, content_type=ct, note=TAG)167 rec("uploads.parts.create", "POST", f"/v1/uploads/{UP_ID}/parts", st, part, id=part.get("id"), object=part.get("object"))168 st, done = oai("POST", f"/v1/uploads/{UP_ID}/complete", {"part_ids": [part.get("id")]}, note=TAG)169 rec("uploads.complete", "POST", f"/v1/uploads/{UP_ID}/complete", st, done, status_field=done.get("status"),170 file_id=(done.get("file") or {}).get("id"), file_purpose=(done.get("file") or {}).get("purpose"),171 file_bytes=(done.get("file") or {}).get("bytes"))172 if (done.get("file") or {}).get("id"):173 CREATED["files"].append(done["file"]["id"])174 # second upload: cancel path175 st, up2 = oai("POST", "/v1/uploads", {"filename": f"{TAG}-{RUN}-cancel.txt", "purpose": up.get("purpose"),176 "bytes": 10, "mime_type": "text/plain"}, note=TAG)177 rec("uploads.create(for cancel)", "POST", "/v1/uploads", st, up2, id=up2.get("id"), status_field=up2.get("status"))178 if st == 200:179 st, c2 = oai("POST", f"/v1/uploads/{up2['id']}/cancel", note=TAG)180 rec("uploads.cancel", "POST", f"/v1/uploads/{up2['id']}/cancel", st, c2, status_field=c2.get("status"))181 st, c3 = oai("POST", f"/v1/uploads/{up2['id']}/parts", data=data, content_type=ct, note=f"{TAG} part after cancel (expect error)")182 rec("uploads.parts.create(after cancel)", "POST", f"/v1/uploads/{up2['id']}/parts", st, c3,183 error=(c3.get("error") or {}).get("message") if isinstance(c3, dict) else None)184185# ---------------------------------------------------------------- (d) Vector stores186st, vs = oai("POST", "/v1/vector_stores", {"name": f"{TAG}-{RUN}", "description": "API Atlas live probe",187 "metadata": {"owner": TAG}, "expires_after": {"anchor": "last_active_at", "days": 1}},188 note=TAG)189rec("vector_stores.create", "POST", "/v1/vector_stores", st, vs, id=vs.get("id"), status_field=vs.get("status"),190 file_counts=vs.get("file_counts"), expires_at=vs.get("expires_at"), usage_bytes=vs.get("usage_bytes"))191VS_ID = vs.get("id")192if VS_ID:193 CREATED["vector_stores"].append(VS_ID)194 st, vsf = oai("POST", f"/v1/vector_stores/{VS_ID}/files",195 {"file_id": TXT_ID, "attributes": {"atlas": "platform", "n": 1, "draft": False},196 "chunking_strategy": {"type": "static", "static": {"max_chunk_size_tokens": 100, "chunk_overlap_tokens": 20}}},197 note=TAG)198 rec("vector_stores.files.create(static 100/20)", "POST", f"/v1/vector_stores/{VS_ID}/files", st, vsf, id=vsf.get("id"),199 status_field=vsf.get("status"), chunking_strategy=vsf.get("chunking_strategy"), attributes=vsf.get("attributes"),200 usage_bytes=vsf.get("usage_bytes"))201 t1 = time.time(); tl = []202 for i in range(30):203 st, vsf = oai("GET", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}", note=f"{TAG} poll")204 tl.append({"t": round(time.time() - t1, 1), "status": vsf.get("status")})205 if vsf.get("status") != "in_progress":206 break207 time.sleep(2)208 rec("vector_stores.files.retrieve(poll)", "GET", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}", st, vsf,209 timeline=tl, final_status=vsf.get("status"), usage_bytes=vsf.get("usage_bytes"), last_error=vsf.get("last_error"))210 st, b = oai("GET", f"/v1/vector_stores/{VS_ID}/files?filter=completed&limit=10", note=TAG)211 rec("vector_stores.files.list(filter=completed)", "GET", f"/v1/vector_stores/{VS_ID}/files", st, b,212 n=len(b.get("data", [])), keys=sorted(b.keys()))213 st, b = oai("GET", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}/content", note=TAG)214 rec("vector_stores.files.content", "GET", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}/content", st, b,215 object=b.get("object"), n_chunks=len(b.get("data", [])), first_chunk=(b.get("data") or [{}])[0].get("text", "")[:80])216 st, vsr = oai("GET", f"/v1/vector_stores/{VS_ID}", note=TAG)217 rec("vector_stores.retrieve", "GET", f"/v1/vector_stores/{VS_ID}", st, vsr, status_field=vsr.get("status"),218 usage_bytes=vsr.get("usage_bytes"), file_counts=vsr.get("file_counts"), last_active_at=vsr.get("last_active_at"))219 st, s = oai("POST", f"/v1/vector_stores/{VS_ID}/search",220 {"query": "How many woodchucks per passenger?", "max_num_results": 3, "rewrite_query": True,221 "filters": {"type": "and", "filters": [{"type": "eq", "key": "atlas", "value": "platform"},222 {"type": "gte", "key": "n", "value": 1}]},223 "ranking_options": {"ranker": "auto", "score_threshold": 0.0}}, note=TAG)224 rec("vector_stores.search(filters+rewrite)", "POST", f"/v1/vector_stores/{VS_ID}/search", st, s,225 object=s.get("object"), search_query=s.get("search_query"), n=len(s.get("data", [])),226 top=(s.get("data") or [{}])[0].get("score"), result_keys=sorted((s.get("data") or [{}])[0].keys()))227 st, s2 = oai("POST", f"/v1/vector_stores/{VS_ID}/search",228 {"query": "woodchucks", "filters": {"type": "eq", "key": "atlas", "value": "nomatch"}}, note=TAG)229 rec("vector_stores.search(filter no match)", "POST", f"/v1/vector_stores/{VS_ID}/search", st, s2, n=len(s2.get("data", [])))230 st, s3 = oai("POST", f"/v1/vector_stores/{VS_ID}/search",231 {"query": ["woodchuck limit", "return policy"], "max_num_results": 2, "ranking_options": {"ranker": "none"}}, note=TAG)232 rec("vector_stores.search(query array, ranker none)", "POST", f"/v1/vector_stores/{VS_ID}/search", st, s3,233 n=len(s3.get("data", [])), search_query=s3.get("search_query"), error=(s3.get("error") or {}).get("message") if isinstance(s3, dict) else None)234 st, u = oai("POST", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}", {"attributes": {"atlas": "platform", "n": 2, "lang": "en"}}, note=TAG)235 rec("vector_stores.files.update(attributes)", "POST", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}", st, u, attributes=u.get("attributes"))236 st, um = oai("POST", f"/v1/vector_stores/{VS_ID}", {"name": f"{TAG}-{RUN}-renamed", "metadata": {"owner": TAG, "phase": "2"}}, note=TAG)237 rec("vector_stores.update", "POST", f"/v1/vector_stores/{VS_ID}", st, um, name=um.get("name"), metadata=um.get("metadata"))238 # file batch with a second file239 st, f2 = upload_file(f"{TAG}-{RUN}-second.txt", b"Second Atlas file: refunds are processed within 5 business days. " * 3, "assistants")240 rec("files.create(assistants)", "POST", "/v1/files", st, f2, id=f2.get("id"), purpose=f2.get("purpose"))241 F2 = f2.get("id")242 st, fb = oai("POST", f"/v1/vector_stores/{VS_ID}/file_batches",243 {"files": [{"file_id": F2, "attributes": {"atlas": "platform", "n": 2}, "chunking_strategy": {"type": "auto"}}]}, note=TAG)244 rec("vector_stores.file_batches.create(files[])", "POST", f"/v1/vector_stores/{VS_ID}/file_batches", st, fb, id=fb.get("id"),245 object=fb.get("object"), status_field=fb.get("status"), file_counts=fb.get("file_counts"),246 error=(fb.get("error") or {}).get("message") if isinstance(fb, dict) else None)247 FB_ID = fb.get("id")248 if FB_ID:249 tl = []250 for i in range(30):251 st, fb = oai("GET", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}", note=f"{TAG} poll")252 tl.append({"t": i * 2, "status": fb.get("status")})253 if fb.get("status") != "in_progress":254 break255 time.sleep(2)256 rec("vector_stores.file_batches.retrieve(poll)", "GET", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}", st, fb,257 timeline=tl, final_status=fb.get("status"), file_counts=fb.get("file_counts"))258 st, b = oai("GET", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}/files?limit=10", note=TAG)259 rec("vector_stores.file_batches.list_files", "GET", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}/files", st, b,260 n=len(b.get("data", [])), chunking=[(x.get("chunking_strategy") or {}).get("type") for x in b.get("data", [])],261 attrs=[x.get("attributes") for x in b.get("data", [])])262 st, c = oai("POST", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}/cancel", note=f"{TAG} cancel completed batch (expect error or no-op)")263 rec("vector_stores.file_batches.cancel(after completion)", "POST", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}/cancel", st, c,264 status_field=c.get("status") if isinstance(c, dict) else None, error=(c.get("error") or {}).get("message") if isinstance(c, dict) else None)265 st, b = oai("GET", "/v1/vector_stores?limit=5", note=TAG)266 rec("vector_stores.list", "GET", "/v1/vector_stores", st, b, n=len(b.get("data", [])))267 # cleanup vs files, store268 for fid in [TXT_ID, F2]:269 if fid:270 st, d = oai("DELETE", f"/v1/vector_stores/{VS_ID}/files/{fid}", note=f"{TAG} cleanup")271 rec("vector_stores.files.delete", "DELETE", f"/v1/vector_stores/{VS_ID}/files/{fid}", st, d, deleted=d.get("deleted"), object=d.get("object"))272 st, d = oai("DELETE", f"/v1/vector_stores/{VS_ID}", note=f"{TAG} cleanup")273 rec("vector_stores.delete", "DELETE", f"/v1/vector_stores/{VS_ID}", st, d, deleted=d.get("deleted"), object=d.get("object"))274 if d.get("deleted"):275 CREATED["vector_stores"].remove(VS_ID)276277# ---------------------------------------------------------------- (e) Fine-tuning + graders278st, b = oai("GET", "/v1/fine_tuning/jobs?limit=3", note=TAG)279rec("fine_tuning.jobs.list", "GET", "/v1/fine_tuning/jobs", st, b, n=len(b.get("data", [])) if isinstance(b, dict) else None,280 keys=sorted(b.keys()) if isinstance(b, dict) else None, error=(b.get("error") or {}) if isinstance(b, dict) else None)281st, b = oai("GET", "/v1/fine_tuning/jobs/ftjob-atlasbogus000", note=f"{TAG} bogus id")282rec("fine_tuning.jobs.retrieve(bogus)", "GET", "/v1/fine_tuning/jobs/ftjob-atlasbogus000", st, b, error=b.get("error"))283st, b = oai("GET", "/v1/fine_tuning/checkpoints/ft:gpt-4.1-nano-2025-04-14:atlas::bogus:ckpt-step-1/permissions", note=f"{TAG} bogus checkpoint")284rec("fine_tuning.checkpoints.permissions.list(bogus)", "GET", "/v1/fine_tuning/checkpoints/{ckpt}/permissions", st, b, error=b.get("error"))285grader = {"type": "string_check", "name": "atlas_exact", "input": "{{sample.output_text}}", "reference": "{{item.expected}}", "operation": "eq"}286st, b = oai("POST", "/v1/fine_tuning/alpha/graders/validate", {"grader": grader}, note=TAG)287rec("graders.validate(string_check)", "POST", "/v1/fine_tuning/alpha/graders/validate", st, b, keys=sorted(b.keys()) if isinstance(b, dict) else None, error=b.get("error"))288st, b = oai("POST", "/v1/fine_tuning/alpha/graders/run", {"grader": grader, "item": {"expected": "OK"}, "model_sample": "OK"}, note=TAG)289rec("graders.run(string_check)", "POST", "/v1/fine_tuning/alpha/graders/run", st, b, reward=b.get("reward") if isinstance(b, dict) else None,290 keys=sorted(b.keys()) if isinstance(b, dict) else None, metadata_keys=sorted((b.get("metadata") or {}).keys()) if isinstance(b, dict) else None, error=b.get("error"))291tsg = {"type": "text_similarity", "name": "atlas_fuzzy", "input": "{{sample.output_text}}", "reference": "{{item.expected}}", "evaluation_metric": "fuzzy_match"}292st, b = oai("POST", "/v1/fine_tuning/alpha/graders/run", {"grader": tsg, "item": {"expected": "fuzzy wuzzy had no hair"}, "model_sample": "fuzzy wuzzy was a bear"}, note=TAG)293rec("graders.run(text_similarity)", "POST", "/v1/fine_tuning/alpha/graders/run", st, b, reward=b.get("reward") if isinstance(b, dict) else None, error=b.get("error"))294pyg = {"type": "python", "name": "atlas_py", "source": "def grade(sample, item):\n return 1.0 if sample['output_text'].strip() == item['expected'] else 0.0\n"}295st, b = oai("POST", "/v1/fine_tuning/alpha/graders/validate", {"grader": pyg}, note=TAG)296rec("graders.validate(python)", "POST", "/v1/fine_tuning/alpha/graders/validate", st, b, error=b.get("error"), keys=sorted(b.keys()) if isinstance(b, dict) else None)297st, b = oai("POST", "/v1/fine_tuning/alpha/graders/run", {"grader": pyg, "item": {"expected": "OK"}, "model_sample": "OK"}, note=TAG)298rec("graders.run(python)", "POST", "/v1/fine_tuning/alpha/graders/run", st, b, reward=b.get("reward") if isinstance(b, dict) else None,299 error=b.get("error"), exec_time=(b.get("metadata") or {}).get("execution_time") if isinstance(b, dict) else None)300multi = {"type": "multi", "name": "atlas_multi", "graders": {"exact": grader, "fuzzy": tsg}, "calculate_output": "0.5 * exact + 0.5 * fuzzy"}301st, b = oai("POST", "/v1/fine_tuning/alpha/graders/run", {"grader": multi, "item": {"expected": "OK"}, "model_sample": "OK"}, note=TAG)302rec("graders.run(multi)", "POST", "/v1/fine_tuning/alpha/graders/run", st, b, reward=b.get("reward") if isinstance(b, dict) else None,303 sub_rewards=b.get("sub_rewards") if isinstance(b, dict) else None, error=b.get("error"))304bad = {"type": "string_check", "name": "bad", "input": "{{sample.output_text}}", "reference": "x", "operation": "contains"}305st, b = oai("POST", "/v1/fine_tuning/alpha/graders/validate", {"grader": bad}, note=f"{TAG} invalid op (expect 400)")306rec("graders.validate(invalid operation)", "POST", "/v1/fine_tuning/alpha/graders/validate", st, b, error=b.get("error"))307308# ---------------------------------------------------------------- (f) Evals309st, ev = oai("POST", "/v1/evals", {"name": f"{TAG}-{RUN}", "metadata": {"owner": TAG},310 "data_source_config": {"type": "custom", "include_sample_schema": True,311 "item_schema": {"type": "object", "properties": {"q": {"type": "string"}, "expected": {"type": "string"}}, "required": ["q", "expected"]}},312 "testing_criteria": [{"type": "string_check", "name": "exact", "input": "{{sample.output_text}}", "reference": "{{item.expected}}", "operation": "eq"},313 {"type": "text_similarity", "name": "fuzzy", "input": "{{sample.output_text}}", "reference": "{{item.expected}}", "evaluation_metric": "fuzzy_match", "pass_threshold": 0.8}]},314 note=TAG)315rec("evals.create(custom+string_check+text_similarity)", "POST", "/v1/evals", st, ev, id=ev.get("id"), object=ev.get("object"),316 dsc_keys=sorted((ev.get("data_source_config") or {}).keys()), criteria_ids=[c.get("id") for c in ev.get("testing_criteria", [])], error=ev.get("error"))317EVAL_ID = ev.get("id")318if EVAL_ID:319 CREATED["evals"].append(EVAL_ID)320 st, b = oai("GET", f"/v1/evals/{EVAL_ID}", note=TAG)321 rec("evals.retrieve", "GET", f"/v1/evals/{EVAL_ID}", st, b, name=b.get("name"))322 st, b = oai("POST", f"/v1/evals/{EVAL_ID}", {"metadata": {"owner": TAG, "phase": "updated"}, "name": f"{TAG}-{RUN}-renamed"}, note=TAG)323 rec("evals.update", "POST", f"/v1/evals/{EVAL_ID}", st, b, name=b.get("name"), metadata=b.get("metadata"))324 st, b = oai("GET", "/v1/evals?limit=5&order=desc&order_by=created_at", note=TAG)325 rec("evals.list", "GET", "/v1/evals", st, b, n=len(b.get("data", [])), keys=sorted(b.keys()))326 run_body = {"name": f"{TAG}-{RUN}-run", "metadata": {"owner": TAG},327 "data_source": {"type": "jsonl", "source": {"type": "file_content", "content": [328 {"item": {"q": "ping", "expected": "OK"}, "sample": {"output_text": "OK"}},329 {"item": {"q": "ping2", "expected": "OK"}, "sample": {"output_text": "KO"}}]}}}330 st, run = oai("POST", f"/v1/evals/{EVAL_ID}/runs", run_body, note=f"{TAG} jsonl file_content, no sampling")331 rec("evals.runs.create(jsonl file_content)", "POST", f"/v1/evals/{EVAL_ID}/runs", st, run, id=run.get("id"), status_field=run.get("status"),332 report_url=bool(run.get("report_url")), model=run.get("model"), error=run.get("error"))333 RUN_ID = run.get("id")334 if RUN_ID:335 tl = []; t2 = time.time()336 for i in range(40):337 st, run = oai("GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", note=f"{TAG} poll")338 tl.append({"t": round(time.time() - t2, 1), "status": run.get("status")})339 if run.get("status") in ("completed", "failed", "canceled", "cancelled"):340 break341 time.sleep(3)342 rec("evals.runs.retrieve(poll)", "GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", st, run, timeline=tl, final_status=run.get("status"),343 result_counts=run.get("result_counts"), per_testing_criteria_results=run.get("per_testing_criteria_results"),344 per_model_usage=run.get("per_model_usage"), error=run.get("error"))345 st, b = oai("GET", f"/v1/evals/{EVAL_ID}/runs?limit=5", note=TAG)346 rec("evals.runs.list", "GET", f"/v1/evals/{EVAL_ID}/runs", st, b, n=len(b.get("data", [])))347 st, b = oai("GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items?limit=10", note=TAG)348 items = b.get("data", [])349 rec("evals.runs.output_items.list", "GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items", st, b, n=len(items),350 statuses=[x.get("status") for x in items], results=[[(r.get("name"), r.get("type"), r.get("score"), r.get("passed")) for r in x.get("results", [])] for x in items],351 item_keys=sorted(items[0].keys()) if items else None, sample_keys=sorted((items[0].get("sample") or {}).keys()) if items else None)352 st, b = oai("GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items?status=fail", note=TAG)353 rec("evals.runs.output_items.list(status=fail)", "GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items", st, b, n=len(b.get("data", [])))354 if items:355 st, b = oai("GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items/{items[0]['id']}", note=TAG)356 rec("evals.runs.output_items.retrieve", "GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items/{{id}}", st, b, status_field=b.get("status"))357 st, b = oai("POST", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", note=f"{TAG} cancel completed run (expect error)")358 rec("evals.runs.cancel(after completion)", "POST", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", st, b, status_field=b.get("status"), error=b.get("error"))359 st, b = oai("DELETE", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", note=f"{TAG} cleanup")360 rec("evals.runs.delete", "DELETE", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", st, b, body_keys=sorted(b.keys()) if isinstance(b, dict) else None, deleted=b.get("deleted"))361 st, b = oai("DELETE", f"/v1/evals/{EVAL_ID}", note=f"{TAG} cleanup")362 rec("evals.delete", "DELETE", f"/v1/evals/{EVAL_ID}", st, b, body_keys=sorted(b.keys()) if isinstance(b, dict) else None, deleted=b.get("deleted"))363 if b.get("deleted"):364 CREATED["evals"].remove(EVAL_ID)365 st, b = oai("GET", f"/v1/evals/{EVAL_ID}", note=f"{TAG} after delete (expect 404)")366 rec("evals.retrieve(after delete)", "GET", f"/v1/evals/{EVAL_ID}", st, b, error=b.get("error"))367368# ---------------------------------------------------------------- cleanup files369for fid in list(dict.fromkeys(CREATED["files"])):370 delete_file(fid)371st, b = oai("GET", f"/v1/files/{TXT_ID}", note=f"{TAG} after delete (expect 404)")372rec("files.retrieve(after delete)", "GET", f"/v1/files/{TXT_ID}", st, b, error=b.get("error"))373374# final leftover check375st, files = oai("GET", "/v1/files?limit=100", note=f"{TAG} leftover check")376st2, stores = oai("GET", "/v1/vector_stores?limit=100", note=f"{TAG} leftover check")377st3, evals = oai("GET", "/v1/evals?limit=100", note=f"{TAG} leftover check")378left = {"files": [f["id"] for f in files.get("data", []) if TAG in f.get("filename", "")],379 "vector_stores": [v["id"] for v in stores.get("data", []) if TAG in (v.get("name") or "")],380 "evals": [e["id"] for e in evals.get("data", []) if TAG in (e.get("name") or "")],381 "all_files_count": len(files.get("data", [])), "all_stores": len(stores.get("data", [])), "all_evals": len(evals.get("data", []))}382rec("leftover_check", "GET", "/v1/{files,vector_stores,evals}", 200, left, **left)383save_sanitized(SUMMARY, OUT / "summary.json")384print("DONE", json.dumps(left))385