#!/usr/bin/env python3
"""Live verification of Files / Uploads / Vector stores / Batch / Fine-tuning / Graders / Evals.
Budget: < $0.05. Everything created is prefixed atlas-platform-agent and deleted at the end.
Raw sanitized responses -> tmp-live/platform/*.json ; summary -> stdout (JSON lines).
"""
from __future__ import annotations
import json, sys, time, uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(ROOT))
from scripts.live import openai_request, save_sanitized  # noqa: E402

OUT = ROOT / "tmp-live" / "platform"
OUT.mkdir(parents=True, exist_ok=True)
TAG = "atlas-platform-agent"
RUN = uuid.uuid4().hex[:6]
SUMMARY: list[dict] = []
CREATED = {"files": [], "vector_stores": [], "evals": [], "uploads": []}


def rec(step, method, path, st, body, note="", **extra):
    save_sanitized({"status": st, "body": body if not isinstance(body, bytes) else body.decode("utf-8", "replace")},
                   OUT / f"{len(SUMMARY):02d}-{step}.json")
    row = {"step": step, "method": method, "path": path, "status": st, "note": note, **extra}
    SUMMARY.append(row)
    print(json.dumps(row, default=str), flush=True)
    return body


def multipart(fields: dict[str, str], files: dict[str, tuple[str, bytes, str]]):
    b = f"----atlas{uuid.uuid4().hex}"
    out = bytearray()
    for k, v in fields.items():
        out += f"--{b}\r\nContent-Disposition: form-data; name=\"{k}\"\r\n\r\n{v}\r\n".encode()
    for k, (fn, data, ct) in files.items():
        out += f"--{b}\r\nContent-Disposition: form-data; name=\"{k}\"; filename=\"{fn}\"\r\nContent-Type: {ct}\r\n\r\n".encode()
        out += data + b"\r\n"
    out += f"--{b}--\r\n".encode()
    return bytes(out), f"multipart/form-data; boundary={b}"


def oai(method, path, json_body=None, *, data=None, content_type="application/json", note="", cost=0.0):
    st, body, hdrs = openai_request(method, path, json_body, data=data, content_type=content_type, note=note, est_cost_usd=cost)
    return st, body


def upload_file(name: str, content: bytes, purpose: str, mime="text/plain", extra_fields=None):
    fields = {"purpose": purpose, **(extra_fields or {})}
    data, ct = multipart(fields, {"file": (name, content, mime)})
    st, body = oai("POST", "/v1/files", data=data, content_type=ct, note=f"{TAG} upload {purpose}")
    if st == 200:
        CREATED["files"].append(body["id"])
    return st, body


def delete_file(fid):
    st, body = oai("DELETE", f"/v1/files/{fid}", note=f"{TAG} cleanup")
    rec("files.delete", "DELETE", f"/v1/files/{fid}", st, body, deleted=body.get("deleted") if isinstance(body, dict) else None)
    if st == 200 and fid in CREATED["files"]:
        CREATED["files"].remove(fid)
    return st


# ---------------------------------------------------------------- (a) Files
txt = (f"{TAG} {RUN} - The Atlas woodchuck policy: each passenger may carry at most two woodchucks. "
       "Woodchucks must be contained during transport. Return window is 30 days.").encode()
txt = (txt + b" " * 200)[:200]
st, f_txt = upload_file(f"{TAG}-{RUN}.txt", txt, "user_data",
                        extra_fields={"expires_after[anchor]": "created_at", "expires_after[seconds]": "3600"})
rec("files.create(user_data,expires_after)", "POST", "/v1/files", st, f_txt,
    id=f_txt.get("id"), bytes=f_txt.get("bytes"), status_field=f_txt.get("status"), expires_at=f_txt.get("expires_at"))
TXT_ID = f_txt.get("id")

st, b = oai("GET", "/v1/files?purpose=user_data&limit=5", note=TAG)
rec("files.list(purpose=user_data)", "GET", "/v1/files", st, b, n=len(b.get("data", [])), has_more=b.get("has_more"))
st, b = oai("GET", f"/v1/files/{TXT_ID}", note=TAG)
rec("files.retrieve", "GET", f"/v1/files/{TXT_ID}", st, b, purpose=b.get("purpose"))
st, b = oai("GET", f"/v1/files/{TXT_ID}/content", content_type=None, note=TAG)
rec("files.content", "GET", f"/v1/files/{TXT_ID}/content", st, b, bytes=len(b) if isinstance(b, bytes) else None,
    roundtrip_ok=(b == txt))

# batch input jsonl
line = {"custom_id": f"{TAG}-req-1", "method": "POST", "url": "/v1/responses",
        "body": {"model": "gpt-5.4-nano", "input": "Reply with OK.", "max_output_tokens": 16}}
jsonl = (json.dumps(line) + "\n").encode()
st, f_batch = upload_file(f"{TAG}-{RUN}-batch.jsonl", jsonl, "batch", mime="application/jsonl")
rec("files.create(batch)", "POST", "/v1/files", st, f_batch, id=f_batch.get("id"), expires_at=f_batch.get("expires_at"),
    purpose=f_batch.get("purpose"))
BATCH_FILE_ID = f_batch.get("id")

# evals purpose file (format check only)
ev_jsonl = (json.dumps({"item": {"q": "ping", "expected": "OK"}}) + "\n").encode()
st, f_ev = upload_file(f"{TAG}-{RUN}-evals.jsonl", ev_jsonl, "evals", mime="application/jsonl")
rec("files.create(evals)", "POST", "/v1/files", st, f_ev, id=f_ev.get("id"), purpose=f_ev.get("purpose"))
EV_FILE_ID = f_ev.get("id")

# ---------------------------------------------------------------- (b) Batch
t0 = time.time()
st, batch = oai("POST", "/v1/batches", {"input_file_id": BATCH_FILE_ID, "endpoint": "/v1/responses",
                                        "completion_window": "24h", "metadata": {"owner": TAG, "run": RUN},
                                        "output_expires_after": {"anchor": "created_at", "seconds": 3600}},
                note=f"{TAG} 1 request gpt-5.4-nano", cost=0.0001)
rec("batches.create", "POST", "/v1/batches", st, batch, id=batch.get("id"), status_field=batch.get("status"),
    request_counts=batch.get("request_counts"), expires_at=batch.get("expires_at"))
BATCH_ID = batch.get("id")
st, b = oai("GET", "/v1/batches?limit=3", note=TAG)
rec("batches.list", "GET", "/v1/batches", st, b, n=len(b.get("data", [])), has_more=b.get("has_more"))

timeline = []
final = None
if BATCH_ID:
    deadline = time.time() + 190
    while time.time() < deadline:
        st, b = oai("GET", f"/v1/batches/{BATCH_ID}", note=f"{TAG} poll")
        timeline.append({"t": round(time.time() - t0, 1), "status": b.get("status"), "counts": b.get("request_counts")})
        if b.get("status") in ("completed", "failed", "expired", "cancelled"):
            final = b
            break
        time.sleep(20)
    rec("batches.retrieve(poll)", "GET", f"/v1/batches/{BATCH_ID}", st, b, timeline=timeline, final_status=b.get("status"),
        output_file_id=b.get("output_file_id"), error_file_id=b.get("error_file_id"), usage=b.get("usage"))
    if final is None:
        st, c = oai("POST", f"/v1/batches/{BATCH_ID}/cancel", note=TAG)
        rec("batches.cancel", "POST", f"/v1/batches/{BATCH_ID}/cancel", st, c, status_field=c.get("status"),
            cancelling_at=c.get("cancelling_at"))
        for i in range(4):
            time.sleep(10)
            st, c = oai("GET", f"/v1/batches/{BATCH_ID}", note=f"{TAG} poll after cancel")
            timeline.append({"t": round(time.time() - t0, 1), "status": c.get("status")})
            if c.get("status") == "cancelled":
                break
        rec("batches.retrieve(after cancel)", "GET", f"/v1/batches/{BATCH_ID}", st, c, status_field=c.get("status"),
            timeline=timeline[-5:], output_file_id=c.get("output_file_id"), error_file_id=c.get("error_file_id"))
        final = c
    for key in ("output_file_id", "error_file_id"):
        fid = (final or {}).get(key)
        if fid:
            st, content = oai("GET", f"/v1/files/{fid}/content", content_type=None, note=f"{TAG} batch {key}")
            text = content.decode() if isinstance(content, bytes) else json.dumps(content)
            shape = None
            try:
                first = json.loads(text.splitlines()[0])
                shape = {k: (type(v).__name__ if k != "response" else {kk: type(vv).__name__ for kk, vv in (v or {}).items()})
                         for k, v in first.items()}
            except Exception as e:  # noqa: BLE001
                shape = f"unparsed: {e}"
            rec(f"files.content({key})", "GET", f"/v1/files/{fid}/content", st, text, line_shape=shape)
            st, meta = oai("GET", f"/v1/files/{fid}", note=TAG)
            rec(f"files.retrieve({key})", "GET", f"/v1/files/{fid}", st, meta, purpose=meta.get("purpose"),
                expires_at=meta.get("expires_at"))
            CREATED["files"].append(fid)

# ---------------------------------------------------------------- (c) Uploads
small = (f"{TAG} {RUN} upload part content\n" * 4).encode()
for purpose in ("user_data", "assistants"):
    st, up = oai("POST", "/v1/uploads", {"filename": f"{TAG}-{RUN}-upload.txt", "purpose": purpose,
                                         "bytes": len(small), "mime_type": "text/plain"}, note=f"{TAG} purpose={purpose}")
    rec(f"uploads.create(purpose={purpose})", "POST", "/v1/uploads", st, up, id=up.get("id"), status_field=up.get("status"),
        expires_at=up.get("expires_at"), created_at=up.get("created_at"))
    if st == 200:
        break
UP_ID = up.get("id") if st == 200 else None
if UP_ID:
    CREATED["uploads"].append(UP_ID)
    data, ct = multipart({}, {"data": ("part", small, "application/octet-stream")})
    st, part = oai("POST", f"/v1/uploads/{UP_ID}/parts", data=data, content_type=ct, note=TAG)
    rec("uploads.parts.create", "POST", f"/v1/uploads/{UP_ID}/parts", st, part, id=part.get("id"), object=part.get("object"))
    st, done = oai("POST", f"/v1/uploads/{UP_ID}/complete", {"part_ids": [part.get("id")]}, note=TAG)
    rec("uploads.complete", "POST", f"/v1/uploads/{UP_ID}/complete", st, done, status_field=done.get("status"),
        file_id=(done.get("file") or {}).get("id"), file_purpose=(done.get("file") or {}).get("purpose"),
        file_bytes=(done.get("file") or {}).get("bytes"))
    if (done.get("file") or {}).get("id"):
        CREATED["files"].append(done["file"]["id"])
    # second upload: cancel path
    st, up2 = oai("POST", "/v1/uploads", {"filename": f"{TAG}-{RUN}-cancel.txt", "purpose": up.get("purpose"),
                                          "bytes": 10, "mime_type": "text/plain"}, note=TAG)
    rec("uploads.create(for cancel)", "POST", "/v1/uploads", st, up2, id=up2.get("id"), status_field=up2.get("status"))
    if st == 200:
        st, c2 = oai("POST", f"/v1/uploads/{up2['id']}/cancel", note=TAG)
        rec("uploads.cancel", "POST", f"/v1/uploads/{up2['id']}/cancel", st, c2, status_field=c2.get("status"))
        st, c3 = oai("POST", f"/v1/uploads/{up2['id']}/parts", data=data, content_type=ct, note=f"{TAG} part after cancel (expect error)")
        rec("uploads.parts.create(after cancel)", "POST", f"/v1/uploads/{up2['id']}/parts", st, c3,
            error=(c3.get("error") or {}).get("message") if isinstance(c3, dict) else None)

# ---------------------------------------------------------------- (d) Vector stores
st, vs = oai("POST", "/v1/vector_stores", {"name": f"{TAG}-{RUN}", "description": "API Atlas live probe",
                                           "metadata": {"owner": TAG}, "expires_after": {"anchor": "last_active_at", "days": 1}},
             note=TAG)
rec("vector_stores.create", "POST", "/v1/vector_stores", st, vs, id=vs.get("id"), status_field=vs.get("status"),
    file_counts=vs.get("file_counts"), expires_at=vs.get("expires_at"), usage_bytes=vs.get("usage_bytes"))
VS_ID = vs.get("id")
if VS_ID:
    CREATED["vector_stores"].append(VS_ID)
    st, vsf = oai("POST", f"/v1/vector_stores/{VS_ID}/files",
                  {"file_id": TXT_ID, "attributes": {"atlas": "platform", "n": 1, "draft": False},
                   "chunking_strategy": {"type": "static", "static": {"max_chunk_size_tokens": 100, "chunk_overlap_tokens": 20}}},
                  note=TAG)
    rec("vector_stores.files.create(static 100/20)", "POST", f"/v1/vector_stores/{VS_ID}/files", st, vsf, id=vsf.get("id"),
        status_field=vsf.get("status"), chunking_strategy=vsf.get("chunking_strategy"), attributes=vsf.get("attributes"),
        usage_bytes=vsf.get("usage_bytes"))
    t1 = time.time(); tl = []
    for i in range(30):
        st, vsf = oai("GET", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}", note=f"{TAG} poll")
        tl.append({"t": round(time.time() - t1, 1), "status": vsf.get("status")})
        if vsf.get("status") != "in_progress":
            break
        time.sleep(2)
    rec("vector_stores.files.retrieve(poll)", "GET", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}", st, vsf,
        timeline=tl, final_status=vsf.get("status"), usage_bytes=vsf.get("usage_bytes"), last_error=vsf.get("last_error"))
    st, b = oai("GET", f"/v1/vector_stores/{VS_ID}/files?filter=completed&limit=10", note=TAG)
    rec("vector_stores.files.list(filter=completed)", "GET", f"/v1/vector_stores/{VS_ID}/files", st, b,
        n=len(b.get("data", [])), keys=sorted(b.keys()))
    st, b = oai("GET", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}/content", note=TAG)
    rec("vector_stores.files.content", "GET", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}/content", st, b,
        object=b.get("object"), n_chunks=len(b.get("data", [])), first_chunk=(b.get("data") or [{}])[0].get("text", "")[:80])
    st, vsr = oai("GET", f"/v1/vector_stores/{VS_ID}", note=TAG)
    rec("vector_stores.retrieve", "GET", f"/v1/vector_stores/{VS_ID}", st, vsr, status_field=vsr.get("status"),
        usage_bytes=vsr.get("usage_bytes"), file_counts=vsr.get("file_counts"), last_active_at=vsr.get("last_active_at"))
    st, s = oai("POST", f"/v1/vector_stores/{VS_ID}/search",
                {"query": "How many woodchucks per passenger?", "max_num_results": 3, "rewrite_query": True,
                 "filters": {"type": "and", "filters": [{"type": "eq", "key": "atlas", "value": "platform"},
                                                        {"type": "gte", "key": "n", "value": 1}]},
                 "ranking_options": {"ranker": "auto", "score_threshold": 0.0}}, note=TAG)
    rec("vector_stores.search(filters+rewrite)", "POST", f"/v1/vector_stores/{VS_ID}/search", st, s,
        object=s.get("object"), search_query=s.get("search_query"), n=len(s.get("data", [])),
        top=(s.get("data") or [{}])[0].get("score"), result_keys=sorted((s.get("data") or [{}])[0].keys()))
    st, s2 = oai("POST", f"/v1/vector_stores/{VS_ID}/search",
                 {"query": "woodchucks", "filters": {"type": "eq", "key": "atlas", "value": "nomatch"}}, note=TAG)
    rec("vector_stores.search(filter no match)", "POST", f"/v1/vector_stores/{VS_ID}/search", st, s2, n=len(s2.get("data", [])))
    st, s3 = oai("POST", f"/v1/vector_stores/{VS_ID}/search",
                 {"query": ["woodchuck limit", "return policy"], "max_num_results": 2, "ranking_options": {"ranker": "none"}}, note=TAG)
    rec("vector_stores.search(query array, ranker none)", "POST", f"/v1/vector_stores/{VS_ID}/search", st, s3,
        n=len(s3.get("data", [])), search_query=s3.get("search_query"), error=(s3.get("error") or {}).get("message") if isinstance(s3, dict) else None)
    st, u = oai("POST", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}", {"attributes": {"atlas": "platform", "n": 2, "lang": "en"}}, note=TAG)
    rec("vector_stores.files.update(attributes)", "POST", f"/v1/vector_stores/{VS_ID}/files/{TXT_ID}", st, u, attributes=u.get("attributes"))
    st, um = oai("POST", f"/v1/vector_stores/{VS_ID}", {"name": f"{TAG}-{RUN}-renamed", "metadata": {"owner": TAG, "phase": "2"}}, note=TAG)
    rec("vector_stores.update", "POST", f"/v1/vector_stores/{VS_ID}", st, um, name=um.get("name"), metadata=um.get("metadata"))
    # file batch with a second file
    st, f2 = upload_file(f"{TAG}-{RUN}-second.txt", b"Second Atlas file: refunds are processed within 5 business days. " * 3, "assistants")
    rec("files.create(assistants)", "POST", "/v1/files", st, f2, id=f2.get("id"), purpose=f2.get("purpose"))
    F2 = f2.get("id")
    st, fb = oai("POST", f"/v1/vector_stores/{VS_ID}/file_batches",
                 {"files": [{"file_id": F2, "attributes": {"atlas": "platform", "n": 2}, "chunking_strategy": {"type": "auto"}}]}, note=TAG)
    rec("vector_stores.file_batches.create(files[])", "POST", f"/v1/vector_stores/{VS_ID}/file_batches", st, fb, id=fb.get("id"),
        object=fb.get("object"), status_field=fb.get("status"), file_counts=fb.get("file_counts"),
        error=(fb.get("error") or {}).get("message") if isinstance(fb, dict) else None)
    FB_ID = fb.get("id")
    if FB_ID:
        tl = []
        for i in range(30):
            st, fb = oai("GET", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}", note=f"{TAG} poll")
            tl.append({"t": i * 2, "status": fb.get("status")})
            if fb.get("status") != "in_progress":
                break
            time.sleep(2)
        rec("vector_stores.file_batches.retrieve(poll)", "GET", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}", st, fb,
            timeline=tl, final_status=fb.get("status"), file_counts=fb.get("file_counts"))
        st, b = oai("GET", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}/files?limit=10", note=TAG)
        rec("vector_stores.file_batches.list_files", "GET", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}/files", st, b,
            n=len(b.get("data", [])), chunking=[(x.get("chunking_strategy") or {}).get("type") for x in b.get("data", [])],
            attrs=[x.get("attributes") for x in b.get("data", [])])
        st, c = oai("POST", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}/cancel", note=f"{TAG} cancel completed batch (expect error or no-op)")
        rec("vector_stores.file_batches.cancel(after completion)", "POST", f"/v1/vector_stores/{VS_ID}/file_batches/{FB_ID}/cancel", st, c,
            status_field=c.get("status") if isinstance(c, dict) else None, error=(c.get("error") or {}).get("message") if isinstance(c, dict) else None)
    st, b = oai("GET", "/v1/vector_stores?limit=5", note=TAG)
    rec("vector_stores.list", "GET", "/v1/vector_stores", st, b, n=len(b.get("data", [])))
    # cleanup vs files, store
    for fid in [TXT_ID, F2]:
        if fid:
            st, d = oai("DELETE", f"/v1/vector_stores/{VS_ID}/files/{fid}", note=f"{TAG} cleanup")
            rec("vector_stores.files.delete", "DELETE", f"/v1/vector_stores/{VS_ID}/files/{fid}", st, d, deleted=d.get("deleted"), object=d.get("object"))
    st, d = oai("DELETE", f"/v1/vector_stores/{VS_ID}", note=f"{TAG} cleanup")
    rec("vector_stores.delete", "DELETE", f"/v1/vector_stores/{VS_ID}", st, d, deleted=d.get("deleted"), object=d.get("object"))
    if d.get("deleted"):
        CREATED["vector_stores"].remove(VS_ID)

# ---------------------------------------------------------------- (e) Fine-tuning + graders
st, b = oai("GET", "/v1/fine_tuning/jobs?limit=3", note=TAG)
rec("fine_tuning.jobs.list", "GET", "/v1/fine_tuning/jobs", st, b, n=len(b.get("data", [])) if isinstance(b, dict) else None,
    keys=sorted(b.keys()) if isinstance(b, dict) else None, error=(b.get("error") or {}) if isinstance(b, dict) else None)
st, b = oai("GET", "/v1/fine_tuning/jobs/ftjob-atlasbogus000", note=f"{TAG} bogus id")
rec("fine_tuning.jobs.retrieve(bogus)", "GET", "/v1/fine_tuning/jobs/ftjob-atlasbogus000", st, b, error=b.get("error"))
st, b = oai("GET", "/v1/fine_tuning/checkpoints/ft:gpt-4.1-nano-2025-04-14:atlas::bogus:ckpt-step-1/permissions", note=f"{TAG} bogus checkpoint")
rec("fine_tuning.checkpoints.permissions.list(bogus)", "GET", "/v1/fine_tuning/checkpoints/{ckpt}/permissions", st, b, error=b.get("error"))
grader = {"type": "string_check", "name": "atlas_exact", "input": "{{sample.output_text}}", "reference": "{{item.expected}}", "operation": "eq"}
st, b = oai("POST", "/v1/fine_tuning/alpha/graders/validate", {"grader": grader}, note=TAG)
rec("graders.validate(string_check)", "POST", "/v1/fine_tuning/alpha/graders/validate", st, b, keys=sorted(b.keys()) if isinstance(b, dict) else None, error=b.get("error"))
st, b = oai("POST", "/v1/fine_tuning/alpha/graders/run", {"grader": grader, "item": {"expected": "OK"}, "model_sample": "OK"}, note=TAG)
rec("graders.run(string_check)", "POST", "/v1/fine_tuning/alpha/graders/run", st, b, reward=b.get("reward") if isinstance(b, dict) else None,
    keys=sorted(b.keys()) if isinstance(b, dict) else None, metadata_keys=sorted((b.get("metadata") or {}).keys()) if isinstance(b, dict) else None, error=b.get("error"))
tsg = {"type": "text_similarity", "name": "atlas_fuzzy", "input": "{{sample.output_text}}", "reference": "{{item.expected}}", "evaluation_metric": "fuzzy_match"}
st, b = oai("POST", "/v1/fine_tuning/alpha/graders/run", {"grader": tsg, "item": {"expected": "fuzzy wuzzy had no hair"}, "model_sample": "fuzzy wuzzy was a bear"}, note=TAG)
rec("graders.run(text_similarity)", "POST", "/v1/fine_tuning/alpha/graders/run", st, b, reward=b.get("reward") if isinstance(b, dict) else None, error=b.get("error"))
pyg = {"type": "python", "name": "atlas_py", "source": "def grade(sample, item):\n    return 1.0 if sample['output_text'].strip() == item['expected'] else 0.0\n"}
st, b = oai("POST", "/v1/fine_tuning/alpha/graders/validate", {"grader": pyg}, note=TAG)
rec("graders.validate(python)", "POST", "/v1/fine_tuning/alpha/graders/validate", st, b, error=b.get("error"), keys=sorted(b.keys()) if isinstance(b, dict) else None)
st, b = oai("POST", "/v1/fine_tuning/alpha/graders/run", {"grader": pyg, "item": {"expected": "OK"}, "model_sample": "OK"}, note=TAG)
rec("graders.run(python)", "POST", "/v1/fine_tuning/alpha/graders/run", st, b, reward=b.get("reward") if isinstance(b, dict) else None,
    error=b.get("error"), exec_time=(b.get("metadata") or {}).get("execution_time") if isinstance(b, dict) else None)
multi = {"type": "multi", "name": "atlas_multi", "graders": {"exact": grader, "fuzzy": tsg}, "calculate_output": "0.5 * exact + 0.5 * fuzzy"}
st, b = oai("POST", "/v1/fine_tuning/alpha/graders/run", {"grader": multi, "item": {"expected": "OK"}, "model_sample": "OK"}, note=TAG)
rec("graders.run(multi)", "POST", "/v1/fine_tuning/alpha/graders/run", st, b, reward=b.get("reward") if isinstance(b, dict) else None,
    sub_rewards=b.get("sub_rewards") if isinstance(b, dict) else None, error=b.get("error"))
bad = {"type": "string_check", "name": "bad", "input": "{{sample.output_text}}", "reference": "x", "operation": "contains"}
st, b = oai("POST", "/v1/fine_tuning/alpha/graders/validate", {"grader": bad}, note=f"{TAG} invalid op (expect 400)")
rec("graders.validate(invalid operation)", "POST", "/v1/fine_tuning/alpha/graders/validate", st, b, error=b.get("error"))

# ---------------------------------------------------------------- (f) Evals
st, ev = oai("POST", "/v1/evals", {"name": f"{TAG}-{RUN}", "metadata": {"owner": TAG},
                                   "data_source_config": {"type": "custom", "include_sample_schema": True,
                                                          "item_schema": {"type": "object", "properties": {"q": {"type": "string"}, "expected": {"type": "string"}}, "required": ["q", "expected"]}},
                                   "testing_criteria": [{"type": "string_check", "name": "exact", "input": "{{sample.output_text}}", "reference": "{{item.expected}}", "operation": "eq"},
                                                        {"type": "text_similarity", "name": "fuzzy", "input": "{{sample.output_text}}", "reference": "{{item.expected}}", "evaluation_metric": "fuzzy_match", "pass_threshold": 0.8}]},
             note=TAG)
rec("evals.create(custom+string_check+text_similarity)", "POST", "/v1/evals", st, ev, id=ev.get("id"), object=ev.get("object"),
    dsc_keys=sorted((ev.get("data_source_config") or {}).keys()), criteria_ids=[c.get("id") for c in ev.get("testing_criteria", [])], error=ev.get("error"))
EVAL_ID = ev.get("id")
if EVAL_ID:
    CREATED["evals"].append(EVAL_ID)
    st, b = oai("GET", f"/v1/evals/{EVAL_ID}", note=TAG)
    rec("evals.retrieve", "GET", f"/v1/evals/{EVAL_ID}", st, b, name=b.get("name"))
    st, b = oai("POST", f"/v1/evals/{EVAL_ID}", {"metadata": {"owner": TAG, "phase": "updated"}, "name": f"{TAG}-{RUN}-renamed"}, note=TAG)
    rec("evals.update", "POST", f"/v1/evals/{EVAL_ID}", st, b, name=b.get("name"), metadata=b.get("metadata"))
    st, b = oai("GET", "/v1/evals?limit=5&order=desc&order_by=created_at", note=TAG)
    rec("evals.list", "GET", "/v1/evals", st, b, n=len(b.get("data", [])), keys=sorted(b.keys()))
    run_body = {"name": f"{TAG}-{RUN}-run", "metadata": {"owner": TAG},
                "data_source": {"type": "jsonl", "source": {"type": "file_content", "content": [
                    {"item": {"q": "ping", "expected": "OK"}, "sample": {"output_text": "OK"}},
                    {"item": {"q": "ping2", "expected": "OK"}, "sample": {"output_text": "KO"}}]}}}
    st, run = oai("POST", f"/v1/evals/{EVAL_ID}/runs", run_body, note=f"{TAG} jsonl file_content, no sampling")
    rec("evals.runs.create(jsonl file_content)", "POST", f"/v1/evals/{EVAL_ID}/runs", st, run, id=run.get("id"), status_field=run.get("status"),
        report_url=bool(run.get("report_url")), model=run.get("model"), error=run.get("error"))
    RUN_ID = run.get("id")
    if RUN_ID:
        tl = []; t2 = time.time()
        for i in range(40):
            st, run = oai("GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", note=f"{TAG} poll")
            tl.append({"t": round(time.time() - t2, 1), "status": run.get("status")})
            if run.get("status") in ("completed", "failed", "canceled", "cancelled"):
                break
            time.sleep(3)
        rec("evals.runs.retrieve(poll)", "GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", st, run, timeline=tl, final_status=run.get("status"),
            result_counts=run.get("result_counts"), per_testing_criteria_results=run.get("per_testing_criteria_results"),
            per_model_usage=run.get("per_model_usage"), error=run.get("error"))
        st, b = oai("GET", f"/v1/evals/{EVAL_ID}/runs?limit=5", note=TAG)
        rec("evals.runs.list", "GET", f"/v1/evals/{EVAL_ID}/runs", st, b, n=len(b.get("data", [])))
        st, b = oai("GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items?limit=10", note=TAG)
        items = b.get("data", [])
        rec("evals.runs.output_items.list", "GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items", st, b, n=len(items),
            statuses=[x.get("status") for x in items], results=[[(r.get("name"), r.get("type"), r.get("score"), r.get("passed")) for r in x.get("results", [])] for x in items],
            item_keys=sorted(items[0].keys()) if items else None, sample_keys=sorted((items[0].get("sample") or {}).keys()) if items else None)
        st, b = oai("GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items?status=fail", note=TAG)
        rec("evals.runs.output_items.list(status=fail)", "GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items", st, b, n=len(b.get("data", [])))
        if items:
            st, b = oai("GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items/{items[0]['id']}", note=TAG)
            rec("evals.runs.output_items.retrieve", "GET", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}/output_items/{{id}}", st, b, status_field=b.get("status"))
        st, b = oai("POST", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", note=f"{TAG} cancel completed run (expect error)")
        rec("evals.runs.cancel(after completion)", "POST", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", st, b, status_field=b.get("status"), error=b.get("error"))
        st, b = oai("DELETE", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", note=f"{TAG} cleanup")
        rec("evals.runs.delete", "DELETE", f"/v1/evals/{EVAL_ID}/runs/{RUN_ID}", st, b, body_keys=sorted(b.keys()) if isinstance(b, dict) else None, deleted=b.get("deleted"))
    st, b = oai("DELETE", f"/v1/evals/{EVAL_ID}", note=f"{TAG} cleanup")
    rec("evals.delete", "DELETE", f"/v1/evals/{EVAL_ID}", st, b, body_keys=sorted(b.keys()) if isinstance(b, dict) else None, deleted=b.get("deleted"))
    if b.get("deleted"):
        CREATED["evals"].remove(EVAL_ID)
    st, b = oai("GET", f"/v1/evals/{EVAL_ID}", note=f"{TAG} after delete (expect 404)")
    rec("evals.retrieve(after delete)", "GET", f"/v1/evals/{EVAL_ID}", st, b, error=b.get("error"))

# ---------------------------------------------------------------- cleanup files
for fid in list(dict.fromkeys(CREATED["files"])):
    delete_file(fid)
st, b = oai("GET", f"/v1/files/{TXT_ID}", note=f"{TAG} after delete (expect 404)")
rec("files.retrieve(after delete)", "GET", f"/v1/files/{TXT_ID}", st, b, error=b.get("error"))

# final leftover check
st, files = oai("GET", "/v1/files?limit=100", note=f"{TAG} leftover check")
st2, stores = oai("GET", "/v1/vector_stores?limit=100", note=f"{TAG} leftover check")
st3, evals = oai("GET", "/v1/evals?limit=100", note=f"{TAG} leftover check")
left = {"files": [f["id"] for f in files.get("data", []) if TAG in f.get("filename", "")],
        "vector_stores": [v["id"] for v in stores.get("data", []) if TAG in (v.get("name") or "")],
        "evals": [e["id"] for e in evals.get("data", []) if TAG in (e.get("name") or "")],
        "all_files_count": len(files.get("data", [])), "all_stores": len(stores.get("data", [])), "all_evals": len(evals.get("data", []))}
rec("leftover_check", "GET", "/v1/{files,vector_stores,evals}", 200, left, **left)
save_sanitized(SUMMARY, OUT / "summary.json")
print("DONE", json.dumps(left))