TeX 59%
Python 40.8%
1#!/usr/bin/env python32"""Submit / monitor / collect the extraction batches.34 .venv/bin/python scripts/04_run_batches.py submit [img_haiku img_sonnet img_opus txt_haiku txt_sonnet txt_opus]5 .venv/bin/python scripts/04_run_batches.py status6 .venv/bin/python scripts/04_run_batches.py collect78Design (configs/data.yaml, configs/models.yaml): Haiku 5.5 sees up to 12 images per listing (image-count sensitivity),9Sonnet 5.5 and Opus 5.5 the first 10 (cost control; comparisons between tiers use the common first 10).10custom_id = "<tier>-<uid with ':'→'__'>-<image_index or 'T'>" (letters, digits, '_', '-' only).11"""12import json13import sys14from pathlib import Path1516import pandas as pd1718sys.path.insert(0, str(Path(__file__).resolve().parents[1]))19from src.extraction import client as C # noqa: E40220from src.extraction.textprep import build_text # noqa: E40221from src.feature_schema.schema import IMAGE_SCHEMA, SCHEMA_VERSION, TEXT_SCHEMA # noqa: E4022223ROOT = Path(__file__).resolve().parents[1]24RAW = ROOT / "results" / "raw" / "batches"25IMG_CAP = {"haiku": 12, "sonnet": 10, "opus": 10}262728def cid(tier: str, uid: str, idx) -> str:29 return f"{tier}-{uid.replace(':', '__')}-{idx}"303132def build(name: str, smp: pd.DataFrame) -> list[tuple[str, dict]]:33 kind, tier = name.split("_")34 rows = smp[smp[f"tier_{tier}"]]35 reqs = []36 if kind == "img":37 for r in rows.itertuples():38 for i, url in enumerate(list(r.image_urls_used)[:IMG_CAP[tier]]):39 reqs.append((cid(tier, r.uid, i), C.image_request(tier, url, IMAGE_SCHEMA)))40 else:41 stats = []42 for r in rows.itertuples():43 text, st = build_text(r._asdict())44 st["uid"] = r.uid45 stats.append(st)46 reqs.append((cid(tier, r.uid, "T"), C.text_request(tier, text, TEXT_SCHEMA)))47 pd.DataFrame(stats).to_csv(RAW / f"{name}{SUFFIX}.text_stats.csv", index=False)48 return reqs495051import os52SAMPLE_FILE = os.environ.get("WP20_SAMPLE", "sample.parquet")53SUFFIX = os.environ.get("WP20_BATCH_SUFFIX", "")545556def submit(names):57 smp = pd.read_parquet(ROOT / "data" / "processed" / SAMPLE_FILE)58 meta = {"schema_version": SCHEMA_VERSION, "prompt_version": C.PROMPTS["prompt_version"], "models": C.MODELS["tiers"],59 "image_caps": IMG_CAP, "n_listings": int(len(smp))}60 json.dump(meta, open(RAW / "run_meta.json", "w"), indent=2, default=str)61 for name in names:62 out_name = name + SUFFIX63 if (RAW / f"{out_name}.batch_ids.txt").exists():64 print(f"{out_name}: already submitted, skipping")65 continue66 reqs = build(name, smp)67 ids = C.batch_submit(out_name, reqs)68 print(f"{out_name}: {len(reqs)} requests → batches {ids}")697071def all_batches():72 out = []73 for f in sorted(RAW.glob("*.batch_ids.txt")):74 name = f.name.replace(".batch_ids.txt", "")75 for line in open(f):76 bid = line.split("\t")[0].strip()77 if bid:78 out.append((name, bid))79 return out808182def status():83 for name, bid in all_batches():84 b = C.batch_status(bid)85 rc = b.request_counts86 print(f"{name:12s} {bid} {b.processing_status:10s} ok={rc.succeeded} err={rc.errored} proc={rc.processing} exp={rc.expired} can={rc.canceled}")878889def collect():90 for name, bid in all_batches():91 b = C.batch_status(bid)92 if b.processing_status == "ended":93 n = C.batch_collect(name, bid)94 print(f"{name}: collected {n} new results from {bid}")95 else:96 print(f"{name}: {bid} still {b.processing_status}")979899if __name__ == "__main__":100 cmd = sys.argv[1]101 if cmd == "submit":102 submit(sys.argv[2:] or ["img_haiku", "img_sonnet", "img_opus", "txt_haiku", "txt_sonnet", "txt_opus"])103 elif cmd == "status":104 status()105 elif cmd == "collect":106 collect()107