#!/usr/bin/env python3
"""WP44 — API increment: works with any CS topic published on/after --from that may be missing from the 2026-10-14
snapshot. Cursor paging needs an API key (none configured), so the harvest is sliced by publication day and uses basic
`page=` paging (max 10,000 results per slice; a day with more results is split by primary field CS / non-CS).
Public API without key: $0.10/day shared budget, list+filter = $0.0001 per call. Raw pages are stored verbatim
(immutable) under <out>/api_increment/pages/<day>/ with a JSON ledger; resumable across days."""
import argparse, json, os, time, datetime as dt, requests

BASE = "https://api.openalex.org/works"
UA = "WP44-UQO-systematic-mapping/0.1 (mailto:spbou4@icloud.com)"
SELECT = ("id,doi,title,display_name,publication_year,publication_date,created_date,updated_date,type,type_crossref,language,ids,"
          "primary_location,best_oa_location,locations,open_access,authorships,is_authors_truncated,primary_topic,topics,keywords,"
          "referenced_works,referenced_works_count,cited_by_count,counts_by_year,abstract_inverted_index,is_retracted,is_paratext,is_xpac,"
          "fwci,citation_normalized_percentile,awards,funders,indexed_in,has_fulltext,apc_paid")


def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--out", required=True); ap.add_argument("--from", dest="frm", default="2026-10-01")
    ap.add_argument("--to", default="2026-11-02"); ap.add_argument("--per-page", type=int, default=200); ap.add_argument("--max-cost", type=float, default=0.085)
    a = ap.parse_args(); root = os.path.join(a.out, "api_increment"); os.makedirs(root, exist_ok=True)
    led_p = os.path.join(root, "ledger.json")
    led = json.load(open(led_p)) if os.path.exists(led_p) else {"slices": {}, "calls": 0, "cost_usd_total": 0.0}
    s = requests.Session(); spent = 0.0
    day = dt.date.fromisoformat(a.frm); end = dt.date.fromisoformat(a.to)
    while day <= end:
        for sub in ("cs_primary", "cs_secondary"):
            key = f"{day}_{sub}"
            st = led["slices"].get(key, {"pages": 0, "records": 0, "done": False, "count": None})
            if st["done"]:
                continue
            d = os.path.join(root, "pages", key); os.makedirs(d, exist_ok=True)
            flt = f"topics.field.id:17,from_publication_date:{day},to_publication_date:{day}," + ("primary_topic.field.id:17" if sub == "cs_primary" else "primary_topic.field.id:!17")
            page = st["pages"] + 1
            while True:
                if spent >= a.max_cost:
                    led["slices"][key] = st; json.dump(led, open(led_p, "w"), indent=1)
                    print(f"[{dt.datetime.now():%H:%M:%S}] budget reached (${spent:.4f}); resume tomorrow", flush=True); return
                r = s.get(BASE, params={"filter": flt, "per_page": a.per_page, "page": page, "select": SELECT, "mailto": "spbou4@icloud.com"},
                          headers={"User-Agent": UA}, timeout=120)
                led["calls"] += 1
                if r.status_code in (429, 403) or r.status_code >= 500:
                    led["slices"][key] = st; json.dump(led, open(led_p, "w"), indent=1)
                    print(f"[{dt.datetime.now():%H:%M:%S}] HTTP {r.status_code}: {r.text[:160]} — stop", flush=True); return
                if r.status_code != 200:
                    print("HTTP", r.status_code, r.text[:300]); return
                j = r.json(); meta = j.get("meta", {}); res = j.get("results", [])
                c = float(meta.get("cost_usd") or 0.0001); spent += c; led["cost_usd_total"] += c
                st["count"] = meta.get("count")
                if res:
                    with open(os.path.join(d, f"page_{page:04d}.json"), "w") as fh:
                        json.dump(j, fh)
                    st["pages"] = page; st["records"] += len(res)
                if not res or len(res) < a.per_page or page * a.per_page >= 10000:
                    st["done"] = True
                    if st["count"] and st["count"] > 10000:
                        st["truncated"] = True
                    break
                page += 1; time.sleep(0.12)
            led["slices"][key] = st; json.dump(led, open(led_p, "w"), indent=1)
            print(f"[{dt.datetime.now():%H:%M:%S}] {key}: {st['records']}/{st['count']} records, {st['pages']} pages, spent ${spent:.4f}", flush=True)
        day += dt.timedelta(days=1)
    print("all slices done; total calls", led["calls"], "cost", round(led["cost_usd_total"], 4))


if __name__ == "__main__":
    main()