1#!/usr/bin/env python32"""WP44 — API increment: works with any CS topic published on/after --from that may be missing from the 2026-10-143snapshot. Cursor paging needs an API key (none configured), so the harvest is sliced by publication day and uses basic4`page=` paging (max 10,000 results per slice; a day with more results is split by primary field CS / non-CS).5Public API without key: $0.10/day shared budget, list+filter = $0.0001 per call. Raw pages are stored verbatim6(immutable) under <out>/api_increment/pages/<day>/ with a JSON ledger; resumable across days."""7import argparse, json, os, time, datetime as dt, requests89BASE = "https://api.openalex.org/works"10UA = "WP44-UQO-systematic-mapping/0.1 (mailto:spbou4@icloud.com)"11SELECT = ("id,doi,title,display_name,publication_year,publication_date,created_date,updated_date,type,type_crossref,language,ids,"12 "primary_location,best_oa_location,locations,open_access,authorships,is_authors_truncated,primary_topic,topics,keywords,"13 "referenced_works,referenced_works_count,cited_by_count,counts_by_year,abstract_inverted_index,is_retracted,is_paratext,is_xpac,"14 "fwci,citation_normalized_percentile,awards,funders,indexed_in,has_fulltext,apc_paid")151617def main():18 ap = argparse.ArgumentParser(); ap.add_argument("--out", required=True); ap.add_argument("--from", dest="frm", default="2026-10-01")19 ap.add_argument("--to", default="2026-11-02"); ap.add_argument("--per-page", type=int, default=200); ap.add_argument("--max-cost", type=float, default=0.085)20 a = ap.parse_args(); root = os.path.join(a.out, "api_increment"); os.makedirs(root, exist_ok=True)21 led_p = os.path.join(root, "ledger.json")22 led = json.load(open(led_p)) if os.path.exists(led_p) else {"slices": {}, "calls": 0, "cost_usd_total": 0.0}23 s = requests.Session(); spent = 0.024 day = dt.date.fromisoformat(a.frm); end = dt.date.fromisoformat(a.to)25 while day <= end:26 for sub in ("cs_primary", "cs_secondary"):27 key = f"{day}_{sub}"28 st = led["slices"].get(key, {"pages": 0, "records": 0, "done": False, "count": None})29 if st["done"]:30 continue31 d = os.path.join(root, "pages", key); os.makedirs(d, exist_ok=True)32 flt = f"topics.field.id:17,from_publication_date:{day},to_publication_date:{day}," + ("primary_topic.field.id:17" if sub == "cs_primary" else "primary_topic.field.id:!17")33 page = st["pages"] + 134 while True:35 if spent >= a.max_cost:36 led["slices"][key] = st; json.dump(led, open(led_p, "w"), indent=1)37 print(f"[{dt.datetime.now():%H:%M:%S}] budget reached (${spent:.4f}); resume tomorrow", flush=True); return38 r = s.get(BASE, params={"filter": flt, "per_page": a.per_page, "page": page, "select": SELECT, "mailto": "spbou4@icloud.com"},39 headers={"User-Agent": UA}, timeout=120)40 led["calls"] += 141 if r.status_code in (429, 403) or r.status_code >= 500:42 led["slices"][key] = st; json.dump(led, open(led_p, "w"), indent=1)43 print(f"[{dt.datetime.now():%H:%M:%S}] HTTP {r.status_code}: {r.text[:160]} — stop", flush=True); return44 if r.status_code != 200:45 print("HTTP", r.status_code, r.text[:300]); return46 j = r.json(); meta = j.get("meta", {}); res = j.get("results", [])47 c = float(meta.get("cost_usd") or 0.0001); spent += c; led["cost_usd_total"] += c48 st["count"] = meta.get("count")49 if res:50 with open(os.path.join(d, f"page_{page:04d}.json"), "w") as fh:51 json.dump(j, fh)52 st["pages"] = page; st["records"] += len(res)53 if not res or len(res) < a.per_page or page * a.per_page >= 10000:54 st["done"] = True55 if st["count"] and st["count"] > 10000:56 st["truncated"] = True57 break58 page += 1; time.sleep(0.12)59 led["slices"][key] = st; json.dump(led, open(led_p, "w"), indent=1)60 print(f"[{dt.datetime.now():%H:%M:%S}] {key}: {st['records']}/{st['count']} records, {st['pages']} pages, spent ${spent:.4f}", flush=True)61 day += dt.timedelta(days=1)62 print("all slices done; total calls", led["calls"], "cost", round(led["cost_usd_total"], 4))636465if __name__ == "__main__":66 main()67