1#!/usr/bin/env python32"""Download the small OpenAlex entity snapshots (topics, subfields, fields, domains,3sources, institutions, publishers, funders, concepts) as JSONL and convert them to4parquet-friendly JSONL.zst files with a SHA-256 ledger. Authors are NOT downloaded5(embedded authorship data is used); keywords are optional (--with-keywords)."""6import argparse, hashlib, json, os, subprocess, time, datetime as dt78S3_HTTP = "https://openalex.s3.amazonaws.com/"9ENTITIES = ["topics", "subfields", "fields", "domains", "sources", "institutions", "publishers", "funders", "concepts"]101112def main():13 ap = argparse.ArgumentParser(); ap.add_argument("--out", required=True); ap.add_argument("--with-keywords", action="store_true")14 a = ap.parse_args()15 ents = ENTITIES + (["keywords"] if a.with_keywords else [])16 ledger_path = os.path.join(a.out, "entities_ledger.json")17 ledger = json.load(open(ledger_path)) if os.path.exists(ledger_path) else {}18 for e in ents:19 man = json.load(open(os.path.join(a.out, f"{e}_manifest.json")))20 d = os.path.join(a.out, e); os.makedirs(d, exist_ok=True)21 for f in man["files"]:22 url = f["url"]; ln = f["meta"]["content_length"]23 name = url.split(f"/{e}/")[1].replace("/", "__")24 path = os.path.join(d, name)25 if ledger.get(url, {}).get("status") == "done" and os.path.exists(path):26 continue27 for attempt in range(5):28 subprocess.run(["curl", "-sS", "-L", "--retry", "5", "-C", "-", "-o", path, url.replace("s3://openalex/", S3_HTTP)])29 if os.path.exists(path) and os.path.getsize(path) == ln:30 break31 time.sleep(3)32 ok = os.path.exists(path) and os.path.getsize(path) == ln33 sha = hashlib.sha256(open(path, "rb").read()).hexdigest() if ok else None34 ledger[url] = {"entity": e, "path": path, "bytes": ln, "sha256": sha, "status": "done" if ok else "error",35 "records": f["meta"].get("record_count"), "when": dt.datetime.now(dt.UTC).isoformat()}36 json.dump(ledger, open(ledger_path, "w"), indent=1)37 print(e, name, "ok" if ok else "ERROR", flush=True)383940if __name__ == "__main__":41 main()42