#!/usr/bin/env python3
"""Flatten the OpenAlex topics snapshot (4,516 topics) into a CSV used by the taxonomy."""
import csv, glob, gzip, json, sys
out, src = sys.argv[1], sys.argv[2]
rows = []
for f in sorted(glob.glob(src + "/*.gz")):
    for line in gzip.open(f, "rt"):
        t = json.loads(line)
        kw = [k if isinstance(k, str) else (k.get("display_name") or k.get("keyword") or "") for k in (t.get("keywords") or [])]
        rows.append(dict(topic_id=t["id"].rsplit("/", 1)[1], display_name=t["display_name"], subfield_id=t["subfield"]["id"].rsplit("/", 1)[1],
                         subfield=t["subfield"]["display_name"], field_id=t["field"]["id"].rsplit("/", 1)[1], field=t["field"]["display_name"],
                         domain=t["domain"]["display_name"], works_count=t.get("works_count"), cited_by_count=t.get("cited_by_count"),
                         keywords="; ".join(kw), description=(t.get("description") or "").replace("\n", " ")))
rows.sort(key=lambda r: (int(r["field_id"]), int(r["subfield_id"]), int(r["topic_id"][1:])))
with open(out, "w", newline="") as fh:
    w = csv.DictWriter(fh, fieldnames=list(rows[0].keys())); w.writeheader(); w.writerows(rows)
print(len(rows), "topics;", sum(1 for r in rows if r["field_id"] == "17"), "in CS")