spb/wpai0002_uqo

1#!/usr/bin/env python32"""Recompute the coarse discipline label (subject_area) of PMC documents in a parsed corpus after a taxonomy fix,3in place, keeping a change log. Pairs must be re-validated afterwards so that their metadata follows."""4from __future__ import annotations56import argparse7import sys8from collections import Counter9from pathlib import Path1011ROOT = Path(__file__).resolve().parents[1]12sys.path.insert(0, str(ROOT / "src"))13from htr.data.taxonomy import discipline_from_pmc  # noqa: E40214from htr.utils.io import read_jsonl, write_json, write_jsonl  # noqa: E40215from htr.utils.provenance import run_header  # noqa: E402161718def main() -> None:19    ap = argparse.ArgumentParser()20    ap.add_argument("--corpus", required=True)21    a = ap.parse_args()22    corpus = Path(a.corpus)23    docs = list(read_jsonl(corpus / "documents.jsonl"))24    before, after, changes = Counter(), Counter(), []25    for d in docs:26        if d["source"] != "pmc":27            after[d["subject_area"]] += 128            before[d["subject_area"]] += 129            continue30        old = d.get("subject_area")31        new = discipline_from_pmc(d.get("venue"), d.get("subject_raw", []))32        before[old] += 133        after[new] += 134        if new != old:35            changes.append({"doc_id": d["doc_id"], "venue": d.get("venue"), "old": old, "new": new})36            d["subject_area"] = new37    write_jsonl(corpus / "documents.jsonl", docs)38    write_json(corpus / "relabel_log.json", {**run_header(), "n_changed": len(changes), "before": dict(before), "after": dict(after), "changes": changes})39    print(f"changed {len(changes)} of {len(docs)}; after: {dict(after.most_common())}")404142if __name__ == "__main__":43    main()44