1#!/usr/bin/env python32"""WP44 — parse the ACL Anthology BibTeX export (anthology+abstracts.bib.gz) into Parquet."""3import argparse, gzip, hashlib, json, os, re, datetime as dt4import pyarrow as pa, pyarrow.parquet as pq56FIELDS = ["title", "author", "editor", "booktitle", "journal", "month", "year", "address", "publisher", "url", "doi", "pages", "abstract", "volume", "number"]7SCHEMA = pa.schema([("acl_id", pa.string()), ("entry_type", pa.string())] + [(f, pa.string()) for f in FIELDS])8910def entries(txt):11 i = 0; n = len(txt)12 while True:13 j = txt.find("@", i)14 if j < 0:15 return16 k = txt.find("{", j)17 et = txt[j + 1:k].strip().lower()18 depth = 0; p = k19 while p < n:20 c = txt[p]21 if c == "{":22 depth += 123 elif c == "}":24 depth -= 125 if depth == 0:26 break27 p += 128 body = txt[k + 1:p]; i = p + 129 if et in ("comment", "string", "preamble"):30 continue31 yield et, body323334def parse_body(body):35 key, _, rest = body.partition(",")36 out = {"acl_id": key.strip()}37 # fields of the form name = {value} or name = "value", values may contain nested braces38 pos = 039 while True:40 m = re.search(r"(\w+)\s*=\s*", rest[pos:])41 if not m:42 break43 name = m.group(1).lower(); p = pos + m.end()44 if p >= len(rest):45 break46 if rest[p] == "{":47 depth = 0; q = p48 while q < len(rest):49 if rest[q] == "{": depth += 150 elif rest[q] == "}":51 depth -= 152 if depth == 0: break53 q += 154 val = rest[p + 1:q]; pos = q + 155 elif rest[p] == '"':56 q = rest.find('"', p + 1); val = rest[p + 1:q]; pos = q + 157 else:58 q = rest.find(",", p); q = len(rest) if q < 0 else q; val = rest[p:q]; pos = q59 val = re.sub(r"\s+", " ", val).strip()60 val = val.replace("{", "").replace("}", "")61 if name in FIELDS:62 out[name] = val63 return out646566def main():67 ap = argparse.ArgumentParser(); ap.add_argument("--bib", required=True); ap.add_argument("--out", required=True)68 a = ap.parse_args(); os.makedirs(a.out, exist_ok=True)69 raw = open(a.bib, "rb").read(); sha = hashlib.sha256(raw).hexdigest()70 txt = gzip.decompress(raw).decode("utf-8", "replace")71 rows = {n: [] for n in SCHEMA.names}; n = 0; types = {}72 for et, body in entries(txt):73 d = parse_body(body); d["entry_type"] = et; types[et] = types.get(et, 0) + 174 for k in SCHEMA.names:75 rows[k].append(d.get(k))76 n += 177 pq.write_table(pa.Table.from_pydict(rows, schema=SCHEMA), os.path.join(a.out, "acl_anthology.parquet"), compression="zstd")78 rep = {"input": a.bib, "sha256": sha, "bytes": len(raw), "entries": n, "types": types, "parsed_at": dt.datetime.now(dt.UTC).isoformat()}79 json.dump(rep, open(os.path.join(a.out, "acl_parse_report.json"), "w"), indent=1); print(json.dumps(rep, indent=1))808182if __name__ == "__main__":83 main()84