1#!/usr/bin/env python32"""WP44 — H-1: draw the record-linkage validation sample (500 DBLP↔OpenAlex pairs stratified by linkage method) with3automatic agreement checks (normalised title similarity, year difference, first-author surname match). The PI reviews a4200-pair subsample (column pi_review = TRUE). Output: data/metadata/linkage_sample.csv and linkage_sample_summary.json."""5import argparse, os, json, duckdb, difflib, re, unicodedata678def norm_name(x):9 """last token, lower-cased, diacritics stripped, DBLP disambiguation suffix (e.g. '0001') removed"""10 if not x: return ""11 x = re.sub(r"\s+\d{4}$", "", str(x).strip()); x = unicodedata.normalize("NFKD", x).encode("ascii", "ignore").decode()12 return x.split()[-1].lower() if x.split() else ""131415def norm_title(x):16 x = unicodedata.normalize("NFKD", str(x or "")).encode("ascii", "ignore").decode().lower()17 return re.sub(r"[^a-z0-9]+", " ", x).strip()181920def main():21 ap = argparse.ArgumentParser(); ap.add_argument("--root", required=True); ap.add_argument("--seed", type=int, default=44); a = ap.parse_args(); R = a.root22 con = duckdb.connect(os.path.join(R, "data/processed/wp44.duckdb"), read_only=True)23 rows = con.execute(f"""24 SELECT l.method, l.dblp_key, l.oa_id, d.title AS dblp_title, d.year AS dblp_year, d.authors[1] AS dblp_first_author, d.doi AS dblp_doi,25 o.title AS oa_title, o.publication_year AS oa_year, o.doi AS oa_doi,26 (SELECT author_name FROM oa_authorships au WHERE au.work_id = o.id ORDER BY position LIMIT 1) AS oa_first_author27 FROM (SELECT *, row_number() OVER (PARTITION BY method ORDER BY hash(dblp_key || '{a.seed}')) rn FROM link_dblp_oa) l28 JOIN dblp_raw d ON d.key = l.dblp_key JOIN oa_works o ON o.id = l.oa_id29 WHERE (l.method = 'doi' AND rn <= 250) OR (l.method = 'title_year' AND rn <= 200) OR (l.method = 'arxiv' AND rn <= 50)""").fetchall()30 out = []; agree = {}31 for i, r in enumerate(rows):32 method, key, oa, dt, dy, da, ddoi, ot, oy, odoi, oa_fa = r33 sim = difflib.SequenceMatcher(None, norm_title(dt), norm_title(ot)).ratio()34 fa = norm_name(da); ofa = norm_name(oa_fa)35 auto_ok = sim >= 0.9 and (dy is None or oy is None or abs(dy - oy) <= 1) and (not fa or not ofa or fa == ofa)36 out.append(dict(method=method, dblp_key=key, oa_id=oa, dblp_title=dt, oa_title=ot, dblp_year=dy, oa_year=oy, dblp_doi=ddoi, oa_doi=odoi,37 dblp_first_author=da, oa_first_author=oa_fa, title_similarity=round(sim, 3), auto_ok=auto_ok, pi_review=(i % 5 < 2), pi_verdict=""))38 agree.setdefault(method, [0, 0]); agree[method][0] += 1; agree[method][1] += int(auto_ok)39 import csv40 with open(os.path.join(R, "data/metadata/linkage_sample.csv"), "w", newline="") as fh:41 w = csv.DictWriter(fh, fieldnames=list(out[0].keys())); w.writeheader(); w.writerows(out)42 summ = {m: {"n": n, "auto_ok": k, "auto_precision": round(k / n, 4)} for m, (n, k) in agree.items()}43 json.dump(summ, open(os.path.join(R, "data/metadata/linkage_sample_summary.json"), "w"), indent=1); print(summ)444546if __name__ == "__main__":47 main()48