1#!/usr/bin/env python32"""Agent E, WP44 — build manuscript/references_part05.bib + references_verification_part05.csv.3Each entry is resolved by a single Crossref (DOI) or arXiv (id) lookup; the identifier comes from the ai_ml landmark CSV4(column doi / arxiv_id) or from an explicit override when the corpus record has none. Nothing is written without a5successful lookup. Usage: python3 make_bib_part05.py <repo_root>"""6import csv, json, sys, re, time, unicodedata, urllib.request, xml.etree.ElementTree as ET7R = sys.argv[1]8L = {r["work_id"]: r for r in csv.DictReader(open(f"{R}/literature_synthesis/landmarks/ai_ml_landmarks.csv"))}9MAIL = "spbou4@icloud.com"10# work_id: (keyword, override identifier or None)11W = {12"7124974472": ("rapidadoption", None), "4323655724": ("chatgptgood", None), "2990704537": ("superglue", "arxiv:1905.00537"),13"4393138529": ("peftsurvey", "arxiv:2403.14608"), "4410551352": ("aurora", None), "4392947521": ("uni", None), "4402220065": ("chief", None),14"4401397240": ("biomedgpt", None), "4411932262": ("centaur", None), "7133220561": ("instructblip", "arxiv:2305.06500"),15"7133250524": ("laion5b", "arxiv:2210.08402"), "2995460200": ("vlbert", "arxiv:1908.08530"), "4393148714": ("t2iadapter", None),16"4289785095": ("textualinversion", "arxiv:2208.01618"), "7164501397": ("textexploit", None), "4396599905": ("lowqualityfusion", None),17"4288099666": ("cfg", "arxiv:2207.12598"), "4406472463": ("mattergen", None), "4400770292": ("harmlearning", None),18"4406728221": ("agenticsurvey", None), "7161734976": ("robin", None), "4412709685": ("virtuallab", None), "4407570917": ("agentsociety", None),19"4413427262": ("agentsvsagentic", None), "4412203333": ("agentmemory", None), "7130481397": ("clinicalagents", None),20"2781726626": ("sac", "arxiv:1801.01290"), "2963923407": ("td3", "arxiv:1802.09477"), "2982316857": ("alphastar", None),21"2761873684": ("rainbow", None), "7133226216": ("mappo", "arxiv:2103.01955"), "4409147637": ("dreamerv3", None),22"2964083594": ("curiosity", "arxiv:1808.04355"), "2991046523": ("marloverview", None), "2945976633": ("stopexplaining", None),23"2439568532": ("mythos", None), "2999615587": ("treeexplainer", None), "2999309192": ("mcc", None), "3041133507": ("transfersurvey", None),24"2436219157": ("optimization", None), "4382203079": ("transformerstsf", None), "3177318507": ("informer", None),25"4385763767": ("tstransformers", None), "4406170795": ("tabpfn", None), "4396821501": ("kan", "arxiv:2404.19756"),26"2907492528": ("gnnsurvey", None), "3152893301": ("gnnreview", None), "2963399829": ("mixup", "arxiv:1710.09412"),27"2981731882": ("xai", None), "2969896603": ("biasfairness", None), "2670253439": ("explanation", None), "4366262984": ("xaitrust", None),28"4402698709": ("riskrepository", None), "4410609100": ("pair", None), "4406152291": ("poisoning", None), "7128782475": ("unsafeanswers", None),29"4411542009": ("stigma", None), "4406063062": ("raigovernance", None), "4406571986": ("hepolicies", None), "4366420437": ("rapidreview", None),30"4384464487": ("students", None), "4383312437": ("policyframework", None), "4389437528": ("chatgptedu", None), "4317910584": ("assessment", None),31"4322617812": ("ragnarok", None), "4413447271": ("radfm", None), "4415913530": ("titan", None), "4392947532": ("conch", None),32"4385948838": ("plip", None), "4411099958": ("panderm", None), "7128029160": ("brainiac", None), "7119541039": ("earthfm", None),33"4384071683": ("clinical", None), "3185341429": ("prompting", None), "4319083882": ("priorities", None), "4390730779": ("agentsdefinitions", None),34"7133236347": ("trism", None), "4407135816": ("fmradiology", None), "4390690043": ("datacentric", None), "4406472599": ("learningfrommodels", None),35"4410882873": ("multimodalmedicine", None), "4399790131": ("overreliance", None), "4398243313": ("assessmentreview", None),36"4409269614": ("integrity", None), "4387355843": ("timellm", "arxiv:2310.01728"), "4393153153": ("timesurl", None), "4402727669": ("mplugowl2", None),37"7148475777": ("gpt5medical", "arxiv:2508.08224"), "7127319731": ("medr1", None), "4414991289": ("dive", None), "4414023998": ("agenticreview", None),38"4416209823": ("agenticparadigms", None), "7128165235": ("multiagentai", None), "4408314709": ("biashealthcare", None),39"7140170686": ("pedagogical", None), "7135409568": ("agentichealthcare", None), "3137147200": ("quantization", None), "4361229539": ("llamaadapter", "arxiv:2303.16199"),40"7133233472": None, # QLoRA already in references.bib (dettmers2023qlora)41}42def get(url):43 req = urllib.request.Request(url, headers={"User-Agent": f"WP44-agentE (mailto:{MAIL})"})44 return urllib.request.urlopen(req, timeout=40).read()45def clean(s):46 s = unicodedata.normalize("NFKC", s)47 s = s.replace("‐", "-").replace("‑", "-").replace("‒", "-").replace("–", "--").replace("—", "---").replace("‘", "'").replace("’", "'").replace("“", "``").replace("”", "''").replace(" ", " ").replace("−", "-")48 s = "".join(c if ord(c) < 0x250 else unicodedata.normalize("NFKD", c)[0] if unicodedata.normalize("NFKD", c)[0].isascii() else "" for c in s)49 s = s.replace("&", "\\&").replace("%", "\\%").replace("#", "\\#").replace("$", "\\$")50 return re.sub(r"\s+", " ", s).strip()51def toks(s): return set(re.findall(r"[a-z0-9]+", unicodedata.normalize("NFKD", s).lower()))52def crossref(doi):53 m = json.loads(get(f"https://api.crossref.org/works/{doi}?mailto={MAIL}"))["message"]54 auth = [a for a in m.get("author", []) if a.get("family")]55 if not auth: # Crossref record without authors: second verified lookup on OpenAlex by DOI56 oa = json.loads(get(f"https://api.openalex.org/works/https://doi.org/{doi}?mailto={MAIL}"))57 auth = [{"family": a["author"]["display_name"].split()[-1], "given": " ".join(a["author"]["display_name"].split()[:-1])} for a in oa.get("authorships", [])]58 m["oa_authors"] = True59 year = (m.get("issued") or m.get("published") or {}).get("date-parts", [[None]])[0][0]60 return {"type": m.get("type"), "title": m.get("title", [""])[0], "subtitle": (m.get("subtitle") or [""])[0], "authors": [(a.get("family", ""), a.get("given", "")) for a in auth], "container": (m.get("container-title") or [""])[0],61 "volume": m.get("volume"), "issue": m.get("issue"), "page": m.get("page"), "year": year, "publisher": m.get("publisher"), "doi": m["DOI"], "event": (m.get("event") or {}).get("name"), "oa_authors": m.get("oa_authors", False)}62def arxiv(aid):63 for attempt in range(4): # arXiv asks for >= 3 s between requests and returns an empty feed when throttled64 time.sleep(3.5)65 root = ET.fromstring(get(f"https://export.arxiv.org/api/query?id_list={aid}"))66 ns = {"a": "http://www.w3.org/2005/Atom"}; e = root.find("a:entry", ns)67 if e is not None and e.find("a:title", ns) is not None and not e.find("a:title", ns).text.strip().startswith("Error"): break68 else: raise RuntimeError("arXiv lookup failed " + aid)69 title = re.sub(r"\s+", " ", e.find("a:title", ns).text.strip())70 authors = [a.find("a:name", ns).text for a in e.findall("a:author", ns)]71 year = int(e.find("a:published", ns).text[:4])72 return {"type": "arxiv", "title": title, "authors": [(n.split()[-1], " ".join(n.split()[:-1])) for n in authors], "year": year, "arxiv": aid, "doi": None}73def bib(key, m):74 A = " and ".join(f"{clean(f)}, {clean(g)}" if g else clean(f) for f, g in m["authors"]) or "Anonymous"75 T = clean(m["title"] + (": " + m["subtitle"] if m.get("subtitle") else ""))76 f = [f" title = {{{{{T}}}}}", f" author = {{{A}}}", f" year = {{{m['year']}}}"]77 t = m["type"]78 if t == "arxiv":79 et = "misc"; f += [f" eprint = {{{m['arxiv']}}}", " archiveprefix = {arXiv}", f" howpublished = {{arXiv preprint arXiv:{m['arxiv']}}}", " note = {Preprint, not peer reviewed}"]80 elif t == "journal-article":81 et = "article"; f += [f" journal = {{{clean(m['container'])}}}"]82 if m.get("volume"): f.append(f" volume = {{{m['volume']}}}")83 if m.get("issue"): f.append(f" number = {{{m['issue']}}}")84 if m.get("page"): f.append(f" pages = {{{clean(m['page']).replace('-', '--').replace('----', '--')}}}")85 elif t == "proceedings-article":86 et = "inproceedings"; f += [f" booktitle = {{{clean(m['container'] or m.get('event') or 'Proceedings')}}}"]87 if m.get("page"): f.append(f" pages = {{{clean(m['page']).replace('-', '--').replace('----', '--')}}}")88 if m.get("publisher"): f.append(f" publisher = {{{clean(m['publisher'])}}}")89 elif t == "book-chapter":90 et = "incollection"; f += [f" booktitle = {{{clean(m['container'])}}}"]91 if m.get("publisher"): f.append(f" publisher = {{{clean(m['publisher'])}}}")92 if m.get("page"): f.append(f" pages = {{{clean(m['page']).replace('-', '--').replace('----', '--')}}}")93 elif t == "posted-content":94 et = "misc"; f += [f" howpublished = {{Preprint ({clean(m.get('publisher') or 'repository')})}}", " note = {Preprint, not peer reviewed}"]95 elif t in ("book", "monograph", "edited-book", "reference-book"):96 et = "book"; f += [f" publisher = {{{clean(m.get('publisher') or '')}}}"]97 elif t == "report":98 et = "techreport"; f += [f" institution = {{{clean(m.get('publisher') or '')}}}"]99 else:100 et = "misc"; f += [f" howpublished = {{{clean(m.get('container') or m.get('publisher') or '')}}}"]101 if m.get("doi"): f.append(f" doi = {{{m['doi']}}}")102 return f"@{et}{{{key},\n" + ",\n".join(f) + "\n}\n"103entries, ver, keys = [], [], {}104for wid, spec in W.items():105 if spec is None: continue106 kw, ovr = spec; r = L[wid]107 ident = ovr or (f"doi:{r['doi']}" if r["doi"] else (f"arxiv:{r['arxiv_id']}" if r["arxiv_id"] else None))108 if not ident: print("NO IDENTIFIER", wid, r["title"][:60]); continue109 try:110 m = crossref(ident[4:]) if ident.startswith("doi:") else arxiv(ident[6:])111 except Exception as e:112 print("LOOKUP FAILED", wid, ident, e); continue113 surname = re.sub(r"[^a-z]", "", unicodedata.normalize("NFKD", m["authors"][0][0]).encode("ascii", "ignore").decode().lower()) if m["authors"] else "anon"114 key = f"{surname}{m['year']}{kw}"115 j = len(toks(r["title"]) & toks(m["title"])) / max(1, len(toks(r["title"]) | toks(m["title"])))116 match = "yes" if j >= 0.5 else f"check (jaccard={j:.2f})"117 keys[wid] = key; entries.append(bib(key, m))118 ver.append({"key": key, "openalex_id": wid, "doi_or_arxiv": ident, "verified_via": ("crossref+openalex" if (ident.startswith("doi:") and m.get("oa_authors")) else "crossref") if ident.startswith("doi:") else "arxiv", "verified_title": clean(m["title"])[:160], "corpus_title": clean(r["title"])[:160], "match": match, "entry_type": m["type"]})119 print(key, "|", match, "|", clean(m["title"])[:70]); time.sleep(0.3)120open(f"{R}/manuscript/references_part05.bib", "w").write("% references_part05.bib — Part V (The Artificial Intelligence Transformation), WP44. Generated by\n% literature_synthesis/agent_E/make_bib_part05.py on 2026-11-03: every entry resolved by one Crossref (DOI) or arXiv lookup;\n% see references_verification_part05.csv. Keys of manuscript/references.bib are not redefined here.\n\n" + "\n".join(entries))121with open(f"{R}/manuscript/references_verification_part05.csv", "w", newline="") as f:122 w = csv.DictWriter(f, fieldnames=list(ver[0].keys())); w.writeheader(); w.writerows(ver)123json.dump(keys, open(f"{R}/literature_synthesis/agent_E/bib_keys.json", "w"), indent=1)124print(len(entries), "entries written")125