1"""Phase B — Crossref (cursor pagination, polite pool) and arXiv discovery, run in parallel with Semantic Scholar.2Outputs: data/metadata/crossref_records.jsonl, data/metadata/arxiv_records.jsonl, search_log.csv (appended)."""3import csv, json, time, urllib.parse, urllib.request, xml.etree.ElementTree as ET4from datetime import date5from pathlib import Path67ROOT = Path(__file__).resolve().parents[2]; META = ROOT / "data" / "metadata"8MAILTO = "simon-pierre.boucher@uqo.ca"9CR_QUERIES = ["hedonic housing prices", "hedonic house price model", "hedonic property valuation residential", "residential property price determinants regression",10 "implicit prices housing attributes", "spatial hedonic housing", "house prices proximity hedonic", "apartment rent hedonic", "housing price machine learning valuation",11 "automated valuation model residential", "prix hédoniques logement", "precios hedónicos vivienda", "hedonische Preise Wohnungen", "preços hedônicos imóveis"]12AX_QUERIES = ["hedonic house price", "house price prediction machine learning", "real estate valuation deep learning", "property valuation computer vision",13 "housing price text listing", "automated valuation model", "hedonic regression housing"]141516def get(url, tries=6):17 for i in range(tries):18 try:19 with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": f"wp21-uqo (mailto:{MAILTO})"}), timeout=120) as r:20 return r.read()21 except Exception as e: # noqa22 err = e; time.sleep(10 * (i + 1))23 raise err242526def crossref(q, out, log, max_rows=4000):27 cursor = "*"; n = 0; total = None28 while n < max_rows:29 url = (f"https://api.crossref.org/works?query.bibliographic={urllib.parse.quote(q)}&filter=from-pub-date:1970-01-01,until-pub-date:2026-10-20"30 f"&rows=500&cursor={urllib.parse.quote(cursor)}&select=DOI,title,author,issued,container-title,abstract,type,is-referenced-by-count,URL,subject&mailto={MAILTO}")31 d = json.loads(get(url))["message"]; total = d.get("total-results", total)32 items = d.get("items", [])33 if not items:34 break35 for it in items:36 it["_query"] = q; it["_source"] = "crossref"; it["_retrieved"] = str(date.today())37 out.write(json.dumps(it, ensure_ascii=False) + "\n"); n += 138 cursor = d.get("next-cursor"); time.sleep(0.5)39 if not cursor:40 break41 log.writerow([str(date.today()), "crossref", "bibliographic", q, total, n]); print(f"crossref {q!r}: {total} total, {n} retrieved", flush=True)424344def arxiv(q, out, log, max_rows=2000):45 n = 0; start = 0; total = None46 ns = {"a": "http://www.w3.org/2005/Atom", "o": "http://a9.com/-/spec/opensearch/1.1/"}47 while start < max_rows:48 url = f"http://export.arxiv.org/api/query?search_query=all:{urllib.parse.quote(q)}&start={start}&max_results=200"49 root = ET.fromstring(get(url)); total = int(root.find("o:totalResults", ns).text)50 entries = root.findall("a:entry", ns)51 if not entries:52 break53 for e in entries:54 rec = {"id": e.find("a:id", ns).text, "title": " ".join(e.find("a:title", ns).text.split()), "abstract": " ".join((e.find("a:summary", ns).text or "").split()),55 "published": e.find("a:published", ns).text, "authors": [a.find("a:name", ns).text for a in e.findall("a:author", ns)],56 "categories": [c.get("term") for c in e.findall("a:category", ns)], "doi": (e.find("{http://arxiv.org/schemas/atom}doi") is not None and e.find("{http://arxiv.org/schemas/atom}doi").text) or "",57 "_query": q, "_source": "arxiv", "_retrieved": str(date.today())}58 out.write(json.dumps(rec, ensure_ascii=False) + "\n"); n += 159 start += 200; time.sleep(3.5)60 if start >= total:61 break62 log.writerow([str(date.today()), "arxiv", "all", q, total, n]); print(f"arxiv {q!r}: {total} total, {n} retrieved", flush=True)636465def main():66 with open(META / "crossref_records.jsonl", "w", encoding="utf-8") as out, open(META / "search_log.csv", "a", newline="") as lf:67 log = csv.writer(lf)68 for q in CR_QUERIES:69 try: crossref(q, out, log)70 except Exception as e: log.writerow([str(date.today()), "crossref", "bibliographic", q, "ERROR", repr(e)]); print("crossref failed", q, e)71 with open(META / "arxiv_records.jsonl", "w", encoding="utf-8") as out, open(META / "search_log.csv", "a", newline="") as lf:72 log = csv.writer(lf)73 for q in AX_QUERIES:74 try: arxiv(q, out, log)75 except Exception as e: log.writerow([str(date.today()), "arxiv", "all", q, "ERROR", repr(e)]); print("arxiv failed", q, e)767778if __name__ == "__main__":79 main()80