1"""Phase B — Semantic Scholar bulk discovery (primary source: OpenAlex without a key is budget-capped, see search_audit).2Runs every query in queries.yaml with the bulk endpoint (token pagination, up to 1,000 records per page, 1 request/s),3then forward-snowballs the foundational studies / competing reviews via /paper/{id}/citations.4Outputs: data/metadata/s2_records.jsonl, data/metadata/search_log.csv (appended), data/metadata/retrieval_failures.csv.5"""6import csv, json, time, sys, urllib.parse, urllib.request7from datetime import date8from pathlib import Path9import yaml1011ROOT = Path(__file__).resolve().parents[2]12META = ROOT / "data" / "metadata"; META.mkdir(parents=True, exist_ok=True)13FIELDS = "title,year,externalIds,abstract,venue,publicationVenue,citationCount,referenceCount,openAccessPdf,authors,publicationTypes,publicationDate,fieldsOfStudy,isOpenAccess"14UA = "wp21-uqo meta-analysis (mailto:simon-pierre.boucher@uqo.ca)"15SNOWBALL_DOIS = { # foundational studies and competing reviews (verified via Crossref before use)16 "sirmans2005": "10.1080/10835547.2005.12090154", "sirmans2006": "10.1007/s11146-006-9983-5", "rosen1974": "10.1086/260169",17}181920def get(url, tries=12):21 """GET with patient backoff: the unauthenticated Semantic Scholar pool returns 429 often; wait up to ~5 min per attempt."""22 err = None23 for i in range(tries):24 try:25 with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": UA}), timeout=120) as r:26 return json.load(r)27 except urllib.error.HTTPError as e:28 err = e29 if e.code == 404:30 return None31 wait = min(300, 15 * (2 ** min(i, 4)))32 print(f" HTTP {e.code}; waiting {wait}s (attempt {i + 1}/{tries})", flush=True); time.sleep(wait)33 except Exception as e: # noqa34 err = e; time.sleep(10 * (i + 1))35 raise err363738def bulk(query, out, log, year="1970-2026", max_records=20000):39 token = None; n = 0; total = None40 while True:41 url = (f"https://api.semanticscholar.org/graph/v1/paper/search/bulk?query={urllib.parse.quote(query)}&fields={FIELDS}&year={year}"42 + (f"&token={token}" if token else ""))43 d = get(url)44 if d is None:45 break46 total = d.get("total", total)47 for p in d.get("data", []):48 p["_query"] = query; p["_source"] = "semanticscholar_bulk"; p["_retrieved"] = str(date.today())49 out.write(json.dumps(p, ensure_ascii=False) + "\n"); n += 150 token = d.get("token")51 time.sleep(3.0)52 if not token or n >= max_records:53 break54 log.writerow([str(date.today()), "semanticscholar", "bulk", query, total, n]); print(f"{query!r}: total {total}, retrieved {n}", flush=True)55 return n565758def citations(doi, key, out, log):59 n = 0; offset = 060 while True:61 d = get(f"https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}/citations?fields={FIELDS}&offset={offset}&limit=1000")62 if not d or not d.get("data"):63 break64 for c in d["data"]:65 p = c.get("citingPaper") or {}66 if not p.get("paperId"):67 continue68 p["_query"] = f"cites:{key}"; p["_source"] = "semanticscholar_citations"; p["_retrieved"] = str(date.today())69 out.write(json.dumps(p, ensure_ascii=False) + "\n"); n += 170 offset = d.get("next")71 time.sleep(3.0)72 if offset is None:73 break74 log.writerow([str(date.today()), "semanticscholar", "citations", f"cites:{key} ({doi})", None, n]); print(f"cites {key}: {n}", flush=True)757677def main():78 qs = yaml.safe_load((ROOT / "scripts" / "discovery" / "queries.yaml").read_text())79 # boolean bulk queries (S2 bulk syntax): broaden the English set80 boolean = [81 "hedonic + (housing | house | houses | residential | apartment | apartments | condominium | dwelling | rent | rental | property | real estate)",82 "(\"implicit price\" | \"implicit prices\") + (housing | house | residential | property)",83 "(\"house prices\" | \"housing prices\" | \"home prices\" | \"property prices\" | \"property values\" | \"home values\") + (regression | determinants | characteristics | attributes | amenities | proximity | distance)",84 "(\"automated valuation\" | \"mass appraisal\") + (residential | housing | house | property)",85 "(\"machine learning\" | \"deep learning\" | \"neural network\" | \"random forest\" | \"gradient boosting\" | XGBoost) + (\"house price\" | \"housing price\" | \"property valuation\" | \"real estate valuation\" | \"property price\")",86 "(\"computer vision\" | \"street view\" | \"satellite imagery\" | \"images\") + (\"house price\" | \"housing price\" | \"property value\" | \"real estate\")",87 "(\"text mining\" | \"natural language processing\" | \"textual\" | \"listing descriptions\") + (\"house price\" | \"housing price\" | \"real estate\" | \"property price\")",88 "(\"spatial hedonic\" | \"spatial econometric\" | \"spatial lag\" | \"spatial error\" | \"geographically weighted regression\") + (housing | house | residential | property)",89 "(\"meta-analysis\" | \"meta analysis\" | \"meta-regression\") + (hedonic | \"house prices\" | \"housing prices\" | \"property values\")",90 "(\"willingness to pay\" | capitalization | capitalisation) + hedonic + (housing | house | residential | property)",91 ]92 done = set()93 if (META / "search_log.csv").exists():94 for r in csv.DictReader(open(META / "search_log.csv")):95 if r["source"] == "semanticscholar" and r["retrieved"] not in ("", None):96 done.add(r["query"])97 with open(META / "s2_records.jsonl", "a", encoding="utf-8") as out, open(META / "search_log.csv", "a", newline="") as lf:98 log = csv.writer(lf)99 if lf.tell() == 0:100 log.writerow(["date", "source", "mode", "query", "hits", "retrieved"])101 for q in boolean + qs["english"]:102 if q not in done: bulk(q, out, log)103 for q in qs["multilingual"]:104 if q not in done: bulk(q, out, log, max_records=3000)105 for key, doi in SNOWBALL_DOIS.items():106 if key == "rosen1974":107 continue # ~10k citations, mostly outside housing; handled through keyword queries108 citations(doi, key, out, log)109110111if __name__ == "__main__":112 main()113