1"""Phase B — OpenAlex discovery. Runs every query in queries.yaml (title/abstract search, 1970-2026), pages with cursors,2stores raw records (JSONL) and a search log. Polite pool via mailto. Usage: python scripts/discovery/openalex_search.py [--max N]3"""4import argparse, csv, json, time, sys, urllib.parse, urllib.request5from datetime import date6from pathlib import Path7import yaml89ROOT = Path(__file__).resolve().parents[2]10META = ROOT / "data" / "metadata"; META.mkdir(parents=True, exist_ok=True)11MAILTO = "simon-pierre.boucher@uqo.ca"12FIELDS = "id,doi,title,display_name,publication_year,publication_date,type,language,primary_location,open_access,best_oa_location,authorships,cited_by_count,referenced_works,abstract_inverted_index,topics,is_retracted"131415def get(url, tries=5):16 for i in range(tries):17 try:18 with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": f"wp21-uqo (mailto:{MAILTO})"}), timeout=60) as r:19 return json.load(r)20 except Exception as e: # noqa21 time.sleep(2 * (i + 1)); err = e22 raise err232425def abstract_from_index(idx):26 if not idx:27 return ""28 pos = {}29 for w, ps in idx.items():30 for p_ in ps:31 pos[p_] = w32 return " ".join(pos[k] for k in sorted(pos))333435def run_query(q, out, log, max_records, mode="title_and_abstract.search"):36 cursor = "*"; n = 0; hits = None37 base = f"https://api.openalex.org/works?filter={mode}:{urllib.parse.quote(q)},publication_year:1970-2026&per-page=200&select={FIELDS}&mailto={MAILTO}"38 while cursor and n < max_records:39 d = get(base + f"&cursor={cursor}")40 hits = d["meta"]["count"] if hits is None else hits41 for w in d["results"]:42 w["abstract"] = abstract_from_index(w.pop("abstract_inverted_index", None))43 w["_query"] = q; w["_mode"] = mode; w["_retrieved"] = str(date.today())44 out.write(json.dumps(w, ensure_ascii=False) + "\n"); n += 145 cursor = d["meta"].get("next_cursor")46 time.sleep(0.15)47 log.writerow([str(date.today()), "openalex", mode, q, hits, n]); print(f"{q!r}: {hits} hits, {n} retrieved", flush=True)48 return n495051def main():52 ap = argparse.ArgumentParser(); ap.add_argument("--max", type=int, default=3000); a = ap.parse_args()53 qs = yaml.safe_load((ROOT / "scripts" / "discovery" / "queries.yaml").read_text())54 with open(META / "openalex_records.jsonl", "w", encoding="utf-8") as out, open(META / "search_log.csv", "w", newline="") as lf:55 log = csv.writer(lf); log.writerow(["date", "source", "mode", "query", "hits", "retrieved"])56 for q in qs["english"]:57 run_query(q, out, log, a.max)58 for q in qs["multilingual"]:59 run_query(q, out, log, 1000)60 # forward snowballing: works citing the foundational studies / main reviews (OpenAlex IDs resolved from DOIs)61 for doi in ["10.1080/10835547.2005.12090154", "10.1007/s11146-006-9983-5", "10.1086/260169"]:62 try:63 w = get(f"https://api.openalex.org/works/https://doi.org/{doi}?mailto={MAILTO}")64 oid = w["id"].split("/")[-1]65 run_query(oid, out, log, 5000, mode="cites")66 except Exception as e:67 log.writerow([str(date.today()), "openalex", "cites", doi, "ERROR", repr(e)]); print("cites failed", doi, e)686970if __name__ == "__main__":71 main()72