"""Phase B — OpenAlex discovery. Runs every query in queries.yaml (title/abstract search, 1970-2026), pages with cursors,
stores raw records (JSONL) and a search log. Polite pool via mailto. Usage: python scripts/discovery/openalex_search.py [--max N]
"""
import argparse, csv, json, time, sys, urllib.parse, urllib.request
from datetime import date
from pathlib import Path
import yaml

ROOT = Path(__file__).resolve().parents[2]
META = ROOT / "data" / "metadata"; META.mkdir(parents=True, exist_ok=True)
MAILTO = "simon-pierre.boucher@uqo.ca"
FIELDS = "id,doi,title,display_name,publication_year,publication_date,type,language,primary_location,open_access,best_oa_location,authorships,cited_by_count,referenced_works,abstract_inverted_index,topics,is_retracted"


def get(url, tries=5):
    for i in range(tries):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": f"wp21-uqo (mailto:{MAILTO})"}), timeout=60) as r:
                return json.load(r)
        except Exception as e:  # noqa
            time.sleep(2 * (i + 1)); err = e
    raise err


def abstract_from_index(idx):
    if not idx:
        return ""
    pos = {}
    for w, ps in idx.items():
        for p_ in ps:
            pos[p_] = w
    return " ".join(pos[k] for k in sorted(pos))


def run_query(q, out, log, max_records, mode="title_and_abstract.search"):
    cursor = "*"; n = 0; hits = None
    base = f"https://api.openalex.org/works?filter={mode}:{urllib.parse.quote(q)},publication_year:1970-2026&per-page=200&select={FIELDS}&mailto={MAILTO}"
    while cursor and n < max_records:
        d = get(base + f"&cursor={cursor}")
        hits = d["meta"]["count"] if hits is None else hits
        for w in d["results"]:
            w["abstract"] = abstract_from_index(w.pop("abstract_inverted_index", None))
            w["_query"] = q; w["_mode"] = mode; w["_retrieved"] = str(date.today())
            out.write(json.dumps(w, ensure_ascii=False) + "\n"); n += 1
        cursor = d["meta"].get("next_cursor")
        time.sleep(0.15)
    log.writerow([str(date.today()), "openalex", mode, q, hits, n]); print(f"{q!r}: {hits} hits, {n} retrieved", flush=True)
    return n


def main():
    ap = argparse.ArgumentParser(); ap.add_argument("--max", type=int, default=3000); a = ap.parse_args()
    qs = yaml.safe_load((ROOT / "scripts" / "discovery" / "queries.yaml").read_text())
    with open(META / "openalex_records.jsonl", "w", encoding="utf-8") as out, open(META / "search_log.csv", "w", newline="") as lf:
        log = csv.writer(lf); log.writerow(["date", "source", "mode", "query", "hits", "retrieved"])
        for q in qs["english"]:
            run_query(q, out, log, a.max)
        for q in qs["multilingual"]:
            run_query(q, out, log, 1000)
        # forward snowballing: works citing the foundational studies / main reviews (OpenAlex IDs resolved from DOIs)
        for doi in ["10.1080/10835547.2005.12090154", "10.1007/s11146-006-9983-5", "10.1086/260169"]:
            try:
                w = get(f"https://api.openalex.org/works/https://doi.org/{doi}?mailto={MAILTO}")
                oid = w["id"].split("/")[-1]
                run_query(oid, out, log, 5000, mode="cites")
            except Exception as e:
                log.writerow([str(date.today()), "openalex", "cites", doi, "ERROR", repr(e)]); print("cites failed", doi, e)


if __name__ == "__main__":
    main()