"""Phase B — Semantic Scholar bulk discovery (primary source: OpenAlex without a key is budget-capped, see search_audit).
Runs every query in queries.yaml with the bulk endpoint (token pagination, up to 1,000 records per page, 1 request/s),
then forward-snowballs the foundational studies / competing reviews via /paper/{id}/citations.
Outputs: data/metadata/s2_records.jsonl, data/metadata/search_log.csv (appended), data/metadata/retrieval_failures.csv.
"""
import csv, json, time, sys, urllib.parse, urllib.request
from datetime import date
from pathlib import Path
import yaml

ROOT = Path(__file__).resolve().parents[2]
META = ROOT / "data" / "metadata"; META.mkdir(parents=True, exist_ok=True)
FIELDS = "title,year,externalIds,abstract,venue,publicationVenue,citationCount,referenceCount,openAccessPdf,authors,publicationTypes,publicationDate,fieldsOfStudy,isOpenAccess"
UA = "wp21-uqo meta-analysis (mailto:simon-pierre.boucher@uqo.ca)"
SNOWBALL_DOIS = {  # foundational studies and competing reviews (verified via Crossref before use)
    "sirmans2005": "10.1080/10835547.2005.12090154", "sirmans2006": "10.1007/s11146-006-9983-5", "rosen1974": "10.1086/260169",
}


def get(url, tries=12):
    """GET with patient backoff: the unauthenticated Semantic Scholar pool returns 429 often; wait up to ~5 min per attempt."""
    err = None
    for i in range(tries):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": UA}), timeout=120) as r:
                return json.load(r)
        except urllib.error.HTTPError as e:
            err = e
            if e.code == 404:
                return None
            wait = min(300, 15 * (2 ** min(i, 4)))
            print(f"  HTTP {e.code}; waiting {wait}s (attempt {i + 1}/{tries})", flush=True); time.sleep(wait)
        except Exception as e:  # noqa
            err = e; time.sleep(10 * (i + 1))
    raise err


def bulk(query, out, log, year="1970-2026", max_records=20000):
    token = None; n = 0; total = None
    while True:
        url = (f"https://api.semanticscholar.org/graph/v1/paper/search/bulk?query={urllib.parse.quote(query)}&fields={FIELDS}&year={year}"
               + (f"&token={token}" if token else ""))
        d = get(url)
        if d is None:
            break
        total = d.get("total", total)
        for p in d.get("data", []):
            p["_query"] = query; p["_source"] = "semanticscholar_bulk"; p["_retrieved"] = str(date.today())
            out.write(json.dumps(p, ensure_ascii=False) + "\n"); n += 1
        token = d.get("token")
        time.sleep(3.0)
        if not token or n >= max_records:
            break
    log.writerow([str(date.today()), "semanticscholar", "bulk", query, total, n]); print(f"{query!r}: total {total}, retrieved {n}", flush=True)
    return n


def citations(doi, key, out, log):
    n = 0; offset = 0
    while True:
        d = get(f"https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}/citations?fields={FIELDS}&offset={offset}&limit=1000")
        if not d or not d.get("data"):
            break
        for c in d["data"]:
            p = c.get("citingPaper") or {}
            if not p.get("paperId"):
                continue
            p["_query"] = f"cites:{key}"; p["_source"] = "semanticscholar_citations"; p["_retrieved"] = str(date.today())
            out.write(json.dumps(p, ensure_ascii=False) + "\n"); n += 1
        offset = d.get("next")
        time.sleep(3.0)
        if offset is None:
            break
    log.writerow([str(date.today()), "semanticscholar", "citations", f"cites:{key} ({doi})", None, n]); print(f"cites {key}: {n}", flush=True)


def main():
    qs = yaml.safe_load((ROOT / "scripts" / "discovery" / "queries.yaml").read_text())
    # boolean bulk queries (S2 bulk syntax): broaden the English set
    boolean = [
        "hedonic + (housing | house | houses | residential | apartment | apartments | condominium | dwelling | rent | rental | property | real estate)",
        "(\"implicit price\" | \"implicit prices\") + (housing | house | residential | property)",
        "(\"house prices\" | \"housing prices\" | \"home prices\" | \"property prices\" | \"property values\" | \"home values\") + (regression | determinants | characteristics | attributes | amenities | proximity | distance)",
        "(\"automated valuation\" | \"mass appraisal\") + (residential | housing | house | property)",
        "(\"machine learning\" | \"deep learning\" | \"neural network\" | \"random forest\" | \"gradient boosting\" | XGBoost) + (\"house price\" | \"housing price\" | \"property valuation\" | \"real estate valuation\" | \"property price\")",
        "(\"computer vision\" | \"street view\" | \"satellite imagery\" | \"images\") + (\"house price\" | \"housing price\" | \"property value\" | \"real estate\")",
        "(\"text mining\" | \"natural language processing\" | \"textual\" | \"listing descriptions\") + (\"house price\" | \"housing price\" | \"real estate\" | \"property price\")",
        "(\"spatial hedonic\" | \"spatial econometric\" | \"spatial lag\" | \"spatial error\" | \"geographically weighted regression\") + (housing | house | residential | property)",
        "(\"meta-analysis\" | \"meta analysis\" | \"meta-regression\") + (hedonic | \"house prices\" | \"housing prices\" | \"property values\")",
        "(\"willingness to pay\" | capitalization | capitalisation) + hedonic + (housing | house | residential | property)",
    ]
    done = set()
    if (META / "search_log.csv").exists():
        for r in csv.DictReader(open(META / "search_log.csv")):
            if r["source"] == "semanticscholar" and r["retrieved"] not in ("", None):
                done.add(r["query"])
    with open(META / "s2_records.jsonl", "a", encoding="utf-8") as out, open(META / "search_log.csv", "a", newline="") as lf:
        log = csv.writer(lf)
        if lf.tell() == 0:
            log.writerow(["date", "source", "mode", "query", "hits", "retrieved"])
        for q in boolean + qs["english"]:
            if q not in done: bulk(q, out, log)
        for q in qs["multilingual"]:
            if q not in done: bulk(q, out, log, max_records=3000)
        for key, doi in SNOWBALL_DOIS.items():
            if key == "rosen1974":
                continue  # ~10k citations, mostly outside housing; handled through keyword queries
            citations(doi, key, out, log)


if __name__ == "__main__":
    main()