"""Phase B — Crossref (cursor pagination, polite pool) and arXiv discovery, run in parallel with Semantic Scholar.
Outputs: data/metadata/crossref_records.jsonl, data/metadata/arxiv_records.jsonl, search_log.csv (appended)."""
import csv, json, time, urllib.parse, urllib.request, xml.etree.ElementTree as ET
from datetime import date
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]; META = ROOT / "data" / "metadata"
MAILTO = "simon-pierre.boucher@uqo.ca"
CR_QUERIES = ["hedonic housing prices", "hedonic house price model", "hedonic property valuation residential", "residential property price determinants regression",
              "implicit prices housing attributes", "spatial hedonic housing", "house prices proximity hedonic", "apartment rent hedonic", "housing price machine learning valuation",
              "automated valuation model residential", "prix hédoniques logement", "precios hedónicos vivienda", "hedonische Preise Wohnungen", "preços hedônicos imóveis"]
AX_QUERIES = ["hedonic house price", "house price prediction machine learning", "real estate valuation deep learning", "property valuation computer vision",
              "housing price text listing", "automated valuation model", "hedonic regression housing"]


def get(url, tries=6):
    for i in range(tries):
        try:
            with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": f"wp21-uqo (mailto:{MAILTO})"}), timeout=120) as r:
                return r.read()
        except Exception as e:  # noqa
            err = e; time.sleep(10 * (i + 1))
    raise err


def crossref(q, out, log, max_rows=4000):
    cursor = "*"; n = 0; total = None
    while n < max_rows:
        url = (f"https://api.crossref.org/works?query.bibliographic={urllib.parse.quote(q)}&filter=from-pub-date:1970-01-01,until-pub-date:2026-10-20"
               f"&rows=500&cursor={urllib.parse.quote(cursor)}&select=DOI,title,author,issued,container-title,abstract,type,is-referenced-by-count,URL,subject&mailto={MAILTO}")
        d = json.loads(get(url))["message"]; total = d.get("total-results", total)
        items = d.get("items", [])
        if not items:
            break
        for it in items:
            it["_query"] = q; it["_source"] = "crossref"; it["_retrieved"] = str(date.today())
            out.write(json.dumps(it, ensure_ascii=False) + "\n"); n += 1
        cursor = d.get("next-cursor"); time.sleep(0.5)
        if not cursor:
            break
    log.writerow([str(date.today()), "crossref", "bibliographic", q, total, n]); print(f"crossref {q!r}: {total} total, {n} retrieved", flush=True)


def arxiv(q, out, log, max_rows=2000):
    n = 0; start = 0; total = None
    ns = {"a": "http://www.w3.org/2005/Atom", "o": "http://a9.com/-/spec/opensearch/1.1/"}
    while start < max_rows:
        url = f"http://export.arxiv.org/api/query?search_query=all:{urllib.parse.quote(q)}&start={start}&max_results=200"
        root = ET.fromstring(get(url)); total = int(root.find("o:totalResults", ns).text)
        entries = root.findall("a:entry", ns)
        if not entries:
            break
        for e in entries:
            rec = {"id": e.find("a:id", ns).text, "title": " ".join(e.find("a:title", ns).text.split()), "abstract": " ".join((e.find("a:summary", ns).text or "").split()),
                   "published": e.find("a:published", ns).text, "authors": [a.find("a:name", ns).text for a in e.findall("a:author", ns)],
                   "categories": [c.get("term") for c in e.findall("a:category", ns)], "doi": (e.find("{http://arxiv.org/schemas/atom}doi") is not None and e.find("{http://arxiv.org/schemas/atom}doi").text) or "",
                   "_query": q, "_source": "arxiv", "_retrieved": str(date.today())}
            out.write(json.dumps(rec, ensure_ascii=False) + "\n"); n += 1
        start += 200; time.sleep(3.5)
        if start >= total:
            break
    log.writerow([str(date.today()), "arxiv", "all", q, total, n]); print(f"arxiv {q!r}: {total} total, {n} retrieved", flush=True)


def main():
    with open(META / "crossref_records.jsonl", "w", encoding="utf-8") as out, open(META / "search_log.csv", "a", newline="") as lf:
        log = csv.writer(lf)
        for q in CR_QUERIES:
            try: crossref(q, out, log)
            except Exception as e: log.writerow([str(date.today()), "crossref", "bibliographic", q, "ERROR", repr(e)]); print("crossref failed", q, e)
    with open(META / "arxiv_records.jsonl", "w", encoding="utf-8") as out, open(META / "search_log.csv", "a", newline="") as lf:
        log = csv.writer(lf)
        for q in AX_QUERIES:
            try: arxiv(q, out, log)
            except Exception as e: log.writerow([str(date.today()), "arxiv", "all", q, "ERROR", repr(e)]); print("arxiv failed", q, e)


if __name__ == "__main__":
    main()