#!/usr/bin/env python3
"""Aggregate raw directory harvests into a deduplicated candidate-domain list.

Reads every data/raw/*.jsonl produced by harvest.py, extracts outbound
business websites, normalizes domains, merges provenance, and writes
data/enriched/candidates.jsonl (one record per unique domain).
"""
import glob
import json
import os
import re
from collections import defaultdict
from urllib.parse import urlparse

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RAW = os.path.join(ROOT, "data", "raw")
OUT = os.path.join(ROOT, "data", "enriched")
os.makedirs(OUT, exist_ok=True)

# Domains that are never an independent Quebec store website
BLOCK = re.compile(r"""(?xi)
  (google\.|gouv\.qc|quebec\.ca$|canada\.ca$|wikipedia|facebook|instagram|linkedin|
   youtube|tiktok|twitter|x\.com$|pinterest|etsy\.com$|amazon\.|ebay\.|walmart|
   saq\.com|iga\.net|metro\.ca|sobeys|provigo|maxi\.ca|costco|canadiantire|
   eventbrite|mailchimp|list-manage|linktr\.ee|bit\.ly|goo\.gl|shopify\.com$|
   squarespace\.com$|wix\.com$|wordpress\.com$|godaddy|weebly\.com$|
   tourisme|bonjourquebec|alimentsduquebec|lesproduitsduquebec|metiersdart|
   vinsduquebec|cidreduquebec|fromagesdici|erabledici|artisansaloeuvre|
   marchespublicsduquebec|signelocal|ideecadeauquebec|acheterquebecois|
   distilleriesduquebec|ambq\.ca|terroiretsaveurs|upa\.qc|mapaq|
   lapresse|radio-canada|journaldemontreal|journaldequebec|tvanouvelles|
   lesaffaires|ledevoir|huffington|narcity|
   ticketmaster|opentable|doordash|ubereats|skipthedishes|
   yelp\.|tripadvisor|booking\.|airbnb|expedia|
   paypal|stripe|square\.site$|interac|
   vimeo|flickr|issuu|calameo|soundcloud|spotify|
   apple\.com|play\.google|microsoft|adobe\.|
   fondationdesgourmands|recettes?\.|allrecipes|ricardocuisine|
   zone\.coop$|desjardins|banquenationale|bmo\.|rbc\.)""")

QC_HINT_TLD = (".quebec", ".qc.ca")


def norm_domain(url):
    try:
        host = urlparse(url).netloc.lower()
    except Exception:
        return None
    host = host.split(":")[0].removeprefix("www.")
    if not host or "." not in host:
        return None
    return host


def main():
    cands = {}
    per_source_counts = defaultdict(int)
    for path in sorted(glob.glob(os.path.join(RAW, "*.jsonl"))):
        source = os.path.basename(path)[:-6]
        with open(path) as f:
            for line in f:
                try:
                    rec = json.loads(line)
                except Exception:
                    continue
                if rec.get("error"):
                    continue
                # schéma "search sweep" : {name, domain, url, region_hint, category_hint, evidence, query}
                if "websites" not in rec and rec.get("domain"):
                    rec = {
                        "url": rec.get("query", ""),
                        "h1": rec.get("name", ""),
                        "title": rec.get("name", ""),
                        "websites": [rec.get("url") or f"https://{rec['domain']}"],
                        "socials": [],
                        "postal_prefix": None,
                        "phone": None,
                        "regions_mentioned": [rec["region_hint"]] if rec.get("region_hint") else [],
                        "text_sample": rec.get("evidence", ""),
                        "category_hint": rec.get("category_hint", ""),
                    }
                name = (rec.get("h1") or rec.get("title") or "").strip()
                for w in rec.get("websites", []):
                    dom = norm_domain(w)
                    if not dom or BLOCK.search(dom) or BLOCK.search(w):
                        continue
                    c = cands.setdefault(dom, {
                        "domain": dom,
                        "urls": [],
                        "names": [],
                        "sources": [],
                        "source_pages": [],
                        "socials": [],
                        "postal_prefix": None,
                        "phone": None,
                        "regions_mentioned": [],
                    })
                    if w not in c["urls"]:
                        c["urls"].append(w)
                    if name and name not in c["names"]:
                        c["names"].append(name)
                    if source not in c["sources"]:
                        c["sources"].append(source)
                        per_source_counts[source] += 1
                    if rec["url"] not in c["source_pages"]:
                        c["source_pages"].append(rec["url"])
                    for s in rec.get("socials", []):
                        if s not in c["socials"]:
                            c["socials"].append(s)
                    c["postal_prefix"] = c["postal_prefix"] or rec.get("postal_prefix")
                    c["phone"] = c["phone"] or rec.get("phone")
                    for r in rec.get("regions_mentioned", []):
                        if r not in c["regions_mentioned"]:
                            c["regions_mentioned"].append(r)

    # signelocal vendors (no website yet -> candidates by name only, kept separately)
    sl_path = os.path.join(RAW, "signelocal_vendors.json")
    vendors = []
    if os.path.exists(sl_path):
        vendors = json.load(open(sl_path))

    out_path = os.path.join(OUT, "candidates.jsonl")
    with open(out_path, "w") as f:
        for dom in sorted(cands):
            f.write(json.dumps(cands[dom], ensure_ascii=False) + "\n")
    print(f"unique candidate domains: {len(cands)}")
    print("contributions per source (unique domains added):")
    for s, n in sorted(per_source_counts.items(), key=lambda x: -x[1]):
        print(f"  {n:6d}  {s}")
    print(f"signelocal vendors (name-only candidates): {len(vendors)}")


if __name__ == "__main__":
    main()