#!/usr/bin/env python3
"""Generic directory harvester for the Quebec store discovery pipeline.

For each configured source: collect entity-page URLs (from sitemaps or listing
pages), fetch each entity page (with on-disk cache), and extract structured
signals: business name, outbound website links, social links, phone, postal
code, city/region mentions, categories.

Output: data/raw/<source>.jsonl  (one record per entity page)
"""
import concurrent.futures as cf
import hashlib
import json
import os
import re
import sys
import time
from urllib.parse import urlparse, urljoin

import requests
from bs4 import BeautifulSoup

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CACHE = os.path.join(ROOT, "data", "cache")
RAW = os.path.join(ROOT, "data", "raw")
os.makedirs(CACHE, exist_ok=True)
os.makedirs(RAW, exist_ok=True)

HDRS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
        "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}

SOCIAL_HOSTS = ("facebook.com", "instagram.com", "twitter.com", "x.com", "linkedin.com",
                "youtube.com", "tiktok.com", "pinterest.")
JUNK_HOSTS = ("google.", "goo.gl", "maps.app", "apple.com", "wp.me", "bit.ly", "mailchi",
              "eepurl", "linktr.ee", "youtu.be", "vimeo.com", "flickr.com", "issuu.com",
              "addtoany", "sharethis", "twitter.com", "cloudfront", "list-manage",
              "doubleclick", "shopify.com", "squarespace.com", "wordpress.org", "wix.com")

POSTAL_RE = re.compile(r"\b([GHJ]\d[A-Z])\s?\d[A-Z]\d\b")
PHONE_RE = re.compile(r"\(?\b(418|514|450|819|579|581|438|873|367|263|354)\)?[\s.\-]?\d{3}[\s.\-]?\d{4}\b")
REGIONS = ["Bas-Saint-Laurent", "Saguenay", "Lac-Saint-Jean", "Capitale-Nationale", "Mauricie",
           "Estrie", "Montréal", "Montreal", "Outaouais", "Abitibi", "Témiscamingue", "Côte-Nord",
           "Nord-du-Québec", "Gaspésie", "Îles-de-la-Madeleine", "Chaudière-Appalaches", "Laval",
           "Lanaudière", "Laurentides", "Montérégie", "Centre-du-Québec", "Cantons-de-l'Est"]


def cache_path(url):
    h = hashlib.sha1(url.encode()).hexdigest()
    return os.path.join(CACHE, h[:2], h + ".html")


def fetch(url, delay=0.0, timeout=25):
    p = cache_path(url)
    if os.path.exists(p):
        with open(p, encoding="utf-8", errors="replace") as f:
            return f.read()
    try:
        r = requests.get(url, headers=HDRS, timeout=timeout)
        if r.status_code != 200:
            return ""
        os.makedirs(os.path.dirname(p), exist_ok=True)
        with open(p, "w", encoding="utf-8") as f:
            f.write(r.text)
        if delay:
            time.sleep(delay)
        return r.text
    except Exception:
        return ""


def sitemap_urls(root, depth=0):
    xml = fetch(root)
    locs = re.findall(r"<loc>\s*(.*?)\s*</loc>", xml)
    if "<sitemapindex" in xml and depth < 2:
        out = []
        for l in locs:
            out += sitemap_urls(l, depth + 1)
        return out
    return locs


def extract_entity(url, html, source_host):
    soup = BeautifulSoup(html, "html.parser")
    title = soup.title.get_text(" ", strip=True) if soup.title else ""
    h1 = soup.h1.get_text(" ", strip=True) if soup.h1 else ""
    text = soup.get_text(" ", strip=True)[:20000]

    websites, socials = [], []
    for a in soup.find_all("a", href=True):
        href = a["href"].strip()
        if href.startswith("mailto:") or href.startswith("tel:") or href.startswith("#"):
            continue
        absu = urljoin(url, href)
        host = urlparse(absu).netloc.lower().replace("www.", "")
        if not host or source_host in host:
            continue
        if any(s in host for s in SOCIAL_HOSTS):
            socials.append(absu)
        elif not any(j in host for j in JUNK_HOSTS):
            websites.append(absu)

    postal = POSTAL_RE.search(text)
    phone = PHONE_RE.search(text)
    regions = sorted({r for r in REGIONS if r.lower() in text.lower()})

    return {
        "url": url,
        "title": title,
        "h1": h1,
        "websites": list(dict.fromkeys(websites))[:15],
        "socials": list(dict.fromkeys(socials))[:8],
        "postal_prefix": postal.group(1) if postal else None,
        "phone": phone.group(0) if phone else None,
        "regions_mentioned": regions,
        "text_sample": text[:1200],
    }


def harvest(source, urls, workers=8, delay=0.05):
    host = urlparse(urls[0]).netloc.lower().replace("www.", "") if urls else ""
    out_path = os.path.join(RAW, f"{source}.jsonl")
    done = set()
    if os.path.exists(out_path):
        with open(out_path) as f:
            for line in f:
                try:
                    done.add(json.loads(line)["url"])
                except Exception:
                    pass
    todo = [u for u in urls if u not in done]
    print(f"[{source}] {len(urls)} urls, {len(done)} cached, {len(todo)} to fetch")
    with open(out_path, "a", encoding="utf-8") as out:
        def work(u):
            html = fetch(u, delay=delay)
            if not html:
                return None
            try:
                return extract_entity(u, html, host)
            except Exception as e:
                return {"url": u, "error": str(e)[:200]}
        with cf.ThreadPoolExecutor(workers) as ex:
            n = 0
            for rec in ex.map(work, todo):
                if rec:
                    out.write(json.dumps(rec, ensure_ascii=False) + "\n")
                n += 1
                if n % 100 == 0:
                    print(f"  [{source}] {n}/{len(todo)}", flush=True)
    print(f"[{source}] done")


SOURCES = {
    # source: (sitemap_root, entity url regex)
    "alimentsduquebec_ent": ("https://alimentsduquebec.com/sitemaps-1-sitemap.xml",
                             r"alimentsduquebec\.com/entreprises/[a-z0-9\-]+$"),
    "lesproduitsduquebec_ent": ("https://lesproduitsduquebec.com/sitemap.xml",
                                r"lesproduitsduquebec\.com/entreprises-adherentes/[^/]+$"),
    "lesproduitsduquebec_det": ("https://lesproduitsduquebec.com/sitemap.xml",
                                r"lesproduitsduquebec\.com/detaillants/[^/]+$"),
    "metiersdart": ("https://www.metiersdart.ca/sitemap.xml",
                    r"metiersdart\.ca/repertoire_artisan\.php/.+"),
    "vinsduquebec": ("https://vinsduquebec.com/sitemap.xml",
                     r"vinsduquebec\.com/adresses/[^/]+/?$"),
    "cidreduquebec": ("https://cidreduquebec.com/wp-sitemap.xml",
                      r"cidreduquebec\.com/(cidreries|producteur-adherent)/[^/]+/?$"),
    "fromagesdici": ("https://www.fromagesdici.com/sitemap.xml",
                     r"fromagesdici\.com/fr/fromageries/[^/]+"),
    "erabledici": ("https://www.erabledici.ca/sitemap_index.xml",
                   r"erabledici\.ca/fr/erabliere/[^/]+"),
    "artisansaloeuvre": ("https://artisansaloeuvre.com/sitemap_index.xml",
                         r"artisansaloeuvre\.com/artisans/[^/]+/?$"),
    "marchespublics": ("https://www.marchespublicsduquebec.ca/sitemap.xml",
                       r"marchespublicsduquebec\.ca/marches-publics/[^/]+/?$"),
    "zoneboreale": ("https://zoneboreale.com/sitemap.xml",
                    r"zoneboreale\.com/entreprises/[^/]+/?$"),
    "createursdesaveurs": ("https://createursdesaveurs.com/sitemap.xml",
                           r"createursdesaveurs\.com/createurs/[^/]+/?$"),
    "croquezoutaouais": ("https://www.croquezoutaouais.com/sitemap_index.xml",
                         r"croquezoutaouais\.com/entreprises/[^/]+/?$"),
    "arretsgourmands": ("https://arretsgourmands.ca/sitemap.xml",
                        r"arretsgourmands\.ca/arrets-gourmands/[^/]+/?$"),
    "goutezy": ("https://goutezy.com/sitemap_index.xml",
                r"goutezy\.com/entreprise/[^/]+/?$"),
    "laurentidesjenmange": ("https://laurentidesjenmange.ca/sitemap_index.xml",
                            r"laurentidesjenmange\.ca/membre/[^/]+/?$"),
    "goutcotenord": ("https://legoutdelacotenord.ca/sitemap_index.xml",
                     r"legoutdelacotenord\.ca/artisan/[^/]+/?$"),
    "charlevoix": ("https://www.tourisme-charlevoix.com/sitemap.xml",
                   r"tourisme-charlevoix\.com/fr/entreprises/[^/]+/?$"),
    "gaspesiegourmande": ("https://gaspesiegourmande.com/sitemap.xml",
                          r"gaspesiegourmande\.com/(producteurs-transformateurs|complices)\?id=\d+"),
    "saveursbsl": ("https://saveursbsl.com/wp-sitemap.xml",
                   r"saveursbsl\.com/product/[^/]+/?$"),
    "goutezat": ("https://goutezat.com/sitemap_index.xml",
                 r"goutezat\.com/produit/[^/]+/?$"),
}


# sources whose entity URLs come from a pre-built list file in data/raw/
LIST_SOURCES = {
    "alimentsduquebec_ent": "data/raw/adq_enterprise_urls.txt",
    "metiersdart_fiches": "data/raw/metiersdart_fiche_urls.txt",
    "ideecadeauquebec": "data/raw/icq_urls.txt",
    "acheterquebecois": "data/raw/acheterquebecois_urls.txt",
    "distilleriesduquebec": "data/raw/distilleries_urls.txt",
    "ambq_profils": "data/raw/ambq_profile_urls.txt",
}


if __name__ == "__main__":
    wanted = sys.argv[1:] or list(SOURCES)
    for src in wanted:
        if src in LIST_SOURCES:
            with open(os.path.join(ROOT, LIST_SOURCES[src])) as f:
                urls = [l.strip() for l in f if l.strip().startswith("http")]
        else:
            sm, pat = SOURCES[src]
            urls = [u for u in sitemap_urls(sm) if re.search(pat, u)]
        urls = list(dict.fromkeys(urls))
        harvest(src, urls)