spb/fabri-ka Public

1#!/usr/bin/env python32"""Aggregate raw directory harvests into a deduplicated candidate-domain list.34Reads every data/raw/*.jsonl produced by harvest.py, extracts outbound5business websites, normalizes domains, merges provenance, and writes6data/enriched/candidates.jsonl (one record per unique domain).7"""8import glob9import json10import os11import re12from collections import defaultdict13from urllib.parse import urlparse1415ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))16RAW = os.path.join(ROOT, "data", "raw")17OUT = os.path.join(ROOT, "data", "enriched")18os.makedirs(OUT, exist_ok=True)1920# Domains that are never an independent Quebec store website21BLOCK = re.compile(r"""(?xi)22  (google\.|gouv\.qc|quebec\.ca$|canada\.ca$|wikipedia|facebook|instagram|linkedin|23   youtube|tiktok|twitter|x\.com$|pinterest|etsy\.com$|amazon\.|ebay\.|walmart|24   saq\.com|iga\.net|metro\.ca|sobeys|provigo|maxi\.ca|costco|canadiantire|25   eventbrite|mailchimp|list-manage|linktr\.ee|bit\.ly|goo\.gl|shopify\.com$|26   squarespace\.com$|wix\.com$|wordpress\.com$|godaddy|weebly\.com$|27   tourisme|bonjourquebec|alimentsduquebec|lesproduitsduquebec|metiersdart|28   vinsduquebec|cidreduquebec|fromagesdici|erabledici|artisansaloeuvre|29   marchespublicsduquebec|signelocal|ideecadeauquebec|acheterquebecois|30   distilleriesduquebec|ambq\.ca|terroiretsaveurs|upa\.qc|mapaq|31   lapresse|radio-canada|journaldemontreal|journaldequebec|tvanouvelles|32   lesaffaires|ledevoir|huffington|narcity|33   ticketmaster|opentable|doordash|ubereats|skipthedishes|34   yelp\.|tripadvisor|booking\.|airbnb|expedia|35   paypal|stripe|square\.site$|interac|36   vimeo|flickr|issuu|calameo|soundcloud|spotify|37   apple\.com|play\.google|microsoft|adobe\.|38   fondationdesgourmands|recettes?\.|allrecipes|ricardocuisine|39   zone\.coop$|desjardins|banquenationale|bmo\.|rbc\.)""")4041QC_HINT_TLD = (".quebec", ".qc.ca")424344def norm_domain(url):45    try:46        host = urlparse(url).netloc.lower()47    except Exception:48        return None49    host = host.split(":")[0].removeprefix("www.")50    if not host or "." not in host:51        return None52    return host535455def main():56    cands = {}57    per_source_counts = defaultdict(int)58    for path in sorted(glob.glob(os.path.join(RAW, "*.jsonl"))):59        source = os.path.basename(path)[:-6]60        with open(path) as f:61            for line in f:62                try:63                    rec = json.loads(line)64                except Exception:65                    continue66                if rec.get("error"):67                    continue68                # schéma "search sweep" : {name, domain, url, region_hint, category_hint, evidence, query}69                if "websites" not in rec and rec.get("domain"):70                    rec = {71                        "url": rec.get("query", ""),72                        "h1": rec.get("name", ""),73                        "title": rec.get("name", ""),74                        "websites": [rec.get("url") or f"https://{rec['domain']}"],75                        "socials": [],76                        "postal_prefix": None,77                        "phone": None,78                        "regions_mentioned": [rec["region_hint"]] if rec.get("region_hint") else [],79                        "text_sample": rec.get("evidence", ""),80                        "category_hint": rec.get("category_hint", ""),81                    }82                name = (rec.get("h1") or rec.get("title") or "").strip()83                for w in rec.get("websites", []):84                    dom = norm_domain(w)85                    if not dom or BLOCK.search(dom) or BLOCK.search(w):86                        continue87                    c = cands.setdefault(dom, {88                        "domain": dom,89                        "urls": [],90                        "names": [],91                        "sources": [],92                        "source_pages": [],93                        "socials": [],94                        "postal_prefix": None,95                        "phone": None,96                        "regions_mentioned": [],97                    })98                    if w not in c["urls"]:99                        c["urls"].append(w)100                    if name and name not in c["names"]:101                        c["names"].append(name)102                    if source not in c["sources"]:103                        c["sources"].append(source)104                        per_source_counts[source] += 1105                    if rec["url"] not in c["source_pages"]:106                        c["source_pages"].append(rec["url"])107                    for s in rec.get("socials", []):108                        if s not in c["socials"]:109                            c["socials"].append(s)110                    c["postal_prefix"] = c["postal_prefix"] or rec.get("postal_prefix")111                    c["phone"] = c["phone"] or rec.get("phone")112                    for r in rec.get("regions_mentioned", []):113                        if r not in c["regions_mentioned"]:114                            c["regions_mentioned"].append(r)115116    # signelocal vendors (no website yet -> candidates by name only, kept separately)117    sl_path = os.path.join(RAW, "signelocal_vendors.json")118    vendors = []119    if os.path.exists(sl_path):120        vendors = json.load(open(sl_path))121122    out_path = os.path.join(OUT, "candidates.jsonl")123    with open(out_path, "w") as f:124        for dom in sorted(cands):125            f.write(json.dumps(cands[dom], ensure_ascii=False) + "\n")126    print(f"unique candidate domains: {len(cands)}")127    print("contributions per source (unique domains added):")128    for s, n in sorted(per_source_counts.items(), key=lambda x: -x[1]):129        print(f"  {n:6d}  {s}")130    print(f"signelocal vendors (name-only candidates): {len(vendors)}")131132133if __name__ == "__main__":134    main()135