spb/fabri-ka Public

1#!/usr/bin/env python32"""Verification pipeline: probe each candidate domain.34For each domain from data/enriched/candidates.jsonl:5  - fetch homepage (https, with UA), record status + final URL6  - detect ecommerce platform + whether its public catalog endpoint responds7    (Shopify /products.json, WooCommerce /wp-json/wc/store/v1/products,8     Squarespace ?format=json)9  - detect cart/checkout signals10  - detect Quebec-location signals (postal codes GHJ, area codes, mentions)11  - detect language, social links, "made in Quebec" wording12Results cached per-domain in data/verify_cache/. Output:13data/enriched/verified.jsonl14"""15import concurrent.futures as cf16import json17import os18import re19import sys20import time21from urllib.parse import urlparse2223import requests2425ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))26CACHE = os.path.join(ROOT, "data", "verify_cache")27OUT = os.path.join(ROOT, "data", "enriched")28os.makedirs(CACHE, exist_ok=True)2930HDRS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 FabriKaBot/1.0 (+contact@spboucher.ai)",31        "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}3233POSTAL_RE = re.compile(r"\b[GHJ]\d[A-Z]\s?\d[A-Z]\d\b")34AREA_RE = re.compile(r"\(?\b(418|514|450|819|579|581|438|873|367)\)?[\s.\-]?\d{3}[\s.\-]?\d{4}\b")35MADE_RE = re.compile(r"(?i)(fabriqu[ée]s?\s+(au|ici\s+au)\s+qu[ée]bec|fait(es)?\s+(au|ici\s+au)\s+qu[ée]bec|"36                     r"made\s+in\s+qu[ée]bec|con[çc]u\s+(et\s+fabriqu[ée]?\s+)?au\s+qu[ée]bec|"37                     r"produit\s+du\s+qu[ée]bec|fait\s+[àa]\s+la\s+main\s+au\s+qu[ée]bec|"38                     r"artisanal|fait\s+ici|100\s?%\s?qu[ée]b[ée]cois)")39QC_WORD_RE = re.compile(r"(?i)\bqu[ée]bec\b|\bqc\b")40CART_RE = re.compile(r"(?i)(add[\s_\-]?to[\s_\-]?cart|ajouter\s+au\s+panier|/cart\b|/panier\b|"41                     r"checkout|caisse|mon\s+panier|shopping[\s_\-]?cart|data-product-form)")4243SOCIAL_RE = re.compile(r'https?://(?:www\.)?(facebook\.com|instagram\.com)/[A-Za-z0-9_.\-/%]+')444546def detect_platform(html, headers):47    h = html.lower()48    hdr = " ".join(f"{k}:{v}" for k, v in headers.items()).lower()49    if "cdn.shopify" in h or "shopify.theme" in h or "x-shopid" in hdr or "myshopify.com" in h:50        return "shopify"51    if "woocommerce" in h:52        return "woocommerce"53    if "wixstatic.com" in h or "wix.com" in h and "wixsite" in h:54        return "wix"55    if "squarespace" in h:56        return "squarespace"57    if "prestashop" in h:58        return "prestashop"59    if "bigcommerce" in h:60        return "bigcommerce"61    if "webshopapp" in h or "lightspeed" in h and "cart" in h:62        return "lightspeed"63    if "ecwid" in h:64        return "ecwid"65    if "snipcart" in h:66        return "snipcart"67    if "sumup" in h and "store" in h:68        return "sumup"69    if "square.site" in h or "squareup.com" in h and "cart" in h:70        return "square"71    if "magento" in h or "mage/cookies" in h:72        return "magento"73    if "wp-content" in h:74        return "wordpress"75    return ""767778def probe_catalog(domain, platform, sess):79    """Check whether the public catalog endpoint responds; return (endpoint, count_hint)."""80    try:81        if platform == "shopify":82            r = sess.get(f"https://{domain}/products.json?limit=1", headers=HDRS, timeout=15)83            if r.status_code == 200 and "products" in r.text[:200]:84                return "/products.json", None85        elif platform in ("woocommerce", "wordpress"):86            r = sess.get(f"https://{domain}/wp-json/wc/store/v1/products?per_page=1", headers=HDRS, timeout=15)87            if r.status_code == 200 and r.text.strip().startswith("["):88                total = r.headers.get("X-WP-Total")89                return "/wp-json/wc/store/v1/products", int(total) if total else None90        elif platform == "squarespace":91            for path in ("/shop", "/boutique", "/store"):92                r = sess.get(f"https://{domain}{path}?format=json-pretty", headers=HDRS, timeout=15)93                if r.status_code == 200 and '"items"' in r.text[:5000]:94                    return path + "?format=json", None95    except Exception:96        pass97    return None, None9899100def verify_domain(domain):101    cpath = os.path.join(CACHE, domain + ".json")102    if os.path.exists(cpath):103        return json.load(open(cpath))104    rec = {"domain": domain, "checked_at": time.strftime("%Y-%m-%d")}105    sess = requests.Session()106    html, final_url, status = "", "", None107    for scheme_host in (f"https://{domain}", f"https://www.{domain}", f"http://{domain}"):108        try:109            r = sess.get(scheme_host, headers=HDRS, timeout=20, allow_redirects=True)110            status, final_url, html = r.status_code, r.url, r.text111            resp_headers = dict(r.headers)112            if status == 200:113                break114        except Exception as e:115            rec.setdefault("errors", []).append(str(e)[:120])116    rec["status"] = status117    rec["final_url"] = final_url118    if not html or status != 200:119        rec["active"] = False120        json.dump(rec, open(cpath, "w"))121        return rec122123    rec["active"] = True124    final_dom = urlparse(final_url).netloc.lower().removeprefix("www.")125    rec["final_domain"] = final_dom126    text = html[:400000]127    title_m = re.search(r"<title[^>]*>(.*?)</title>", text, re.S | re.I)128    rec["title"] = re.sub(r"\s+", " ", title_m.group(1)).strip()[:200] if title_m else ""129    platform = detect_platform(text, resp_headers)130    rec["platform"] = platform131    endpoint, count = probe_catalog(final_dom, platform, sess)132    rec["catalog_endpoint"] = endpoint133    rec["catalog_count_hint"] = count134    rec["has_cart"] = bool(CART_RE.search(text))135    rec["made_in_qc_wording"] = bool(MADE_RE.search(text))136    postal = POSTAL_RE.search(text)137    rec["qc_postal"] = postal.group(0) if postal else None138    area = AREA_RE.search(text)139    rec["qc_phone"] = area.group(0) if area else None140    rec["mentions_quebec"] = bool(QC_WORD_RE.search(text))141    rec["tld_quebec"] = final_dom.endswith(".quebec") or final_dom.endswith(".qc.ca")142    rec["socials"] = list(dict.fromkeys(SOCIAL_RE.findall(text)))[:4]143    fr_hits = len(re.findall(r"(?i)\b(nous|votre|panier|boutique|livraison|accueil|produits)\b", text))144    en_hits = len(re.findall(r"(?i)\b(shop|cart|shipping|home|about us|products)\b", text))145    rec["language"] = ("fr" if fr_hits >= en_hits else "en") if (fr_hits + en_hits) > 3 else None146    json.dump(rec, open(cpath, "w"))147    return rec148149150def main():151    src = sys.argv[1] if len(sys.argv) > 1 else os.path.join(OUT, "candidates.jsonl")152    domains = []153    with open(src) as f:154        for line in f:155            domains.append(json.loads(line)["domain"])156    domains = list(dict.fromkeys(domains))157    print(f"verifying {len(domains)} domains")158    results = []159    with cf.ThreadPoolExecutor(24) as ex:160        for i, rec in enumerate(ex.map(verify_domain, domains)):161            results.append(rec)162            if (i + 1) % 100 == 0:163                print(f"  {i+1}/{len(domains)}", flush=True)164    out = os.path.join(OUT, "verified.jsonl")165    with open(out, "w") as f:166        for r in results:167            f.write(json.dumps(r, ensure_ascii=False) + "\n")168    active = sum(1 for r in results if r.get("active"))169    plat = {}170    for r in results:171        if r.get("active"):172            plat[r.get("platform") or "unknown"] = plat.get(r.get("platform") or "unknown", 0) + 1173    connectable = sum(1 for r in results if r.get("catalog_endpoint"))174    print(f"active: {active}/{len(results)}  connectable-catalog: {connectable}")175    print("platforms:", json.dumps(plat, indent=1))176177178if __name__ == "__main__":179    main()180