spb/fabri-ka Public

1#!/usr/bin/env python32"""Generic directory harvester for the Quebec store discovery pipeline.34For each configured source: collect entity-page URLs (from sitemaps or listing5pages), fetch each entity page (with on-disk cache), and extract structured6signals: business name, outbound website links, social links, phone, postal7code, city/region mentions, categories.89Output: data/raw/<source>.jsonl  (one record per entity page)10"""11import concurrent.futures as cf12import hashlib13import json14import os15import re16import sys17import time18from urllib.parse import urlparse, urljoin1920import requests21from bs4 import BeautifulSoup2223ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))24CACHE = os.path.join(ROOT, "data", "cache")25RAW = os.path.join(ROOT, "data", "raw")26os.makedirs(CACHE, exist_ok=True)27os.makedirs(RAW, exist_ok=True)2829HDRS = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",30        "Accept-Language": "fr-CA,fr;q=0.9,en;q=0.8"}3132SOCIAL_HOSTS = ("facebook.com", "instagram.com", "twitter.com", "x.com", "linkedin.com",33                "youtube.com", "tiktok.com", "pinterest.")34JUNK_HOSTS = ("google.", "goo.gl", "maps.app", "apple.com", "wp.me", "bit.ly", "mailchi",35              "eepurl", "linktr.ee", "youtu.be", "vimeo.com", "flickr.com", "issuu.com",36              "addtoany", "sharethis", "twitter.com", "cloudfront", "list-manage",37              "doubleclick", "shopify.com", "squarespace.com", "wordpress.org", "wix.com")3839POSTAL_RE = re.compile(r"\b([GHJ]\d[A-Z])\s?\d[A-Z]\d\b")40PHONE_RE = re.compile(r"\(?\b(418|514|450|819|579|581|438|873|367|263|354)\)?[\s.\-]?\d{3}[\s.\-]?\d{4}\b")41REGIONS = ["Bas-Saint-Laurent", "Saguenay", "Lac-Saint-Jean", "Capitale-Nationale", "Mauricie",42           "Estrie", "Montréal", "Montreal", "Outaouais", "Abitibi", "Témiscamingue", "Côte-Nord",43           "Nord-du-Québec", "Gaspésie", "Îles-de-la-Madeleine", "Chaudière-Appalaches", "Laval",44           "Lanaudière", "Laurentides", "Montérégie", "Centre-du-Québec", "Cantons-de-l'Est"]454647def cache_path(url):48    h = hashlib.sha1(url.encode()).hexdigest()49    return os.path.join(CACHE, h[:2], h + ".html")505152def fetch(url, delay=0.0, timeout=25):53    p = cache_path(url)54    if os.path.exists(p):55        with open(p, encoding="utf-8", errors="replace") as f:56            return f.read()57    try:58        r = requests.get(url, headers=HDRS, timeout=timeout)59        if r.status_code != 200:60            return ""61        os.makedirs(os.path.dirname(p), exist_ok=True)62        with open(p, "w", encoding="utf-8") as f:63            f.write(r.text)64        if delay:65            time.sleep(delay)66        return r.text67    except Exception:68        return ""697071def sitemap_urls(root, depth=0):72    xml = fetch(root)73    locs = re.findall(r"<loc>\s*(.*?)\s*</loc>", xml)74    if "<sitemapindex" in xml and depth < 2:75        out = []76        for l in locs:77            out += sitemap_urls(l, depth + 1)78        return out79    return locs808182def extract_entity(url, html, source_host):83    soup = BeautifulSoup(html, "html.parser")84    title = soup.title.get_text(" ", strip=True) if soup.title else ""85    h1 = soup.h1.get_text(" ", strip=True) if soup.h1 else ""86    text = soup.get_text(" ", strip=True)[:20000]8788    websites, socials = [], []89    for a in soup.find_all("a", href=True):90        href = a["href"].strip()91        if href.startswith("mailto:") or href.startswith("tel:") or href.startswith("#"):92            continue93        absu = urljoin(url, href)94        host = urlparse(absu).netloc.lower().replace("www.", "")95        if not host or source_host in host:96            continue97        if any(s in host for s in SOCIAL_HOSTS):98            socials.append(absu)99        elif not any(j in host for j in JUNK_HOSTS):100            websites.append(absu)101102    postal = POSTAL_RE.search(text)103    phone = PHONE_RE.search(text)104    regions = sorted({r for r in REGIONS if r.lower() in text.lower()})105106    return {107        "url": url,108        "title": title,109        "h1": h1,110        "websites": list(dict.fromkeys(websites))[:15],111        "socials": list(dict.fromkeys(socials))[:8],112        "postal_prefix": postal.group(1) if postal else None,113        "phone": phone.group(0) if phone else None,114        "regions_mentioned": regions,115        "text_sample": text[:1200],116    }117118119def harvest(source, urls, workers=8, delay=0.05):120    host = urlparse(urls[0]).netloc.lower().replace("www.", "") if urls else ""121    out_path = os.path.join(RAW, f"{source}.jsonl")122    done = set()123    if os.path.exists(out_path):124        with open(out_path) as f:125            for line in f:126                try:127                    done.add(json.loads(line)["url"])128                except Exception:129                    pass130    todo = [u for u in urls if u not in done]131    print(f"[{source}] {len(urls)} urls, {len(done)} cached, {len(todo)} to fetch")132    with open(out_path, "a", encoding="utf-8") as out:133        def work(u):134            html = fetch(u, delay=delay)135            if not html:136                return None137            try:138                return extract_entity(u, html, host)139            except Exception as e:140                return {"url": u, "error": str(e)[:200]}141        with cf.ThreadPoolExecutor(workers) as ex:142            n = 0143            for rec in ex.map(work, todo):144                if rec:145                    out.write(json.dumps(rec, ensure_ascii=False) + "\n")146                n += 1147                if n % 100 == 0:148                    print(f"  [{source}] {n}/{len(todo)}", flush=True)149    print(f"[{source}] done")150151152SOURCES = {153    # source: (sitemap_root, entity url regex)154    "alimentsduquebec_ent": ("https://alimentsduquebec.com/sitemaps-1-sitemap.xml",155                             r"alimentsduquebec\.com/entreprises/[a-z0-9\-]+$"),156    "lesproduitsduquebec_ent": ("https://lesproduitsduquebec.com/sitemap.xml",157                                r"lesproduitsduquebec\.com/entreprises-adherentes/[^/]+$"),158    "lesproduitsduquebec_det": ("https://lesproduitsduquebec.com/sitemap.xml",159                                r"lesproduitsduquebec\.com/detaillants/[^/]+$"),160    "metiersdart": ("https://www.metiersdart.ca/sitemap.xml",161                    r"metiersdart\.ca/repertoire_artisan\.php/.+"),162    "vinsduquebec": ("https://vinsduquebec.com/sitemap.xml",163                     r"vinsduquebec\.com/adresses/[^/]+/?$"),164    "cidreduquebec": ("https://cidreduquebec.com/wp-sitemap.xml",165                      r"cidreduquebec\.com/(cidreries|producteur-adherent)/[^/]+/?$"),166    "fromagesdici": ("https://www.fromagesdici.com/sitemap.xml",167                     r"fromagesdici\.com/fr/fromageries/[^/]+"),168    "erabledici": ("https://www.erabledici.ca/sitemap_index.xml",169                   r"erabledici\.ca/fr/erabliere/[^/]+"),170    "artisansaloeuvre": ("https://artisansaloeuvre.com/sitemap_index.xml",171                         r"artisansaloeuvre\.com/artisans/[^/]+/?$"),172    "marchespublics": ("https://www.marchespublicsduquebec.ca/sitemap.xml",173                       r"marchespublicsduquebec\.ca/marches-publics/[^/]+/?$"),174    "zoneboreale": ("https://zoneboreale.com/sitemap.xml",175                    r"zoneboreale\.com/entreprises/[^/]+/?$"),176    "createursdesaveurs": ("https://createursdesaveurs.com/sitemap.xml",177                           r"createursdesaveurs\.com/createurs/[^/]+/?$"),178    "croquezoutaouais": ("https://www.croquezoutaouais.com/sitemap_index.xml",179                         r"croquezoutaouais\.com/entreprises/[^/]+/?$"),180    "arretsgourmands": ("https://arretsgourmands.ca/sitemap.xml",181                        r"arretsgourmands\.ca/arrets-gourmands/[^/]+/?$"),182    "goutezy": ("https://goutezy.com/sitemap_index.xml",183                r"goutezy\.com/entreprise/[^/]+/?$"),184    "laurentidesjenmange": ("https://laurentidesjenmange.ca/sitemap_index.xml",185                            r"laurentidesjenmange\.ca/membre/[^/]+/?$"),186    "goutcotenord": ("https://legoutdelacotenord.ca/sitemap_index.xml",187                     r"legoutdelacotenord\.ca/artisan/[^/]+/?$"),188    "charlevoix": ("https://www.tourisme-charlevoix.com/sitemap.xml",189                   r"tourisme-charlevoix\.com/fr/entreprises/[^/]+/?$"),190    "gaspesiegourmande": ("https://gaspesiegourmande.com/sitemap.xml",191                          r"gaspesiegourmande\.com/(producteurs-transformateurs|complices)\?id=\d+"),192    "saveursbsl": ("https://saveursbsl.com/wp-sitemap.xml",193                   r"saveursbsl\.com/product/[^/]+/?$"),194    "goutezat": ("https://goutezat.com/sitemap_index.xml",195                 r"goutezat\.com/produit/[^/]+/?$"),196}197198199# sources whose entity URLs come from a pre-built list file in data/raw/200LIST_SOURCES = {201    "alimentsduquebec_ent": "data/raw/adq_enterprise_urls.txt",202    "metiersdart_fiches": "data/raw/metiersdart_fiche_urls.txt",203    "ideecadeauquebec": "data/raw/icq_urls.txt",204    "acheterquebecois": "data/raw/acheterquebecois_urls.txt",205    "distilleriesduquebec": "data/raw/distilleries_urls.txt",206    "ambq_profils": "data/raw/ambq_profile_urls.txt",207}208209210if __name__ == "__main__":211    wanted = sys.argv[1:] or list(SOURCES)212    for src in wanted:213        if src in LIST_SOURCES:214            with open(os.path.join(ROOT, LIST_SOURCES[src])) as f:215                urls = [l.strip() for l in f if l.strip().startswith("http")]216        else:217            sm, pat = SOURCES[src]218            urls = [u for u in sitemap_urls(sm) if re.search(pat, u)]219        urls = list(dict.fromkeys(urls))220        harvest(src, urls)221