spb/fabri-ka Public

1"""Build URL lists for list-based sources."""2import requests, re, time, json3HDRS={"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/126 Safari/537.36"}4def get(u):5    try: return requests.get(u,headers=HDRS,timeout=25).text6    except Exception as e: print("ERR",u,e); return ""78# --- metiersdart: expand section pages -> artisan fiche links9import sys; sys.path.insert(0,'scripts')10from harvest import sitemap_urls11secs=[u for u in sitemap_urls("https://www.metiersdart.ca/sitemap.xml") if "repertoire_artisan" in u]12fiches=set()13for s in secs:14    html=get(s)15    for m in re.findall(r'href="((?:https?://www\.metiersdart\.ca)?/?repertoire_artisan\.php\?[^"]*id=\d+[^"]*)"', html):16        u=m if m.startswith("http") else "https://www.metiersdart.ca/"+m.lstrip("/")17        fiches.add(u.replace("&","&"))18    time.sleep(0.1)19print("metiersdart fiches:",len(fiches))20open("data/raw/metiersdart_fiche_urls.txt","w").write("\n".join(sorted(fiches)))2122# --- ideecadeauquebec: boutique pages from sitemap23icq=[u for u in sitemap_urls("https://www.ideecadeauquebec.com/sitemap.xml") if "/boutique/" in u]24print("icq boutiques:",len(icq))25open("data/raw/icq_urls.txt","w").write("\n".join(sorted(set(icq))))2627# --- acheterquebecois: product-for-sale posts (each = a business listing)28aq=[u for u in sitemap_urls("https://acheterquebecois.ca/sitemap.xml") if "/produits-quebecois-a-vendre/" in u]29print("acheterquebecois:",len(aq))30open("data/raw/acheterquebecois_urls.txt","w").write("\n".join(sorted(set(aq))))3132# --- distilleriesduquebec: fr pages that are distillery profiles33dd=[u for u in sitemap_urls("https://distilleriesduquebec.com/sitemap.xml") if "/fr/" in u]34print("distilleries fr pages:",len(dd))35open("data/raw/distilleries_urls.txt","w").write("\n".join(sorted(set(dd))))36