"""Build URL lists for list-based sources."""
import requests, re, time, json
HDRS={"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/126 Safari/537.36"}
def get(u):
    try: return requests.get(u,headers=HDRS,timeout=25).text
    except Exception as e: print("ERR",u,e); return ""

# --- metiersdart: expand section pages -> artisan fiche links
import sys; sys.path.insert(0,'scripts')
from harvest import sitemap_urls
secs=[u for u in sitemap_urls("https://www.metiersdart.ca/sitemap.xml") if "repertoire_artisan" in u]
fiches=set()
for s in secs:
    html=get(s)
    for m in re.findall(r'href="((?:https?://www\.metiersdart\.ca)?/?repertoire_artisan\.php\?[^"]*id=\d+[^"]*)"', html):
        u=m if m.startswith("http") else "https://www.metiersdart.ca/"+m.lstrip("/")
        fiches.add(u.replace("&","&"))
    time.sleep(0.1)
print("metiersdart fiches:",len(fiches))
open("data/raw/metiersdart_fiche_urls.txt","w").write("\n".join(sorted(fiches)))

# --- ideecadeauquebec: boutique pages from sitemap
icq=[u for u in sitemap_urls("https://www.ideecadeauquebec.com/sitemap.xml") if "/boutique/" in u]
print("icq boutiques:",len(icq))
open("data/raw/icq_urls.txt","w").write("\n".join(sorted(set(icq))))

# --- acheterquebecois: product-for-sale posts (each = a business listing)
aq=[u for u in sitemap_urls("https://acheterquebecois.ca/sitemap.xml") if "/produits-quebecois-a-vendre/" in u]
print("acheterquebecois:",len(aq))
open("data/raw/acheterquebecois_urls.txt","w").write("\n".join(sorted(set(aq))))

# --- distilleriesduquebec: fr pages that are distillery profiles
dd=[u for u in sitemap_urls("https://distilleriesduquebec.com/sitemap.xml") if "/fr/" in u]
print("distilleries fr pages:",len(dd))
open("data/raw/distilleries_urls.txt","w").write("\n".join(sorted(set(dd))))