import requests, concurrent.futures as cf, re
HDRS={"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36"}
SITES = ["alimentsduquebec.com","lesproduitsduquebec.com","www.signelocal.com","www.metiersdart.ca",
"vinsduquebec.com","cidreduquebec.com","ambq.ca","www.fromagesdici.com","www.erabledici.ca",
"terroiretsaveurs.com","www.marchespublicsduquebec.ca","mazonequebec.com","artisansaloeuvre.com"]
def get(u):
    try:
        r=requests.get(u,headers=HDRS,timeout=15)
        return r.status_code, r.text[:3000]
    except Exception as e: return "ERR", str(e)[:100]
for s in SITES:
    print("="*70); print(s)
    sc,txt = get(f"https://{s}/robots.txt")
    print("robots:",sc)
    if sc==200:
        for line in txt.splitlines():
            if "itemap" in line or "isallow" in line.lower()[:12]: print("  ",line.strip())
    sc,txt = get(f"https://{s}/sitemap.xml")
    print("sitemap.xml:",sc, txt[:400].replace("\n"," ") if sc==200 else "")