spb/fabri-ka Public

1import requests, concurrent.futures as cf, re2HDRS={"User-Agent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126 Safari/537.36"}3SITES = ["alimentsduquebec.com","lesproduitsduquebec.com","www.signelocal.com","www.metiersdart.ca",4"vinsduquebec.com","cidreduquebec.com","ambq.ca","www.fromagesdici.com","www.erabledici.ca",5"terroiretsaveurs.com","www.marchespublicsduquebec.ca","mazonequebec.com","artisansaloeuvre.com"]6def get(u):7    try:8        r=requests.get(u,headers=HDRS,timeout=15)9        return r.status_code, r.text[:3000]10    except Exception as e: return "ERR", str(e)[:100]11for s in SITES:12    print("="*70); print(s)13    sc,txt = get(f"https://{s}/robots.txt")14    print("robots:",sc)15    if sc==200:16        for line in txt.splitlines():17            if "itemap" in line or "isallow" in line.lower()[:12]: print("  ",line.strip())18    sc,txt = get(f"https://{s}/sitemap.xml")19    print("sitemap.xml:",sc, txt[:400].replace("\n"," ") if sc==200 else "")20