spb/fabri-ka

1# -----------------------------------------------------------------------------2# Fabri-Ka — Agrégateur de produits québécois3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/generic.py : connecteur universel par sitemap + extraction du5#   balisage produit rendu côté serveur (JSON-LD schema.org Product, microdata,6#   Open Graph product, blobs PrestaShop/Magento). Couvre PrestaShop, Magento,7#   BigCommerce, WordPress non-Woo et sites ecommerce sur mesure. Scrapfly en8#   secours pour l'anti-bot. Rendu client-only (Ecwid) non couvert ; Square9#   Online est couvert par connectors/square.py (vague 2).10# -----------------------------------------------------------------------------11from __future__ import annotations1213import concurrent.futures as cf14import json15import re16from html import unescape17from urllib.parse import urlparse1819from ..schema import Product, parse_price20from .base import BaseConnector2122PRODUCT_URL_RE = re.compile(23    r"(/produits?/|/product/|/products/|/boutique/|/shop/|/store/|/p/|/item/"24    r"|/\d+-[a-z0-9]|/achat/|/produit-|-p\d+\.html|\.html$)", re.I)25NON_PRODUCT_RE = re.compile(26    r"(/blog|/blogue|/category|/categorie|/tag/|/page/|/compte|/account|/cart|"27    r"/panier|/checkout|/contact|/a-propos|/about|/cms|/content/|/faq|/policies|"28    r"/politique|sitemap|\.(?:jpg|png|pdf|css|js)$)", re.I)29LOC_RE = re.compile(r"<loc>\s*(?:<!\[CDATA\[)?\s*(.*?)\s*(?:\]\]>)?\s*</loc>", re.I | re.S)30IP_HOST_RE = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?$")313233def _clean(s):34    return unescape(re.sub(r"<[^>]+>", " ", re.sub(r"\s+", " ", s or ""))).strip()353637def _walk_jsonld(node, out):38    if isinstance(node, list):39        for x in node:40            _walk_jsonld(x, out)41    elif isinstance(node, dict):42        t = node.get("@type")43        types = t if isinstance(t, list) else [t]44        if any(str(x).endswith("Product") for x in types if x):45            out.append(node)46        for v in node.values():47            if isinstance(v, (list, dict)):48                _walk_jsonld(v, out)495051def _jsonld_images(img) -> list[str]:52    """Normalise le champ image JSON-LD (str | dict | liste mixte) en URLs."""53    out: list[str] = []54    items = img if isinstance(img, list) else [img]55    for x in items:56        if isinstance(x, dict):57            x = x.get("url") or x.get("contentUrl")58        if isinstance(x, str) and x.startswith("http") and x not in out:59            out.append(x)60    return out[:10]616263def extract_product(url, html):64    """Retourne un dict {title, price, price_max, image, images, description,65    currency, available, brand, sku, gtin, rating, review_count} ou None."""66    title = price = price_max = image = desc = None67    brand = sku = gtin = rating = review_count = None68    images: list[str] = []69    currency = "CAD"70    available = None7172    # 1) JSON-LD Product73    for block in re.findall(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>',74                            html, re.S | re.I):75        try:76            data = json.loads(block.strip())77        except Exception:78            continue79        prods = []80        _walk_jsonld(data, prods)81        for p in prods:82            if not isinstance(p, dict):83                continue84            offers = p.get("offers") or {}85            if isinstance(offers, list):86                offers = next((o for o in offers if isinstance(o, dict)), {})87            if not isinstance(offers, dict):88                offers = {}89            spec = offers.get("priceSpecification") or {}90            if isinstance(spec, list):91                spec = next((s for s in spec if isinstance(s, dict)), {})92            if not isinstance(spec, dict):93                spec = {}94            pr = parse_price(offers.get("price") or offers.get("lowPrice")95                             or spec.get("price"))96            if pr:97                title = title or _clean(p.get("name"))98                price = price or pr99                price_max = price_max or parse_price(offers.get("highPrice"))100                currency = offers.get("priceCurrency") or currency101                if not images:102                    images = _jsonld_images(p.get("image"))103                image = image or (images[0] if images else None)104                desc = desc or _clean(p.get("description"))105                av = str(offers.get("availability") or "")106                available = ("InStock" in av) if av else available107                b = p.get("brand")108                if isinstance(b, dict):109                    b = b.get("name")110                if isinstance(b, str) and b.strip():111                    brand = brand or _clean(b)112                if p.get("sku"):113                    sku = sku or str(p["sku"])[:80]114                for gk in ("gtin13", "gtin", "gtin12", "gtin8", "mpn"):115                    if p.get(gk):116                        gtin = gtin or str(p[gk])[:40]117                        break118                ar = p.get("aggregateRating") or {}119                if isinstance(ar, dict) and ar.get("ratingValue"):120                    try:121                        rating = rating or float(ar["ratingValue"])122                        rc = ar.get("reviewCount") or ar.get("ratingCount")123                        if rc:124                            review_count = review_count or int(float(rc))125                    except (TypeError, ValueError):126                        pass127128    # 2) Open Graph product / meta129    if not price:130        m = re.search(r'<meta[^>]+(?:og:price:amount|product:price:amount)"[^>]*content="([^"]+)"', html, re.I) \131            or re.search(r'<meta[^>]+content="([^"]+)"[^>]*(?:og:price:amount|product:price:amount)"', html, re.I)132        if m:133            price = parse_price(m.group(1))134    # 3) microdata itemprop=price135    if not price:136        m = re.search(r'itemprop="price"[^>]*content="([^"]+)"', html, re.I) \137            or re.search(r'content="([^"]+)"[^>]*itemprop="price"', html, re.I)138        if m:139            price = parse_price(m.group(1))140    if not title:141        m = re.search(r'<meta[^>]+property="og:title"[^>]*content="([^"]+)"', html, re.I)142        title = _clean(m.group(1)) if m else None143        if not title:144            m = re.search(r"<title[^>]*>(.*?)</title>", html, re.S | re.I)145            title = _clean(m.group(1)) if m else None146    if not image:147        m = re.search(r'<meta[^>]+property="og:image"[^>]*content="([^"]+)"', html, re.I)148        image = m.group(1) if m else None149        if image and image not in images:150            images.insert(0, image)151    if not desc:152        m = re.search(r'<meta[^>]+(?:name|property)="(?:description|og:description)"[^>]*content="([^"]+)"', html, re.I)153        desc = _clean(m.group(1)) if m else None154    if not brand:155        m = re.search(r'<meta[^>]+property="(?:og:brand|product:brand)"[^>]*content="([^"]+)"', html, re.I)156        brand = _clean(m.group(1)) if m else None157158    if not (title and price):159        return None160    return {"title": title, "price": price, "price_max": price_max,161            "image": image, "images": images or ([image] if image else []),162            "description": desc, "currency": currency, "available": available,163            "brand": brand, "sku": sku, "gtin": gtin,164            "rating": rating, "review_count": review_count}165166167class GenericConnector(BaseConnector):168    platform = "generic"169    request_delay = 0.2170    max_products = 800171    use_scrapfly = True      # False pendant la détection (vitesse)172173    def _rehost(self, u):174        # certains CMS mal configurés (ex. boreale.com, Craft CMS, 2026-09-08)175        # émettent des <loc> sur l'IP brute du serveur → cert TLS invalide.176        # On ramène ces URLs sur le domaine de la boutique, qui sert les177        # mêmes chemins (sitemaps enfants et pages produit vérifiés 200).178        # Variante 2026-09-22 (boreale.com encore) : <loc> émis sur un domaine179        # parasite (sitemaps.instead.beer) — « sitemap » dans l'hôte faisait en180        # plus exclure chaque URL produit via NON_PRODUCT_RE. Réécriture OPT-IN181        # par fiche registre (rehost_hosts), jamais globale : d'autres boutiques182        # génériques pointent légitimement vers un domaine tiers (CDN, miroir).183        p = urlparse(u)184        bad_hosts = {h.lower() for h in (self.store.get("rehost_hosts") or [])}185        if IP_HOST_RE.match(p.netloc) or p.netloc.lower() in bad_hosts:186            return f"{self.base}{p.path}" + (f"?{p.query}" if p.query else "")187        return u188189    def _sitemap_products(self):190        from . import scrapfly191        seen, out = set(), []192        roots = [f"{self.base}/sitemap.xml", f"{self.base}/sitemap_index.xml",193                 f"{self.base}/wp-sitemap.xml", f"{self.base}/1_fr_0_sitemap.xml",194                 f"{self.base}/sitemap/sitemap-index.xml", f"{self.base}/media/sitemap.xml",195                 f"{self.base}/pub/media/sitemap.xml", f"{self.base}/sitemap1.xml",196                 f"{self.base}/en/sitemap.xml", f"{self.base}/fr/sitemap.xml"]197        queue, depth_left = list(roots), 3198        fetched_roots = 0199        while queue and fetched_roots < 60:200            u = queue.pop(0)201            if u in seen:202                continue203            seen.add(u)204            try:205                r = self.session.get(u, timeout=self.timeout)206                xml = r.text if r.status_code == 200 else ""207            except Exception:208                xml = ""209            if not xml and self.use_scrapfly and scrapfly.available() and u == roots[0]:210                try:211                    _, xml = scrapfly.scrapfly_get(u)212                except Exception:213                    xml = ""214            if not xml:215                continue216            fetched_roots += 1217            locs = [self._rehost(l.strip()) for l in LOC_RE.findall(xml)]218            child_maps = [l for l in locs if l.endswith(".xml") or "sitemap" in l.lower()]219            if child_maps and depth_left > 0:220                queue = child_maps + queue221                depth_left -= 0222            for l in locs:223                if l.endswith(".xml"):224                    continue225                if PRODUCT_URL_RE.search(l) and not NON_PRODUCT_RE.search(l):226                    out.append(l)227            if len(out) >= self.max_products * 2:228                break229        # sites bilingues (ex. maisondherbes.com, refonte Next.js fr/en) : le230        # sitemap liste chaque produit en double sous un préfixe de langue231        # (/en/…). Exclusion OPT-IN par fiche registre (skip_url_prefixes),232        # jamais globale : d'autres boutiques génériques ont leur catalogue233        # légitimement sous /en/ (ex. pierresdailleurs.ca, 771 produits).234        skip = tuple(self.store.get("skip_url_prefixes") or [])235        if skip:236            out = [u for u in out if not urlparse(u).path.startswith(skip)]237        # dédup en gardant l'ordre238        return list(dict.fromkeys(out))[: self.max_products]239240    def _fetch_html(self, url):241        from . import scrapfly242        try:243            r = self.session.get(url, timeout=self.timeout)244            if r.status_code == 200 and len(r.text) > 500:245                return r.text246        except Exception:247            pass248        if self.use_scrapfly and scrapfly.available():249            try:250                st, content = scrapfly.scrapfly_get(url)251                if st == 200:252                    return content253            except Exception:254                pass255        return ""256257    def fetch(self) -> list[Product]:258        urls = self._sitemap_products()259        if not urls:260            return []261        out: list[Product] = []262        base_host = urlparse(self.base).netloc.lower().replace("www.", "")263264        def work(u):265            html = self._fetch_html(u)266            if not html:267                return None268            info = extract_product(u, html)269            if not info:270                return None271            det: dict = {}272            if info.get("sku"):273                det["sku"] = info["sku"]274            if info.get("gtin"):275                det["gtin"] = info["gtin"]276            if info.get("rating"):277                det["average_rating"] = info["rating"]278            if info.get("review_count"):279                det["review_count"] = info["review_count"]280            return Product(281                store_id=self.store_id,282                external_id=u.rstrip("/").split("/")[-1][:80] or u,283                url=u, title=info["title"], description=info.get("description") or "",284                price=info["price"],285                price_max=info.get("price_max") or info["price"],286                currency=info.get("currency") or "CAD",287                images=info.get("images") or ([info["image"]] if info.get("image") else []),288                vendor=info.get("brand") or "",289                available=info.get("available"),290                details=det)291292        with cf.ThreadPoolExecutor(6) as ex:293            for rec in ex.map(work, urls):294                if rec:295                    out.append(rec)296        return out297298299def probe_generic(domain, session, sample=6):300    """Teste si une boutique est récoltable en générique. Retourne (ok, n_urls, n_hits)."""301    store = {"id": domain, "url": f"https://{domain}"}302    conn = GenericConnector(store)303    conn.session = session304    urls = conn._sitemap_products()305    if not urls:306        return False, 0, 0307    hits = 0308    for u in urls[:sample]:309        html = conn._fetch_html(u)310        if html and extract_product(u, html):311            hits += 1312    return (hits >= max(2, sample // 2)), len(urls), hits313