spb/ora-ka Public

1# -----------------------------------------------------------------------------2# Fabri-Ka — Agrégateur de produits québécois3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/generic.py : connecteur universel par sitemap + extraction du5#   balisage produit rendu côté serveur (JSON-LD schema.org Product, microdata,6#   Open Graph product, blobs PrestaShop/Magento). Couvre PrestaShop, Magento,7#   BigCommerce, WordPress non-Woo et sites ecommerce sur mesure. Scrapfly en8#   secours pour l'anti-bot. Rendu client-only (Square, Ecwid) non couvert.9# -----------------------------------------------------------------------------10from __future__ import annotations1112import concurrent.futures as cf13import json14import re15from html import unescape16from urllib.parse import urlparse1718from ..schema import Product, parse_price19from .base import BaseConnector2021PRODUCT_URL_RE = re.compile(22    r"(/produits?/|/product/|/products/|/boutique/|/shop/|/store/|/p/|/item/"23    r"|/\d+-[a-z0-9]|/achat/|/produit-|-p\d+\.html|\.html$)", re.I)24NON_PRODUCT_RE = re.compile(25    r"(/blog|/blogue|/category|/categorie|/tag/|/page/|/compte|/account|/cart|"26    r"/panier|/checkout|/contact|/a-propos|/about|/cms|/content/|/faq|/policies|"27    r"/politique|sitemap|\.(?:jpg|png|pdf|css|js)$)", re.I)28LOC_RE = re.compile(r"<loc>\s*(?:<!\[CDATA\[)?\s*(.*?)\s*(?:\]\]>)?\s*</loc>", re.I | re.S)293031def _clean(s):32    return unescape(re.sub(r"<[^>]+>", " ", re.sub(r"\s+", " ", s or ""))).strip()333435def _walk_jsonld(node, out):36    if isinstance(node, list):37        for x in node:38            _walk_jsonld(x, out)39    elif isinstance(node, dict):40        t = node.get("@type")41        types = t if isinstance(t, list) else [t]42        if any(str(x).endswith("Product") for x in types if x):43            out.append(node)44        for v in node.values():45            if isinstance(v, (list, dict)):46                _walk_jsonld(v, out)474849def extract_product(url, html):50    """Retourne un dict {title, price, image, description, currency, available} ou None."""51    title = price = image = desc = None52    currency = "CAD"53    available = None5455    # 1) JSON-LD Product56    for block in re.findall(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>',57                            html, re.S | re.I):58        try:59            data = json.loads(block.strip())60        except Exception:61            continue62        prods = []63        _walk_jsonld(data, prods)64        for p in prods:65            if not isinstance(p, dict):66                continue67            offers = p.get("offers") or {}68            if isinstance(offers, list):69                offers = next((o for o in offers if isinstance(o, dict)), {})70            if not isinstance(offers, dict):71                offers = {}72            spec = offers.get("priceSpecification") or {}73            if isinstance(spec, list):74                spec = next((s for s in spec if isinstance(s, dict)), {})75            if not isinstance(spec, dict):76                spec = {}77            pr = parse_price(offers.get("price") or offers.get("lowPrice")78                             or spec.get("price"))79            if pr:80                title = title or _clean(p.get("name"))81                price = price or pr82                currency = offers.get("priceCurrency") or currency83                img = p.get("image")84                if isinstance(img, list):85                    img = img[0] if img else None86                if isinstance(img, dict):87                    img = img.get("url")88                image = image or img89                desc = desc or _clean(p.get("description"))90                av = str(offers.get("availability") or "")91                available = ("InStock" in av) if av else available9293    # 2) Open Graph product / meta94    if not price:95        m = re.search(r'<meta[^>]+(?:og:price:amount|product:price:amount)"[^>]*content="([^"]+)"', html, re.I) \96            or re.search(r'<meta[^>]+content="([^"]+)"[^>]*(?:og:price:amount|product:price:amount)"', html, re.I)97        if m:98            price = parse_price(m.group(1))99    # 3) microdata itemprop=price100    if not price:101        m = re.search(r'itemprop="price"[^>]*content="([^"]+)"', html, re.I) \102            or re.search(r'content="([^"]+)"[^>]*itemprop="price"', html, re.I)103        if m:104            price = parse_price(m.group(1))105    if not title:106        m = re.search(r'<meta[^>]+property="og:title"[^>]*content="([^"]+)"', html, re.I)107        title = _clean(m.group(1)) if m else None108        if not title:109            m = re.search(r"<title[^>]*>(.*?)</title>", html, re.S | re.I)110            title = _clean(m.group(1)) if m else None111    if not image:112        m = re.search(r'<meta[^>]+property="og:image"[^>]*content="([^"]+)"', html, re.I)113        image = m.group(1) if m else None114    if not desc:115        m = re.search(r'<meta[^>]+(?:name|property)="(?:description|og:description)"[^>]*content="([^"]+)"', html, re.I)116        desc = _clean(m.group(1)) if m else None117118    if not (title and price):119        return None120    return {"title": title, "price": price, "image": image, "description": desc,121            "currency": currency, "available": available}122123124class GenericConnector(BaseConnector):125    platform = "generic"126    request_delay = 0.2127    max_products = 800128    use_scrapfly = True      # False pendant la détection (vitesse)129130    def _sitemap_products(self):131        from . import scrapfly132        seen, out = set(), []133        roots = [f"{self.base}/sitemap.xml", f"{self.base}/sitemap_index.xml",134                 f"{self.base}/wp-sitemap.xml", f"{self.base}/1_fr_0_sitemap.xml",135                 f"{self.base}/sitemap/sitemap-index.xml", f"{self.base}/media/sitemap.xml",136                 f"{self.base}/pub/media/sitemap.xml", f"{self.base}/sitemap1.xml",137                 f"{self.base}/en/sitemap.xml", f"{self.base}/fr/sitemap.xml"]138        queue, depth_left = list(roots), 3139        fetched_roots = 0140        while queue and fetched_roots < 60:141            u = queue.pop(0)142            if u in seen:143                continue144            seen.add(u)145            try:146                r = self.session.get(u, timeout=self.timeout)147                xml = r.text if r.status_code == 200 else ""148            except Exception:149                xml = ""150            if not xml and self.use_scrapfly and scrapfly.available() and u == roots[0]:151                try:152                    _, xml = scrapfly.scrapfly_get(u)153                except Exception:154                    xml = ""155            if not xml:156                continue157            fetched_roots += 1158            locs = [l.strip() for l in LOC_RE.findall(xml)]159            child_maps = [l for l in locs if l.endswith(".xml") or "sitemap" in l.lower()]160            if child_maps and depth_left > 0:161                queue = child_maps + queue162                depth_left -= 0163            for l in locs:164                if l.endswith(".xml"):165                    continue166                if PRODUCT_URL_RE.search(l) and not NON_PRODUCT_RE.search(l):167                    out.append(l)168            if len(out) >= self.max_products * 2:169                break170        # dédup en gardant l'ordre171        return list(dict.fromkeys(out))[: self.max_products]172173    def _fetch_html(self, url):174        from . import scrapfly175        try:176            r = self.session.get(url, timeout=self.timeout)177            if r.status_code == 200 and len(r.text) > 500:178                return r.text179        except Exception:180            pass181        if self.use_scrapfly and scrapfly.available():182            try:183                st, content = scrapfly.scrapfly_get(url)184                if st == 200:185                    return content186            except Exception:187                pass188        return ""189190    def fetch(self) -> list[Product]:191        urls = self._sitemap_products()192        if not urls:193            return []194        out: list[Product] = []195        base_host = urlparse(self.base).netloc.lower().replace("www.", "")196197        def work(u):198            html = self._fetch_html(u)199            if not html:200                return None201            info = extract_product(u, html)202            if not info:203                return None204            return Product(205                store_id=self.store_id,206                external_id=u.rstrip("/").split("/")[-1][:80] or u,207                url=u, title=info["title"], description=info.get("description") or "",208                price=info["price"], price_max=info["price"],209                currency=info.get("currency") or "CAD",210                images=[info["image"]] if info.get("image") else [],211                available=info.get("available"))212213        with cf.ThreadPoolExecutor(6) as ex:214            for rec in ex.map(work, urls):215                if rec:216                    out.append(rec)217        return out218219220def probe_generic(domain, session, sample=6):221    """Teste si une boutique est récoltable en générique. Retourne (ok, n_urls, n_hits)."""222    store = {"id": domain, "url": f"https://{domain}"}223    conn = GenericConnector(store)224    conn.session = session225    urls = conn._sitemap_products()226    if not urls:227        return False, 0, 0228    hits = 0229    for u in urls[:sample]:230        html = conn._fetch_html(u)231        if html and extract_product(u, html):232            hits += 1233    return (hits >= max(2, sample // 2)), len(urls), hits234