# ----------------------------------------------------------------------------- # Food-Ka — Agrégateur de produits d'épicerie (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/_metro.py : socle commun Metro / Super C (même plateforme) # Tuiles produit rendues côté serveur (data-product-code) mais site derrière # un anti-bot agressif -> Scrapfly (ASP). Pagination : « -page-N ». # ----------------------------------------------------------------------------- from __future__ import annotations import re from bs4 import BeautifulSoup from ..schema import Product, parse_price from .base import BaseConnector _PER_KG_RE = re.compile(r"(\d+(?:,\d+)?)\s*\$\s*/\s*kg") # (slug d'allée, catégorie canonique) AISLES = [ ("fruits-et-legumes", "Fruits et légumes"), ("viandes-et-volailles", "Viandes et volailles"), ("poissons-et-fruits-de-mer", "Poissons et fruits de mer"), ("produits-laitiers-et-oeufs", "Produits laitiers et œufs"), ("pains-et-patisseries", "Boulangerie"), ("produits-surgeles", "Surgelés"), ("garde-manger", "Garde-manger"), ("boissons", "Boissons"), ("collations", "Collations et confiseries"), ] class MetroPlatformConnector(BaseConnector): """Base Metro/Super C — sous-classes : définir source_id et domain.""" domain: str = "" # ex. "www.metro.ca" aisle_prefix: str = "/epicerie-en-ligne/allees" # superc : "/allees" pages_per_aisle: int = 2 # ~45 produits/page def _parse_tiles(self, html: str, category: str) -> list[Product]: soup = BeautifulSoup(html, "html.parser") products: list[Product] = [] for tile in soup.select("[data-product-code]"): code = tile.get("data-product-code") name = tile.get("data-product-name") or "" if not code or not name: continue link_el = tile.select_one("a.product-details-link[href]") link = link_el["href"] if link_el else "" img_el = tile.select_one(".pt__visual img[src]") image = (img_el["src"] if img_el else "") or "" brand_el = tile.select_one(".head__brand") size_el = tile.select_one(".head__unit-details") price_el = tile.select_one(".price-update") before_el = tile.select_one(".pricing__before-price") unit_el = tile.select_one(".pricing__secondary-price") price_label = price_el.get_text(" ", strip=True) if price_el else "" unit_label = unit_el.get_text(" ", strip=True) if unit_el else "" before_txt = before_el.get_text(" ", strip=True) if before_el else "" price = None regular = parse_price(before_txt) if before_txt else None # Produit au poids : le prix affiché est une estimation à l'unité # mais le « prix régulier » est au kg — on compare tout au kg. if "kg" in before_txt: m_cur = _PER_KG_RE.search(unit_label) m_reg = _PER_KG_RE.search(before_txt) if m_cur: price = float(m_cur.group(1).replace(",", ".")) price_label = f"{m_cur.group(1)} $ / kg" regular = (float(m_reg.group(1).replace(",", ".")) if m_reg else None) else: # pas de prix au kg comparable : ne rien inventer regular = None on_sale = bool(tile.select_one(".pi-price-promo, .promo-price")) or regular is not None products.append(Product( source=self.source_id, external_id=str(code), url=(f"https://{self.domain}{link}" if link.startswith("/") else link), name=name, brand=brand_el.get_text(strip=True) if brand_el else "", category=category, category_raw=tile.get("data-product-category") or "", size_label=size_el.get_text(" ", strip=True) if size_el else "", price=price, price_label=price_label, regular_price=regular, on_sale=on_sale, unit_price_label=unit_label, images=[image] if image.startswith("http") else [], )) return products def fetch(self) -> list[Product]: products: list[Product] = [] seen: set[str] = set() for slug, category in AISLES: for page in range(1, self.pages_per_aisle + 1): suffix = slug if page == 1 else f"{slug}-page-{page}" url = f"https://{self.domain}{self.aisle_prefix}/{suffix}" try: html = self.get_scrapfly(url) except Exception: # une allée qui casse ne bloque pas le reste break batch = self._parse_tiles(html, category) new = 0 for prod in batch: if prod.uid not in seen: seen.add(prod.uid) products.append(prod) new += 1 if new == 0: # page vide ou répétée : allée épuisée break return products