# ----------------------------------------------------------------------------- # Food-Ka — connecteur Walmart Canada (walmart.ca, rayon épicerie) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # Site Next.js derrière Akamai/PerimeterX (curl direct → HTTP 418) : on lit la # grille produits embarquée dans ', re.S) # format dans le nom du produit : "567 g", "2 L", "12 x 355 ml"… _SIZE_IN_NAME = re.compile( r"((?:\d+\s*[x×]\s*)?\d+(?:[.,]\d+)?\s*(?:kg|g|ml|l|lb|oz|un))\b", re.IGNORECASE) # rayons épicerie Walmart : (slug décoratif, id de catégorie, catégorie canonique) CATEGORIES = [ ("fruits-et-legumes-frais", "6000194327370", "Fruits et légumes"), ("charcuteries-et-repas-prepares", "6000194327356", "Charcuteries et fromages"), ("produits-laitiers-oeufs", "6000194327369", "Produits laitiers et œufs"), ("pains-et-boulangerie", "6000194327359", "Boulangerie"), ("aliments-surgeles", "6000194326337", "Surgelés"), ("garde-manger", "6000194326346", "Garde-manger"), ("boissons", "6000194326336", "Boissons"), ("collations-bonbons", "6000194328523", "Collations et confiseries"), ] class WalmartConnector(BaseConnector): source_id = "walmart" pages_per_category = 2 # 55 produits/page → ~880 produits/sync # -- récupération ---------------------------------------------------------- def _items(self, url: str) -> list[dict]: """Tuiles produit d'une page de rayon (JSON __NEXT_DATA__).""" html = self.get_scrapfly(url) m = _NEXT_RE.search(html) if not m: return [] data = json.loads(m.group(1)) try: sr = data["props"]["pageProps"]["initialData"]["searchResult"] except (KeyError, TypeError): return [] items: list[dict] = [] for stack in sr.get("itemStacks") or []: items.extend(stack.get("items") or []) return items # -- transformation -------------------------------------------------------- def _item_to_product(self, item: dict, category: str) -> Product | None: if item.get("__typename") not in (None, "Product"): return None pid = item.get("usItemId") or item.get("id") name = item.get("name") or "" if not pid or not name: return None info = item.get("priceInfo") or {} # `price` est le prix courant numérique ; linePrice le libellé affiché price = parse_price(item.get("price")) if price is None: price = parse_price(info.get("linePrice")) was = parse_price(info.get("wasPrice")) link = item.get("canonicalUrl") or "" image = (item.get("imageInfo") or {}).get("thumbnailUrl") \ or item.get("image") or "" cat_path = [p.get("name") for p in ((item.get("category") or {}).get("path") or []) if isinstance(p, dict) and p.get("name")] size_m = _SIZE_IN_NAME.search(name) out_of_stock = item.get("isOutOfStock") return Product( source=self.source_id, external_id=str(pid), url=f"https://www.walmart.ca{link}" if link.startswith("/") else link, name=name, brand=item.get("brand") or "", category=category, category_raw=" > ".join(cat_path), size_label=size_m.group(1) if size_m else "", price=price, regular_price=was, price_label=info.get("linePriceDisplay") or info.get("linePrice") or "", on_sale=bool(was and price and was > price), in_stock=(not out_of_stock) if out_of_stock is not None else None, unit_price_label=info.get("unitPrice") or "", details={"sales_unit": item.get("salesUnitType"), "seller": item.get("sellerName")}, images=[image] if image else [], ) # -- contrat --------------------------------------------------------------- def fetch(self) -> list[Product]: products: list[Product] = [] seen: set[str] = set() for slug, cat_id, category in CATEGORIES: for page in range(1, self.pages_per_category + 1): url = (f"https://www.walmart.ca/fr/browse/epicerie/" f"{slug}/10019_{cat_id}?page={page}") try: items = self._items(url) except Exception: # un rayon qui casse ne bloque pas le reste break if not items: break new = 0 for item in items: try: prod = self._item_to_product(item, category) except Exception: continue if prod and prod.uid not in seen: seen.add(prod.uid) products.append(prod) new += 1 if new == 0: # page sans nouveauté = fin du rayon break return products