spb/fabri-ka

1# -----------------------------------------------------------------------------2# Fabri-Ka — Agrégateur de produits québécois (marketplace de découverte)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# schema.py : modèles standardisés Product / Store + normalisation5# -----------------------------------------------------------------------------6"""Chaque connecteur, peu importe la plateforme ecommerce de la boutique,7produit des objets `Product` conformes à ce schéma. Le pipeline (ingest.py)8fait le diff avec la base et le frontend consomme un format unique."""9from __future__ import annotations1011import hashlib12import json13import re14import unicodedata15from dataclasses import dataclass, field, asdict1617REGIONS = [18    "Bas-Saint-Laurent", "Saguenay–Lac-Saint-Jean", "Capitale-Nationale",19    "Mauricie", "Estrie", "Montréal", "Outaouais", "Abitibi-Témiscamingue",20    "Côte-Nord", "Nord-du-Québec", "Gaspésie–Îles-de-la-Madeleine",21    "Chaudière-Appalaches", "Laval", "Lanaudière", "Laurentides",22    "Montérégie", "Centre-du-Québec",23]2425# Taxonomie produit Fabri-Ka (clé canonique -> libellé FR + mots-clés de match)26CATEGORIES: dict[str, tuple[str, list[str]]] = {27    "erable": ("Érable & miel", ["erable", "maple", "sirop", "miel", "honey", "hydromel"]),28    "epicerie": ("Épicerie fine", ["confiture", "jam", "sauce", "epice", "spice", "condiment",29                                    "vinaigre", "huile", "moutarde", "ketchup", "marinade",30                                    "tartinade", "sel ", "sucre", "farine", "grain", "cereale",31                                    "granola", "collation", "snack", "chip", "craquelin",32                                    "sans gluten", "vrac", "conserve", "pesto", "bouillon"]),33    "cafe_the": ("Café & thé", ["cafe", "coffee", "the ", "tisane", "tea", "infusion", "espresso", "matcha"]),34    "chocolat": ("Chocolat & confiseries", ["chocolat", "chocolate", "bonbon", "candy", "caramel",35                                             "confiserie", "sucrerie", "guimauve", "fudge"]),36    "boulangerie": ("Boulangerie & pâtisserie", ["pain", "bread", "patisserie", "biscuit", "cookie",37                                                  "gateau", "brioche", "viennoiserie", "tarte"]),38    "viande_poisson": ("Viandes & poissons", ["viande", "meat", "boucherie", "charcuterie", "saucisse",39                                               "jerky", "poisson", "fish", "fruits de mer", "seafood",40                                               "fume", "smoked", "canard", "porc", "boeuf", "bison", "cerf"]),41    "fromage_laitier": ("Fromages & produits laitiers", ["fromage", "cheese", "laitier", "dairy",42                                                          "yogourt", "beurre", "creme glacee"]),43    "boissons": ("Boissons", ["kombucha", "jus", "juice", "limonade", "boisson", "drink", "cocktail",44                               "sirop simple", "tonic", "soda", "eau petillante"]),45    "alcool": ("Bières, vins & spiritueux", ["biere", "beer", "vin ", "wine", "cidre", "cider", "gin",46                                              "vodka", "whisky", "spiritueux", "hydromel", "brasserie"]),47    "sante_beaute": ("Beauté & soins", ["savon", "soap", "cosmetique", "creme", "skincare", "shampoing",48                                         "supplement", "vitamine", "huile essentielle", "aromatherapie",49                                         "hydrolat", "herboristerie", "probiotique",50                                         "barbe", "beard", "baume", "lotion", "deodorant", "parfum",51                                         "bain", "bath", "chandelle de massage", "serum", "soin"]),52    "maison": ("Maison & déco", ["chandelle", "bougie", "candle", "deco", "coussin", "vaisselle",53                                  "ceramique", "poterie", "pottery", "verre", "planche", "ustensile",54                                  "tablier", "linge", "literie", "couverture", "lampe", "meuble",55                                  "furniture", "bois", "woodwork", "menuiserie"]),56    "mode": ("Mode & accessoires", ["vetement", "clothing", "t-shirt", "tshirt", "chandail", "hoodie",57                                     "soulier", "sandale", "chaussure", "botte", "espadrille",58                                     "valise", "bagage",59                                     "tuque", "casquette", "chapeau", "foulard", "mitaine", "bas ",60                                     "chaussette", "manteau", "robe", "jupe", "pantalon", "legging",61                                     "sac ", "handbag", "cuir", "leather", "portefeuille", "ceinture"]),62    "bijoux": ("Bijoux", ["bijou", "jewel", "collier", "necklace", "bracelet", "bague", "ring",63                           "montre", "watch", "horloger",64                           "boucle", "earring", "pendentif", "argent sterling",65                           "plaque or", "vermeil", "swarovski", "pierre fine"]),66    "art": ("Art & artisanat", ["oeuvre", "art ", "print", "affiche", "poster", "illustration",67                                 "vitrail", "carte de voeux", "carte de vœux",68                                 "peinture", "sculpture", "photographie", "carte de souhait",69                                 "papeterie", "stationery", "carnet", "sticker", "autocollant", "macrame"]),70    "enfants": ("Enfants & bébés", ["bebe", "baby", "enfant", "kid", "jouet", "toy", "doudou",71                                     "suce ", "biberon", "poussette",72                                     "hochet", "couche", "puericulture", "peluche", "figurine",73                                     "casse-tete", "puzzle", "bricolage", "jeu de societe",74                                     "jeux de societe", "lego", "poupee"]),75    "animaux": ("Animaux", ["chien", "dog", "chat", "cat", "animaux", "pet ", "gaterie", "laisse",76                             "collier pour", "litiere"]),77    "plein_air": ("Plein air & sport", ["plein air", "outdoor", "camping", "randonnee", "velo", "bike",78                                         "peche", "chasse", "kayak", "ski", "raquette", "sport"]),79    "jardin": ("Jardin & agriculture", ["jardin", "garden", "semence", "seed", "plante", "compost",80                                         "serre", "potager", "engrais"]),81    "industriel": ("Manufacturier & spécialisé", ["outil", "tool", "equipement", "industriel",82                                                   "machine", "piece", "quincaillerie", "electronique",83                                                   "remorque", "trailer", "essieu", "attache", "pneu",84                                                   "frein", "garde-boue", "treuil", "moteur", "hitch",85                                                   "auto", "camion", "vehicule", "soudure", "boulon"]),86    "autre": ("Autre", []),87}888990def strip_accents(s: str) -> str:91    return "".join(c for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn")929394def infer_category(*texts: str) -> str:95    """Devine la catégorie canonique à partir de textes bruts (type, tags, titre)."""96    blob = " " + strip_accents(" ".join(t for t in texts if t).lower()) + " "97    best, best_hits = "autre", 098    for key, (_, kws) in CATEGORIES.items():99        hits = sum(1 for kw in kws if kw in blob)100        if hits > best_hits:101            best, best_hits = key, hits102    return best103104105def details_signal_text(details) -> str:106    """Signaux de catégorisation tirés du JSON `details` : noms + valeurs des107    options Shopify et des attributs WooCommerce (matériaux, formats…)."""108    if not isinstance(details, dict) or not details:109        return ""110    parts: list[str] = []111    for o in details.get("options") or []:        # Shopify112        if isinstance(o, dict):113            parts.append(str(o.get("name") or ""))114            parts.extend(str(v) for v in (o.get("values") or [])[:12])115    for a in details.get("attributes") or []:     # WooCommerce116        if isinstance(a, dict):117            parts.append(str(a.get("name") or ""))118            parts.extend(str(t) for t in (a.get("terms") or [])[:12])119    return " ".join(p for p in parts if p)120121122# --- Règles de pertinence & de qualité (Phase 2 — enrichissement 2026-08) ----123# Un agrégateur de FABRICANTS n'affiche ni cartes-cadeaux, ni ateliers/cours,124# ni abonnements, ni billets d'événements : ces items restent en base125# (marqués, comptés, réintégrables) mais ne sont pas publiés.126PRICE_SANE_MAX = 100_000.0     # au-delà : prix suspect -> quarantaine127128_GIFT_POS = re.compile(r"(carte|certificat)s?[ -]cadeaux?|gift ?card|gift certificate")129_GIFT_NEG = re.compile(r"porte[ -]cartes?|pochette|enveloppe|presentoir|support|boite|etui")130_WORK_T = re.compile(r"^ateliers?\b|\bateliers? (de|d.|creatif|culinaire|priv|decouverte)"131                     r"|\bcours (de|d.|en ligne|priv)|^cours\b|^formation|\bformation (en|de|d.)"132                     r"|\bworkshop\b|\bmasterclass\b|\bwebinaire\b|\bvisite guidee\b"133                     r"|\bcamp de jour\b|\bdegustation (guidee|priv)")134_WORK_PT = re.compile(r"^(ateliers?|cours|formations?|workshops?|classes?|events?|evenements?)$")135_SUB_T = re.compile(r"^abonnements?\b|\babonnements? (a|au|aux|mensuel|annuel|d.un|de)"136                    r"|\bsubscription box|^subscription")137_SUB_PT = re.compile(r"^abonnements?$|^subscriptions?$|^adhesions?$|^memberships?$")138_TICK_T = re.compile(r"^billets?\b|\bbillets? (de|d.|pour)|^tickets?\b|\bdroit d.acces"139                     r"|\bdroits? d.entree|\blaissez-passer\b|^admission\b|^reservation s?\W*$")140_TICK_PT = re.compile(r"^(billets?|billetterie|tickets?|events?|evenements?|spectacles?)$")141_TEST_T = re.compile(r"^tests? ?(product|produit)?\W*$")142143144def classify_listing(title: str, product_type: str = "") -> str:145    """'published' ou 'excluded:<raison>' selon le titre/type normalisés.146147    Règles calibrées sur la base du 2026-08-19 (623 cartes-cadeaux,148    518 ateliers/cours, 88 abonnements, 23 billets — précision > rappel)."""149    t = strip_accents((title or "").lower())150    pt = strip_accents((product_type or "").lower()).strip()151    if _GIFT_POS.search(t) and not _GIFT_NEG.search(t):152        return "excluded:carte_cadeau"153    if _WORK_T.search(t) or _WORK_PT.match(pt):154        return "excluded:atelier_cours"155    if _SUB_T.search(t) or _SUB_PT.match(pt):156        return "excluded:abonnement"157    if _TICK_T.search(t) or _TICK_PT.match(pt):158        return "excluded:billet"159    if _TEST_T.match(t):160        return "excluded:test"161    return "published"162163164def is_placeholder_price(v: float | None) -> bool:165    """999 999 $, 99 999 $… : placeholders « sur devis » (ecksand & cie)."""166    if v is None or v < 99_999:167        return False168    digits = str(int(v))169    return set(digits) == {"9"}170171172def parse_price(raw) -> float | None:173    """'24,95 $' | '$24.95' | 24.95 -> 24.95 (CAD)."""174    if raw is None:175        return None176    if isinstance(raw, (int, float)):177        return float(raw) if raw > 0 else None178    s = re.sub(r"[^\d,.]", "", str(raw))179    if not s:180        return None181    if "," in s and "." not in s:182        s = s.replace(",", ".")183    else:184        s = s.replace(",", "")185    try:186        v = float(s)187        return v if v > 0 else None188    except ValueError:189        return None190191192@dataclass193class Product:194    """Produit standardisé Fabri-Ka."""195196    store_id: str                    # domaine canonique de la boutique197    external_id: str                 # identifiant chez la source198    url: str                         # page produit chez la boutique199    title: str = ""200    description: str = ""            # texte court, sans HTML201    price: float | None = None       # CAD, le plus bas des variantes202    price_max: float | None = None   # CAD, le plus haut des variantes203    compare_at_price: float | None = None204    currency: str = "CAD"205    images: list[str] = field(default_factory=list)206    category: str = ""               # clé canonique (CATEGORIES)207    product_type: str = ""           # valeur brute de la source208    tags: list[str] = field(default_factory=list)209    vendor: str = ""                 # marque affichée par la boutique210    available: bool | None = None211    details: dict = field(default_factory=dict)  # avis, dimensions, variantes… (JSON)212    listing_status: str = "published"   # published | excluded:<raison> | quarantine:<raison>213    price_on_request: int = 0           # 1 = sur devis / prix non publié par la boutique214215    @property216    def uid(self) -> str:217        return hashlib.sha1(f"{self.store_id}::{self.external_id}".encode()).hexdigest()[:20]218219    def finalize(self) -> "Product":220        import html as _html221        self.title = _html.unescape(re.sub(r"\s+", " ", self.title or "")).strip()[:300]222        # cap 5 000 car. (l'ancien cap 600 tronquait 34 % des fiches — matériaux223        # et dimensions vivent souvent dans la description longue)224        self.description = re.sub(r"(?is)<(script|style)[^>]*>.*?</\1>", " ", self.description or "")225        self.description = re.sub(r"<[^>]+>", " ", self.description)226        self.description = _html.unescape(re.sub(r"\s+", " ", self.description)).strip()[:5000]227        if not self.category:228            self.category = infer_category(self.product_type, " ".join(self.tags),229                                           self.title, self.description[:200],230                                           details_signal_text(self.details))231        self.images = [i for i in self.images if isinstance(i, str) and i.startswith("http")][:15]232        # prix : 0 $ n'est pas un prix (Woo renvoie parfois 0 pour « sur demande »)233        if self.price is not None and self.price <= 0:234            self.price = None235        if self.price_max is not None and self.price_max <= 0:236            self.price_max = None237        # placeholders « sur devis » neutralisés, jamais publiés comme nombres238        if is_placeholder_price(self.price) or is_placeholder_price(self.price_max):239            self.details = dict(self.details or {})240            self.details["price_placeholder"] = self.price241            self.price = self.price_max = self.compare_at_price = None242        if self.price is None:243            self.price_on_request = 1244        # pertinence + quarantaine (aberrations à l'ingestion)245        status = classify_listing(self.title, self.product_type)246        if status == "published":247            if not self.title:248                status = "quarantine:titre_vide"249            elif self.price is not None and self.price > PRICE_SANE_MAX:250                status = "quarantine:prix_hors_bornes"251        self.listing_status = status252        return self253254    def content_hash(self) -> str:255        # len(description)/len(images) : force la mise à jour quand les caps256        # changent (600→5000, 8→15) ou qu'une galerie s'enrichit (Square)257        basis = json.dumps([self.title, self.price, self.price_max, self.available,258                            self.images[:1], len(self.images), self.description[:200],259                            len(self.description), self.details or None],260                           ensure_ascii=False, sort_keys=True)261        return hashlib.sha1(basis.encode()).hexdigest()[:16]262263    def to_row(self) -> dict:264        d = asdict(self)265        d["uid"] = self.uid266        return d267