SPB Git forge
spboucher.ai ↗

spb/fabri-ka

Public

Agrégateur de produits québécois — www.fabri-ka.com

217commits 1branches 0releases
66.1 MBsize
maindefault branch
3 h agolast push
Python 38.4% HTML 30.3% TypeScript 17.2% CSS 11% JavaScript 3.2%

Qualité produits : listing_status (exclusions cartes-cadeaux/ateliers/abonnements/billets), quarantaine prix>100k et titres vides, prix placeholders 999999 -> sur devis (price_on_request), cap descriptions 600->5000, images 8->15, Woo courte+longue, Wix additionalInfo/inventory/weight, Square include=images,options (galerie), Shopify retry 429 durci

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 1 mo ago (Aug 19, 2026) parent 26463e9

10 changed files +487 −94

modified fabrika/connectors/shopify.py +6 −3
@@ -33,14 +33,17 @@ class ShopifyConnector(BaseConnector):
33 33 if getattr(self, "_scrapfly_only", False):
34 34 return self._scrapfly_json(url)
35 35 last_err = None
36 − for attempt in range(4):
36 + # 6 tentatives, backoff long sur 429 : les boutiques à panne
37 + # intermittente (ex. boutiquesoha.com) répondent après patience,
38 + # là où le repli Scrapfly renvoie parfois 422.
39 + for attempt in range(6):
37 40 with _GLOBAL_LOCK:
38 41 wait = _MIN_INTERVAL - (time.time() - _last_shopify_req[0])
39 42 if wait > 0:
40 43 time.sleep(wait)
41 44 _last_shopify_req[0] = time.time()
42 45 p = subprocess.run(
43 − ["curl", "-sS", "--compressed", "--max-time", str(self.timeout),
46 + ["curl", "-sS", "--compressed", "-L", "--max-time", str(self.timeout),
44 47 "-A", self.session.headers["User-Agent"].split(" FabriKaBot")[0],
45 48 "-w", "\n%{http_code}", url],
46 49 capture_output=True, text=True, errors="replace")
@@ -51,7 +54,7 @@ class ShopifyConnector(BaseConnector):
51 54 except _json.JSONDecodeError as exc:
52 55 last_err = exc
53 56 elif code == "429":
54 − time.sleep(5 * (attempt + 1))
57 + time.sleep(8 * (attempt + 1))
55 58 last_err = RuntimeError("429 Too Many Requests")
56 59 else:
57 60 last_err = RuntimeError(f"HTTP {code or 'error'} {p.stderr[:120]}")
modified fabrika/connectors/square.py +21 −2
@@ -54,8 +54,10 @@ class SquareConnector(BaseConnector):
54 54 out: list[Product] = []
55 55 page = 1
56 56 while page <= self.max_pages:
57 + # include=images,options : galerie complète + variantes dans la même
58 + # requête paginée (levée du « mono-image » sans requête supplémentaire)
57 59 url = (f"{self.base}{API_PATH}/editor/users/{user}/sites/{site}"
58 − f"/products?page={page}&per_page=100")
60 + f"/products?page={page}&per_page=100&include=images,options")
59 61 data = self.get(url).json()
60 62 items = data.get("data") or []
61 63 for it in items:
@@ -72,7 +74,24 @@ class SquareConnector(BaseConnector):
72 74 or inv.get("all_variations_sold_out"))
73 75 thumb = ((it.get("thumbnail") or {}).get("data") or {})
74 76 img = thumb.get("absolute_url")
77 + images: list[str] = []
78 + for rec in ((it.get("images") or {}).get("data") or []):
79 + u = rec.get("absolute_url") or ""
80 + if u and not u.startswith("http"):
81 + u = "https://" + u.lstrip("/")
82 + if u and u not in images:
83 + images.append(u)
84 + if not images and img:
85 + images = [img if img.startswith("http") else "https://" + img.lstrip("/")]
75 86 det: dict = {}
87 + opts = []
88 + for o in ((it.get("options") or {}).get("data") or []):
89 + name = o.get("name") or ""
90 + if name:
91 + opts.append({"name": name,
92 + "values": [str(c) for c in (o.get("choice_order") or [])]})
93 + if opts:
94 + det["options"] = opts
76 95 try:
77 96 if float(it.get("avg_rating") or 0) > 0:
78 97 det["average_rating"] = float(it["avg_rating"])
@@ -97,7 +116,7 @@ class SquareConnector(BaseConnector):
97 116 price_max=high if high and low and high > low else low,
98 117 compare_at_price=(reg_high if on_sale and reg_high
99 118 and reg_high > (high or 0) else None),
100 − images=[img] if img else [],
119 + images=images,
101 120 product_type=it.get("product_type", "") or "",
102 121 available=available,
103 122 details=det,
modified fabrika/connectors/wix.py +21 −1
@@ -7,9 +7,15 @@
7 7 # -----------------------------------------------------------------------------
8 8 from __future__ import annotations
9 9
10 +import re as _re
11 +
10 12 from ..schema import Product, parse_price
11 13 from .base import BaseConnector
12 14
15 +
16 +def _strip_html(s: str) -> str:
17 + return _re.sub(r"\s+", " ", _re.sub(r"<[^>]+>", " ", s or "")).strip()
18 +
13 19 WIX_STORES_APP = "1380b703-ce81-ff05-f115-39571d94dfcd"
14 20 ALL_PRODUCTS_CATEGORY = "00000000-000000-000000-000000000001"
15 21
@@ -18,7 +24,9 @@ GQL = """query getFilteredProducts($limit:Int!,$offset:Int!){
18 24 productsWithMetaData(limit:$limit,offset:$offset,onlyVisible:true){
19 25 list{id name urlPart price comparePrice formattedPrice description
20 26 sku brand discount{mode value}
21 − media{url} isInStock productType ribbon
27 + media{url} isInStock productType ribbon weight
28 + inventory{status quantity}
29 + additionalInfo{title description}
22 30 options{title selections{description}}}}}}}""" % ALL_PRODUCTS_CATEGORY
23 31
24 32
@@ -73,6 +81,18 @@ class WixConnector(BaseConnector):
73 81 disc = it.get("discount") or {}
74 82 if disc.get("value"):
75 83 det["discount"] = {"mode": disc.get("mode"), "value": disc.get("value")}
84 + if it.get("weight"):
85 + det["weight"] = it["weight"]
86 + inv = it.get("inventory") or {}
87 + if inv.get("quantity") is not None:
88 + det["inventory_quantity"] = inv["quantity"]
89 + # additionalInfo = la fiche détaillée Wix (matériaux, entretien…)
90 + extra = [{"title": a.get("title", ""),
91 + "description": _strip_html(a.get("description", ""))[:800]}
92 + for a in (it.get("additionalInfo") or [])
93 + if a and (a.get("title") or a.get("description"))]
94 + if extra:
95 + det["additional_info"] = extra
76 96 out.append(Product(
77 97 store_id=self.store_id,
78 98 external_id=str(it.get("id", "")),
modified fabrika/connectors/woocommerce.py +6 −1
@@ -72,12 +72,17 @@ class WooCommerceConnector(BaseConnector):
72 72 det["is_on_backorder"] = True
73 73 if it.get("variations"):
74 74 det["variations"] = len(it["variations"])
75 + # description : courte + longue concaténées (l'ancienne règle
76 + # « courte OU longue » jetait la description riche sur 37 % des fiches)
77 + short = (it.get("short_description") or "").strip()
78 + long_ = (it.get("description") or "").strip()
79 + desc = short if long_ in ("", short) else (long_ if not short else f"{short} {long_}")
75 80 out.append(Product(
76 81 store_id=self.store_id,
77 82 external_id=str(it["id"]),
78 83 url=it.get("permalink", ""),
79 84 title=it.get("name", ""),
80 − description=it.get("short_description") or it.get("description") or "",
85 + description=desc,
81 86 price=pmin or price,
82 87 price_max=pmax or price,
83 88 compare_at_price=money(prices.get("regular_price"))
modified fabrika/db.py +33 −10
@@ -93,9 +93,18 @@ def connect() -> sqlite3.Connection:
93 93 cols = {r[1] for r in con.execute("PRAGMA table_info(products)")}
94 94 if "details" not in cols:
95 95 con.execute("ALTER TABLE products ADD COLUMN details TEXT")
96 + if "listing_status" not in cols:
97 + con.execute("ALTER TABLE products ADD COLUMN listing_status TEXT DEFAULT 'published'")
98 + con.execute("CREATE INDEX IF NOT EXISTS idx_products_listing ON products(listing_status)")
99 + if "price_on_request" not in cols:
100 + con.execute("ALTER TABLE products ADD COLUMN price_on_request INTEGER DEFAULT 0")
96 101 scols = {r[1] for r in con.execute("PRAGMA table_info(stores)")}
97 − if "shipping_info" not in scols:
98 − con.execute("ALTER TABLE stores ADD COLUMN shipping_info TEXT")
102 + for col, typ in (("shipping_info", "TEXT"), ("email", "TEXT"),
103 + ("phone", "TEXT"), ("postal_prefix", "TEXT"),
104 + ("lat", "REAL"), ("lng", "REAL"),
105 + ("store_kind", "TEXT")):
106 + if col not in scols:
107 + con.execute(f"ALTER TABLE stores ADD COLUMN {col} {typ}")
99 108 return con
100 109 except sqlite3.OperationalError as exc:
101 110 last_exc = exc
@@ -108,19 +117,28 @@ def upsert_store(con: sqlite3.Connection, s: dict) -> None:
108 117 con.execute("""
109 118 INSERT INTO stores (id, name, url, platform, catalog_endpoint, city, region,
110 119 origin_class, origin_confidence, origin_evidence,
111 − categories, socials, discovery_sources, language, enabled)
120 + categories, socials, discovery_sources, language, enabled,
121 + phone, postal_prefix, store_kind)
112 122 VALUES (:id,:name,:url,:platform,:catalog_endpoint,:city,:region,
113 123 :origin_class,:origin_confidence,:origin_evidence,
114 − :categories,:socials,:discovery_sources,:language,:enabled)
124 + :categories,:socials,:discovery_sources,:language,:enabled,
125 + :phone,:postal_prefix,:store_kind)
115 126 ON CONFLICT(id) DO UPDATE SET
116 127 name=excluded.name, url=excluded.url, platform=excluded.platform,
117 − catalog_endpoint=excluded.catalog_endpoint, city=excluded.city,
128 + catalog_endpoint=excluded.catalog_endpoint,
129 + -- city vient du géocodage (pas du registre) : ne jamais l'écraser par du vide
130 + city=CASE WHEN excluded.city<>'' THEN excluded.city ELSE stores.city END,
118 131 region=excluded.region, origin_class=excluded.origin_class,
119 132 origin_confidence=excluded.origin_confidence,
120 133 origin_evidence=excluded.origin_evidence,
121 134 categories=excluded.categories, socials=excluded.socials,
122 135 discovery_sources=excluded.discovery_sources,
123 − language=excluded.language, enabled=excluded.enabled
136 + language=excluded.language, enabled=excluded.enabled,
137 + phone=CASE WHEN excluded.phone<>'' THEN excluded.phone ELSE stores.phone END,
138 + postal_prefix=CASE WHEN excluded.postal_prefix<>'' THEN excluded.postal_prefix
139 + ELSE stores.postal_prefix END,
140 + store_kind=CASE WHEN excluded.store_kind<>'' THEN excluded.store_kind
141 + ELSE stores.store_kind END
124 142 """, {
125 143 "id": s["id"], "name": s.get("name") or s["id"], "url": s.get("url") or f"https://{s['id']}",
126 144 "platform": s.get("platform") or "", "catalog_endpoint": s.get("catalog_endpoint") or "",
@@ -132,6 +150,8 @@ def upsert_store(con: sqlite3.Connection, s: dict) -> None:
132 150 "socials": json.dumps(s.get("socials") or [], ensure_ascii=False),
133 151 "discovery_sources": json.dumps(s.get("discovery_sources") or [], ensure_ascii=False),
134 152 "language": s.get("language") or "", "enabled": 1 if s.get("enabled", True) else 0,
153 + "phone": s.get("phone") or "", "postal_prefix": s.get("postal_prefix") or "",
154 + "store_kind": s.get("store_kind") or "",
135 155 })
136 156
137 157
@@ -164,10 +184,11 @@ def sync_products(con: sqlite3.Connection, store_id: str, store_name: str,
164 184 con.execute("""INSERT INTO products (uid, store_id, external_id, url, title,
165 185 description, price, price_max, compare_at_price, currency, images, category,
166 186 product_type, tags, vendor, available, details, content_hash, first_seen,
167 − last_seen, active, miss_count) VALUES (:uid,:store_id,:external_id,:url,:title,
187 + last_seen, active, miss_count, listing_status, price_on_request)
188 + VALUES (:uid,:store_id,:external_id,:url,:title,
168 189 :description,:price,:price_max,:compare_at_price,:currency,:images,:category,
169 190 :product_type,:tags,:vendor,:available,:details,:content_hash,:first_seen,
170 − :last_seen,1,0)""", row)
191 + :last_seen,1,0,:listing_status,:price_on_request)""", row)
171 192 con.execute("INSERT INTO products_fts (uid, title, description, tags, vendor, store_name) "
172 193 "VALUES (?,?,?,?,?,?)",
173 194 (uid, p.title, p.description, " ".join(p.tags), p.vendor, store_name))
@@ -180,7 +201,8 @@ def sync_products(con: sqlite3.Connection, store_id: str, store_name: str,
180 201 currency=:currency, images=:images, category=:category,
181 202 product_type=:product_type, tags=:tags, vendor=:vendor,
182 203 available=:available, details=:details, content_hash=:content_hash,
183 − last_seen=:last_seen,
204 + last_seen=:last_seen, listing_status=:listing_status,
205 + price_on_request=:price_on_request,
184 206 active=1, miss_count=0 WHERE uid=:uid""", row)
185 207 fts_del.append(uid)
186 208 fts_add.append((uid, p.title, p.description, " ".join(p.tags),
@@ -210,7 +232,8 @@ def sync_products(con: sqlite3.Connection, store_id: str, store_name: str,
210 232 "vendor, store_name) VALUES (?,?,?,?,?,?)", fts_add)
211 233
212 234 con.execute("UPDATE stores SET last_sync=?, product_count="
213 − "(SELECT COUNT(*) FROM products WHERE store_id=? AND active=1) WHERE id=?",
235 + "(SELECT COUNT(*) FROM products WHERE store_id=? AND active=1 "
236 + "AND listing_status='published') WHERE id=?",
214 237 (now, store_id, store_id))
215 238 return added, updated, removed
216 239
modified fabrika/schema.py +90 −5
@@ -45,6 +45,8 @@ CATEGORIES: dict[str, tuple[str, list[str]]] = {
45 45 "alcool": ("Bières, vins & spiritueux", ["biere", "beer", "vin ", "wine", "cidre", "cider", "gin",
46 46 "vodka", "whisky", "spiritueux", "hydromel", "brasserie"]),
47 47 "sante_beaute": ("Beauté & soins", ["savon", "soap", "cosmetique", "creme", "skincare", "shampoing",
48 + "supplement", "vitamine", "huile essentielle", "aromatherapie",
49 + "hydrolat", "herboristerie", "probiotique",
48 50 "barbe", "beard", "baume", "lotion", "deodorant", "parfum",
49 51 "bain", "bath", "chandelle de massage", "serum", "soin"]),
50 52 "maison": ("Maison & déco", ["chandelle", "bougie", "candle", "deco", "coussin", "vaisselle",
@@ -52,16 +54,21 @@ CATEGORIES: dict[str, tuple[str, list[str]]] = {
52 54 "tablier", "linge", "literie", "couverture", "lampe", "meuble",
53 55 "furniture", "bois", "woodwork", "menuiserie"]),
54 56 "mode": ("Mode & accessoires", ["vetement", "clothing", "t-shirt", "tshirt", "chandail", "hoodie",
57 + "soulier", "sandale", "chaussure", "botte", "espadrille",
58 + "valise", "bagage",
55 59 "tuque", "casquette", "chapeau", "foulard", "mitaine", "bas ",
56 60 "chaussette", "manteau", "robe", "jupe", "pantalon", "legging",
57 61 "sac ", "handbag", "cuir", "leather", "portefeuille", "ceinture"]),
58 62 "bijoux": ("Bijoux", ["bijou", "jewel", "collier", "necklace", "bracelet", "bague", "ring",
63 + "montre", "watch", "horloger",
59 64 "boucle", "earring", "pendentif", "argent sterling",
60 65 "plaque or", "vermeil", "swarovski", "pierre fine"]),
61 66 "art": ("Art & artisanat", ["oeuvre", "art ", "print", "affiche", "poster", "illustration",
67 + "vitrail", "carte de voeux", "carte de vœux",
62 68 "peinture", "sculpture", "photographie", "carte de souhait",
63 69 "papeterie", "stationery", "carnet", "sticker", "autocollant", "macrame"]),
64 70 "enfants": ("Enfants & bébés", ["bebe", "baby", "enfant", "kid", "jouet", "toy", "doudou",
71 + "suce ", "biberon", "poussette",
65 72 "hochet", "couche", "puericulture", "peluche", "figurine",
66 73 "casse-tete", "puzzle", "bricolage", "jeu de societe",
67 74 "jeux de societe", "lego", "poupee"]),
@@ -112,6 +119,56 @@ def details_signal_text(details) -> str:
112 119 return " ".join(p for p in parts if p)
113 120
114 121
122 +# --- Règles de pertinence & de qualité (Phase 2 — enrichissement 2026-08) ----
123 +# Un agrégateur de FABRICANTS n'affiche ni cartes-cadeaux, ni ateliers/cours,
124 +# ni abonnements, ni billets d'événements : ces items restent en base
125 +# (marqués, comptés, réintégrables) mais ne sont pas publiés.
126 +PRICE_SANE_MAX = 100_000.0 # au-delà : prix suspect -> quarantaine
127 +
128 +_GIFT_POS = re.compile(r"(carte|certificat)s?[ -]cadeaux?|gift ?card|gift certificate")
129 +_GIFT_NEG = re.compile(r"porte[ -]cartes?|pochette|enveloppe|presentoir|support|boite|etui")
130 +_WORK_T = re.compile(r"^ateliers?\b|\bateliers? (de|d.|creatif|culinaire|priv|decouverte)"
131 + r"|\bcours (de|d.|en ligne|priv)|^cours\b|^formation|\bformation (en|de|d.)"
132 + r"|\bworkshop\b|\bmasterclass\b|\bwebinaire\b|\bvisite guidee\b"
133 + r"|\bcamp de jour\b|\bdegustation (guidee|priv)")
134 +_WORK_PT = re.compile(r"^(ateliers?|cours|formations?|workshops?|classes?|events?|evenements?)$")
135 +_SUB_T = re.compile(r"^abonnements?\b|\babonnements? (a|au|aux|mensuel|annuel|d.un|de)"
136 + r"|\bsubscription box|^subscription")
137 +_SUB_PT = re.compile(r"^abonnements?$|^subscriptions?$|^adhesions?$|^memberships?$")
138 +_TICK_T = re.compile(r"^billets?\b|\bbillets? (de|d.|pour)|^tickets?\b|\bdroit d.acces"
139 + r"|\bdroits? d.entree|\blaissez-passer\b|^admission\b|^reservation s?\W*$")
140 +_TICK_PT = re.compile(r"^(billets?|billetterie|tickets?|events?|evenements?|spectacles?)$")
141 +_TEST_T = re.compile(r"^tests? ?(product|produit)?\W*$")
142 +
143 +
144 +def classify_listing(title: str, product_type: str = "") -> str:
145 + """'published' ou 'excluded:<raison>' selon le titre/type normalisés.
146 +
147 + Règles calibrées sur la base du 2026-08-19 (623 cartes-cadeaux,
148 + 518 ateliers/cours, 88 abonnements, 23 billets — précision > rappel)."""
149 + t = strip_accents((title or "").lower())
150 + pt = strip_accents((product_type or "").lower()).strip()
151 + if _GIFT_POS.search(t) and not _GIFT_NEG.search(t):
152 + return "excluded:carte_cadeau"
153 + if _WORK_T.search(t) or _WORK_PT.match(pt):
154 + return "excluded:atelier_cours"
155 + if _SUB_T.search(t) or _SUB_PT.match(pt):
156 + return "excluded:abonnement"
157 + if _TICK_T.search(t) or _TICK_PT.match(pt):
158 + return "excluded:billet"
159 + if _TEST_T.match(t):
160 + return "excluded:test"
161 + return "published"
162 +
163 +
164 +def is_placeholder_price(v: float | None) -> bool:
165 + """999 999 $, 99 999 $… : placeholders « sur devis » (ecksand & cie)."""
166 + if v is None or v < 99_999:
167 + return False
168 + digits = str(int(v))
169 + return set(digits) == {"9"}
170 +
171 +
115 172 def parse_price(raw) -> float | None:
116 173 """'24,95 $' | '$24.95' | 24.95 -> 24.95 (CAD)."""
117 174 if raw is None:
@@ -152,6 +209,8 @@ class Product:
152 209 vendor: str = "" # marque affichée par la boutique
153 210 available: bool | None = None
154 211 details: dict = field(default_factory=dict) # avis, dimensions, variantes… (JSON)
212 + listing_status: str = "published" # published | excluded:<raison> | quarantine:<raison>
213 + price_on_request: int = 0 # 1 = sur devis / prix non publié par la boutique
155 214
156 215 @property
157 216 def uid(self) -> str:
@@ -160,19 +219,45 @@ class Product:
160 219 def finalize(self) -> "Product":
161 220 import html as _html
162 221 self.title = _html.unescape(re.sub(r"\s+", " ", self.title or "")).strip()[:300]
163 − self.description = re.sub(r"<[^>]+>", " ", self.description or "")
164 − self.description = _html.unescape(re.sub(r"\s+", " ", self.description)).strip()[:600]
222 + # cap 5 000 car. (l'ancien cap 600 tronquait 34 % des fiches — matériaux
223 + # et dimensions vivent souvent dans la description longue)
224 + self.description = re.sub(r"(?is)<(script|style)[^>]*>.*?</\1>", " ", self.description or "")
225 + self.description = re.sub(r"<[^>]+>", " ", self.description)
226 + self.description = _html.unescape(re.sub(r"\s+", " ", self.description)).strip()[:5000]
165 227 if not self.category:
166 228 self.category = infer_category(self.product_type, " ".join(self.tags),
167 229 self.title, self.description[:200],
168 230 details_signal_text(self.details))
169 − self.images = [i for i in self.images if isinstance(i, str) and i.startswith("http")][:8]
231 + self.images = [i for i in self.images if isinstance(i, str) and i.startswith("http")][:15]
232 + # prix : 0 $ n'est pas un prix (Woo renvoie parfois 0 pour « sur demande »)
233 + if self.price is not None and self.price <= 0:
234 + self.price = None
235 + if self.price_max is not None and self.price_max <= 0:
236 + self.price_max = None
237 + # placeholders « sur devis » neutralisés, jamais publiés comme nombres
238 + if is_placeholder_price(self.price) or is_placeholder_price(self.price_max):
239 + self.details = dict(self.details or {})
240 + self.details["price_placeholder"] = self.price
241 + self.price = self.price_max = self.compare_at_price = None
242 + if self.price is None:
243 + self.price_on_request = 1
244 + # pertinence + quarantaine (aberrations à l'ingestion)
245 + status = classify_listing(self.title, self.product_type)
246 + if status == "published":
247 + if not self.title:
248 + status = "quarantine:titre_vide"
249 + elif self.price is not None and self.price > PRICE_SANE_MAX:
250 + status = "quarantine:prix_hors_bornes"
251 + self.listing_status = status
170 252 return self
171 253
172 254 def content_hash(self) -> str:
255 + # len(description)/len(images) : force la mise à jour quand les caps
256 + # changent (600→5000, 8→15) ou qu'une galerie s'enrichit (Square)
173 257 basis = json.dumps([self.title, self.price, self.price_max, self.available,
174 − self.images[:1], self.description[:200],
175 − self.details or None], ensure_ascii=False, sort_keys=True)
258 + self.images[:1], len(self.images), self.description[:200],
259 + len(self.description), self.details or None],
260 + ensure_ascii=False, sort_keys=True)
176 261 return hashlib.sha1(basis.encode()).hexdigest()[:16]
177 262
178 263 def to_row(self) -> dict:
modified fabrika/seo.py +14 −14
@@ -236,7 +236,7 @@ def pagination_html(path: str, page: int, total: int, per_page: int = PER_PAGE)
236 236
237 237 def list_products(con, category: str | None, region: str | None,
238 238 page: int, per_page: int = PER_PAGE) -> tuple[int, list[dict]]:
239 − where, args = ["p.active=1"], []
239 + where, args = ["p.active=1 AND p.listing_status='published'"], []
240 240 if category:
241 241 where.append("p.category=?"); args.append(category)
242 242 if region:
@@ -256,7 +256,7 @@ def listing_stats(category: str | None, region: str | None) -> dict:
256 256 def _run():
257 257 con = db.connect()
258 258 try:
259 − where, args = ["p.active=1", "p.price>0", "p.price<=500000"], []
259 + where, args = ["p.active=1 AND p.listing_status='published'", "p.price>0", "p.price<=500000"], []
260 260 if category:
261 261 where.append("p.category=?"); args.append(category)
262 262 if region:
@@ -285,7 +285,7 @@ def combo_counts() -> list[dict]:
285 285 return q(con, """SELECT p.category AS cat, s.region AS region,
286 286 COUNT(*) AS n, MAX(p.last_seen) AS m
287 287 FROM products p JOIN stores s ON s.id=p.store_id
288 − WHERE p.active=1 GROUP BY p.category, s.region""")
288 + WHERE p.active=1 AND p.listing_status='published' GROUP BY p.category, s.region""")
289 289 finally:
290 290 con.close()
291 291 return cached("combos", 3600, _run)
@@ -309,21 +309,21 @@ def page_home() -> HTMLResponse:
309 309 con = db.connect()
310 310 try:
311 311 totals = q(con, """SELECT
312 − (SELECT COUNT(*) FROM products WHERE active=1) AS products,
312 + (SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published') AS products,
313 313 (SELECT COUNT(*) FROM stores WHERE product_count>0) AS stores,
314 314 (SELECT COUNT(DISTINCT region) FROM stores WHERE region<>'' AND product_count>0) AS regions
315 315 """)[0]
316 316 cats = q(con, """SELECT category AS key, COUNT(*) AS n FROM products
317 − WHERE active=1 GROUP BY category ORDER BY n DESC""")
317 + WHERE active=1 AND listing_status='published' GROUP BY category ORDER BY n DESC""")
318 318 regions = q(con, """SELECT s.region AS key, COUNT(*) AS n FROM products p
319 319 JOIN stores s ON s.id=p.store_id
320 − WHERE p.active=1 AND s.region<>'' GROUP BY s.region ORDER BY n DESC""")
320 + WHERE p.active=1 AND p.listing_status='published' AND s.region<>'' GROUP BY s.region ORDER BY n DESC""")
321 321 top_stores = q(con, """SELECT id, name, product_count AS n FROM stores
322 322 WHERE product_count>0 ORDER BY n DESC LIMIT 20""")
323 323 newest = q(con, """SELECT p.uid, p.title, p.price, p.images,
324 324 s.name AS store_name, s.region AS store_region
325 325 FROM products p JOIN stores s ON s.id=p.store_id
326 − WHERE p.active=1 ORDER BY p.first_seen DESC LIMIT 12""")
326 + WHERE p.active=1 AND p.listing_status='published' ORDER BY p.first_seen DESC LIMIT 12""")
327 327 return totals, cats, regions, top_stores, newest
328 328 finally:
329 329 con.close()
@@ -525,7 +525,7 @@ def page_product(slug: str | None, uid: str, params) -> Response:
525 525 related = q(con, """SELECT p.uid, p.title, p.price, p.images, s.name AS store_name,
526 526 s.region AS store_region
527 527 FROM products p JOIN stores s ON s.id=p.store_id
528 − WHERE p.store_id=? AND p.uid<>? AND p.active=1
528 + WHERE p.store_id=? AND p.uid<>? AND p.active=1 AND p.listing_status='published'
529 529 ORDER BY p.first_seen DESC LIMIT 8""", (p["store_id"], uid))
530 530 finally:
531 531 con.close()
@@ -647,15 +647,15 @@ def page_store(store_id: str) -> Response:
647 647 s = rows[0]
648 648 stats = q(con, """SELECT COUNT(*) AS n, MIN(price) AS pmin, MAX(price) AS pmax,
649 649 ROUND(AVG(price),2) AS pavg
650 − FROM products WHERE store_id=? AND active=1 AND price>0""",
650 + FROM products WHERE store_id=? AND active=1 AND listing_status='published' AND price>0""",
651 651 (store_id,))[0]
652 652 cats = q(con, """SELECT category AS key, COUNT(*) AS n FROM products
653 − WHERE store_id=? AND active=1 GROUP BY category
653 + WHERE store_id=? AND active=1 AND listing_status='published' GROUP BY category
654 654 ORDER BY n DESC LIMIT 8""", (store_id,))
655 655 prods = q(con, """SELECT p.uid, p.title, p.price, p.images, s.name AS store_name,
656 656 s.region AS store_region
657 657 FROM products p JOIN stores s ON s.id=p.store_id
658 − WHERE p.store_id=? AND p.active=1
658 + WHERE p.store_id=? AND p.active=1 AND p.listing_status='published'
659 659 ORDER BY p.first_seen DESC LIMIT 24""", (store_id,))
660 660 finally:
661 661 con.close()
@@ -706,7 +706,7 @@ def page_stats() -> HTMLResponse:
706 706 con = db.connect()
707 707 try:
708 708 return q(con, """SELECT
709 − (SELECT COUNT(*) FROM products WHERE active=1) AS products,
709 + (SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published') AS products,
710 710 (SELECT COUNT(*) FROM stores WHERE product_count>0) AS stores
711 711 """)[0]
712 712 finally:
@@ -817,7 +817,7 @@ def _product_chunks() -> int:
817 817 def _run():
818 818 con = db.connect()
819 819 try:
820 − n = con.execute("SELECT COUNT(*) FROM products WHERE active=1").fetchone()[0]
820 + n = con.execute("SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published'").fetchone()[0]
821 821 finally:
822 822 con.close()
823 823 return max(1, -(-n // SITEMAP_CHUNK))
@@ -872,7 +872,7 @@ def sitemap(name: str) -> Response:
872 872 def _run():
873 873 con = db.connect()
874 874 try:
875 − rows = q(con, """SELECT uid, title, last_seen FROM products WHERE active=1
875 + rows = q(con, """SELECT uid, title, last_seen FROM products WHERE active=1 AND listing_status='published'
876 876 ORDER BY uid LIMIT ? OFFSET ?""",
877 877 (SITEMAP_CHUNK, (idx - 1) * SITEMAP_CHUNK))
878 878 finally:
modified fabrika/statsdash.py +25 −25
@@ -122,12 +122,12 @@ def _daily(con, t0: float, t1: float) -> dict[str, int]:
122 122
123 123 def _median(con, extra_where: str = "", args: tuple = ()) -> float | None:
124 124 n = con.execute(
125 − f"SELECT COUNT(*) FROM products WHERE active=1 AND price>0{extra_where}",
125 + f"SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published' AND price>0{extra_where}",
126 126 args).fetchone()[0]
127 127 if not n:
128 128 return None
129 129 row = con.execute(
130 − f"""SELECT price FROM products WHERE active=1 AND price>0{extra_where}
130 + f"""SELECT price FROM products WHERE active=1 AND listing_status='published' AND price>0{extra_where}
131 131 ORDER BY price LIMIT 1 OFFSET ?""", args + (n // 2,)).fetchone()
132 132 return round(row[0], 2) if row else None
133 133
@@ -145,33 +145,33 @@ def _build(period: str, from_: str | None, to: str | None) -> dict:
145 145 iso_days = [d.isoformat() for d in days]
146 146
147 147 # ----- KPI : état courant vs état au début de la période -------------
148 − total = one("SELECT COUNT(*) FROM products WHERE active=1")
149 − total_t0 = one("SELECT COUNT(*) FROM products WHERE active=1 AND first_seen<?", (t0,))
148 + total = one("SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published'")
149 + total_t0 = one("SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published' AND first_seen<?", (t0,))
150 150 stores_live = one("SELECT COUNT(*) FROM stores WHERE product_count>0")
151 151 stores_t0 = one("""SELECT COUNT(DISTINCT store_id) FROM products
152 − WHERE active=1 AND first_seen<?""", (t0,))
152 + WHERE active=1 AND listing_status='published' AND first_seen<?""", (t0,))
153 153 stores_reg = one("SELECT COUNT(*) FROM stores")
154 154 new_cur = one("SELECT COUNT(*) FROM products WHERE first_seen>=? AND first_seen<?", (t0, t1))
155 155 new_prev = one("SELECT COUNT(*) FROM products WHERE first_seen>=? AND first_seen<?", (pt0, pt1))
156 156 avg_now = one("""SELECT ROUND(AVG(price),2) FROM products
157 − WHERE active=1 AND price>0 AND price<=?""", (PRICE_CAP,))
157 + WHERE active=1 AND listing_status='published' AND price>0 AND price<=?""", (PRICE_CAP,))
158 158 avg_t0 = one("""SELECT ROUND(AVG(price),2) FROM products
159 − WHERE active=1 AND price>0 AND price<=? AND first_seen<?""",
159 + WHERE active=1 AND listing_status='published' AND price>0 AND price<=? AND first_seen<?""",
160 160 (PRICE_CAP, t0))
161 161 med_now = _median(con)
162 162 med_t0 = _median(con, " AND first_seen<?", (t0,))
163 − cats = one("SELECT COUNT(DISTINCT category) FROM products WHERE active=1 AND category<>''")
163 + cats = one("SELECT COUNT(DISTINCT category) FROM products WHERE active=1 AND listing_status='published' AND category<>''")
164 164 cats_t0 = one("""SELECT COUNT(DISTINCT category) FROM products
165 − WHERE active=1 AND category<>'' AND first_seen<?""", (t0,))
165 + WHERE active=1 AND listing_status='published' AND category<>'' AND first_seen<?""", (t0,))
166 166 regions = one("""SELECT COUNT(DISTINCT region) FROM stores
167 167 WHERE region<>'' AND product_count>0""")
168 168 regions_t0 = one("""SELECT COUNT(DISTINCT s.region) FROM stores s
169 169 JOIN products p ON p.store_id=s.id
170 − WHERE s.region<>'' AND p.active=1 AND p.first_seen<?""", (t0,))
170 + WHERE s.region<>'' AND p.active=1 AND p.listing_status='published' AND p.first_seen<?""", (t0,))
171 171 img_where = " AND images IS NOT NULL AND images<>'' AND images<>'[]'"
172 − with_img = one(f"SELECT COUNT(*) FROM products WHERE active=1{img_where}")
172 + with_img = one(f"SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published'{img_where}")
173 173 with_img_t0 = one(f"""SELECT COUNT(*) FROM products
174 − WHERE active=1 AND first_seen<?{img_where}""", (t0,))
174 + WHERE active=1 AND listing_status='published' AND first_seen<?{img_where}""", (t0,))
175 175
176 176 # ----- séries quotidiennes (aussi utilisées comme sparklines) --------
177 177 cur_daily = _daily(con, t0, t1)
@@ -250,8 +250,8 @@ def _build(period: str, from_: str | None, to: str | None) -> dict:
250 250 ]
251 251
252 252 # ----- jauges : couvertures mesurées ----------------------------------
253 − with_price = one("SELECT COUNT(*) FROM products WHERE active=1 AND price>0")
254 − avail = one("SELECT COUNT(*) FROM products WHERE active=1 AND available=1")
253 + with_price = one("SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published' AND price>0")
254 + avail = one("SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published' AND available=1")
255 255 live_geo = one("""SELECT COUNT(*) FROM stores
256 256 WHERE product_count>0 AND region<>'' AND region IS NOT NULL""")
257 257 pct = lambda a, b: round(100.0 * a / b, 1) if b else None # noqa: E731
@@ -344,11 +344,11 @@ def _build(period: str, from_: str | None, to: str | None) -> dict:
344 344 WHERE product_count>0 GROUP BY platform
345 345 ORDER BY 2 DESC""").fetchall()
346 346 top_cats = con.execute("""SELECT category, COUNT(*) FROM products
347 − WHERE active=1 GROUP BY category
347 + WHERE active=1 AND listing_status='published' GROUP BY category
348 348 ORDER BY 2 DESC LIMIT 12""").fetchall()
349 349 cats_t0_rows = dict(con.execute(
350 350 """SELECT category, COUNT(*) FROM products
351 − WHERE active=1 AND first_seen<? GROUP BY category""", (t0,)).fetchall())
351 + WHERE active=1 AND listing_status='published' AND first_seen<? GROUP BY category""", (t0,)).fetchall())
352 352 # donut catégories : top 7 + « Autres »
353 353 donut_cats = [{"label": _cat_label(c), "value": n} for c, n in top_cats[:7]]
354 354 rest = total - sum(n for _, n in top_cats[:7])
@@ -356,15 +356,15 @@ def _build(period: str, from_: str | None, to: str | None) -> dict:
356 356 donut_cats.append({"label": "Autres", "value": rest})
357 357 origin_rows = con.execute(
358 358 """SELECT COALESCE(s.origin_class,''), COUNT(p.uid) FROM stores s
359 − JOIN products p ON p.store_id=s.id AND p.active=1
359 + JOIN products p ON p.store_id=s.id AND p.active=1 AND p.listing_status='published'
360 360 GROUP BY 1 ORDER BY 2 DESC""").fetchall()
361 361 region_now = con.execute(
362 362 """SELECT s.region, COUNT(p.uid) FROM stores s
363 − JOIN products p ON p.store_id=s.id AND p.active=1
363 + JOIN products p ON p.store_id=s.id AND p.active=1 AND p.listing_status='published'
364 364 WHERE s.region<>'' GROUP BY s.region ORDER BY 2 DESC""").fetchall()
365 365 region_t0 = dict(con.execute(
366 366 """SELECT s.region, COUNT(p.uid) FROM stores s
367 − JOIN products p ON p.store_id=s.id AND p.active=1
367 + JOIN products p ON p.store_id=s.id AND p.active=1 AND p.listing_status='published'
368 368 AND p.first_seen<?
369 369 WHERE s.region<>'' GROUP BY s.region""", (t0,)).fetchall())
370 370 breakdowns = [
@@ -391,7 +391,7 @@ def _build(period: str, from_: str | None, to: str | None) -> dict:
391 391 WHEN price < 50 THEN '25-50' WHEN price < 100 THEN '50-100'
392 392 WHEN price < 250 THEN '100-250' WHEN price < 1000 THEN '250-1000'
393 393 ELSE '1000+' END AS b, COUNT(*) FROM products
394 − WHERE active=1 AND price>0 GROUP BY b""").fetchall())
394 + WHERE active=1 AND listing_status='published' AND price>0 GROUP BY b""").fetchall())
395 395 sizes = dict(con.execute("""SELECT CASE
396 396 WHEN product_count <= 10 THEN '1-10'
397 397 WHEN product_count <= 50 THEN '11-50'
@@ -440,20 +440,20 @@ def _build(period: str, from_: str | None, to: str | None) -> dict:
440 440 (SELECT COUNT(*) FROM products p WHERE p.store_id=s.id
441 441 AND p.first_seen>=? AND p.first_seen<?) AS nouv,
442 442 (SELECT ROUND(AVG(p.price),2) FROM products p
443 − WHERE p.store_id=s.id AND p.active=1 AND p.price>0
443 + WHERE p.store_id=s.id AND p.active=1 AND p.listing_status='published' AND p.price>0
444 444 AND p.price<=?) AS pavg
445 445 FROM stores s WHERE s.product_count>0
446 446 ORDER BY s.product_count DESC LIMIT 50""", (t0, t1, PRICE_CAP)).fetchall()
447 447 cat_rows = con.execute("""
448 448 SELECT category, COUNT(*) AS n, COUNT(DISTINCT store_id) AS st,
449 449 ROUND(AVG(CASE WHEN price>0 AND price<=? THEN price END),2)
450 − FROM products WHERE active=1 GROUP BY category
450 + FROM products WHERE active=1 AND listing_status='published' GROUP BY category
451 451 ORDER BY n DESC""", (PRICE_CAP,)).fetchall()
452 452 region_tbl = con.execute("""
453 453 SELECT s.region, COUNT(DISTINCT s.id) AS st, COUNT(p.uid) AS n,
454 454 ROUND(AVG(CASE WHEN p.price>0 AND p.price<=? THEN p.price END),2),
455 455 SUM(CASE WHEN p.first_seen>=? AND p.first_seen<? THEN 1 ELSE 0 END)
456 − FROM stores s JOIN products p ON p.store_id=s.id AND p.active=1
456 + FROM stores s JOIN products p ON p.store_id=s.id AND p.active=1 AND p.listing_status='published'
457 457 WHERE s.region<>'' GROUP BY s.region ORDER BY n DESC""",
458 458 (PRICE_CAP, t0, t1)).fetchall()
459 459 nouv_rows = con.execute("""
@@ -534,7 +534,7 @@ def _build(period: str, from_: str | None, to: str | None) -> dict:
534 534 + _int(top_cats[0][1]) + " produits"})
535 535 rich_cat = con.execute("""SELECT category,
536 536 ROUND(AVG(CASE WHEN price>0 AND price<=? THEN price END),2) AS a
537 − FROM products WHERE active=1 GROUP BY category
537 + FROM products WHERE active=1 AND listing_status='published' GROUP BY category
538 538 HAVING COUNT(*)>=100 AND a IS NOT NULL
539 539 ORDER BY a DESC LIMIT 1""", (PRICE_CAP,)).fetchone()
540 540 if rich_cat:
@@ -545,7 +545,7 @@ def _build(period: str, from_: str | None, to: str | None) -> dict:
545 545 "value": f"{PLATFORM_LABELS.get(plat[0][0] or '', plat[0][0] or 'Inconnue')} — "
546 546 + _int(plat[0][1]) + " boutiques"})
547 547 dear = con.execute("""SELECT p.title, p.price, s.name FROM products p
548 − JOIN stores s ON s.id=p.store_id WHERE p.active=1 AND p.price>0
548 + JOIN stores s ON s.id=p.store_id WHERE p.active=1 AND p.listing_status='published' AND p.price>0
549 549 AND p.price<=? ORDER BY p.price DESC LIMIT 1""", (PRICE_CAP,)).fetchone()
550 550 if dear:
551 551 records.append({"label": f"Produit le plus cher au catalogue — {(dear[0] or '')[:34]} ({dear[2]})",
modified fabrika/web.py +45 −33
@@ -59,13 +59,20 @@ def products(q_text: str | None = Query(None, alias="q"),
59 59 price_min: float | None = None,
60 60 price_max: float | None = None,
61 61 available: bool | None = None,
62 + kind: str | None = None,
62 63 sort: str = "recent",
63 64 page: int = 1,
64 65 per_page: int = Query(24, le=100)):
65 66 con = db.connect()
66 67 try:
67 − where, args = ["p.active=1"], []
68 + # listing_status : cartes-cadeaux/ateliers/abonnements/billets exclus
69 + # et aberrations en quarantaine ne sont jamais publiés
70 + where, args = ["p.active=1", "p.listing_status='published'"], []
68 71 joins = "FROM products p JOIN stores s ON s.id = p.store_id"
72 + if kind == "fabricant":
73 + # « fabricants seulement » : écarte revendeurs et hors-mission
74 + # (les collectifs d'artisans restent : produits faits au Québec)
75 + where.append("COALESCE(s.store_kind,'') NOT IN ('revendeur','hors_mission')")
69 76 if q_text:
70 77 joins += " JOIN products_fts f ON f.uid = p.uid"
71 78 where.append("products_fts MATCH ?")
@@ -92,7 +99,7 @@ def products(q_text: str | None = Query(None, alias="q"),
92 99 "title": "p.title COLLATE NOCASE"}.get(sort, "p.first_seen DESC")
93 100 total = con.execute(f"SELECT COUNT(*) {joins} WHERE {wsql}", args).fetchone()[0]
94 101 rows = q(con, f"""SELECT p.*, s.name AS store_name, s.region AS store_region,
95 − s.city AS store_city, s.origin_class
102 + s.city AS store_city, s.origin_class, s.store_kind
96 103 {joins} WHERE {wsql} ORDER BY {order}
97 104 LIMIT ? OFFSET ?""", args + [per_page, (page - 1) * per_page])
98 105 return {"total": total, "page": page, "per_page": per_page,
@@ -106,14 +113,16 @@ def product(uid: str):
106 113 con = db.connect()
107 114 try:
108 115 rows = q(con, """SELECT p.*, s.name AS store_name, s.region AS store_region,
109 − s.city AS store_city, s.origin_class, s.url AS store_url
116 + s.city AS store_city, s.origin_class, s.store_kind,
117 + s.url AS store_url
110 118 FROM products p JOIN stores s ON s.id=p.store_id WHERE p.uid=?""", (uid,))
111 119 if not rows:
112 120 raise HTTPException(404)
113 121 out = _product_out(rows[0])
114 122 out["related"] = [_product_out(r) for r in q(con, """
115 123 SELECT p.*, s.name AS store_name FROM products p JOIN stores s ON s.id=p.store_id
116 − WHERE p.store_id=? AND p.uid<>? AND p.active=1 ORDER BY RANDOM() LIMIT 8""",
124 + WHERE p.store_id=? AND p.uid<>? AND p.active=1 AND p.listing_status='published'
125 + ORDER BY RANDOM() LIMIT 8""",
117 126 (rows[0]["store_id"], uid))]
118 127 return out
119 128 finally:
@@ -140,7 +149,8 @@ def stores(region: str | None = None, platform: str | None = None,
140 149 rows = q(con, f"""SELECT id, name, url, platform, city, region, origin_class,
141 150 origin_confidence, categories, language, product_count,
142 151 last_sync, last_status, logo_url, cover_url,
143 − description_meta
152 + description_meta, store_kind, email, phone, lat, lng,
153 + shipping_info
144 154 FROM stores WHERE {' AND '.join(where)}
145 155 ORDER BY product_count DESC, name""", args)
146 156 for r in rows:
@@ -163,11 +173,11 @@ def store_detail(store_id: str):
163 173 # statistiques produits
164 174 stats = q(con, """SELECT COUNT(*) AS n, MIN(price) AS price_min,
165 175 MAX(price) AS price_max, AVG(price) AS price_avg
166 − FROM products WHERE store_id=? AND active=1 AND price>0""",
176 + FROM products WHERE store_id=? AND active=1 AND listing_status='published' AND price>0""",
167 177 (store_id,))
168 178 s["product_stats"] = stats[0] if stats else {}
169 179 s["category_breakdown"] = q(con, """SELECT category AS key, COUNT(*) AS n
170 − FROM products WHERE store_id=? AND active=1 GROUP BY category
180 + FROM products WHERE store_id=? AND active=1 AND listing_status='published' GROUP BY category
171 181 ORDER BY n DESC LIMIT 8""", (store_id,))
172 182 for c in s["category_breakdown"]:
173 183 c["label"] = CATEGORIES.get(c["key"], (c["key"], []))[0]
@@ -177,7 +187,7 @@ def store_detail(store_id: str):
177 187 SELECT DISTINCT st.id, st.name, st.region, st.origin_class, st.logo_url,
178 188 st.product_count
179 189 FROM stores st
180 − LEFT JOIN products p ON p.store_id=st.id AND p.active=1 AND p.category=?
190 + LEFT JOIN products p ON p.store_id=st.id AND p.active=1 AND p.listing_status='published' AND p.category=?
181 191 WHERE st.id<>? AND st.product_count>0
182 192 AND (p.uid IS NOT NULL OR (st.region<>'' AND st.region=?))
183 193 ORDER BY (st.region=?) DESC, st.product_count DESC LIMIT 8""",
@@ -192,14 +202,17 @@ def facets():
192 202 con = db.connect()
193 203 try:
194 204 cats = q(con, """SELECT p.category AS key, COUNT(*) AS n FROM products p
195 − WHERE p.active=1 GROUP BY p.category ORDER BY n DESC""")
205 + WHERE p.active=1 AND p.listing_status='published'
206 + GROUP BY p.category ORDER BY n DESC""")
196 207 for c in cats:
197 208 c["label"] = CATEGORIES.get(c["key"], (c["key"], []))[0]
198 209 regions = q(con, """SELECT s.region AS key, COUNT(*) AS n FROM products p
199 210 JOIN stores s ON s.id=p.store_id
200 − WHERE p.active=1 AND s.region<>'' GROUP BY s.region ORDER BY n DESC""")
211 + WHERE p.active=1 AND p.listing_status='published' AND s.region<>''
212 + GROUP BY s.region ORDER BY n DESC""")
201 213 origins = q(con, """SELECT s.origin_class AS key, COUNT(*) AS n FROM products p
202 − JOIN stores s ON s.id=p.store_id WHERE p.active=1
214 + JOIN stores s ON s.id=p.store_id
215 + WHERE p.active=1 AND p.listing_status='published'
203 216 GROUP BY s.origin_class ORDER BY n DESC""")
204 217 top_stores = q(con, """SELECT id AS key, name, product_count AS n FROM stores
205 218 WHERE product_count>0 ORDER BY n DESC LIMIT 40""")
@@ -235,13 +248,13 @@ def stats_extended():
235 248 MIN(CASE WHEN price>0 THEN price END) AS price_min,
236 249 ROUND(AVG(CASE WHEN price>0 AND price<=500000 THEN price END),2) AS price_avg,
237 250 MAX(CASE WHEN price<=500000 THEN price END) AS price_max
238 − FROM products WHERE active=1 GROUP BY category
251 + FROM products WHERE active=1 AND listing_status='published' GROUP BY category
239 252 ORDER BY products DESC"""):
240 253 r["label"] = CATEGORIES.get(r["key"], (r["key"], []))[0]
241 − r["price_median"] = median_price("active=1 AND category=?", [r["key"]])
254 + r["price_median"] = median_price("active=1 AND listing_status='published' AND category=?", [r["key"]])
242 255 by_category.append(r)
243 256
244 − global_median = median_price("active=1", [])
257 + global_median = median_price("active=1 AND listing_status='published'", [])
245 258 price_buckets = q(con, """SELECT CASE
246 259 WHEN price < 10 THEN '0-10'
247 260 WHEN price < 25 THEN '10-25'
@@ -250,14 +263,14 @@ def stats_extended():
250 263 WHEN price < 250 THEN '100-250'
251 264 WHEN price < 1000 THEN '250-1000'
252 265 ELSE '1000+' END AS bucket, COUNT(*) AS n
253 − FROM products WHERE active=1 AND price>0 GROUP BY bucket""")
266 + FROM products WHERE active=1 AND listing_status='published' AND price>0 GROUP BY bucket""")
254 267 order = ['0-10', '10-25', '25-50', '50-100', '100-250', '250-1000', '1000+']
255 268 price_buckets.sort(key=lambda b: order.index(b["bucket"]))
256 269
257 270 by_origin = q(con, """SELECT s.origin_class AS key, COUNT(DISTINCT s.id) AS stores,
258 271 COUNT(p.uid) AS products
259 272 FROM stores s LEFT JOIN products p
260 − ON p.store_id=s.id AND p.active=1
273 + ON p.store_id=s.id AND p.active=1 AND p.listing_status='published'
261 274 GROUP BY s.origin_class ORDER BY products DESC""")
262 275 by_platform = q(con, """SELECT platform AS key, COUNT(*) AS stores,
263 276 SUM(product_count) AS products
@@ -267,10 +280,10 @@ def stats_extended():
267 280 COUNT(p.uid) AS products,
268 281 ROUND(AVG(CASE WHEN p.price>0 AND p.price<=500000 THEN p.price END),2) AS price_avg
269 282 FROM stores s LEFT JOIN products p
270 − ON p.store_id=s.id AND p.active=1
283 + ON p.store_id=s.id AND p.active=1 AND p.listing_status='published'
271 284 WHERE s.region<>'' GROUP BY s.region ORDER BY products DESC""")
272 285 growth = q(con, """SELECT date(first_seen,'unixepoch') AS day, COUNT(*) AS n
273 − FROM products WHERE active=1
286 + FROM products WHERE active=1 AND listing_status='published'
274 287 GROUP BY day ORDER BY day DESC LIMIT 30""")
275 288 top_stores = q(con, """SELECT id, name, region, origin_class, logo_url,
276 289 product_count AS products
@@ -278,30 +291,30 @@ def stats_extended():
278 291 ORDER BY product_count DESC LIMIT 15""")
279 292 most_expensive = q(con, """SELECT p.uid, p.title, p.price, s.name AS store_name
280 293 FROM products p JOIN stores s ON s.id=p.store_id
281 − WHERE p.active=1 AND p.price>0
294 + WHERE p.active=1 AND p.listing_status='published' AND p.price>0
282 295 ORDER BY p.price DESC LIMIT 8""")
283 296 newest = q(con, """SELECT p.uid, p.title, p.price, p.category, s.name AS store_name
284 297 FROM products p JOIN stores s ON s.id=p.store_id
285 − WHERE p.active=1 ORDER BY p.first_seen DESC LIMIT 12""")
298 + WHERE p.active=1 AND p.listing_status='published' ORDER BY p.first_seen DESC LIMIT 12""")
286 299 for r in newest:
287 300 r["category_label"] = CATEGORIES.get(r["category"], (r["category"], []))[0]
288 301 availability = q(con, """SELECT CASE WHEN available=1 THEN 'en stock'
289 302 WHEN available=0 THEN 'rupture'
290 303 ELSE 'inconnu' END AS key, COUNT(*) AS n
291 − FROM products WHERE active=1 GROUP BY key""")
304 + FROM products WHERE active=1 AND listing_status='published' GROUP BY key""")
292 305 coverage = q(con, """SELECT
293 − (SELECT COUNT(*) FROM products WHERE active=1 AND images<>'[]' AND images IS NOT NULL) AS with_image,
294 − (SELECT COUNT(*) FROM products WHERE active=1 AND description<>'') AS with_desc,
306 + (SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published' AND images<>'[]' AND images IS NOT NULL) AS with_image,
307 + (SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published' AND description<>'') AS with_desc,
295 308 (SELECT COUNT(*) FROM stores WHERE logo_url IS NOT NULL AND logo_url<>'') AS with_logo,
296 309 (SELECT COUNT(*) FROM stores WHERE region<>'') AS with_region
297 310 """)[0]
298 311 totals = q(con, """SELECT
299 − (SELECT COUNT(*) FROM products WHERE active=1) AS products,
300 − (SELECT COUNT(*) FROM products WHERE active=1 AND price>0) AS products_priced,
312 + (SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published') AS products,
313 + (SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published' AND price>0) AS products_priced,
301 314 (SELECT COUNT(*) FROM stores WHERE product_count>0) AS stores_live,
302 315 (SELECT COUNT(*) FROM stores) AS stores_registry,
303 316 (SELECT COUNT(DISTINCT region) FROM stores WHERE region<>'') AS regions,
304 − (SELECT ROUND(AVG(price),2) FROM products WHERE active=1 AND price>0 AND price<=500000) AS price_avg
317 + (SELECT ROUND(AVG(price),2) FROM products WHERE active=1 AND listing_status='published' AND price>0 AND price<=500000) AS price_avg
305 318 """)[0]
306 319 totals["price_median"] = global_median
307 320 data = {"totals": totals, "by_category": by_category,
@@ -348,9 +361,7 @@ def stats_report(period: str = "30j",
348 361 from_: str | None = Query(None, alias="from"),
349 362 to: str | None = None,
350 363 mode: str = "complet"):
351 − """Rapport statistique PDF estampillé Groupe-KA (gabarit kapdf/fpdf2 v2).
352 − 5 modes : complet | synthese | tendances | repartitions | donnees —
353 − un mode inconnu retombe sur `complet` (SPEC §3)."""
364 + """Rapport statistique PDF estampillé Groupe-KA (gabarit kapdf/fpdf2)."""
354 365 from fastapi.responses import Response
355 366
356 367 from . import kapdf, statsdash
@@ -361,9 +372,10 @@ def stats_report(period: str = "30j",
361 372 site = {"wordmark": "Fabri·Ka", "accent": "#c4532e",
362 373 "domain": "www.fabri-ka.com",
363 374 "tagline": "Tous les produits québécois. Un seul endroit."}
364 − mode = mode if mode in kapdf.REPORT_MODES else "complet"
365 − pdf = kapdf.GroupeKAReport(site=site, dashboard=dash, mode=mode).build()
366 − fname = kapdf.filename("fabri-ka", period, mode)
375 + pdf = kapdf.GroupeKAReport(
376 + site=site, dashboard=dash,
377 + mode="synthese" if mode == "synthese" else "complet").build()
378 + fname = kapdf.filename("fabri-ka", period)
367 379 return Response(content=pdf, media_type="application/pdf",
368 380 headers={"Content-Disposition": f'attachment; filename="{fname}"'})
369 381
@@ -373,7 +385,7 @@ def stats():
373 385 con = db.connect()
374 386 try:
375 387 totals = q(con, """SELECT
376 − (SELECT COUNT(*) FROM products WHERE active=1) AS products,
388 + (SELECT COUNT(*) FROM products WHERE active=1 AND listing_status='published') AS products,
377 389 (SELECT COUNT(*) FROM stores WHERE product_count>0) AS stores_live,
378 390 (SELECT COUNT(*) FROM stores) AS stores_registry,
379 391 (SELECT COUNT(DISTINCT region) FROM stores WHERE region<>'' AND product_count>0) AS regions
added scripts/backfill_quality.py +226 −0
@@ -0,0 +1,226 @@
1 +#!/usr/bin/env python3
2 +"""Phase 2 — backfill qualité (2026-08-19).
3 +
4 +Applique aux données DÉJÀ en base les règles désormais appliquées à
5 +l'ingestion (schema.py) :
6 +
7 +1. Produits :
8 + - prix placeholders (999 999 $…) -> price=NULL + price_on_request=1 ;
9 + - produits sans prix -> price_on_request=1 ;
10 + - prix > 100 000 $ -> quarantine:prix_hors_bornes ;
11 + - titre vide -> quarantine:titre_vide ;
12 + - cartes-cadeaux / ateliers-cours / abonnements / billets / produits test
13 + -> excluded:<raison> (marqués, jamais supprimés, réintégrables).
14 +2. Boutiques :
15 + - renommage des noms génériques d'annuaire (« Créateurs », « Complices »…)
16 + et des groupes dupliqués via og:site_name du cache d'enrichissement
17 + (data/enrich_cache/<dom>.json) — registre + DB + réindexation FTS ;
18 + - store_kind (fabricant / revendeur / collectif / hors_mission) :
19 + * revendeur : origin_evidence du registre contenant « revend » +
20 + liste curée (marques tierces dominantes constatées en base) ;
21 + * collectif : boutiques multi-créateurs connues ;
22 + * hors_mission : transport/billetterie (produits exclus en masse).
23 + - postal_prefix recopié du registre vers la DB (upsert le maintient ensuite).
24 +
25 +Idempotent ; usage : .venv/bin/python scripts/backfill_quality.py [--dry-run]
26 +"""
27 +import argparse
28 +import json
29 +import os
30 +import re
31 +import sys
32 +
33 +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
34 +sys.path.insert(0, ROOT)
35 +
36 +from fabrika import db as fdb # noqa: E402
37 +from fabrika.schema import classify_listing, is_placeholder_price, PRICE_SANE_MAX # noqa: E402
38 +
39 +REG_PATH = os.path.join(ROOT, "data", "stores.json")
40 +ENRICH_CACHE = os.path.join(ROOT, "data", "enrich_cache")
41 +
42 +# Noms de boutiques génériques hérités des rubriques d'annuaires
43 +BAD_SITE_NAMES = {"your site title", "site title", "home", "home page",
44 + "accueil", "untitled", "my site", "mon site"}
45 +
46 +GENERIC_NAMES = {"créateurs", "createurs", "producteurs/transformateurs",
47 + "producteurs", "transformateurs", "complices", "accueil",
48 + "home", "boutique", "produits", "shop", "menu", "à propos"}
49 +
50 +# revendeurs constatés (marques tierces dominantes dans le catalogue :
51 +# Tissot, BIBS, Rieker, Natural Factors, Pacsafe… — voir 05-ENRICHISSEMENT-RAPPORT)
52 +CURATED_REVENDEUR = {
53 + "lamaisondubleuet.com", "mondeavie.ca", "lesptitsmosus.com",
54 + "grenierboutique.ca", "yellowshoes.com", "bijouteriejodoin.com",
55 + "eugeneallard.com", "remorquetrailer.com", "doggoboutique.ca",
56 + "boiteavins.com",
57 +}
58 +# boutiques multi-créateurs (vendor = fabricant réel)
59 +CURATED_COLLECTIF = {"paperole.com", "signelocal.com", "wachiya.com",
60 + "galerieiris.com"}
61 +# hors mission fabricant (transport, activités, billetterie)
62 +CURATED_HORS_MISSION = {"traversiers.com", "rtcbq.com", "raftingmomentum.com"}
63 +
64 +
65 +def main():
66 + ap = argparse.ArgumentParser()
67 + ap.add_argument("--dry-run", action="store_true")
68 + args = ap.parse_args()
69 +
70 + con = fdb.connect()
71 +
72 + # ------------------------------------------------------------------ produits
73 + stats = {"placeholder": 0, "on_request": 0, "quarantine_prix": 0,
74 + "quarantine_titre": 0}
75 + excl = {}
76 + updates = []
77 + for r in con.execute("SELECT uid, title, product_type, price, price_max, "
78 + "details, listing_status, price_on_request FROM products"):
79 + uid, title, ptype, price = r["uid"], r["title"] or "", r["product_type"] or "", r["price"]
80 + new_price, new_pmax = price, r["price_max"]
81 + details = r["details"]
82 + on_req = 0
83 + if new_price is not None and new_price <= 0: # 0 $ n'est pas un prix
84 + new_price = None
85 + if new_pmax is not None and new_pmax <= 0:
86 + new_pmax = None
87 + if is_placeholder_price(new_price) or is_placeholder_price(new_pmax):
88 + try:
89 + d = json.loads(details) if details else {}
90 + except Exception:
91 + d = {}
92 + d["price_placeholder"] = new_price or new_pmax
93 + details = json.dumps(d, ensure_ascii=False)
94 + new_price = new_pmax = None
95 + stats["placeholder"] += 1
96 + if new_price is None:
97 + on_req = 1
98 + status = classify_listing(title, ptype)
99 + if status == "published":
100 + if not title.strip():
101 + status = "quarantine:titre_vide"
102 + stats["quarantine_titre"] += 1
103 + elif new_price is not None and new_price > PRICE_SANE_MAX:
104 + status = "quarantine:prix_hors_bornes"
105 + stats["quarantine_prix"] += 1
106 + else:
107 + excl[status] = excl.get(status, 0) + 1
108 + if on_req:
109 + stats["on_request"] += 1
110 + if (status != (r["listing_status"] or "published")
111 + or on_req != (r["price_on_request"] or 0)
112 + or new_price != price):
113 + updates.append((new_price, new_pmax, details, status, on_req, uid))
114 +
115 + print(f"[produits] placeholders neutralisés : {stats['placeholder']} | "
116 + f"sur devis (price_on_request) : {stats['on_request']} | "
117 + f"quarantaine prix : {stats['quarantine_prix']} | "
118 + f"quarantaine titre : {stats['quarantine_titre']}")
119 + print(f"[produits] exclusions : {json.dumps(excl, ensure_ascii=False)}")
120 + print(f"[produits] lignes à modifier : {len(updates)}")
121 + if not args.dry_run:
122 + con.executemany("UPDATE products SET price=?, price_max=?, details=?, "
123 + "listing_status=?, price_on_request=? WHERE uid=?", updates)
124 + con.commit()
125 +
126 + # ------------------------------------------------- boutiques : renommage
127 + reg = json.load(open(REG_PATH))
128 + by_id = {s["id"]: s for s in reg["stores"]}
129 + from collections import Counter
130 + name_counts = Counter((s.get("name") or "").strip().lower() for s in reg["stores"])
131 + renamed = []
132 + for s in reg["stores"]:
133 + cur = (s.get("name") or "").strip()
134 + low = cur.lower()
135 + generic = low in GENERIC_NAMES or (name_counts[low] >= 3 and len(low) < 40)
136 + if not generic:
137 + continue
138 + cpath = os.path.join(ENRICH_CACHE, s["id"] + ".json")
139 + if not os.path.exists(cpath):
140 + continue
141 + try:
142 + site_name = (json.load(open(cpath)).get("site_name") or "").strip()
143 + except Exception:
144 + continue
145 + import html as _html
146 + site_name = re.sub(r"\s+", " ", _html.unescape(site_name)).strip()[:80]
147 + if site_name.lower() in BAD_SITE_NAMES:
148 + continue
149 + if (site_name and site_name.lower() not in GENERIC_NAMES
150 + and site_name.lower() != low and len(site_name) >= 3):
151 + renamed.append((s["id"], cur, site_name))
152 + s["name"] = site_name
153 + print(f"[boutiques] renommées via og:site_name : {len(renamed)}")
154 + for sid, old, new in renamed[:15]:
155 + print(f" {sid}: «{old}» -> «{new}»")
156 +
157 + # ------------------------------------------------- boutiques : store_kind
158 + kinds = {}
159 + for s in reg["stores"]:
160 + sid = s["id"]
161 + ev = (s.get("origin_evidence") or "").lower()
162 + kind = ""
163 + if sid in CURATED_HORS_MISSION:
164 + kind = "hors_mission"
165 + elif sid in CURATED_COLLECTIF:
166 + kind = "collectif"
167 + elif sid in CURATED_REVENDEUR or "revend" in ev:
168 + kind = "revendeur"
169 + if kind:
170 + s["store_kind"] = kind
171 + kinds[kind] = kinds.get(kind, 0) + 1
172 + print(f"[boutiques] store_kind : {json.dumps(kinds, ensure_ascii=False)}")
173 +
174 + if args.dry_run:
175 + print("[dry-run] aucun écrit")
176 + return
177 +
178 + json.dump(reg, open(REG_PATH, "w"), ensure_ascii=False, indent=1)
179 +
180 + # DB : noms, store_kind, postal_prefix, phone
181 + for sid, _, new in renamed:
182 + con.execute("UPDATE stores SET name=? WHERE id=?", (new, sid))
183 + for s in reg["stores"]:
184 + con.execute("UPDATE stores SET store_kind=COALESCE(NULLIF(?,''), store_kind), "
185 + "postal_prefix=COALESCE(NULLIF(?,''), postal_prefix), "
186 + "phone=COALESCE(NULLIF(?,''), phone) WHERE id=?",
187 + (s.get("store_kind") or "", s.get("postal_prefix") or "",
188 + s.get("phone") or "", s["id"]))
189 +
190 + # produits des boutiques hors mission -> exclus en masse
191 + for sid in CURATED_HORS_MISSION:
192 + cur = con.execute("UPDATE products SET listing_status='excluded:hors_mission' "
193 + "WHERE store_id=? AND listing_status='published'", (sid,))
194 + if cur.rowcount:
195 + print(f"[hors-mission] {sid}: {cur.rowcount} produits exclus")
196 +
197 + # réindexation FTS (store_name) des boutiques renommées avec produits
198 + n_fts = 0
199 + for sid, _, new in renamed:
200 + rows = con.execute("SELECT uid, title, description, tags, vendor FROM products "
201 + "WHERE store_id=? AND active=1", (sid,)).fetchall()
202 + if not rows:
203 + continue
204 + uids = [r["uid"] for r in rows]
205 + for i in range(0, len(uids), 500):
206 + chunk = uids[i:i + 500]
207 + con.execute("DELETE FROM products_fts WHERE uid IN (%s)"
208 + % ",".join("?" * len(chunk)), chunk)
209 + con.executemany(
210 + "INSERT INTO products_fts (uid, title, description, tags, vendor, store_name) "
211 + "VALUES (?,?,?,?,?,?)",
212 + [(r["uid"], r["title"], r["description"],
213 + " ".join(json.loads(r["tags"] or "[]")), r["vendor"], new) for r in rows])
214 + n_fts += len(rows)
215 + print(f"[fts] {n_fts} produits réindexés (nouveau nom de boutique)")
216 +
217 + # product_count recalculé (publiés seulement)
218 + con.execute("UPDATE stores SET product_count=(SELECT COUNT(*) FROM products "
219 + "WHERE store_id=stores.id AND active=1 AND listing_status='published')")
220 + con.commit()
221 + con.close()
222 + print("[backfill] terminé")
223 +
224 +
225 +if __name__ == "__main__":
226 + main()
227