SPB Git forge
spboucher.ai ↗

spb/fabri-ka

Public

Agrégateur de produits québécois — www.fabri-ka.com

217commits 1branches 0releases
66.1 MBsize
maindefault branch
4 h agolast push
Python 38.4% HTML 30.3% TypeScript 17.2% CSS 11% JavaScript 3.2%

Colonne details (avis/dimensions/variantes) + repli Scrapfly robuste (large_object, sticky)

- schema.py : champ Product.details (dict JSON) inclus dans le content_hash
- db.py : colonne products.details (migration additive) + insert/update
- shopify.py : details (variantes, options, dates) ; Scrapfly collant par
  boutique bloquée + retry — répare ici-la.co (20 077 produits)
- scrapfly.py : suivi des liens large_object (pages >5 Mo, ex. ici-la p.68)
- woocommerce.py : details (avis, sku, poids, dimensions, attributs,
  low_stock, backorder, variations) + vendor depuis la taxonomie brands
- wix.py : GraphQL enrichi (sku, brand, discount) -> details + vendor
- web.py : details décodé en JSON dans les réponses produit

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Simon-Pierre Boucher committed 1 mo ago (Aug 18, 2026) parent 91fd907

7 changed files +108 −7

modified fabrika/connectors/scrapfly.py +7 −1
@@ -41,7 +41,13 @@ def scrapfly_get(url: str, render_js: bool = False, timeout: int = 150) -> tuple
41 41 r = requests.get(API, params=params, timeout=timeout)
42 42 r.raise_for_status()
43 43 res = r.json().get("result", {})
44 − return int(res.get("status_code") or 0), res.get("content") or ""
44 + content = res.get("content") or ""
45 + # réponse volumineuse : Scrapfly renvoie un lien "large_object" à suivre
46 + if content.startswith("https://api.scrapfly.io/scrape/large_object/"):
47 + lo = requests.get(content, params={"key": key}, timeout=timeout)
48 + lo.raise_for_status()
49 + content = lo.text
50 + return int(res.get("status_code") or 0), content
45 51
46 52
47 53 def scrapfly_json(url: str, render_js: bool = False) -> dict | list:
modified fabrika/connectors/shopify.py +36 −1
@@ -28,6 +28,10 @@ class ShopifyConnector(BaseConnector):
28 28 429 de Shopify sous volume ; curl passe). Throttle global + retry."""
29 29 import json as _json
30 30 import subprocess
31 + # boutique déjà identifiée comme bloquée (challenge anti-bot) :
32 + # rester sur Scrapfly pour les pages suivantes, sans re-tenter en direct
33 + if getattr(self, "_scrapfly_only", False):
34 + return self._scrapfly_json(url)
31 35 last_err = None
32 36 for attempt in range(4):
33 37 with _GLOBAL_LOCK:
@@ -56,11 +60,25 @@ class ShopifyConnector(BaseConnector):
56 60 from . import scrapfly
57 61 if scrapfly.available():
58 62 try:
59 − return scrapfly.scrapfly_json(url)
63 + data = self._scrapfly_json(url)
64 + self._scrapfly_only = True
65 + return data
60 66 except Exception as exc:
61 67 last_err = exc
62 68 raise RuntimeError(f"{url}: {last_err}")
63 69
70 + def _scrapfly_json(self, url: str):
71 + """Scrapfly avec retry : le bypass anti-bot échoue parfois par intermittence."""
72 + from . import scrapfly
73 + last_err = None
74 + for attempt in range(3):
75 + try:
76 + return scrapfly.scrapfly_json(url)
77 + except Exception as exc:
78 + last_err = exc
79 + time.sleep(3 * (attempt + 1))
80 + raise RuntimeError(f"{url}: scrapfly: {last_err}")
81 +
64 82 def fetch(self) -> list[Product]:
65 83 out: list[Product] = []
66 84 page = 1
@@ -76,6 +94,22 @@ class ShopifyConnector(BaseConnector):
76 94 compare = [parse_price(v.get("compare_at_price")) for v in variants]
77 95 compare = [c for c in compare if c]
78 96 available = any(v.get("available", True) for v in variants) if variants else None
97 + det: dict = {}
98 + if variants:
99 + det["variants"] = [{"title": v.get("title", ""),
100 + "price": parse_price(v.get("price")),
101 + "sku": v.get("sku") or "",
102 + "grams": v.get("grams"),
103 + "available": v.get("available")}
104 + for v in variants[:20]]
105 + options = [{"name": o.get("name", ""), "values": o.get("values") or []}
106 + for o in (it.get("options") or [])
107 + if (o.get("values") or []) != ["Default Title"]]
108 + if options:
109 + det["options"] = options
110 + for k in ("published_at", "created_at", "updated_at"):
111 + if it.get(k):
112 + det[k] = it[k]
79 113 out.append(Product(
80 114 store_id=self.store_id,
81 115 external_id=str(it["id"]),
@@ -91,6 +125,7 @@ class ShopifyConnector(BaseConnector):
91 125 else [t.strip() for t in (it.get("tags") or "").split(",") if t.strip()]),
92 126 vendor=it.get("vendor", "") or "",
93 127 available=available,
128 + details=det,
94 129 ))
95 130 if len(items) < 250:
96 131 break
modified fabrika/connectors/wix.py +17 −0
@@ -17,6 +17,7 @@ GQL = """query getFilteredProducts($limit:Int!,$offset:Int!){
17 17 catalog{category(categoryId:"%s"){numOfProducts
18 18 productsWithMetaData(limit:$limit,offset:$offset,onlyVisible:true){
19 19 list{id name urlPart price comparePrice formattedPrice description
20 + sku brand discount{mode value}
20 21 media{url} isInStock productType ribbon
21 22 options{title selections{description}}}}}}}""" % ALL_PRODUCTS_CATEGORY
22 23
@@ -58,6 +59,20 @@ class WixConnector(BaseConnector):
58 59 imgs.append(u)
59 60 price = parse_price(it.get("price"))
60 61 compare = parse_price(it.get("comparePrice"))
62 + det: dict = {}
63 + options = [{"title": o.get("title", ""),
64 + "selections": [s.get("description", "")
65 + for s in (o.get("selections") or [])]}
66 + for o in (it.get("options") or []) if o]
67 + if options:
68 + det["options"] = options
69 + if it.get("sku"):
70 + det["sku"] = it["sku"]
71 + if it.get("brand"):
72 + det["brand"] = it["brand"]
73 + disc = it.get("discount") or {}
74 + if disc.get("value"):
75 + det["discount"] = {"mode": disc.get("mode"), "value": disc.get("value")}
61 76 out.append(Product(
62 77 store_id=self.store_id,
63 78 external_id=str(it.get("id", "")),
@@ -70,7 +85,9 @@ class WixConnector(BaseConnector):
70 85 images=imgs,
71 86 product_type=it.get("productType", "") or "",
72 87 tags=[it["ribbon"]] if it.get("ribbon") else [],
88 + vendor=it.get("brand") or "",
73 89 available=bool(it.get("isInStock", True)),
90 + details=det,
74 91 ))
75 92 offset += 100
76 93 return out
modified fabrika/connectors/woocommerce.py +31 −0
@@ -43,6 +43,35 @@ class WooCommerceConnector(BaseConnector):
43 43 pr = prices.get("price_range") or {}
44 44 pmin, pmax = money(pr.get("min_amount")), money(pr.get("max_amount"))
45 45 cats = [c.get("name", "") for c in (it.get("categories") or [])]
46 + brands = [b.get("name", "") for b in (it.get("brands") or []) if b.get("name")]
47 + det: dict = {}
48 + try:
49 + if float(it.get("average_rating") or 0) > 0:
50 + det["average_rating"] = float(it["average_rating"])
51 + except (TypeError, ValueError):
52 + pass
53 + if it.get("review_count"):
54 + det["review_count"] = int(it["review_count"])
55 + if it.get("sku"):
56 + det["sku"] = it["sku"]
57 + if it.get("weight"):
58 + det["weight"] = it["weight"]
59 + det["formatted_weight"] = it.get("formatted_weight") or ""
60 + dims = it.get("dimensions") or {}
61 + if any(dims.get(k) for k in ("length", "width", "height")):
62 + det["dimensions"] = dims
63 + det["formatted_dimensions"] = it.get("formatted_dimensions") or ""
64 + attributes = [{"name": a.get("name", ""),
65 + "terms": [t.get("name", "") for t in (a.get("terms") or [])]}
66 + for a in (it.get("attributes") or [])]
67 + if attributes:
68 + det["attributes"] = attributes
69 + if it.get("low_stock_remaining"):
70 + det["low_stock_remaining"] = it["low_stock_remaining"]
71 + if it.get("is_on_backorder"):
72 + det["is_on_backorder"] = True
73 + if it.get("variations"):
74 + det["variations"] = len(it["variations"])
46 75 out.append(Product(
47 76 store_id=self.store_id,
48 77 external_id=str(it["id"]),
@@ -58,7 +87,9 @@ class WooCommerceConnector(BaseConnector):
58 87 images=[im.get("src", "") for im in (it.get("images") or [])],
59 88 product_type=", ".join(cats),
60 89 tags=[t.get("name", "") for t in (it.get("tags") or [])],
90 + vendor=brands[0] if brands else "",
61 91 available=bool(it.get("is_in_stock", True)),
92 + details=det,
62 93 ))
63 94 if len(items) < 100:
64 95 break
modified fabrika/db.py +12 −4
@@ -56,6 +56,7 @@ CREATE TABLE IF NOT EXISTS products (
56 56 tags TEXT, -- JSON
57 57 vendor TEXT,
58 58 available INTEGER,
59 + details TEXT, -- JSON : avis, dimensions, matériaux, variantes…
59 60 content_hash TEXT,
60 61 first_seen REAL,
61 62 last_seen REAL,
@@ -82,6 +83,10 @@ def connect() -> sqlite3.Connection:
82 83 con.execute("PRAGMA busy_timeout=60000")
83 84 con.row_factory = sqlite3.Row
84 85 con.executescript(_SCHEMA)
86 + # migration additive : bases créées avant l'ajout de la colonne `details`
87 + cols = {r[1] for r in con.execute("PRAGMA table_info(products)")}
88 + if "details" not in cols:
89 + con.execute("ALTER TABLE products ADD COLUMN details TEXT")
85 90 return con
86 91
87 92
@@ -134,16 +139,18 @@ def sync_products(con: sqlite3.Connection, store_id: str, store_name: str,
134 139 seen.add(uid)
135 140 row = p.to_row()
136 141 row.update(images=json.dumps(p.images), tags=json.dumps(p.tags, ensure_ascii=False),
142 + details=json.dumps(p.details, ensure_ascii=False) if p.details else None,
137 143 content_hash=chash, last_seen=now,
138 144 available=None if p.available is None else int(p.available))
139 145 if uid not in existing:
140 146 row["first_seen"] = now
141 147 con.execute("""INSERT INTO products (uid, store_id, external_id, url, title,
142 148 description, price, price_max, compare_at_price, currency, images, category,
143 − product_type, tags, vendor, available, content_hash, first_seen, last_seen,
144 − active, miss_count) VALUES (:uid,:store_id,:external_id,:url,:title,
149 + product_type, tags, vendor, available, details, content_hash, first_seen,
150 + last_seen, active, miss_count) VALUES (:uid,:store_id,:external_id,:url,:title,
145 151 :description,:price,:price_max,:compare_at_price,:currency,:images,:category,
146 − :product_type,:tags,:vendor,:available,:content_hash,:first_seen,:last_seen,1,0)""", row)
152 + :product_type,:tags,:vendor,:available,:details,:content_hash,:first_seen,
153 + :last_seen,1,0)""", row)
147 154 con.execute("INSERT INTO products_fts (uid, title, description, tags, vendor, store_name) "
148 155 "VALUES (?,?,?,?,?,?)",
149 156 (uid, p.title, p.description, " ".join(p.tags), p.vendor, store_name))
@@ -155,7 +162,8 @@ def sync_products(con: sqlite3.Connection, store_id: str, store_name: str,
155 162 price=:price, price_max=:price_max, compare_at_price=:compare_at_price,
156 163 currency=:currency, images=:images, category=:category,
157 164 product_type=:product_type, tags=:tags, vendor=:vendor,
158 − available=:available, content_hash=:content_hash, last_seen=:last_seen,
165 + available=:available, details=:details, content_hash=:content_hash,
166 + last_seen=:last_seen,
159 167 active=1, miss_count=0 WHERE uid=:uid""", row)
160 168 con.execute("DELETE FROM products_fts WHERE uid=?", (uid,))
161 169 con.execute("INSERT INTO products_fts (uid, title, description, tags, vendor, store_name) "
modified fabrika/schema.py +3 −1
@@ -133,6 +133,7 @@ class Product:
133 133 tags: list[str] = field(default_factory=list)
134 134 vendor: str = "" # marque affichée par la boutique
135 135 available: bool | None = None
136 + details: dict = field(default_factory=dict) # avis, dimensions, variantes… (JSON)
136 137
137 138 @property
138 139 def uid(self) -> str:
@@ -151,7 +152,8 @@ class Product:
151 152
152 153 def content_hash(self) -> str:
153 154 basis = json.dumps([self.title, self.price, self.price_max, self.available,
154 − self.images[:1], self.description[:200]], ensure_ascii=False)
155 + self.images[:1], self.description[:200],
156 + self.details or None], ensure_ascii=False, sort_keys=True)
155 157 return hashlib.sha1(basis.encode()).hexdigest()[:16]
156 158
157 159 def to_row(self) -> dict:
modified fabrika/web.py +2 −0
@@ -37,6 +37,8 @@ def q(con: sqlite3.Connection, sql: str, args=()) -> list[dict]:
37 37 def _product_out(r: dict) -> dict:
38 38 r["images"] = json.loads(r.get("images") or "[]")
39 39 r["tags"] = json.loads(r.get("tags") or "[]")
40 + if "details" in r:
41 + r["details"] = json.loads(r["details"]) if r["details"] else None
40 42 return r
41 43
42 44
43 45