Resto·Ka — tous les restaurants du Québec, menus complets et prix réels (famille ·Ka)
Python 69.3%
TypeScript 16.7%
CSS 7.9%
JavaScript 4.7%
HTML 1.4%
1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File: restoka/connectors/siteresto.py4# Desc: Connecteur « sites web des restos » (Palier 3) — capte les MENUS5# MAISON des restaurants découverts par OSM qui publient un site web.6#7# Pipeline par site (budget par passage, incrémental via detail_cache) :8# 1. GET la page d'accueil (requests direct, UA RestoKaBot).9# 2. DÉTECTION DE PLATEFORME (CLAUDE.md §9 P3) : si le site embarque10# UEAT (order.ueat.io/integration/<GUID> ou *.order-online.ai),11# le GUID est consigné dans data/ueat-discovered.json — le12# connecteur UEAT (Palier 1, menu structuré parfait) le couvrira13# au prochain passage. Aucune fiche émise ici.14# 3. Sinon : trouver le lien « menu/carte », récupérer la page15# (Firecrawl en secours pour les sites JS, budget limité) ou le16# PDF du menu.17# 4. Structuration par Claude Haiku 4.5 (restoka/menullm.py) —18# sorties structurées JSON + validation STRICTE des prix.19# 5. Émettre une fiche `site-resto` clonée de la fiche OSM avec le20# menu ; la déduplication masque la fiche OSM derrière elle.21#22# Contexte de prix produit : DINE-IN (menu affiché par le resto23# lui-même — la référence idéale, §6.2). price_source: site-resto.24# Les fiches déjà captées sont RÉÉMISES depuis la base à chaque25# passage (délai de grâce) et re-crawlées après REFRESH_DAYS.26# ==============================================================================27from __future__ import annotations2829import datetime30import json31import os32import re33import sys34import urllib.parse35from pathlib import Path3637from ..hours import parse_jsonld_hours38from ..normalize import normalize_phone39from ..schema import Restaurant40from .base import BaseConnector4142DATA_DIR = Path(__file__).resolve().parent.parent.parent / "data"43UEAT_DISCOVERED = DATA_DIR / "ueat-discovered.json"4445# Budgets par passage (incrémental : la couverture grandit à chaque sync)46MAX_SITES_PER_RUN = int(os.environ.get("SITE_RESTO_BUDGET", "250"))47MAX_FIRECRAWL_PER_RUN = int(os.environ.get("SITE_RESTO_FIRECRAWL", "40"))48WORKERS = int(os.environ.get("SITE_RESTO_WORKERS", "6")) # crawls parallèles49REFRESH_DAYS = 30 # re-crawl des menus captés (cadence §14)5051# verrou d'écriture du fichier des intégrations UEAT découvertes (workers //)52_UEAT_FILE_LOCK = __import__("threading").Lock()5354# plateformes de réservation reconnues (lien « réserver » sur le site du resto)55_RESERVATION_DOMAINS = (56 "opentable.com", "opentable.ca", "libroreserve.com", "restoloco.com",57 "restoloco.ca", "resy.com", "sevenrooms.com", "bookenda.com",58 "zenchef.com", "guestonline.io", "tablein.com",59)6061_UEAT_RE = re.compile(62 r"(?:order\.ueat\.io/integration/|order-online\.ai.{0,200}?integration/)"63 r"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})", re.I)64_ORDER_ONLINE_RE = re.compile(r"https?://([a-z0-9-]+)\.order-online\.ai", re.I)6566# autres plateformes de commande (CLAUDE.md §9 P1) détectables sur le site du67# resto — consignées dans data/<plateforme>-discovered.json, couvertes par68# leurs connecteurs dédiés (gloriafood.py, chownow.py, squareonline.py ;69# Clover : consigné comme candidat, pas de connecteur — SPA opaque)70_GLF_RUID_RE = re.compile(r'data-glf-ruid="([0-9a-f-]{30,40})"', re.I)71_GLF_CUID_RE = re.compile(r'data-glf-cuid="([0-9a-f-]{30,40})"', re.I)72_CHOWNOW_RE = re.compile(73 r"(?:direct\.chownow\.com/order/|ordering\.chownow\.com/order/"74 r"|api\.chownow\.com/api/restaurant/|chownow\.com/order/)(\d+)", re.I)75_SQUARE_SITE_RE = re.compile(r"https?://([a-z0-9-]+\.square\.site)", re.I)76_CLOVER_RE = re.compile(r"clover\.com/online-ordering/([a-z0-9-]+)", re.I)77# plateformes dont le connecteur dédié capte un MENU structuré : quand l'une78# d'elles est détectée, site-resto ne crawle pas le menu maison (comme UEAT)79_MENU_PLATFORMS = ("gloriafood", "chownow", "square")80_MENU_WORD_RE = re.compile(r"menu|carte|nos-plats|nos_plats|food", re.I)81_PRICE_HINT_RE = re.compile(r"\d{1,3}[.,]\d{2}\s*\$|\$\s*\d{1,3}[.,]\d{2}|\d{1,3}\s*\$")828384def _domain(url: str) -> str:85 try:86 return urllib.parse.urlparse(url).netloc.lower().removeprefix("www.")87 except ValueError:88 return ""899091class SiteRestoConnector(BaseConnector):92 source_id = "site-resto"93 request_delay = 1.094 timeout = 2095 use_detail_cache = False # cache géré à la main (statuts par domaine)9697 # -- accès base -------------------------------------------------------------98 def _con(self):99 from .. import db100 if self._detail_con is None:101 self._detail_con = db.connect()102 return self._detail_con103104 def _get_status(self, domain: str) -> tuple[dict | None, float]:105 row = self._con().execute(106 "SELECT payload, fetched_at FROM detail_cache"107 " WHERE source=? AND external_id=?",108 (self.source_id, domain)).fetchone()109 if row is None:110 return None, 0.0111 try:112 return json.loads(row["payload"] or "{}"), row["fetched_at"] or 0.0113 except ValueError:114 return None, 0.0115116 def _set_status(self, domain: str, payload: dict) -> None:117 from .. import db118 db.put_cached_detail(self._con(), self.source_id, domain, "v1", payload)119120 # -- fetch HTML ---------------------------------------------------------------121 def _get_html(self, url: str) -> str:122 try:123 resp = self.get(url, allow_redirects=True)124 except Exception:125 return ""126 ctype = resp.headers.get("content-type", "")127 if "pdf" in ctype:128 return ""129 try:130 return resp.text[:800_000]131 except Exception:132 return ""133134 def _get_pdf(self, url: str) -> bytes:135 try:136 resp = self.get(url, allow_redirects=True)137 if len(resp.content) <= 8_000_000:138 return resp.content139 except Exception:140 pass141 return b""142143 @staticmethod144 def _html_text(html: str) -> str:145 from bs4 import BeautifulSoup146 soup = BeautifulSoup(html, "html.parser")147 for tag in soup(["script", "style", "nav", "footer", "noscript",148 "header", "svg", "form"]):149 tag.decompose()150 return re.sub(r"\n{3,}", "\n\n", soup.get_text("\n", strip=True))151152 def _menu_links(self, html: str, base_url: str) -> tuple[list[str], list[str]]:153 """(pages menu HTML, PDF de menu) trouvés sur la page d'accueil."""154 from bs4 import BeautifulSoup155 soup = BeautifulSoup(html, "html.parser")156 pages, pdfs = [], []157 base_dom = _domain(base_url)158 for a in soup.find_all("a", href=True):159 href = a["href"].strip()160 label = f"{href} {a.get_text(' ', strip=True)}"161 if not _MENU_WORD_RE.search(label):162 continue163 full = urllib.parse.urljoin(base_url, href)164 if full.lower().split("?")[0].endswith(".pdf"):165 if full not in pdfs:166 pdfs.append(full)167 elif _domain(full) == base_dom and full not in pages:168 pages.append(full)169 return pages[:2], pdfs[:2]170171 # -- extraction des photos du resto (page d'accueil) ------------------------------172 @staticmethod173 def _extract_images(html: str, base_url: str) -> list[str]:174 """Jusqu'à 6 photos du resto extraites de sa page d'accueil.175176 og:image/twitter:image d'abord (l'image que le resto met lui-même de177 l'avant), puis les <img> de la page (src/srcset/lazy-load). Filtres :178 logos, icônes, pictos de paiement, placeholders et miniatures179 (suffixe WordPress -LxH < 300 px) sont écartés ; les variantes de180 taille d'une même image sont dédupliquées sur le nom de base.181 """182 from bs4 import BeautifulSoup183 if not html:184 return []185 _BAD_RE = re.compile(186 r"logo|icon|favicon|placeholder|sprite|badge|paiement|payment|"187 r"visa|mastercard|interac|avatar|emoji|pixel|spinner|loader|"188 r"footer|bandeau|widget|captcha", re.I)189 _EXT = r"(?:jpe?g|png|webp|avif)"190 _EXT_RE = re.compile(rf"\.{_EXT}(?:\?|$)", re.I)191 _THUMB_RE = re.compile(rf"-(\d{{2,4}})x(\d{{2,4}})\.{_EXT}", re.I)192193 def _ok(u: str) -> bool:194 if not u.startswith("http") or not _EXT_RE.search(u):195 return False196 if _BAD_RE.search(u):197 return False198 m = _THUMB_RE.search(u)199 if m and max(int(m.group(1)), int(m.group(2))) < 300:200 return False201 return True202203 def _base_key(u: str) -> str:204 # même photo en plusieurs tailles : clé = URL sans suffixe de205 # variante (-LxH WordPress, -p-500 Webflow) ni extension206 u = u.split("?")[0].lower()207 u = re.sub(rf"-\d{{2,4}}x\d{{2,4}}(?=\.{_EXT}$)", "", u)208 u = re.sub(rf"-p-\d{{2,4}}(?=\.{_EXT}$)", "", u)209 return re.sub(rf"\.{_EXT}$", "", u)210211 soup = BeautifulSoup(html, "html.parser")212 found: list[str] = []213 for name, attrs in (("meta", {"property": "og:image"}),214 ("meta", {"name": "twitter:image"})):215 for tag in soup.find_all(name, attrs=attrs):216 u = urllib.parse.urljoin(base_url, (tag.get("content") or "").strip())217 if _ok(u):218 found.append(u)219 for img in soup.find_all("img"):220 try: # pictos et flèches déclarés étroits dans le markup221 if img.get("width") and int(str(img["width"]).rstrip("px")) < 200:222 continue223 except (ValueError, TypeError):224 pass225 cands = [img.get("src") or "", img.get("data-src") or "",226 img.get("data-lazy-src") or ""]227 srcset = img.get("srcset") or img.get("data-srcset") or ""228 if srcset: # prendre la plus grande variante du srcset229 parts = [p.strip().split(" ")[0] for p in srcset.split(",")]230 if parts:231 cands.append(parts[-1])232 for c in cands:233 u = urllib.parse.urljoin(base_url, c.strip())234 if _ok(u):235 found.append(u)236 break237 out: list[str] = []238 seen: set[str] = set()239 for u in found:240 k = _base_key(u)241 if k in seen:242 continue243 seen.add(k)244 out.append(u)245 if len(out) >= 6:246 break247 return out248249 # -- extraction contact/horaires/réservation (page d'accueil) ---------------------250 @staticmethod251 def _extract_contact(html: str, base_url: str) -> dict:252 """{"phone", "hours", "reservation_url"} extraits du HTML d'accueil.253254 - téléphone : lien tel: d'abord (fiable), sinon regex nord-américaine255 sur le texte visible -> E.164 (+1XXXXXXXXXX) ;256 - réservation : premier lien vers une plateforme connue (OpenTable,257 Libro, Resto Loco…) ;258 - horaires : JSON-LD schema.org (LocalBusiness/Restaurant,259 openingHoursSpecification ou openingHours) -> structuré par jour.260 """261 from bs4 import BeautifulSoup262 out = {"phone": "", "hours": None, "reservation_url": ""}263 if not html:264 return out265 soup = BeautifulSoup(html, "html.parser")266 tel_numbers: list[str] = []267 for a in soup.find_all("a", href=True):268 href = a["href"].strip()269 if href.lower().startswith("tel:"):270 p = normalize_phone(href[4:])271 if p and p not in tel_numbers:272 tel_numbers.append(p)273 if not out["reservation_url"] and href.startswith("http"):274 dom = _domain(href)275 if any(dom == d or dom.endswith("." + d)276 for d in _RESERVATION_DOMAINS):277 out["reservation_url"] = href278 # >3 numéros distincts = page « trouver une succursale » d'une chaîne :279 # impossible d'attribuer LE bon numéro à CE resto — on s'abstient280 if 1 <= len(tel_numbers) <= 3:281 out["phone"] = tel_numbers[0]282 # horaires structurés via JSON-LD (LocalBusiness / Restaurant)283 for script in soup.find_all("script", type="application/ld+json"):284 try:285 data = json.loads(script.string or "")286 except (ValueError, TypeError):287 continue288 nodes = data if isinstance(data, list) else [data]289 more = []290 for n in nodes:291 if isinstance(n, dict) and isinstance(n.get("@graph"), list):292 more.extend(n["@graph"])293 for node in nodes + more:294 if not isinstance(node, dict):295 continue296 if not out["hours"]:297 out["hours"] = parse_jsonld_hours(node)298 if not out["phone"]:299 out["phone"] = normalize_phone(node.get("telephone"))300 if out["hours"] and out["phone"]:301 break302 if not out["phone"]: # secours : regex sur le texte visible303 text = SiteRestoConnector._html_text(html)[:20_000]304 m = re.search(r"(?:t[eé]l|phone|appelez|r[eé]servation)[^\n]{0,40}?"305 r"(\(?\d{3}\)?[\s.\-]\d{3}[\s.\-]\d{4})", text, re.I)306 out["phone"] = normalize_phone(m.group(1) if m else "")307 return out308309 # -- traitement d'un resto -------------------------------------------------------310 def _discover_ueat(self, html: str, final_url: str) -> str | None:311 """GUID d'intégration UEAT si le site embarque la plateforme."""312 m = _UEAT_RE.search(html)313 if m:314 return m.group(1)315 m = _ORDER_ONLINE_RE.search(html)316 if m: # lien vers la page de commande marque blanche : suivre pour le GUID317 sub = self._get_html(f"https://{m.group(1)}.order-online.ai/")318 m2 = _UEAT_RE.search(sub)319 if m2:320 return m2.group(1)321 return None322323 def _record_ueat(self, guid: str, name: str, domain: str) -> None:324 with _UEAT_FILE_LOCK:325 self._record_ueat_locked(guid, name, domain)326327 def _discover_platforms(self, html: str) -> list[dict]:328 """Plateformes de commande tierces (hors UEAT) embarquées sur le site."""329 out: list[dict] = []330 m = _GLF_RUID_RE.search(html)331 if m:332 cuid = _GLF_CUID_RE.search(html)333 entry = {"name": "gloriafood", "key": m.group(1)}334 if cuid:335 entry["cuid"] = cuid.group(1)336 out.append(entry)337 m = _CHOWNOW_RE.search(html)338 if m:339 out.append({"name": "chownow", "key": m.group(1)})340 m = _SQUARE_SITE_RE.search(html)341 if m:342 out.append({"name": "square", "key": m.group(1).lower()})343 m = _CLOVER_RE.search(html)344 if m:345 out.append({"name": "clover", "key": m.group(1)})346 return out347348 def _record_platform(self, platform: dict, name: str, domain: str,349 uid: str) -> None:350 """Consigne une intégration découverte dans351 data/<plateforme>-discovered.json (avec l'uid du resto porteur pour352 les connecteurs d'enrichissement gloriafood/square)."""353 path = DATA_DIR / f"{platform['name']}-discovered.json"354 with _UEAT_FILE_LOCK:355 data = {"integrations": []}356 if path.exists():357 try:358 data = json.loads(path.read_text(encoding="utf-8"))359 except ValueError:360 pass361 if any(i.get("key") == platform["key"]362 for i in data.get("integrations", [])):363 return364 entry = {"key": platform["key"], "uid": uid,365 "label": f"{name} (découvert via {domain})"}366 if platform.get("cuid"):367 entry["cuid"] = platform["cuid"]368 data.setdefault("integrations", []).append(entry)369 path.write_text(json.dumps(data, ensure_ascii=False, indent=1),370 encoding="utf-8")371 print(f"[resto-ka] site-resto: plateforme {platform['name']} "372 f"découverte chez {name} ({platform['key']}) -> {path.name}")373374 def _record_ueat_locked(self, guid: str, name: str, domain: str) -> None:375 data = {"integrations": []}376 if UEAT_DISCOVERED.exists():377 try:378 data = json.loads(UEAT_DISCOVERED.read_text(encoding="utf-8"))379 except ValueError:380 pass381 if any(i.get("key") == guid for i in data["integrations"]):382 return383 data["integrations"].append(384 {"key": guid, "label": f"{name} (découvert via {domain})"})385 UEAT_DISCOVERED.write_text(386 json.dumps(data, ensure_ascii=False, indent=1), encoding="utf-8")387 print(f"[resto-ka] site-resto: plateforme UEAT découverte chez {name}"388 f" ({guid}) -> ueat-discovered.json")389390 def _crawl_menu(self, website: str, firecrawl_budget: list[int]) -> dict | None:391 """Retourne {"sections": [...], "url": <page>, "contact": {...}},392 {"ueat": guid, "contact": ...} ou {"contact": ...} (pas de menu mais393 téléphone/horaires/réservation captés) ou None."""394 from .. import menullm395 home = self._get_html(website)396 if not home:397 return None398 contact = self._extract_contact(home, website)399 images = self._extract_images(home, website)400 platforms = self._discover_platforms(home)401 guid = self._discover_ueat(home, website)402 if guid:403 return {"ueat": guid, "contact": contact, "images": images,404 "platforms": platforms}405 if any(p["name"] in _MENU_PLATFORMS for p in platforms):406 # une plateforme à menu structuré couvre ce resto : son connecteur407 # dédié prendra le relais (comme UEAT) — pas de crawl maison408 return {"contact": contact, "images": images,409 "platforms": platforms}410 pages, pdfs = self._menu_links(home, website)411 # 1) pages HTML de menu412 for page_url in pages:413 html = self._get_html(page_url)414 text = self._html_text(html) if html else ""415 if len(_PRICE_HINT_RE.findall(text)) < 3 and firecrawl_budget[0] > 0:416 try: # site JS : rendu via Firecrawl (budget limité)417 firecrawl_budget[0] -= 1418 rendered = self.get_rendered(page_url)419 if rendered:420 text = self._html_text(rendered)421 except Exception:422 pass423 if len(_PRICE_HINT_RE.findall(text)) >= 3:424 menu = menullm.menu_from_text(text)425 if menu:426 return {"sections": menu["sections"], "url": page_url,427 "contact": contact, "images": images,428 "platforms": platforms}429 # 2) PDF de menu430 for pdf_url in pdfs:431 pdf = self._get_pdf(pdf_url)432 if pdf[:5] == b"%PDF-":433 menu = menullm.menu_from_pdf(pdf)434 if menu:435 return {"sections": menu["sections"], "url": pdf_url,436 "contact": contact, "images": images,437 "platforms": platforms}438 # 3) la page d'accueil elle-même contient parfois le menu439 text = self._html_text(home)440 if len(_PRICE_HINT_RE.findall(text)) >= 8:441 menu = menullm.menu_from_text(text)442 if menu:443 return {"sections": menu["sections"], "url": website,444 "contact": contact, "images": images,445 "platforms": platforms}446 if any(v for v in contact.values()) or images or platforms:447 # pas de menu, mais du contact/photos/plateformes captés448 return {"contact": contact, "images": images,449 "platforms": platforms}450 return None451452 # -- reconstruction des fiches déjà captées ------------------------------------------453 def _row_to_restaurant(self, row, menus_by_uid: dict) -> Restaurant:454 menu = menus_by_uid.get(row["uid"])455 return Restaurant(456 source=self.source_id, external_id=row["external_id"],457 name=row["name"], url=row["url"], chain=row["chain"],458 cuisines=json.loads(row["cuisines"] or "[]"),459 establishment_type=row["establishment_type"] or "",460 price_range=row["price_range"] or "",461 address=row["address"] or "", city=row["city"] or "",462 region=row["region"] or "", postal_code=row["postal_code"] or "",463 lat=row["lat"], lng=row["lng"], phone=row["phone"] or "",464 website=row["website"] or "",465 hours=json.loads(row["hours"] or "{}"),466 services=json.loads(row["services"] or "[]"),467 dietary_options=json.loads(row["dietary_options"] or "[]"),468 languages=json.loads(row["languages"] or '["fr"]'),469 images=json.loads(row["images"] or "[]"),470 menu=menu,471 )472473 def _existing(self) -> list[Restaurant]:474 con = self._con()475 menus_by_uid = {}476 for m in con.execute(477 "SELECT uid, price_context, price_source, currency, captured_at,"478 " sections FROM menus WHERE uid LIKE 'site-resto:%'"):479 menus_by_uid[m["uid"]] = {480 "currency": m["currency"] or "CAD",481 "price_context": m["price_context"],482 "price_source": m["price_source"],483 "captured_at": m["captured_at"],484 "sections": json.loads(m["sections"] or "[]"),485 }486 return [self._row_to_restaurant(r, menus_by_uid) for r in con.execute(487 "SELECT * FROM restaurants WHERE source='site-resto'")]488489 # -- fetch --------------------------------------------------------------------------490 def fetch(self) -> list[Restaurant]:491 import time492 con = self._con()493 out: dict[str, Restaurant] = {r.uid: r for r in self._existing()}494495 candidates = con.execute(496 "SELECT * FROM restaurants WHERE source='osm' AND active=1"497 " AND dup_of IS NULL AND website<>''"498 " AND NOT EXISTS (SELECT 1 FROM menus m WHERE m.uid=restaurants.uid)"499 " ORDER BY region<>'' DESC, city").fetchall()500501 firecrawl_budget = [MAX_FIRECRAWL_PER_RUN502 if os.environ.get("FIRECRAWL_API_KEY") else 0]503 stats = {"crawled": 0, "menus": 0, "ueat": 0, "plateformes": 0,504 "vides": 0, "contacts": 0, "photos": 0}505 now = time.time()506 # enrichissements à appliquer APRÈS sync_source (la fiche site-resto507 # n'existe en base qu'après l'upsert) — consommé par ingest.run508 self.pending_enrichment: list[tuple[str, dict]] = []509510 # 1re passe (fil principal) : réémission depuis le cache + sélection511 # des sites à crawler (budget) — aucune E/S réseau ici.512 to_crawl: list[tuple] = []513 for row in candidates:514 website = row["website"]515 if not website.startswith("http"):516 website = "https://" + website517 domain = _domain(website)518 if not domain:519 continue520 status, fetched_at = self._get_status(domain)521 if status is not None and now - fetched_at < REFRESH_DAYS * 86400:522 # menu déjà capté mais pas encore en base (run interrompu) :523 # réémettre depuis le cache — le connecteur est idempotent524 uid = f"{self.source_id}:{row['external_id']}"525 if (status.get("status") == "menu" and status.get("sections")526 and uid not in out):527 resto = self._row_to_restaurant(row, {})528 resto.source = self.source_id529 resto.url = status.get("url") or website530 if status.get("images") and not resto.images:531 resto.images = status["images"]532 resto.menu = {533 "currency": "CAD", "price_context": "dine-in",534 "price_source": self.source_id,535 "captured_at": status.get("captured_at") or536 datetime.datetime.now(datetime.timezone.utc)537 .strftime("%Y-%m-%dT%H:%M:%SZ"),538 "sections": status["sections"],539 }540 resto.cuisines, resto.price_range, resto.dietary_options = [], "", []541 out[uid] = resto542 continue # déjà tenté récemment543 if len(to_crawl) < MAX_SITES_PER_RUN:544 to_crawl.append((row, website, domain))545546 # 2e passe : crawl + structuration Claude en PARALLÈLE (WORKERS fils).547 # Chaque fil a son propre connecteur (session requests dédiée) et ne548 # touche JAMAIS la base — les écritures restent dans le fil principal.549 from concurrent.futures import ThreadPoolExecutor550551 def _crawl_one(args):552 row, website, domain = args553 worker = SiteRestoConnector()554 try:555 return row, domain, website, worker._crawl_menu(website,556 firecrawl_budget)557 except Exception as exc: # un site cassé ne bloque pas le lot558 print(f"[resto-ka] site-resto: {domain} erreur: {exc}",559 file=sys.stderr)560 return row, domain, website, None561562 with ThreadPoolExecutor(max_workers=max(1, WORKERS)) as pool:563 results = pool.map(_crawl_one, to_crawl)564 for row, domain, website, result in results:565 stats["crawled"] += 1566 # téléphone / horaires / lien de réservation captés sur le567 # site : COALESCE conservateur sur la fiche OSM d'origine568 # (fil principal seulement — les workers ne touchent pas la BD)569 contact = (result or {}).get("contact") or {}570 if any(v for v in contact.values()):571 from .. import db as _db572 _db.enrich_contact(573 con, row["uid"], phone=contact.get("phone") or "",574 hours=contact.get("hours"),575 details={"reservation_url":576 contact.get("reservation_url")}577 if contact.get("reservation_url") else None)578 con.commit()579 stats["contacts"] += 1580 # photos captées sur le site du resto : posées sur la fiche581 # OSM d'origine si elle n'en a pas (COALESCE conservateur)582 images = (result or {}).get("images") or []583 if images:584 con.execute(585 "UPDATE restaurants SET images=? WHERE uid=? AND"586 " (images IS NULL OR images='' OR images='[]')",587 (json.dumps(images), row["uid"]))588 con.commit()589 stats["photos"] += 1590 # plateformes de commande découvertes : consignées ici (fil591 # principal seulement — les workers n'écrivent ni BD ni fichiers)592 for p in (result or {}).get("platforms") or []:593 self._record_platform(p, row["name"], domain, row["uid"])594 if result and result.get("ueat"):595 self._record_ueat(result["ueat"], row["name"], domain)596 self._set_status(domain, {"status": "ueat",597 "guid": result["ueat"]})598 stats["ueat"] += 1599 continue600 blocking = [p for p in (result or {}).get("platforms") or []601 if p["name"] in _MENU_PLATFORMS]602 if blocking and not (result and result.get("sections")):603 # menu servi par une plateforme connue : le connecteur604 # dédié (gloriafood/chownow/square) prendra le relais605 self._set_status(domain, {"status": "platform",606 "platforms":607 [p["name"] for p in blocking]})608 stats["plateformes"] += 1609 continue610 if not result or not result.get("sections"):611 self._set_status(domain, {"status": "no_menu"})612 stats["vides"] += 1613 continue614 captured_at = datetime.datetime.now(datetime.timezone.utc) \615 .strftime("%Y-%m-%dT%H:%M:%SZ")616 resto = self._row_to_restaurant(row, {})617 resto.source = self.source_id618 resto.url = result["url"]619 resto.menu = {620 "currency": "CAD",621 "price_context": "dine-in", # menu maison : la référence (§6.2)622 "price_source": self.source_id,623 "captured_at": captured_at,624 "sections": result["sections"],625 }626 resto.cuisines = [] # reclassées avec le texte du menu627 resto.price_range = ""628 resto.dietary_options = []629 if images and not resto.images:630 resto.images = images631 # contact capté sur le site : complète la fiche émise (COALESCE)632 if contact.get("phone") and not (resto.phone or "").strip():633 resto.phone = contact["phone"]634 if contact.get("hours"):635 resto.hours = {**contact["hours"], **(resto.hours or {})}636 if contact.get("reservation_url"):637 self.pending_enrichment.append(638 (resto.uid, {"details": {"reservation_url":639 contact["reservation_url"]}}))640 out[resto.uid] = resto641 self._set_status(domain, {"status": "menu", "url": result["url"],642 "captured_at": captured_at,643 "sections": result["sections"],644 "images": images})645 stats["menus"] += 1646647 print(f"[resto-ka] site-resto: {stats} (workers={WORKERS})")648 return list(out.values())649