Vague 2 : inspections MAPAQ, horaires structurés, contact site-resto, Yelp, fix Overpass 406
- MAPAQ (Données Québec, CC-BY 4.0) : module restoka/inspections.py — import
des 2 994 condamnations (listecondamnation.csv) dans la table liée
inspections, croisement CONSERVATEUR avec les restaurants (nom+lieu avec
CP ou civique exigé ; adresse+nom hors tokens de ville — jamais la mauvaise
succursale d une chaîne, jamais deux voisins d un centre commercial) :
288 condamnations croisées sur 204 restos. details.mapaq + endpoint
GET /api/restaurants/{uid}/inspections + compteurs /api/stats.
- Horaires : restoka/hours.py — parseur opening_hours OSM (plages de jours,
double service, off, 24/7, PH ignoré ; l exotique reste en brut) + JSON-LD
schema.org. Câblé dans le connecteur OSM et backfillé : 2 656/2 939 tags
bruts structurés par jour (90,4 %).
- site-resto : extraction téléphone (tel:/JSON-LD/regex, E.164, garde
anti-annuaire de chaîne), horaires JSON-LD et lien de réservation
(OpenTable, Libro, Resto Loco…) -> COALESCE conservateur sur la fiche OSM
(db.enrich_contact) + details.reservation_url (pending_enrichment).
- Yelp Fusion : connecteur d enrichissement complet (rating, review_count,
price, categories, phone -> details.yelp ; croisement par téléphone puis
/businesses/matches, GPS <300 m si ambigu, budget 450 req/run). Livré en
SkipSource « clé requise » (YELP_API_KEY absente) — nouveau patron
SkipSource dans base.py + .env.example.
- Overpass 406 : overpass-api.de refuse les UA déguisés en navigateur mais
accepte un UA de bot honnête — UA dédié OVERPASS_UA (vérifié live, 200).
- Colonne restaurants.details (migration auto, jamais touchée par
sync_source), registre data/sources.json à 6 sources, /api/stats expose
inspections/inspections_matched/restaurants_with_inspections.
- Tests : 64 au vert (51 existants + 13 nouveaux : hours, inspections,
extraction contact).
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
15 changed files +1,215 −11
added
.env.example
+28 −0
@@ -0,0 +1,28 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: .env.example | |
| 4 | +# Desc: Resto·Ka — gabarit des secrets locaux. Copier vers `.env` (jamais | |
| 5 | +# commité, voir .gitignore) et remplir les clés. | |
| 6 | +# ============================================================================== | |
| 7 | + | |
| 8 | +# Scrapfly — rendu JS + bypass anti-bot (plateformes de livraison, §10/§15) | |
| 9 | +SCRAPFLY_KEY= | |
| 10 | + | |
| 11 | +# Anthropic — structuration des menus maison par Claude (restoka/menullm.py) | |
| 12 | +ANTHROPIC_API_KEY= | |
| 13 | + | |
| 14 | +# Firecrawl — rendu JS de secours pour les sites de restos SPA (site-resto) | |
| 15 | +FIRECRAWL_API_KEY= | |
| 16 | + | |
| 17 | +# Yelp Fusion — enrichissement avis/notes (connecteur yelp, gratuit 500 req/j) | |
| 18 | +# Créer une app : https://www.yelp.com/developers — sans clé, la source est | |
| 19 | +# simplement sautée (SkipSource « clé requise »). | |
| 20 | +YELP_API_KEY= | |
| 21 | + | |
| 22 | +# Budget de requêtes Yelp par passage (quota quotidien 500) | |
| 23 | +# YELP_BUDGET=450 | |
| 24 | + | |
| 25 | +# SSO Groupe KA (« Se connecter avec KA ») + sessions | |
| 26 | +KA_SSO_SECRET= | |
| 27 | +SESSION_SECRET= | |
| 28 | +RESTOKA_BASE_URL=https://www.resto-ka.com | |
modified
data/sources.json
+27 −1
@@ -256,6 +256,32 @@ | ||
| 256 | 256 | "acces_legal": "Site de commande de la chaîne elle-même (prix réels). À industrialiser prudemment (CLAUDE.md §15).", |
| 257 | 257 | "cadence": "hebdomadaire (visé)", |
| 258 | 258 | "status": "à faire" |
| 259 | + }, | |
| 260 | + { | |
| 261 | + "id": "mapaq", | |
| 262 | + "name": "MAPAQ — Inspections alimentaires (condamnations)", | |
| 263 | + "url": "https://www.donneesquebec.ca/recherche/dataset/condamnations-des-etablissements-alimentaires-et-condamnations-concernant-le-bien-etre-des-anim", | |
| 264 | + "platform": "donnees-quebec", | |
| 265 | + "connector": "inspections (module d'enrichissement, pas de fiches)", | |
| 266 | + "tier": 5, | |
| 267 | + "price_context": "aucun", | |
| 268 | + "extraction": "CSV ouvert listecondamnation.csv (Données Québec) -> table liée `inspections` + croisement CONSERVATEUR avec les restaurants (nom normalisé + ville/code postal, ou code postal + civique + similarité de nom). Résumé conformité dans details.mapaq, endpoint GET /api/restaurants/{uid}/inspections.", | |
| 269 | + "acces_legal": "Données ouvertes du gouvernement du Québec, licence CC-BY 4.0 — attribution MAPAQ/Données Québec affichée avec les inspections.", | |
| 270 | + "cadence": "hebdomadaire (guard 6 jours dans ingest.enrich)", | |
| 271 | + "status": "actif" | |
| 272 | + }, | |
| 273 | + { | |
| 274 | + "id": "yelp", | |
| 275 | + "name": "Yelp Fusion (avis et notes)", | |
| 276 | + "url": "https://www.yelp.com/developers", | |
| 277 | + "platform": "yelp", | |
| 278 | + "connector": "yelp", | |
| 279 | + "tier": 5, | |
| 280 | + "price_context": "aucun", | |
| 281 | + "extraction": "API officielle Yelp Fusion (gratuit, 500 req/jour) : rating, review_count, price, categories, phone -> details.yelp. Croisement conservateur par téléphone (/businesses/search/phone, GPS <300 m si ambigu) puis nom+adresse+GPS (/businesses/matches). N'émet aucune fiche.", | |
| 282 | + "acces_legal": "API officielle avec clé, Display Requirements Yelp : attribution affichée avec la note.", | |
| 283 | + "cadence": "hebdomadaire, re-vérification 30 jours", | |
| 284 | + "status": "clé requise (YELP_API_KEY absente de .env — connecteur livré, SkipSource)" | |
| 259 | 285 | } |
| 260 | 286 | ] |
| 261 | −} | |
| \ No newline at end of file | ||
| 287 | +} | |
modified
restoka/connectors/base.py
+8 −0
@@ -23,6 +23,14 @@ FIRECRAWL_API = "https://api.firecrawl.dev/v1/scrape" | ||
| 23 | 23 | SCRAPFLY_API = "https://api.scrapfly.io/scrape" |
| 24 | 24 | |
| 25 | 25 | |
| 26 | +class SkipSource(RuntimeError): | |
| 27 | + """Levée par un connecteur qui ne peut pas tourner (clé API manquante…). | |
| 28 | + | |
| 29 | + Le pipeline (ingest.run) consigne le motif et passe à la source suivante | |
| 30 | + SANS marquer d'échec dans sync_log — ce n'est pas une erreur, la source | |
| 31 | + est simplement en attente (ex. Yelp Fusion sans YELP_API_KEY).""" | |
| 32 | + | |
| 33 | + | |
| 26 | 34 | class BaseConnector: |
| 27 | 35 | """Un connecteur = un adaptateur propre à une source (ou une plateforme). |
| 28 | 36 | |
modified
restoka/connectors/osm.py
+15 −1
@@ -21,6 +21,7 @@ from __future__ import annotations | ||
| 21 | 21 | |
| 22 | 22 | import sys |
| 23 | 23 | |
| 24 | +from ..hours import parse_opening_hours | |
| 24 | 25 | from ..normalize import classify_cuisines |
| 25 | 26 | from ..schema import Restaurant |
| 26 | 27 | from .base import BaseConnector |
@@ -34,6 +35,13 @@ MIRRORS = [ | ||
| 34 | 35 | "https://overpass.private.coffee/api/interpreter", |
| 35 | 36 | "https://overpass.osm.jp/api/interpreter", |
| 36 | 37 | ] |
| 38 | + | |
| 39 | +# User-Agent DÉDIÉ aux miroirs Overpass. Vérifié le 2026-08-18 : | |
| 40 | +# overpass-api.de répond 406 Not Acceptable à tout UA « déguisé en | |
| 41 | +# navigateur » (Mozilla/… Chrome/… — y compris notre UA global qui préfixe | |
| 42 | +# RestoKaBot d'une chaîne Chrome) et à python-requests, mais accepte un UA | |
| 43 | +# de bot honnête et identifiable. On envoie donc le nom nu du bot ici. | |
| 44 | +OVERPASS_UA = "RestoKaBot/1.0 (+https://www.resto-ka.com/bot; contact@spboucher.ai)" | |
| 37 | 45 | # NB : overpass.osm.ch RETIRÉ des miroirs — c'est un extrait SUISSE seulement : |
| 38 | 46 | # il répond 200 avec 0 élément pour l'aire Québec, ce qui empoisonnait les |
| 39 | 47 | # runs (incident des runs #2/#3 : des types entiers « réussis » à 0 POI). |
@@ -135,7 +143,8 @@ class OsmConnector(BaseConnector): | ||
| 135 | 143 | last_exc: Exception | None = None |
| 136 | 144 | for mirror in MIRRORS: |
| 137 | 145 | try: |
| 138 | − resp = self.post(mirror, data={"data": query}) | |
| 146 | + resp = self.post(mirror, data={"data": query}, | |
| 147 | + headers={"User-Agent": OVERPASS_UA}) | |
| 139 | 148 | data = resp.json() |
| 140 | 149 | # Overpass peut répondre 200 avec des résultats TRONQUÉS et un |
| 141 | 150 | # « remark » (timeout/mémoire) — traiter comme un échec de |
@@ -213,6 +222,11 @@ class OsmConnector(BaseConnector): | ||
| 213 | 222 | hours = {} |
| 214 | 223 | if tags.get("opening_hours"): |
| 215 | 224 | hours["osm"] = tags["opening_hours"] |
| 225 | + # horaires STRUCTURÉS par jour quand la syntaxe est courante | |
| 226 | + # (Mo-Fr 11:00-22:00…) ; les cas exotiques restent en brut | |
| 227 | + parsed = parse_opening_hours(tags["opening_hours"]) | |
| 228 | + if parsed: | |
| 229 | + hours.update(parsed) | |
| 216 | 230 | osm_id = f"{el.get('type')}/{el.get('id')}" |
| 217 | 231 | return Restaurant( |
| 218 | 232 | source=self.source_id, |
modified
restoka/connectors/siteresto.py
+110 −7
@@ -34,6 +34,8 @@ import sys | ||
| 34 | 34 | import urllib.parse |
| 35 | 35 | from pathlib import Path |
| 36 | 36 | |
| 37 | +from ..hours import parse_jsonld_hours | |
| 38 | +from ..normalize import normalize_phone | |
| 37 | 39 | from ..schema import Restaurant |
| 38 | 40 | from .base import BaseConnector |
| 39 | 41 | |
@@ -49,6 +51,13 @@ REFRESH_DAYS = 30 # re-crawl des menus captés (cadence §14) | ||
| 49 | 51 | # verrou d'écriture du fichier des intégrations UEAT découvertes (workers //) |
| 50 | 52 | _UEAT_FILE_LOCK = __import__("threading").Lock() |
| 51 | 53 | |
| 54 | +# plateformes de réservation reconnues (lien « réserver » sur le site du resto) | |
| 55 | +_RESERVATION_DOMAINS = ( | |
| 56 | + "opentable.com", "opentable.ca", "libroreserve.com", "restoloco.com", | |
| 57 | + "restoloco.ca", "resy.com", "sevenrooms.com", "bookenda.com", | |
| 58 | + "zenchef.com", "guestonline.io", "tablein.com", | |
| 59 | +) | |
| 60 | + | |
| 52 | 61 | _UEAT_RE = re.compile( |
| 53 | 62 | r"(?:order\.ueat\.io/integration/|order-online\.ai.{0,200}?integration/)" |
| 54 | 63 | r"([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})", re.I) |
@@ -144,6 +153,66 @@ class SiteRestoConnector(BaseConnector): | ||
| 144 | 153 | pages.append(full) |
| 145 | 154 | return pages[:2], pdfs[:2] |
| 146 | 155 | |
| 156 | + # -- extraction contact/horaires/réservation (page d'accueil) --------------------- | |
| 157 | + @staticmethod | |
| 158 | + def _extract_contact(html: str, base_url: str) -> dict: | |
| 159 | + """{"phone", "hours", "reservation_url"} extraits du HTML d'accueil. | |
| 160 | + | |
| 161 | + - téléphone : lien tel: d'abord (fiable), sinon regex nord-américaine | |
| 162 | + sur le texte visible -> E.164 (+1XXXXXXXXXX) ; | |
| 163 | + - réservation : premier lien vers une plateforme connue (OpenTable, | |
| 164 | + Libro, Resto Loco…) ; | |
| 165 | + - horaires : JSON-LD schema.org (LocalBusiness/Restaurant, | |
| 166 | + openingHoursSpecification ou openingHours) -> structuré par jour. | |
| 167 | + """ | |
| 168 | + from bs4 import BeautifulSoup | |
| 169 | + out = {"phone": "", "hours": None, "reservation_url": ""} | |
| 170 | + if not html: | |
| 171 | + return out | |
| 172 | + soup = BeautifulSoup(html, "html.parser") | |
| 173 | + tel_numbers: list[str] = [] | |
| 174 | + for a in soup.find_all("a", href=True): | |
| 175 | + href = a["href"].strip() | |
| 176 | + if href.lower().startswith("tel:"): | |
| 177 | + p = normalize_phone(href[4:]) | |
| 178 | + if p and p not in tel_numbers: | |
| 179 | + tel_numbers.append(p) | |
| 180 | + if not out["reservation_url"] and href.startswith("http"): | |
| 181 | + dom = _domain(href) | |
| 182 | + if any(dom == d or dom.endswith("." + d) | |
| 183 | + for d in _RESERVATION_DOMAINS): | |
| 184 | + out["reservation_url"] = href | |
| 185 | + # >3 numéros distincts = page « trouver une succursale » d'une chaîne : | |
| 186 | + # impossible d'attribuer LE bon numéro à CE resto — on s'abstient | |
| 187 | + if 1 <= len(tel_numbers) <= 3: | |
| 188 | + out["phone"] = tel_numbers[0] | |
| 189 | + # horaires structurés via JSON-LD (LocalBusiness / Restaurant) | |
| 190 | + for script in soup.find_all("script", type="application/ld+json"): | |
| 191 | + try: | |
| 192 | + data = json.loads(script.string or "") | |
| 193 | + except (ValueError, TypeError): | |
| 194 | + continue | |
| 195 | + nodes = data if isinstance(data, list) else [data] | |
| 196 | + more = [] | |
| 197 | + for n in nodes: | |
| 198 | + if isinstance(n, dict) and isinstance(n.get("@graph"), list): | |
| 199 | + more.extend(n["@graph"]) | |
| 200 | + for node in nodes + more: | |
| 201 | + if not isinstance(node, dict): | |
| 202 | + continue | |
| 203 | + if not out["hours"]: | |
| 204 | + out["hours"] = parse_jsonld_hours(node) | |
| 205 | + if not out["phone"]: | |
| 206 | + out["phone"] = normalize_phone(node.get("telephone")) | |
| 207 | + if out["hours"] and out["phone"]: | |
| 208 | + break | |
| 209 | + if not out["phone"]: # secours : regex sur le texte visible | |
| 210 | + text = SiteRestoConnector._html_text(html)[:20_000] | |
| 211 | + m = re.search(r"(?:t[eé]l|phone|appelez|r[eé]servation)[^\n]{0,40}?" | |
| 212 | + r"(\(?\d{3}\)?[\s.\-]\d{3}[\s.\-]\d{4})", text, re.I) | |
| 213 | + out["phone"] = normalize_phone(m.group(1) if m else "") | |
| 214 | + return out | |
| 215 | + | |
| 147 | 216 | # -- traitement d'un resto ------------------------------------------------------- |
| 148 | 217 | def _discover_ueat(self, html: str, final_url: str) -> str | None: |
| 149 | 218 | """GUID d'intégration UEAT si le site embarque la plateforme.""" |
@@ -179,14 +248,17 @@ class SiteRestoConnector(BaseConnector): | ||
| 179 | 248 | f" ({guid}) -> ueat-discovered.json") |
| 180 | 249 | |
| 181 | 250 | def _crawl_menu(self, website: str, firecrawl_budget: list[int]) -> dict | None: |
| 182 | − """Retourne {"sections": [...], "url": <page>} ou None.""" | |
| 251 | + """Retourne {"sections": [...], "url": <page>, "contact": {...}}, | |
| 252 | + {"ueat": guid, "contact": ...} ou {"contact": ...} (pas de menu mais | |
| 253 | + téléphone/horaires/réservation captés) ou None.""" | |
| 183 | 254 | from .. import menullm |
| 184 | 255 | home = self._get_html(website) |
| 185 | 256 | if not home: |
| 186 | 257 | return None |
| 258 | + contact = self._extract_contact(home, website) | |
| 187 | 259 | guid = self._discover_ueat(home, website) |
| 188 | 260 | if guid: |
| 189 | − return {"ueat": guid} | |
| 261 | + return {"ueat": guid, "contact": contact} | |
| 190 | 262 | pages, pdfs = self._menu_links(home, website) |
| 191 | 263 | # 1) pages HTML de menu |
| 192 | 264 | for page_url in pages: |
@@ -203,20 +275,25 @@ class SiteRestoConnector(BaseConnector): | ||
| 203 | 275 | if len(_PRICE_HINT_RE.findall(text)) >= 3: |
| 204 | 276 | menu = menullm.menu_from_text(text) |
| 205 | 277 | if menu: |
| 206 | − return {"sections": menu["sections"], "url": page_url} | |
| 278 | + return {"sections": menu["sections"], "url": page_url, | |
| 279 | + "contact": contact} | |
| 207 | 280 | # 2) PDF de menu |
| 208 | 281 | for pdf_url in pdfs: |
| 209 | 282 | pdf = self._get_pdf(pdf_url) |
| 210 | 283 | if pdf[:5] == b"%PDF-": |
| 211 | 284 | menu = menullm.menu_from_pdf(pdf) |
| 212 | 285 | if menu: |
| 213 | − return {"sections": menu["sections"], "url": pdf_url} | |
| 286 | + return {"sections": menu["sections"], "url": pdf_url, | |
| 287 | + "contact": contact} | |
| 214 | 288 | # 3) la page d'accueil elle-même contient parfois le menu |
| 215 | 289 | text = self._html_text(home) |
| 216 | 290 | if len(_PRICE_HINT_RE.findall(text)) >= 8: |
| 217 | 291 | menu = menullm.menu_from_text(text) |
| 218 | 292 | if menu: |
| 219 | − return {"sections": menu["sections"], "url": website} | |
| 293 | + return {"sections": menu["sections"], "url": website, | |
| 294 | + "contact": contact} | |
| 295 | + if any(v for v in contact.values()): | |
| 296 | + return {"contact": contact} # pas de menu, mais du contact | |
| 220 | 297 | return None |
| 221 | 298 | |
| 222 | 299 | # -- reconstruction des fiches déjà captées ------------------------------------------ |
@@ -270,8 +347,11 @@ class SiteRestoConnector(BaseConnector): | ||
| 270 | 347 | |
| 271 | 348 | firecrawl_budget = [MAX_FIRECRAWL_PER_RUN |
| 272 | 349 | if os.environ.get("FIRECRAWL_API_KEY") else 0] |
| 273 | − stats = {"crawled": 0, "menus": 0, "ueat": 0, "vides": 0} | |
| 350 | + stats = {"crawled": 0, "menus": 0, "ueat": 0, "vides": 0, "contacts": 0} | |
| 274 | 351 | now = time.time() |
| 352 | + # enrichissements à appliquer APRÈS sync_source (la fiche site-resto | |
| 353 | + # n'existe en base qu'après l'upsert) — consommé par ingest.run | |
| 354 | + self.pending_enrichment: list[tuple[str, dict]] = [] | |
| 275 | 355 | |
| 276 | 356 | # 1re passe (fil principal) : réémission depuis le cache + sélection |
| 277 | 357 | # des sites à crawler (budget) — aucune E/S réseau ici. |
@@ -327,13 +407,27 @@ class SiteRestoConnector(BaseConnector): | ||
| 327 | 407 | results = pool.map(_crawl_one, to_crawl) |
| 328 | 408 | for row, domain, website, result in results: |
| 329 | 409 | stats["crawled"] += 1 |
| 410 | + # téléphone / horaires / lien de réservation captés sur le | |
| 411 | + # site : COALESCE conservateur sur la fiche OSM d'origine | |
| 412 | + # (fil principal seulement — les workers ne touchent pas la BD) | |
| 413 | + contact = (result or {}).get("contact") or {} | |
| 414 | + if any(v for v in contact.values()): | |
| 415 | + from .. import db as _db | |
| 416 | + _db.enrich_contact( | |
| 417 | + con, row["uid"], phone=contact.get("phone") or "", | |
| 418 | + hours=contact.get("hours"), | |
| 419 | + details={"reservation_url": | |
| 420 | + contact.get("reservation_url")} | |
| 421 | + if contact.get("reservation_url") else None) | |
| 422 | + con.commit() | |
| 423 | + stats["contacts"] += 1 | |
| 330 | 424 | if result and result.get("ueat"): |
| 331 | 425 | self._record_ueat(result["ueat"], row["name"], domain) |
| 332 | 426 | self._set_status(domain, {"status": "ueat", |
| 333 | 427 | "guid": result["ueat"]}) |
| 334 | 428 | stats["ueat"] += 1 |
| 335 | 429 | continue |
| 336 | − if not result: | |
| 430 | + if not result or not result.get("sections"): | |
| 337 | 431 | self._set_status(domain, {"status": "no_menu"}) |
| 338 | 432 | stats["vides"] += 1 |
| 339 | 433 | continue |
@@ -352,6 +446,15 @@ class SiteRestoConnector(BaseConnector): | ||
| 352 | 446 | resto.cuisines = [] # reclassées avec le texte du menu |
| 353 | 447 | resto.price_range = "" |
| 354 | 448 | resto.dietary_options = [] |
| 449 | + # contact capté sur le site : complète la fiche émise (COALESCE) | |
| 450 | + if contact.get("phone") and not (resto.phone or "").strip(): | |
| 451 | + resto.phone = contact["phone"] | |
| 452 | + if contact.get("hours"): | |
| 453 | + resto.hours = {**contact["hours"], **(resto.hours or {})} | |
| 454 | + if contact.get("reservation_url"): | |
| 455 | + self.pending_enrichment.append( | |
| 456 | + (resto.uid, {"details": {"reservation_url": | |
| 457 | + contact["reservation_url"]}})) | |
| 355 | 458 | out[resto.uid] = resto |
| 356 | 459 | self._set_status(domain, {"status": "menu", "url": result["url"], |
| 357 | 460 | "captured_at": captured_at, |
added
restoka/connectors/yelp.py
+172 −0
@@ -0,0 +1,172 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: restoka/connectors/yelp.py | |
| 4 | +# Desc: Connecteur d'ENRICHISSEMENT Yelp Fusion (avis/notes) — gratuit, | |
| 5 | +# 500 requêtes/jour. N'émet AUCUNE fiche restaurant : il complète les | |
| 6 | +# fiches existantes avec rating, review_count, price, categories et | |
| 7 | +# phone dans details.yelp (colonne d'enrichissement, jamais écrasée | |
| 8 | +# par les re-crawls des sources). | |
| 9 | +# | |
| 10 | +# Croisement CONSERVATEUR (jamais fusionner deux établissements) : | |
| 11 | +# 1. par TÉLÉPHONE (/businesses/search/phone) — signal fort ; si | |
| 12 | +# plusieurs résultats, on exige le GPS le plus proche (<300 m) ; | |
| 13 | +# 2. sinon par NOM+ADRESSE+GPS (/businesses/matches) — l'endpoint | |
| 14 | +# d'appariement officiel de Yelp, seuil par défaut (conservateur). | |
| 15 | +# | |
| 16 | +# Clé requise : YELP_API_KEY dans .env (https://www.yelp.com/developers | |
| 17 | +# — app gratuite). Sans clé, le connecteur lève SkipSource et le | |
| 18 | +# pipeline passe son tour SANS marquer d'échec (statut « clé requise » | |
| 19 | +# dans data/sources.json). | |
| 20 | +# Base légale : API officielle Yelp Fusion, conditions Display | |
| 21 | +# Requirements (attribution Yelp affichée avec la note sur la fiche). | |
| 22 | +# ============================================================================== | |
| 23 | +from __future__ import annotations | |
| 24 | + | |
| 25 | +import datetime | |
| 26 | +import json | |
| 27 | +import math | |
| 28 | +import os | |
| 29 | +import sys | |
| 30 | +import time | |
| 31 | + | |
| 32 | +from ..normalize import normalize_phone | |
| 33 | +from ..schema import Restaurant | |
| 34 | +from .base import BaseConnector, SkipSource | |
| 35 | + | |
| 36 | +API = "https://api.yelp.com/v3" | |
| 37 | +REFRESH_DAYS = 30 # re-vérification des notes captées | |
| 38 | +MAX_REQUESTS_PER_RUN = int(os.environ.get("YELP_BUDGET", "450")) # quota 500/j | |
| 39 | +MAX_PHONE_DISTANCE_M = 300 # ambiguïté téléphone : GPS requis | |
| 40 | + | |
| 41 | + | |
| 42 | +def _haversine_m(lat1, lng1, lat2, lng2) -> float: | |
| 43 | + r = 6371000.0 | |
| 44 | + p1, p2 = math.radians(lat1), math.radians(lat2) | |
| 45 | + dp, dl = math.radians(lat2 - lat1), math.radians(lng2 - lng1) | |
| 46 | + a = math.sin(dp / 2) ** 2 + math.cos(p1) * math.cos(p2) * math.sin(dl / 2) ** 2 | |
| 47 | + return 2 * r * math.asin(math.sqrt(a)) | |
| 48 | + | |
| 49 | + | |
| 50 | +class YelpConnector(BaseConnector): | |
| 51 | + source_id = "yelp" | |
| 52 | + request_delay = 0.35 # politesse API (limite 5 QPS) | |
| 53 | + timeout = 20 | |
| 54 | + use_detail_cache = False | |
| 55 | + enrichment_only = True # n'émet aucune fiche (ingest.run) | |
| 56 | + | |
| 57 | + def _api(self, path: str, params: dict) -> dict: | |
| 58 | + resp = self.get(f"{API}{path}", params=params, | |
| 59 | + headers={"Authorization": f"Bearer {self._key}"}) | |
| 60 | + return resp.json() | |
| 61 | + | |
| 62 | + def _payload(self, biz: dict, matched_by: str) -> dict: | |
| 63 | + return { | |
| 64 | + "id": biz.get("id"), | |
| 65 | + "url": (biz.get("url") or "").split("?")[0], | |
| 66 | + "name": biz.get("name"), | |
| 67 | + "rating": biz.get("rating"), | |
| 68 | + "review_count": biz.get("review_count"), | |
| 69 | + "price": biz.get("price"), | |
| 70 | + "categories": [c.get("title") for c in biz.get("categories") or []], | |
| 71 | + "phone": biz.get("phone"), | |
| 72 | + "matched_by": matched_by, | |
| 73 | + "fetched_at": datetime.datetime.now(datetime.timezone.utc) | |
| 74 | + .strftime("%Y-%m-%dT%H:%M:%SZ"), | |
| 75 | + } | |
| 76 | + | |
| 77 | + def _match_by_phone(self, phone: str, lat, lng) -> dict | None: | |
| 78 | + data = self._api("/businesses/search/phone", {"phone": phone}) | |
| 79 | + businesses = data.get("businesses") or [] | |
| 80 | + if not businesses: | |
| 81 | + return None | |
| 82 | + if len(businesses) == 1: | |
| 83 | + return self._payload(businesses[0], "telephone") | |
| 84 | + if lat is None or lng is None: | |
| 85 | + return None # plusieurs candidats, pas de GPS : on passe | |
| 86 | + best, best_d = None, None | |
| 87 | + for b in businesses: | |
| 88 | + c = b.get("coordinates") or {} | |
| 89 | + if c.get("latitude") is None or c.get("longitude") is None: | |
| 90 | + continue | |
| 91 | + d = _haversine_m(lat, lng, c["latitude"], c["longitude"]) | |
| 92 | + if best_d is None or d < best_d: | |
| 93 | + best, best_d = b, d | |
| 94 | + if best is not None and best_d is not None and best_d <= MAX_PHONE_DISTANCE_M: | |
| 95 | + return self._payload(best, "telephone+gps") | |
| 96 | + return None | |
| 97 | + | |
| 98 | + def _match_by_name(self, row) -> dict | None: | |
| 99 | + if not (row["name"] and row["address"] and row["city"] | |
| 100 | + and row["lat"] is not None and row["lng"] is not None): | |
| 101 | + return None | |
| 102 | + data = self._api("/businesses/matches", { | |
| 103 | + "name": row["name"][:64], "address1": row["address"][:64], | |
| 104 | + "city": row["city"][:64], "state": "QC", "country": "CA", | |
| 105 | + "latitude": row["lat"], "longitude": row["lng"], "limit": 1, | |
| 106 | + }) | |
| 107 | + businesses = data.get("businesses") or [] | |
| 108 | + if not businesses: | |
| 109 | + return None | |
| 110 | + biz = businesses[0] | |
| 111 | + # /matches ne retourne pas rating/review_count : détail requis | |
| 112 | + detail = self._api(f"/businesses/{biz['id']}", {}) | |
| 113 | + return self._payload(detail or biz, "nom+gps") | |
| 114 | + | |
| 115 | + def fetch(self) -> list[Restaurant]: | |
| 116 | + self._key = os.environ.get("YELP_API_KEY", "").strip() | |
| 117 | + if not self._key: | |
| 118 | + raise SkipSource("YELP_API_KEY manquant (.env) — connecteur prêt, " | |
| 119 | + "clé gratuite : https://www.yelp.com/developers") | |
| 120 | + from .. import db | |
| 121 | + con = db.connect() | |
| 122 | + now = time.time() | |
| 123 | + stale = now - REFRESH_DAYS * 86400 | |
| 124 | + budget = MAX_REQUESTS_PER_RUN | |
| 125 | + enriched = skipped = 0 | |
| 126 | + rows = con.execute( | |
| 127 | + "SELECT uid, name, address, city, lat, lng, phone, details" | |
| 128 | + " FROM restaurants WHERE active=1 AND dup_of IS NULL" | |
| 129 | + " ORDER BY phone<>'' DESC, updated_at DESC").fetchall() | |
| 130 | + for row in rows: | |
| 131 | + if budget <= 0: | |
| 132 | + break | |
| 133 | + try: | |
| 134 | + details = json.loads(row["details"] or "{}") | |
| 135 | + except ValueError: | |
| 136 | + details = {} | |
| 137 | + yelp = details.get("yelp") or {} | |
| 138 | + if yelp.get("fetched_at"): | |
| 139 | + try: | |
| 140 | + ts = datetime.datetime.strptime( | |
| 141 | + yelp["fetched_at"], "%Y-%m-%dT%H:%M:%SZ") \ | |
| 142 | + .replace(tzinfo=datetime.timezone.utc).timestamp() | |
| 143 | + if ts > stale: | |
| 144 | + continue # déjà frais (<30 j) | |
| 145 | + except ValueError: | |
| 146 | + pass | |
| 147 | + payload = None | |
| 148 | + try: | |
| 149 | + phone = normalize_phone(row["phone"]) | |
| 150 | + if phone: | |
| 151 | + budget -= 1 | |
| 152 | + payload = self._match_by_phone(phone, row["lat"], row["lng"]) | |
| 153 | + if payload is None: | |
| 154 | + budget -= 2 # /matches + détail | |
| 155 | + payload = self._match_by_name(row) | |
| 156 | + except Exception as exc: # un resto raté ne bloque pas | |
| 157 | + skipped += 1 | |
| 158 | + print(f"[resto-ka] yelp: {row['uid']} erreur: {exc}", | |
| 159 | + file=sys.stderr) | |
| 160 | + continue | |
| 161 | + if payload: | |
| 162 | + db.merge_details(con, row["uid"], {"yelp": payload}) | |
| 163 | + con.commit() | |
| 164 | + enriched += 1 | |
| 165 | + con.commit() | |
| 166 | + con.close() | |
| 167 | + self.enriched_count = enriched | |
| 168 | + self.enrich_message = (f"{enriched} resto(s) enrichis (avis/notes), " | |
| 169 | + f"budget restant {max(budget, 0)} req") | |
| 170 | + print(f"[resto-ka] yelp: {self.enrich_message}" | |
| 171 | + + (f", {skipped} erreurs" if skipped else "")) | |
| 172 | + return [] | |
modified
restoka/db.py
+78 −0
@@ -157,6 +157,30 @@ CREATE TABLE IF NOT EXISTS favorites ( | ||
| 157 | 157 | ts REAL, |
| 158 | 158 | PRIMARY KEY (user_id, uid) |
| 159 | 159 | ); |
| 160 | + | |
| 161 | +-- Inspections alimentaires MAPAQ (« Condamnations des établissements | |
| 162 | +-- alimentaires », Données Québec, licence CC-BY 4.0). Table LIÉE : le | |
| 163 | +-- croisement conservateur avec restaurants remplit `uid` (sinon NULL). | |
| 164 | +CREATE TABLE IF NOT EXISTS inspections ( | |
| 165 | + id INTEGER PRIMARY KEY AUTOINCREMENT, | |
| 166 | + row_hash TEXT UNIQUE, -- anti-doublon au ré-import | |
| 167 | + exploitant TEXT, -- Nom_exploitant (entité légale) | |
| 168 | + etablissement TEXT, -- Raison_sociale (nom commercial) | |
| 169 | + description TEXT, -- Description_infraction | |
| 170 | + adresse TEXT, -- Adresse_lieu_infraction (brute) | |
| 171 | + postal_code TEXT, -- extrait de l'adresse (A1A1A1) | |
| 172 | + type_etablissement TEXT, -- Type_etablissement MAPAQ | |
| 173 | + categorie TEXT, -- regroupement (RESTAURATION, LAIT…) | |
| 174 | + date_infraction TEXT, -- ISO-8601 (date seulement) | |
| 175 | + date_jugement TEXT, | |
| 176 | + date_publication TEXT, | |
| 177 | + montant_amende REAL, | |
| 178 | + loi TEXT, | |
| 179 | + motif TEXT, -- SOC_NOM_ARTCL_INFRC (INSALUBRITE…) | |
| 180 | + uid TEXT, -- restaurants.uid si croisement réussi | |
| 181 | + matched_by TEXT -- nom+ville | adresse+nom (traçabilité) | |
| 182 | +); | |
| 183 | +CREATE INDEX IF NOT EXISTS idx_inspections_uid ON inspections(uid); | |
| 160 | 184 | """ |
| 161 | 185 | |
| 162 | 186 | |
@@ -165,6 +189,9 @@ CREATE TABLE IF NOT EXISTS favorites ( | ||
| 165 | 189 | _MIGRATIONS = { |
| 166 | 190 | "restaurants": { |
| 167 | 191 | "images": "TEXT", |
| 192 | + # enrichissements hors connecteurs (réservation, Yelp, MAPAQ…) — | |
| 193 | + # JAMAIS touché par sync_source : survit aux re-crawls des sources | |
| 194 | + "details": "TEXT", | |
| 168 | 195 | }, |
| 169 | 196 | } |
| 170 | 197 | |
@@ -426,6 +453,57 @@ def log_failure(con: sqlite3.Connection, source: str, message: str) -> None: | ||
| 426 | 453 | con.commit() |
| 427 | 454 | |
| 428 | 455 | |
| 456 | +# --------------------------------------------------------------------------- | |
| 457 | +# Enrichissements hors connecteurs (colonne `details` + COALESCE conservateur) | |
| 458 | +# --------------------------------------------------------------------------- | |
| 459 | + | |
| 460 | +def merge_details(con: sqlite3.Connection, uid: str, patch: dict) -> bool: | |
| 461 | + """Fusionne `patch` dans la colonne JSON `details` du resto (additif : | |
| 462 | + les clés du patch écrasent seulement leurs propres clés). Retourne False | |
| 463 | + si le resto n'existe pas (encore).""" | |
| 464 | + row = con.execute("SELECT details FROM restaurants WHERE uid=?", | |
| 465 | + (uid,)).fetchone() | |
| 466 | + if row is None: | |
| 467 | + return False | |
| 468 | + try: | |
| 469 | + details = json.loads(row["details"] or "{}") | |
| 470 | + except ValueError: | |
| 471 | + details = {} | |
| 472 | + details.update({k: v for k, v in patch.items() if v not in (None, "", {})}) | |
| 473 | + con.execute("UPDATE restaurants SET details=? WHERE uid=?", | |
| 474 | + (json.dumps(details, ensure_ascii=False), uid)) | |
| 475 | + return True | |
| 476 | + | |
| 477 | + | |
| 478 | +def enrich_contact(con: sqlite3.Connection, uid: str, phone: str = "", | |
| 479 | + hours: dict | None = None, details: dict | None = None) -> None: | |
| 480 | + """Enrichissement CONSERVATEUR d'une fiche : ne remplit que les trous. | |
| 481 | + | |
| 482 | + - phone : seulement si la colonne est vide (COALESCE) ; | |
| 483 | + - hours : fusion — n'ajoute que les clés absentes (le connecteur d'origine | |
| 484 | + garde la priorité) ; | |
| 485 | + - details : fusion via merge_details (reservation_url, yelp, mapaq…). | |
| 486 | + """ | |
| 487 | + row = con.execute("SELECT phone, hours FROM restaurants WHERE uid=?", | |
| 488 | + (uid,)).fetchone() | |
| 489 | + if row is None: | |
| 490 | + return | |
| 491 | + if phone and not (row["phone"] or "").strip(): | |
| 492 | + con.execute("UPDATE restaurants SET phone=? WHERE uid=?", (phone, uid)) | |
| 493 | + if hours: | |
| 494 | + try: | |
| 495 | + cur = json.loads(row["hours"] or "{}") | |
| 496 | + except ValueError: | |
| 497 | + cur = {} | |
| 498 | + add = {k: v for k, v in hours.items() if k not in cur} | |
| 499 | + if add: | |
| 500 | + cur.update(add) | |
| 501 | + con.execute("UPDATE restaurants SET hours=? WHERE uid=?", | |
| 502 | + (json.dumps(cur, ensure_ascii=False), uid)) | |
| 503 | + if details: | |
| 504 | + merge_details(con, uid, details) | |
| 505 | + | |
| 506 | + | |
| 429 | 507 | # --------------------------------------------------------------------------- |
| 430 | 508 | # Cache des pages détail (« détail si nouveau/modifié ») |
| 431 | 509 | # --------------------------------------------------------------------------- |
added
restoka/hours.py
+167 −0
@@ -0,0 +1,167 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: restoka/hours.py | |
| 4 | +# Desc: Parseur d'horaires — convertit la syntaxe OSM `opening_hours` | |
| 5 | +# (« Mo-Fr 11:00-22:00; Sa-Su 10:00-23:00 ») et les horaires JSON-LD | |
| 6 | +# schema.org (openingHoursSpecification) en horaires STRUCTURÉS par | |
| 7 | +# jour : {"mon": [["11:00","22:00"]], ...}. Les syntaxes exotiques | |
| 8 | +# (sunrise, mois, conditions) ne sont PAS devinées : on retourne None | |
| 9 | +# et le tag brut reste dans hours["osm"] (jamais de perte). | |
| 10 | +# ============================================================================== | |
| 11 | +from __future__ import annotations | |
| 12 | + | |
| 13 | +import re | |
| 14 | + | |
| 15 | +__all__ = ["parse_opening_hours", "parse_jsonld_hours", "DAYS"] | |
| 16 | + | |
| 17 | +DAYS = ("mon", "tue", "wed", "thu", "fri", "sat", "sun") | |
| 18 | + | |
| 19 | +_OSM_DAY = {"mo": 0, "tu": 1, "we": 2, "th": 3, "fr": 4, "sa": 5, "su": 6} | |
| 20 | + | |
| 21 | +# schema.org : "Monday", "https://schema.org/Monday", "Mo"… | |
| 22 | +_SCHEMA_DAY = { | |
| 23 | + "monday": 0, "tuesday": 1, "wednesday": 2, "thursday": 3, | |
| 24 | + "friday": 4, "saturday": 5, "sunday": 6, | |
| 25 | + "mo": 0, "tu": 1, "we": 2, "th": 3, "fr": 4, "sa": 5, "su": 6, | |
| 26 | +} | |
| 27 | + | |
| 28 | +_TIME_RANGE_RE = re.compile(r"^(\d{1,2}):(\d{2})\s*-\s*(\d{1,2}):(\d{2})$") | |
| 29 | +# marqueurs de syntaxe OSM avancée que l'on refuse de deviner (le brut suffit) | |
| 30 | +_EXOTIC_RE = re.compile( | |
| 31 | + r"sunrise|sunset|dawn|dusk|week\s|\bweek\b|easter|\[|\"|\|\||" | |
| 32 | + r"\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\b", re.I) | |
| 33 | + | |
| 34 | + | |
| 35 | +def _expand_days(spec: str) -> list[int] | None: | |
| 36 | + """« Mo-Fr », « Sa,Su », « Mo,We-Fr » -> indices 0-6 ; None si inconnu.""" | |
| 37 | + out: list[int] = [] | |
| 38 | + for part in spec.split(","): | |
| 39 | + part = part.strip().lower() | |
| 40 | + if not part: | |
| 41 | + continue | |
| 42 | + if part in ("ph", "sh"): # jours fériés/vacances : ignorés | |
| 43 | + continue | |
| 44 | + m = re.match(r"^([a-z]{2})\s*-\s*([a-z]{2})$", part) | |
| 45 | + if m: | |
| 46 | + a, b = _OSM_DAY.get(m.group(1)), _OSM_DAY.get(m.group(2)) | |
| 47 | + if a is None or b is None: | |
| 48 | + return None | |
| 49 | + i = a | |
| 50 | + while True: | |
| 51 | + if i not in out: | |
| 52 | + out.append(i) | |
| 53 | + if i == b: | |
| 54 | + break | |
| 55 | + i = (i + 1) % 7 # Sa-Mo enroule la semaine | |
| 56 | + elif part in _OSM_DAY: | |
| 57 | + if _OSM_DAY[part] not in out: | |
| 58 | + out.append(_OSM_DAY[part]) | |
| 59 | + else: | |
| 60 | + return None | |
| 61 | + return out | |
| 62 | + | |
| 63 | + | |
| 64 | +def _parse_times(spec: str) -> list[list[str]] | None: | |
| 65 | + """« 11:00-14:00,17:00-22:00 » -> [["11:00","14:00"],["17:00","22:00"]].""" | |
| 66 | + if spec.strip().lower() in ("off", "closed"): | |
| 67 | + return [] | |
| 68 | + out: list[list[str]] = [] | |
| 69 | + for chunk in spec.split(","): | |
| 70 | + m = _TIME_RANGE_RE.match(chunk.strip()) | |
| 71 | + if not m: | |
| 72 | + return None | |
| 73 | + h1, m1, h2, m2 = (int(g) for g in m.groups()) | |
| 74 | + if h1 > 24 or h2 > 48 or m1 > 59 or m2 > 59: | |
| 75 | + return None | |
| 76 | + # « 17:00-01:00 » (ferme après minuit) est conservé tel quel | |
| 77 | + out.append([f"{h1:02d}:{m1:02d}", f"{h2 % 24:02d}:{m2:02d}" | |
| 78 | + if h2 > 24 else f"{h2:02d}:{m2:02d}"]) | |
| 79 | + return out | |
| 80 | + | |
| 81 | + | |
| 82 | +def parse_opening_hours(raw: str | None) -> dict[str, list[list[str]]] | None: | |
| 83 | + """Tag OSM `opening_hours` -> {"mon": [["11:00","22:00"]], ...} ou None. | |
| 84 | + | |
| 85 | + Gère les syntaxes courantes : plages de jours (Mo-Fr), listes (Sa,Su), | |
| 86 | + plusieurs plages horaires (11:00-14:00,17:00-22:00), « off », « 24/7 », | |
| 87 | + PH/SH ignorés. Tout le reste (sunrise, mois, semaines, conditions) est | |
| 88 | + refusé — le tag brut reste disponible dans hours["osm"]. | |
| 89 | + """ | |
| 90 | + if not raw or not isinstance(raw, str): | |
| 91 | + return None | |
| 92 | + raw = raw.strip() | |
| 93 | + if raw == "24/7": | |
| 94 | + return {d: [["00:00", "24:00"]] for d in DAYS} | |
| 95 | + if _EXOTIC_RE.search(raw): | |
| 96 | + return None | |
| 97 | + out: dict[str, list[list[str]]] = {} | |
| 98 | + for rule in raw.split(";"): | |
| 99 | + rule = rule.strip().rstrip(",") | |
| 100 | + if not rule: | |
| 101 | + continue | |
| 102 | + # séparer le spécificateur de jours du spécificateur d'heures | |
| 103 | + m = re.match(r"^([A-Za-z ,\-]+?)\s+(.+)$", rule) | |
| 104 | + if m and re.search(r"[a-zA-Z]", m.group(1)): | |
| 105 | + days_spec, times_spec = m.group(1), m.group(2) | |
| 106 | + elif re.match(r"^\d", rule): # heures seules -> tous les jours | |
| 107 | + days_spec, times_spec = "Mo-Su", rule | |
| 108 | + elif rule.strip().lower() in ("ph off", "sh off", "ph closed"): | |
| 109 | + continue | |
| 110 | + else: | |
| 111 | + return None | |
| 112 | + # « PH off » collé à une règle de jours (ex. « PH 10:00-… ») : ignorer | |
| 113 | + if days_spec.strip().lower() in ("ph", "sh"): | |
| 114 | + continue | |
| 115 | + days = _expand_days(days_spec) | |
| 116 | + times = _parse_times(times_spec) | |
| 117 | + if days is None or times is None: | |
| 118 | + return None # syntaxe non couverte : brut | |
| 119 | + for d in days: | |
| 120 | + out[DAYS[d]] = times | |
| 121 | + return out or None | |
| 122 | + | |
| 123 | + | |
| 124 | +def parse_jsonld_hours(node: dict) -> dict[str, list[list[str]]] | None: | |
| 125 | + """Horaires d'un nœud JSON-LD schema.org (LocalBusiness/Restaurant). | |
| 126 | + | |
| 127 | + Gère `openingHoursSpecification` (liste de {dayOfWeek, opens, closes}) | |
| 128 | + et `openingHours` (chaînes façon OSM « Mo-Fr 11:00-22:00 »). | |
| 129 | + """ | |
| 130 | + spec = node.get("openingHoursSpecification") | |
| 131 | + if isinstance(spec, dict): | |
| 132 | + spec = [spec] | |
| 133 | + if isinstance(spec, list) and spec: | |
| 134 | + out: dict[str, list[list[str]]] = {} | |
| 135 | + for entry in spec: | |
| 136 | + if not isinstance(entry, dict): | |
| 137 | + continue | |
| 138 | + days = entry.get("dayOfWeek") or [] | |
| 139 | + if isinstance(days, str): | |
| 140 | + days = [days] | |
| 141 | + opens = str(entry.get("opens") or "")[:5] | |
| 142 | + closes = str(entry.get("closes") or "")[:5] | |
| 143 | + if not re.match(r"^\d{1,2}:\d{2}$", opens) or \ | |
| 144 | + not re.match(r"^\d{1,2}:\d{2}$", closes): | |
| 145 | + continue | |
| 146 | + rng = [f"{int(opens.split(':')[0]):02d}:{opens.split(':')[1]}", | |
| 147 | + f"{int(closes.split(':')[0]):02d}:{closes.split(':')[1]}"] | |
| 148 | + for d in days: | |
| 149 | + key = str(d).rsplit("/", 1)[-1].strip().lower() | |
| 150 | + idx = _SCHEMA_DAY.get(key) | |
| 151 | + if idx is None: | |
| 152 | + continue | |
| 153 | + out.setdefault(DAYS[idx], []).append(rng) | |
| 154 | + if out: | |
| 155 | + return out | |
| 156 | + oh = node.get("openingHours") | |
| 157 | + if isinstance(oh, str): | |
| 158 | + oh = [oh] | |
| 159 | + if isinstance(oh, list) and oh: | |
| 160 | + merged: dict[str, list[list[str]]] = {} | |
| 161 | + for chunk in oh: | |
| 162 | + parsed = parse_opening_hours(str(chunk)) | |
| 163 | + if parsed: | |
| 164 | + merged.update(parsed) | |
| 165 | + if merged: | |
| 166 | + return merged | |
| 167 | + return None | |
modified
restoka/ingest.py
+29 −0
@@ -14,6 +14,7 @@ import traceback | ||
| 14 | 14 | |
| 15 | 15 | from . import db |
| 16 | 16 | from .connectors import CONNECTORS |
| 17 | +from .connectors.base import SkipSource | |
| 17 | 18 | |
| 18 | 19 | |
| 19 | 20 | def run(sources: list[str] | None = None) -> list[dict]: |
@@ -31,6 +32,19 @@ def run(sources: list[str] | None = None) -> list[dict]: | ||
| 31 | 32 | try: |
| 32 | 33 | connector = cls() |
| 33 | 34 | restos = connector.fetch() |
| 35 | + # connecteur d'ENRICHISSEMENT (ex. Yelp) : complète les fiches | |
| 36 | + # existantes sans en émettre — pas de diff, journal dédié | |
| 37 | + if getattr(connector, "enrichment_only", False): | |
| 38 | + n = getattr(connector, "enriched_count", 0) | |
| 39 | + msg = getattr(connector, "enrich_message", "enrichissement") | |
| 40 | + con.execute( | |
| 41 | + "INSERT INTO sync_log (source, ts, found, added, updated," | |
| 42 | + " removed, ok, message) VALUES (?,?,?,0,?,0,1,?)", | |
| 43 | + (sid, time.time(), n, n, msg)) | |
| 44 | + con.commit() | |
| 45 | + results.append({"source": sid, "enriched": n, | |
| 46 | + "seconds": round(time.time() - t0, 1)}) | |
| 47 | + continue | |
| 34 | 48 | finalized, dropped = [], 0 |
| 35 | 49 | for r in restos: |
| 36 | 50 | try: |
@@ -43,6 +57,13 @@ def run(sources: list[str] | None = None) -> list[dict]: | ||
| 43 | 57 | stats = db.sync_source(con, sid, finalized, |
| 44 | 58 | partial=getattr(connector, "partial_run", |
| 45 | 59 | False)) |
| 60 | + # enrichissements différés (ex. site-resto : reservation_url de la | |
| 61 | + # fiche site-resto, qui n'existe en base qu'après l'upsert) | |
| 62 | + for uid, patch in getattr(connector, "pending_enrichment", []) or []: | |
| 63 | + db.enrich_contact(con, uid, phone=patch.get("phone", ""), | |
| 64 | + hours=patch.get("hours"), | |
| 65 | + details=patch.get("details")) | |
| 66 | + con.commit() | |
| 46 | 67 | stats["seconds"] = round(time.time() - t0, 1) |
| 47 | 68 | if dropped: |
| 48 | 69 | stats["dropped"] = dropped |
@@ -50,6 +71,9 @@ def run(sources: list[str] | None = None) -> list[dict]: | ||
| 50 | 71 | print(f"[resto-ka] ⚠ ALERTE {sid} : {stats['alert']}") |
| 51 | 72 | print(f"[resto-ka] {stats}") |
| 52 | 73 | results.append(stats) |
| 74 | + except SkipSource as exc: # source en attente (clé API manquante…) | |
| 75 | + print(f"[resto-ka] {sid} sauté : {exc}") | |
| 76 | + results.append({"source": sid, "skipped": str(exc)}) | |
| 53 | 77 | except Exception as exc: # robustesse : une source ne bloque pas les autres |
| 54 | 78 | db.log_failure(con, sid, f"{exc}") |
| 55 | 79 | traceback.print_exc() |
@@ -75,6 +99,11 @@ def enrich() -> None: | ||
| 75 | 99 | print(f"[resto-ka] dedup: {stats}") |
| 76 | 100 | except Exception as exc: |
| 77 | 101 | print(f"[resto-ka] dedup: erreur non bloquante: {exc}", file=sys.stderr) |
| 102 | + try: # inspections MAPAQ (condamnations) — cadence hebdo, croisement | |
| 103 | + from . import inspections | |
| 104 | + inspections.sync() | |
| 105 | + except Exception as exc: | |
| 106 | + print(f"[resto-ka] mapaq: erreur non bloquante: {exc}", file=sys.stderr) | |
| 78 | 107 | |
| 79 | 108 | |
| 80 | 109 | def watch(interval_seconds: int = 7 * 86400 // 7) -> None: |
added
restoka/inspections.py
+284 −0
@@ -0,0 +1,284 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: restoka/inspections.py | |
| 4 | +# Desc: Connecteur MAPAQ — « Condamnations des établissements alimentaires » | |
| 5 | +# (Données Québec, jeu 515374ee…, CSV listecondamnation.csv, licence | |
| 6 | +# CC-BY 4.0). Inspections alimentaires condamnées : établissement, | |
| 7 | +# adresse, dates, amende, motif (INSALUBRITE…). Alimente la table | |
| 8 | +# LIÉE `inspections` + croisement CONSERVATEUR avec les restaurants | |
| 9 | +# (jamais de fusion hasardeuse : nom normalisé + ville/code postal, | |
| 10 | +# ou code postal + civique + similarité de nom). | |
| 11 | +# | |
| 12 | +# Ce n'est PAS une source de fiches restaurant : c'est un | |
| 13 | +# enrichissement conformité/qualité appelé par ingest.enrich() | |
| 14 | +# (cadence hebdomadaire, guard via sync_log source='mapaq'). | |
| 15 | +# ============================================================================== | |
| 16 | +from __future__ import annotations | |
| 17 | + | |
| 18 | +import csv | |
| 19 | +import hashlib | |
| 20 | +import io | |
| 21 | +import json | |
| 22 | +import re | |
| 23 | +import sys | |
| 24 | +import time | |
| 25 | + | |
| 26 | +import requests | |
| 27 | + | |
| 28 | +from .normalize import parse_price, strip_accents | |
| 29 | + | |
| 30 | +SOURCE_ID = "mapaq" | |
| 31 | +CSV_URL = ("https://www.donneesquebec.ca/recherche/dataset/" | |
| 32 | + "515374ee-ce34-464f-9875-7d1af3fa9b2a/resource/" | |
| 33 | + "40105615-3abf-414b-bcba-182e8f2c5eb2/download/listecondamnation.csv") | |
| 34 | +USER_AGENT = "RestoKaBot/1.0 (+https://www.resto-ka.com/bot; contact@spboucher.ai)" | |
| 35 | +REFRESH_DAYS = 6 # au plus une fois par cycle hebdo du watcher | |
| 36 | + | |
| 37 | +_POSTAL_RE = re.compile(r"\b([A-Z]\d[A-Z])\s?(\d[A-Z]\d)\b") | |
| 38 | +_CIVIC_RE = re.compile(r"^\s*(\d+)") | |
| 39 | +# entité légale « à numéro » (9277-4876 QUEBEC INC.) : inutilisable pour le nom | |
| 40 | +_NUMBERED_CO_RE = re.compile(r"^\d{4}-\d{4}\s+quebec", re.I) | |
| 41 | +_LEGAL_RE = re.compile( | |
| 42 | + r"\b(inc|ltee|ltd|enr|senc|sec|s\.e\.n\.c|s\.a|cie|co|corp|corporation)\b\.?", | |
| 43 | + re.I) | |
| 44 | +_STOPWORDS = {"le", "la", "les", "l", "du", "de", "des", "d", "au", "aux", | |
| 45 | + "et", "un", "une", "chez", "restaurant", "resto", "cafe", "bar", | |
| 46 | + "groupe", "gestion", "quebec", "canada"} | |
| 47 | + | |
| 48 | + | |
| 49 | +def norm_name(name: str) -> str: | |
| 50 | + """Nom commercial normalisé pour le croisement : accents, ponctuation, | |
| 51 | + suffixes légaux et espaces réduits.""" | |
| 52 | + s = strip_accents((name or "").lower()) | |
| 53 | + s = _LEGAL_RE.sub(" ", s) | |
| 54 | + s = re.sub(r"[^a-z0-9]+", " ", s) | |
| 55 | + return re.sub(r"\s+", " ", s).strip() | |
| 56 | + | |
| 57 | + | |
| 58 | +def core_name(norm: str) -> str: | |
| 59 | + """Cœur du nom : tokens significatifs sans mots génériques (« Restaurant | |
| 60 | + Chez Mamy » et « Chez Mamy » -> « mamy »).""" | |
| 61 | + return " ".join(t for t in norm.split() if t not in _STOPWORDS) | |
| 62 | + | |
| 63 | + | |
| 64 | +def _tokens(norm: str) -> set[str]: | |
| 65 | + return {t for t in norm.split() if len(t) >= 3 and t not in _STOPWORDS} | |
| 66 | + | |
| 67 | + | |
| 68 | +def _name_similar(a: str, b: str, exclude: frozenset | set = frozenset()) -> bool: | |
| 69 | + """Similarité conservatrice : contenance mutuelle ou ≥1 token significatif | |
| 70 | + partagé. `exclude` = tokens de l'adresse/ville : un nom qui ne partage que | |
| 71 | + le nom de sa VILLE (« Les Moulins La Fayette St-Hyacinthe » vs « Sushi | |
| 72 | + Taxi — St-Hyacinthe », voisins de la même adresse) n'est PAS similaire.""" | |
| 73 | + if not a or not b: | |
| 74 | + return False | |
| 75 | + ta = _tokens(a) - set(exclude) | |
| 76 | + if not ta: # rien de distinctif hors adresse/ville | |
| 77 | + return False | |
| 78 | + if a == b or a in b or b in a: | |
| 79 | + return True | |
| 80 | + return bool(ta & _tokens(b)) | |
| 81 | + | |
| 82 | + | |
| 83 | +def _iso_date(raw: str) -> str: | |
| 84 | + """« 05/21/2025 00:00:00 » (MM/DD/YYYY) -> « 2025-05-21 ».""" | |
| 85 | + m = re.match(r"^(\d{2})/(\d{2})/(\d{4})", (raw or "").strip()) | |
| 86 | + if not m: | |
| 87 | + return "" | |
| 88 | + mm, dd, yyyy = m.groups() | |
| 89 | + return f"{yyyy}-{mm}-{dd}" | |
| 90 | + | |
| 91 | + | |
| 92 | +def _download() -> list[dict]: | |
| 93 | + resp = requests.get(CSV_URL, headers={"User-Agent": USER_AGENT}, timeout=120) | |
| 94 | + resp.raise_for_status() | |
| 95 | + text = None | |
| 96 | + for enc in ("utf-8-sig", "latin-1"): | |
| 97 | + try: | |
| 98 | + text = resp.content.decode(enc) | |
| 99 | + break | |
| 100 | + except UnicodeDecodeError: | |
| 101 | + continue | |
| 102 | + if text is None: | |
| 103 | + raise RuntimeError("encodage CSV MAPAQ inconnu") | |
| 104 | + return list(csv.DictReader(io.StringIO(text))) | |
| 105 | + | |
| 106 | + | |
| 107 | +def _row_hash(rec: dict) -> str: | |
| 108 | + blob = "|".join((rec.get("Nom_exploitant") or "", | |
| 109 | + rec.get("Description_infraction") or "", | |
| 110 | + rec.get("Adresse_lieu_infraction") or "", | |
| 111 | + rec.get("Date_infraction") or "", | |
| 112 | + rec.get("Date_jugement") or "", | |
| 113 | + rec.get("Amende") or "")) | |
| 114 | + return hashlib.sha256(blob.encode("utf-8")).hexdigest() | |
| 115 | + | |
| 116 | + | |
| 117 | +def import_rows(con, records: list[dict]) -> int: | |
| 118 | + """Insère les condamnations (anti-doublon par row_hash). Retourne le nb | |
| 119 | + de nouvelles lignes.""" | |
| 120 | + added = 0 | |
| 121 | + for rec in records: | |
| 122 | + h = _row_hash(rec) | |
| 123 | + adresse = (rec.get("Adresse_lieu_infraction") or "").strip() | |
| 124 | + m = _POSTAL_RE.search(adresse.upper()) | |
| 125 | + postal = f"{m.group(1)}{m.group(2)}" if m else "" | |
| 126 | + cur = con.execute( | |
| 127 | + """INSERT OR IGNORE INTO inspections | |
| 128 | + (row_hash, exploitant, etablissement, description, adresse, | |
| 129 | + postal_code, type_etablissement, categorie, date_infraction, | |
| 130 | + date_jugement, date_publication, montant_amende, loi, motif) | |
| 131 | + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", | |
| 132 | + (h, | |
| 133 | + (rec.get("Nom_exploitant") or "").strip(), | |
| 134 | + (rec.get("Raison_sociale") or "").strip(), | |
| 135 | + (rec.get("Description_infraction") or "").strip(), | |
| 136 | + adresse, postal, | |
| 137 | + (rec.get("Type_etablissement") or "").strip(), | |
| 138 | + (rec.get("SOC_DESC_REGRP_TYP_ENTT") or "").strip(), | |
| 139 | + _iso_date(rec.get("Date_infraction")), | |
| 140 | + _iso_date(rec.get("Date_jugement")), | |
| 141 | + _iso_date(rec.get("Date_publication")), | |
| 142 | + parse_price(rec.get("Amende")), | |
| 143 | + (rec.get("SOC_NOM_LOI") or "").strip(), | |
| 144 | + (rec.get("SOC_NOM_ARTCL_INFRC") or "").strip())) | |
| 145 | + added += cur.rowcount | |
| 146 | + con.commit() | |
| 147 | + return added | |
| 148 | + | |
| 149 | + | |
| 150 | +def match(con) -> dict: | |
| 151 | + """Croisement CONSERVATEUR inspections <-> restaurants (uid canonique). | |
| 152 | + | |
| 153 | + Règle A « nom+lieu » : nom commercial normalisé IDENTIQUE au nom du resto | |
| 154 | + ET (code postal identique OU ville du resto contenue dans l'adresse | |
| 155 | + d'infraction ET numéro civique identique). Le civique est exigé quand on | |
| 156 | + n'a que la ville : les CHAÎNES ont plusieurs succursales par ville et une | |
| 157 | + condamnation ne doit JAMAIS être épinglée sur la mauvaise succursale. | |
| 158 | + Règle B « adresse+nom » : code postal identique ET numéro civique | |
| 159 | + identique ET similarité de nom (contenance ou token partagé). | |
| 160 | + """ | |
| 161 | + restos = con.execute( | |
| 162 | + "SELECT uid, name, chain, city, postal_code, address FROM restaurants" | |
| 163 | + " WHERE active=1 AND dup_of IS NULL").fetchall() | |
| 164 | + by_name: dict[str, list] = {} | |
| 165 | + by_postal: dict[str, list] = {} | |
| 166 | + for r in restos: | |
| 167 | + info = { | |
| 168 | + "uid": r["uid"], | |
| 169 | + "nname": norm_name(r["name"]), | |
| 170 | + "nchain": norm_name(r["chain"] or ""), | |
| 171 | + "ncity": strip_accents((r["city"] or "").lower()).strip(), | |
| 172 | + "postal": (r["postal_code"] or "").replace(" ", "").upper(), | |
| 173 | + } | |
| 174 | + mm = _CIVIC_RE.match(r["address"] or "") | |
| 175 | + info["civic"] = mm.group(1) if mm else "" | |
| 176 | + if info["nname"]: | |
| 177 | + by_name.setdefault(info["nname"], []).append(info) | |
| 178 | + core = core_name(info["nname"]) | |
| 179 | + if core and core != info["nname"]: | |
| 180 | + by_name.setdefault(core, []).append(info) | |
| 181 | + if info["postal"]: | |
| 182 | + by_postal.setdefault(info["postal"], []).append(info) | |
| 183 | + | |
| 184 | + matched = 0 | |
| 185 | + for row in con.execute( | |
| 186 | + "SELECT id, exploitant, etablissement, adresse, postal_code" | |
| 187 | + " FROM inspections WHERE uid IS NULL").fetchall(): | |
| 188 | + names = [] | |
| 189 | + if row["etablissement"]: | |
| 190 | + names.append(norm_name(row["etablissement"])) | |
| 191 | + expl = row["exploitant"] or "" | |
| 192 | + if expl and not _NUMBERED_CO_RE.match(strip_accents(expl.lower())): | |
| 193 | + names.append(norm_name(expl)) | |
| 194 | + # variantes du nom : normalisé complet + cœur sans mots génériques | |
| 195 | + names = [n for n in names if n] | |
| 196 | + names += [c for c in (core_name(n) for n in names) | |
| 197 | + if c and c not in names] | |
| 198 | + nadresse = strip_accents((row["adresse"] or "").lower()) | |
| 199 | + civic_m = _CIVIC_RE.match(row["adresse"] or "") | |
| 200 | + civic = civic_m.group(1) if civic_m else "" | |
| 201 | + postal = (row["postal_code"] or "").upper() | |
| 202 | + | |
| 203 | + hit, how = None, "" | |
| 204 | + # Règle A : nom exact + ville/code postal | |
| 205 | + for n in names: | |
| 206 | + for info in by_name.get(n, []): | |
| 207 | + same_place = ((postal and info["postal"] == postal) | |
| 208 | + or (info["ncity"] and len(info["ncity"]) >= 4 | |
| 209 | + and info["ncity"] in nadresse | |
| 210 | + and civic and info["civic"] == civic)) | |
| 211 | + if same_place: | |
| 212 | + hit, how = info["uid"], "nom+lieu" | |
| 213 | + break | |
| 214 | + if hit: | |
| 215 | + break | |
| 216 | + # Règle B : code postal + civique + similarité de nom (hors tokens | |
| 217 | + # d'adresse/ville — les voisins d'un même centre commercial partagent | |
| 218 | + # CP + civique, seul un nom distinctif partagé fait foi) | |
| 219 | + if hit is None and postal and civic: | |
| 220 | + addr_tokens = set(re.sub(r"[^a-z0-9]+", " ", nadresse).split()) | |
| 221 | + for info in by_postal.get(postal, []): | |
| 222 | + if info["civic"] != civic: | |
| 223 | + continue | |
| 224 | + if any(_name_similar(n, info["nname"], addr_tokens) | |
| 225 | + or (info["nchain"] | |
| 226 | + and _name_similar(n, info["nchain"], addr_tokens)) | |
| 227 | + for n in names): | |
| 228 | + hit, how = info["uid"], "adresse+nom" | |
| 229 | + break | |
| 230 | + if hit: | |
| 231 | + con.execute("UPDATE inspections SET uid=?, matched_by=? WHERE id=?", | |
| 232 | + (hit, how, row["id"])) | |
| 233 | + matched += 1 | |
| 234 | + con.commit() | |
| 235 | + | |
| 236 | + # résumé conformité par resto -> details.mapaq (affichage fiche) | |
| 237 | + from . import db as _db | |
| 238 | + for agg in con.execute( | |
| 239 | + "SELECT uid, COUNT(*) n, SUM(COALESCE(montant_amende,0)) total," | |
| 240 | + " MAX(date_jugement) derniere FROM inspections" | |
| 241 | + " WHERE uid IS NOT NULL GROUP BY uid"): | |
| 242 | + _db.merge_details(con, agg["uid"], {"mapaq": { | |
| 243 | + "condamnations": agg["n"], | |
| 244 | + "total_amendes": round(agg["total"] or 0, 2), | |
| 245 | + "derniere_condamnation": agg["derniere"], | |
| 246 | + }}) | |
| 247 | + con.commit() | |
| 248 | + total_linked = con.execute( | |
| 249 | + "SELECT COUNT(*) c FROM inspections WHERE uid IS NOT NULL").fetchone()["c"] | |
| 250 | + return {"matched_new": matched, "matched_total": total_linked} | |
| 251 | + | |
| 252 | + | |
| 253 | +def sync(con=None, force: bool = False) -> dict | None: | |
| 254 | + """Télécharge le CSV MAPAQ, importe et croise. Cadence hebdo (guard).""" | |
| 255 | + from . import db | |
| 256 | + own = con is None | |
| 257 | + if own: | |
| 258 | + con = db.connect() | |
| 259 | + try: | |
| 260 | + last = con.execute( | |
| 261 | + "SELECT MAX(ts) ts FROM sync_log WHERE source=? AND ok=1", | |
| 262 | + (SOURCE_ID,)).fetchone()["ts"] | |
| 263 | + if not force and last and time.time() - last < REFRESH_DAYS * 86400: | |
| 264 | + return None # déjà à jour cette semaine | |
| 265 | + records = _download() | |
| 266 | + added = import_rows(con, records) | |
| 267 | + m = match(con) | |
| 268 | + stats = {"rows": len(records), "new_rows": added, **m} | |
| 269 | + con.execute( | |
| 270 | + "INSERT INTO sync_log (source, ts, found, added, updated, removed," | |
| 271 | + " ok, message, stats) VALUES (?,?,?,?,0,0,1,?,?)", | |
| 272 | + (SOURCE_ID, time.time(), len(records), added, | |
| 273 | + f"condamnations importées, {m['matched_total']} croisées", | |
| 274 | + json.dumps(stats, ensure_ascii=False))) | |
| 275 | + con.commit() | |
| 276 | + print(f"[resto-ka] mapaq: {stats}") | |
| 277 | + return stats | |
| 278 | + except Exception as exc: | |
| 279 | + db.log_failure(con, SOURCE_ID, str(exc)) | |
| 280 | + print(f"[resto-ka] mapaq: erreur non bloquante: {exc}", file=sys.stderr) | |
| 281 | + return {"error": str(exc)} | |
| 282 | + finally: | |
| 283 | + if own: | |
| 284 | + con.close() | |
modified
restoka/normalize.py
+19 −1
@@ -17,7 +17,7 @@ from .regions import strip_accents | ||
| 17 | 17 | __all__ = [ |
| 18 | 18 | "strip_accents", "clean_name", "clean_address", "parse_price", |
| 19 | 19 | "classify_cuisines", "classify_establishment", "infer_dietary", |
| 20 | − "price_range_from_menu", "CUISINES", "PRICE_CONTEXTS", | |
| 20 | + "price_range_from_menu", "normalize_phone", "CUISINES", "PRICE_CONTEXTS", | |
| 21 | 21 | ] |
| 22 | 22 | |
| 23 | 23 | # Taxonomie des cuisines (CLAUDE.md §6.1) — multi-valué. |
@@ -60,6 +60,24 @@ def clean_address(address: str) -> str: | ||
| 60 | 60 | return s |
| 61 | 61 | |
| 62 | 62 | |
| 63 | +_PHONE_RE = re.compile( | |
| 64 | + r"(?:\+?1[\s.\-]?)?\(?([2-9]\d{2})\)?[\s.\-]?(\d{3})[\s.\-]?(\d{4})(?!\d)") | |
| 65 | + | |
| 66 | + | |
| 67 | +def normalize_phone(text: str | None) -> str: | |
| 68 | + """Extrait et normalise un numéro nord-américain en E.164 (« +14185551234 »). | |
| 69 | + | |
| 70 | + Retourne "" si aucun numéro à 10 chiffres plausible n'est trouvé. | |
| 71 | + Accepte les formats courants : (418) 555-1234, 418.555.1234, tel:+1418… | |
| 72 | + """ | |
| 73 | + if not text: | |
| 74 | + return "" | |
| 75 | + m = _PHONE_RE.search(str(text)) | |
| 76 | + if not m: | |
| 77 | + return "" | |
| 78 | + return "+1" + "".join(m.groups()) | |
| 79 | + | |
| 80 | + | |
| 63 | 81 | _PRICE_RE = re.compile(r"(\d{1,4}(?:[ ]\d{3})*(?:[.,]\d{1,2})?)\s*\$?") |
| 64 | 82 | |
| 65 | 83 | |
modified
restoka/web.py
+38 −1
@@ -48,6 +48,10 @@ def _row_to_dict(row) -> dict: | ||
| 48 | 48 | for col in ("cuisines", "services", "dietary_options", "languages", "images"): |
| 49 | 49 | d[col] = json.loads(d.get(col) or "[]") |
| 50 | 50 | d["hours"] = json.loads(d.get("hours") or "{}") |
| 51 | + try: # enrichissements (reservation_url, yelp, mapaq…) — voir db.merge_details | |
| 52 | + d["details"] = json.loads(d.get("details") or "{}") | |
| 53 | + except (ValueError, TypeError): | |
| 54 | + d["details"] = {} | |
| 51 | 55 | if d.get("dup_sources"): |
| 52 | 56 | try: |
| 53 | 57 | d["dup_sources"] = json.loads(d["dup_sources"]) |
@@ -250,6 +254,28 @@ def price_history(uid: str): | ||
| 250 | 254 | return {"uid": uid, "prices": rows} |
| 251 | 255 | |
| 252 | 256 | |
| 257 | +@app.get("/api/restaurants/{uid:path}/inspections") | |
| 258 | +def restaurant_inspections(uid: str): | |
| 259 | + """Condamnations MAPAQ croisées avec ce restaurant (inspections | |
| 260 | + alimentaires — Données Québec, licence CC-BY 4.0). Croisement | |
| 261 | + conservateur : voir restoka/inspections.py.""" | |
| 262 | + con = db.connect() | |
| 263 | + if con.execute("SELECT 1 FROM restaurants WHERE uid=?", | |
| 264 | + (uid,)).fetchone() is None: | |
| 265 | + con.close() | |
| 266 | + raise HTTPException(404, "Restaurant introuvable") | |
| 267 | + rows = [dict(r) for r in con.execute( | |
| 268 | + "SELECT etablissement, exploitant, description, adresse," | |
| 269 | + " date_infraction, date_jugement, montant_amende, motif, matched_by" | |
| 270 | + " FROM inspections WHERE uid=? ORDER BY date_jugement DESC", (uid,))] | |
| 271 | + con.close() | |
| 272 | + return {"uid": uid, "total": len(rows), | |
| 273 | + "total_amendes": round(sum(r["montant_amende"] or 0 for r in rows), 2), | |
| 274 | + "source": "MAPAQ — Condamnations des établissements alimentaires " | |
| 275 | + "(Données Québec, CC-BY 4.0)", | |
| 276 | + "inspections": rows} | |
| 277 | + | |
| 278 | + | |
| 253 | 279 | @app.get("/api/restaurants/{uid:path}") |
| 254 | 280 | def get_restaurant(uid: str): |
| 255 | 281 | con = db.connect() |
@@ -421,8 +447,19 @@ def stats(): | ||
| 421 | 447 | log = [dict(r) for r in con.execute( |
| 422 | 448 | "SELECT source, ts, found, added, updated, removed, ok, message" |
| 423 | 449 | " FROM sync_log ORDER BY ts DESC LIMIT 20")] |
| 450 | + insp = dict(con.execute( | |
| 451 | + """SELECT COUNT(*) inspections, | |
| 452 | + SUM(uid IS NOT NULL) inspections_matched, | |
| 453 | + COUNT(DISTINCT uid) restaurants_with_inspections | |
| 454 | + FROM inspections""").fetchone()) | |
| 424 | 455 | con.close() |
| 425 | − return {**head, **m, "by_region": by_region, "by_context": by_context, | |
| 456 | + try: # nb de sources au registre (data/sources.json), actives ou en attente | |
| 457 | + registry = json.loads(SOURCES_PATH.read_text(encoding="utf-8"))["sources"] | |
| 458 | + registered = len(registry) | |
| 459 | + except (ValueError, OSError, KeyError): | |
| 460 | + registered = None | |
| 461 | + return {**head, **m, **insp, "sources_registry": registered, | |
| 462 | + "by_region": by_region, "by_context": by_context, | |
| 426 | 463 | "recent_syncs": log} |
| 427 | 464 | |
| 428 | 465 | |
added
tests/test_hours.py
+60 −0
@@ -0,0 +1,60 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: tests/test_hours.py | |
| 4 | +# Desc: Parseur d'horaires — syntaxes OSM courantes structurées par jour, | |
| 5 | +# cas exotiques refusés (le brut reste dans hours["osm"]), horaires | |
| 6 | +# JSON-LD schema.org (openingHoursSpecification / openingHours). | |
| 7 | +# ============================================================================== | |
| 8 | +from restoka.hours import parse_jsonld_hours, parse_opening_hours | |
| 9 | + | |
| 10 | + | |
| 11 | +def test_plages_de_jours_simples(): | |
| 12 | + h = parse_opening_hours("Mo-Fr 11:00-22:00; Sa-Su 10:00-23:00") | |
| 13 | + assert h["mon"] == [["11:00", "22:00"]] | |
| 14 | + assert h["fri"] == [["11:00", "22:00"]] | |
| 15 | + assert h["sat"] == [["10:00", "23:00"]] | |
| 16 | + assert h["sun"] == [["10:00", "23:00"]] | |
| 17 | + | |
| 18 | + | |
| 19 | +def test_double_service_et_jour_off(): | |
| 20 | + h = parse_opening_hours("Tu-Sa 11:30-14:00,17:00-22:00; Su-Mo off") | |
| 21 | + assert h["tue"] == [["11:30", "14:00"], ["17:00", "22:00"]] | |
| 22 | + assert h["sun"] == [] and h["mon"] == [] | |
| 23 | + | |
| 24 | + | |
| 25 | +def test_24_7_et_liste_de_jours(): | |
| 26 | + assert parse_opening_hours("24/7")["wed"] == [["00:00", "24:00"]] | |
| 27 | + h = parse_opening_hours("Mo,We,Fr 09:00-17:00") | |
| 28 | + assert set(h) == {"mon", "wed", "fri"} | |
| 29 | + | |
| 30 | + | |
| 31 | +def test_heures_seules_et_enroulement_semaine(): | |
| 32 | + assert parse_opening_hours("08:00-16:00")["sun"] == [["08:00", "16:00"]] | |
| 33 | + h = parse_opening_hours("Sa-Mo 10:00-15:00") # enroule Sa, Su, Mo | |
| 34 | + assert set(h) == {"sat", "sun", "mon"} | |
| 35 | + | |
| 36 | + | |
| 37 | +def test_ph_ignore_et_exotique_refuse(): | |
| 38 | + h = parse_opening_hours("Mo-Fr 11:00-22:00; PH off") | |
| 39 | + assert h["mon"] == [["11:00", "22:00"]] and "ph" not in h | |
| 40 | + assert parse_opening_hours("Mo-Fr sunrise-sunset") is None | |
| 41 | + assert parse_opening_hours("Jun-Aug Mo-Su 10:00-22:00") is None | |
| 42 | + assert parse_opening_hours("") is None | |
| 43 | + assert parse_opening_hours(None) is None | |
| 44 | + | |
| 45 | + | |
| 46 | +def test_jsonld_opening_hours_specification(): | |
| 47 | + node = {"@type": "Restaurant", "openingHoursSpecification": [ | |
| 48 | + {"dayOfWeek": ["Monday", "Tuesday"], "opens": "11:00", "closes": "22:00"}, | |
| 49 | + {"dayOfWeek": "https://schema.org/Sunday", "opens": "10:00", | |
| 50 | + "closes": "15:00"}, | |
| 51 | + ]} | |
| 52 | + h = parse_jsonld_hours(node) | |
| 53 | + assert h["mon"] == [["11:00", "22:00"]] | |
| 54 | + assert h["sun"] == [["10:00", "15:00"]] | |
| 55 | + | |
| 56 | + | |
| 57 | +def test_jsonld_opening_hours_chaines(): | |
| 58 | + h = parse_jsonld_hours({"openingHours": ["Mo-Fr 11:00-22:00"]}) | |
| 59 | + assert h["thu"] == [["11:00", "22:00"]] | |
| 60 | + assert parse_jsonld_hours({}) is None | |
added
tests/test_inspections.py
+119 −0
@@ -0,0 +1,119 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: tests/test_inspections.py | |
| 4 | +# Desc: Connecteur MAPAQ (condamnations) — import CSV HORS LIGNE, parsing | |
| 5 | +# des dates/amendes/code postal, croisement CONSERVATEUR avec les | |
| 6 | +# restaurants (nom+lieu, adresse+nom, jamais de fusion hasardeuse), | |
| 7 | +# anti-doublon au ré-import, endpoint et details.mapaq. | |
| 8 | +# ============================================================================== | |
| 9 | +import json | |
| 10 | + | |
| 11 | +from restoka import inspections | |
| 12 | +from restoka.schema import Restaurant | |
| 13 | +from restoka import db as rdb | |
| 14 | + | |
| 15 | + | |
| 16 | +RECORDS = [ | |
| 17 | + { # croisement par nom exact + ville contenue dans l'adresse | |
| 18 | + "Nom_exploitant": "9277-4876 QUEBEC INC.", | |
| 19 | + "Raison_sociale": "RESTAURANT CHEZ MAMY", | |
| 20 | + "Description_infraction": "LIEUX NON PROPRES.", | |
| 21 | + "Adresse_lieu_infraction": "1999 SAINTE-FAMILLE SAGUENAY, (QC) G7X4X5", | |
| 22 | + "Type_etablissement": "CASSE-CROUTE", | |
| 23 | + "Date_infraction": "11/27/2025 00:00:00", | |
| 24 | + "Date_jugement": "05/19/2026 00:00:00", | |
| 25 | + "Date_publication": "05/22/2026 00:00:00", | |
| 26 | + "Amende": "500 $", | |
| 27 | + "SOC_NOM_LOI": "Loi sur les produits alimentaires", | |
| 28 | + "SOC_NOM_ARTCL_INFRC": "INSALUBRITE", | |
| 29 | + "SOC_DESC_REGRP_TYP_ENTT": "RESTAURATION", | |
| 30 | + }, | |
| 31 | + { # croisement par code postal + civique + similarité de nom | |
| 32 | + "Nom_exploitant": "GESTION PIZZA BELLA INC.", | |
| 33 | + "Raison_sociale": "PIZZA BELLA", | |
| 34 | + "Description_infraction": "TEMPERATURE INADEQUATE.", | |
| 35 | + "Adresse_lieu_infraction": "300 RUE PRINCIPALE QUEBEC, (QC) G1K 3Y2", | |
| 36 | + "Type_etablissement": "RESTAURANT", | |
| 37 | + "Date_infraction": "01/15/2026 00:00:00", | |
| 38 | + "Date_jugement": "03/10/2026 00:00:00", | |
| 39 | + "Date_publication": "03/12/2026 00:00:00", | |
| 40 | + "Amende": "1 250 $", | |
| 41 | + "SOC_NOM_LOI": "Loi sur les produits alimentaires", | |
| 42 | + "SOC_NOM_ARTCL_INFRC": "TEMPERATURE", | |
| 43 | + "SOC_DESC_REGRP_TYP_ENTT": "RESTAURATION", | |
| 44 | + }, | |
| 45 | + { # AUCUN croisement : même nom qu'un resto mais autre ville/CP | |
| 46 | + "Nom_exploitant": "FERME LAITIERE X INC.", | |
| 47 | + "Raison_sociale": "CHEZ MAMY", | |
| 48 | + "Description_infraction": "EQUIPEMENTS NON PROPRES.", | |
| 49 | + "Adresse_lieu_infraction": "714 RANG OUEST SAINT-JEAN-DE-DIEU, (QC) G0L3M0", | |
| 50 | + "Type_etablissement": "FERME LAITIERE", | |
| 51 | + "Date_infraction": "05/21/2025 00:00:00", | |
| 52 | + "Date_jugement": "04/14/2026 00:00:00", | |
| 53 | + "Date_publication": "04/17/2026 00:00:00", | |
| 54 | + "Amende": "2 000 $", | |
| 55 | + "SOC_NOM_LOI": "Loi sur les produits alimentaires", | |
| 56 | + "SOC_NOM_ARTCL_INFRC": "INSALUBRITE", | |
| 57 | + "SOC_DESC_REGRP_TYP_ENTT": "LAIT", | |
| 58 | + }, | |
| 59 | +] | |
| 60 | + | |
| 61 | + | |
| 62 | +def _seed_restaurants(con): | |
| 63 | + restos = [ | |
| 64 | + Restaurant(source="osm", external_id="node/1", name="Chez Mamy", | |
| 65 | + address="1999 Rue Sainte-Famille", city="Saguenay", | |
| 66 | + postal_code="G7X 4X5", lat=48.42, lng=-71.06), | |
| 67 | + Restaurant(source="osm", external_id="node/2", name="Bella Pizza Resto", | |
| 68 | + address="300 Rue Principale", city="Québec", | |
| 69 | + postal_code="G1K 3Y2", lat=46.81, lng=-71.21), | |
| 70 | + # piège : même nom que la condamnation #1 mais dans une AUTRE ville | |
| 71 | + Restaurant(source="osm", external_id="node/3", name="Chez Mamy", | |
| 72 | + address="12 Rue Untel", city="Gatineau", | |
| 73 | + postal_code="J8X 1A1", lat=45.47, lng=-75.70), | |
| 74 | + ] | |
| 75 | + rdb.sync_source(con, "osm", [r.finalize() for r in restos]) | |
| 76 | + | |
| 77 | + | |
| 78 | +def test_import_croisement_et_antidoublon(con): | |
| 79 | + _seed_restaurants(con) | |
| 80 | + added = inspections.import_rows(con, RECORDS) | |
| 81 | + assert added == 3 | |
| 82 | + # parsing : date ISO, amende numérique, code postal extrait | |
| 83 | + row = con.execute("SELECT * FROM inspections WHERE etablissement=" | |
| 84 | + "'RESTAURANT CHEZ MAMY'").fetchone() | |
| 85 | + assert row["date_jugement"] == "2026-05-19" | |
| 86 | + assert row["montant_amende"] == 500.0 | |
| 87 | + assert row["postal_code"] == "G7X4X5" | |
| 88 | + | |
| 89 | + stats = inspections.match(con) | |
| 90 | + assert stats["matched_total"] == 2 | |
| 91 | + # #1 : nom+lieu -> le Chez Mamy de SAGUENAY, jamais celui de Gatineau | |
| 92 | + assert con.execute("SELECT uid, matched_by FROM inspections WHERE " | |
| 93 | + "etablissement='RESTAURANT CHEZ MAMY'").fetchone()["uid"] \ | |
| 94 | + == "osm:node/1" | |
| 95 | + # #2 : adresse+nom (CP + civique + token « pizza/bella » partagé) | |
| 96 | + m2 = con.execute("SELECT uid, matched_by FROM inspections WHERE " | |
| 97 | + "etablissement='PIZZA BELLA'").fetchone() | |
| 98 | + assert m2["uid"] == "osm:node/2" and m2["matched_by"] == "adresse+nom" | |
| 99 | + # #3 : la ferme laitière homonyme ne matche RIEN (ville et CP différents) | |
| 100 | + assert con.execute("SELECT uid FROM inspections WHERE " | |
| 101 | + "exploitant='FERME LAITIERE X INC.'").fetchone()["uid"] is None | |
| 102 | + | |
| 103 | + # résumé conformité écrit dans details.mapaq du resto croisé | |
| 104 | + details = json.loads(con.execute( | |
| 105 | + "SELECT details FROM restaurants WHERE uid='osm:node/1'") | |
| 106 | + .fetchone()["details"]) | |
| 107 | + assert details["mapaq"]["condamnations"] == 1 | |
| 108 | + assert details["mapaq"]["total_amendes"] == 500.0 | |
| 109 | + | |
| 110 | + # ré-import : aucun doublon | |
| 111 | + assert inspections.import_rows(con, RECORDS) == 0 | |
| 112 | + assert con.execute("SELECT COUNT(*) c FROM inspections").fetchone()["c"] == 3 | |
| 113 | + | |
| 114 | + | |
| 115 | +def test_norm_et_core_name(): | |
| 116 | + n = inspections.norm_name("RESTAURANT CHEZ MAMY INC.") | |
| 117 | + assert n == "restaurant chez mamy" | |
| 118 | + assert inspections.core_name(n) == "mamy" | |
| 119 | + assert inspections.norm_name("Pâtisserie L'Éclair Ltée") == "patisserie l eclair" | |
added
tests/test_siteresto_contact.py
+61 −0
@@ -0,0 +1,61 @@ | ||
| 1 | +# ============================================================================== | |
| 2 | +# Author: Simon-Pierre Boucher <contact@spboucher.ai> | |
| 3 | +# File: tests/test_siteresto_contact.py | |
| 4 | +# Desc: Extraction téléphone / horaires / lien de réservation depuis la page | |
| 5 | +# d'accueil d'un site de resto (HORS LIGNE) : lien tel:, JSON-LD | |
| 6 | +# LocalBusiness (openingHoursSpecification), plateformes de | |
| 7 | +# réservation reconnues, regex de secours, normalisation E.164. | |
| 8 | +# ============================================================================== | |
| 9 | +from restoka.connectors.siteresto import SiteRestoConnector | |
| 10 | +from restoka.normalize import normalize_phone | |
| 11 | + | |
| 12 | +HTML = """ | |
| 13 | +<html><head> | |
| 14 | +<script type="application/ld+json"> | |
| 15 | +{"@context":"https://schema.org","@type":"Restaurant","name":"Chez Test", | |
| 16 | + "telephone":"+1-418-555-1234", | |
| 17 | + "openingHoursSpecification":[ | |
| 18 | + {"@type":"OpeningHoursSpecification","dayOfWeek":["Monday","Tuesday"], | |
| 19 | + "opens":"11:00","closes":"22:00"}]} | |
| 20 | +</script> | |
| 21 | +</head><body> | |
| 22 | +<a href="tel:+14185551234">Appelez-nous</a> | |
| 23 | +<a href="https://www.opentable.ca/r/chez-test-quebec">Réserver</a> | |
| 24 | +<a href="/menu">Menu</a> | |
| 25 | +</body></html> | |
| 26 | +""" | |
| 27 | + | |
| 28 | +HTML_REGEX_ONLY = """ | |
| 29 | +<html><body><p>Réservation : (514) 555-9876</p> | |
| 30 | +<a href="https://widgets.libroreserve.com/QC012345">Réserver une table</a> | |
| 31 | +</body></html> | |
| 32 | +""" | |
| 33 | + | |
| 34 | + | |
| 35 | +def test_extraction_complete(): | |
| 36 | + c = SiteRestoConnector._extract_contact(HTML, "https://chez-test.ca") | |
| 37 | + assert c["phone"] == "+14185551234" | |
| 38 | + assert c["reservation_url"] == "https://www.opentable.ca/r/chez-test-quebec" | |
| 39 | + assert c["hours"]["mon"] == [["11:00", "22:00"]] | |
| 40 | + assert c["hours"]["tue"] == [["11:00", "22:00"]] | |
| 41 | + | |
| 42 | + | |
| 43 | +def test_extraction_regex_et_libro(): | |
| 44 | + c = SiteRestoConnector._extract_contact(HTML_REGEX_ONLY, | |
| 45 | + "https://exemple.ca") | |
| 46 | + assert c["phone"] == "+15145559876" | |
| 47 | + assert "libroreserve.com" in c["reservation_url"] | |
| 48 | + assert c["hours"] is None | |
| 49 | + | |
| 50 | + | |
| 51 | +def test_extraction_vide(): | |
| 52 | + c = SiteRestoConnector._extract_contact("", "https://exemple.ca") | |
| 53 | + assert c == {"phone": "", "hours": None, "reservation_url": ""} | |
| 54 | + | |
| 55 | + | |
| 56 | +def test_normalize_phone(): | |
| 57 | + assert normalize_phone("tel:418.555.1234") == "+14185551234" | |
| 58 | + assert normalize_phone("(514) 555-9876") == "+15145559876" | |
| 59 | + assert normalize_phone("+1 450 555 0000") == "+14505550000" | |
| 60 | + assert normalize_phone("1234") == "" | |
| 61 | + assert normalize_phone(None) == "" | |
| 62 | ||