Fiabilité sync + enrichissement UEAT/OSM
- Garde anti-fausses-fermetures : garde bootstrap (found < 60 % du max historique => retraits suspendus, active dès le 2e run), signal de run partiel remonté du connecteur au pipeline (sync_source(partial=...)). - OSM : run marqué partiel si un type échoue sur tous les miroirs ; réponses Overpass tronquées (remark timeout) et miroirs sans données (0 élément) traités comme des échecs ; overpass.osm.ch retiré (extrait suisse => 0 POI pour le Québec) ; +2 miroirs planet. - OSM : brand => chain, branch suffixé au nom, wheelchair/payment:*/ capacity => services, diet:*=limited accepté, ~45 cuisines OSM supplémentaires mappées (belgian, turkish, cantonese, peruvian...). - UEAT : phoneNumber + address.latitude/longitude captés a la source (GPS de première partie, COALESCE côté base) ; 20 GUID découverts par site-resto promus dans data/sources.json (50 intégrations). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
5 changed files +327 −40
modified
data/sources.json
+203 −31
@@ -14,36 +14,208 @@ | ||
| 14 | 14 | "cadence": "hebdomadaire", |
| 15 | 15 | "status": "actif", |
| 16 | 16 | "integrations": [ |
| 17 | − { "key": "4268100d-6470-40b9-a755-e11342c91d6d", "label": "Poulet Rouge (chaîne)" }, | |
| 18 | − { "key": "01d1c391-59cf-45a3-b051-7b2f9a623858", "label": "Fairmont Le Château Frontenac (Québec)" }, | |
| 19 | − { "key": "b7282ce3-95cc-43ff-80b5-0cca8a7e4778", "label": "Allô mon Coco (chaîne, déjeuner-brunch)" }, | |
| 20 | − { "key": "b1c2c004-bcff-4eb0-b3b9-d3bc5c8799a3", "label": "Restaurant Impérial (Brossard, Pointe-Claire)" }, | |
| 21 | − { "key": "2f51961a-f378-4812-a31a-a0b45da3f87e", "label": "Pizza Mia (Laval)" }, | |
| 22 | − { "key": "1b432c57-98d6-4681-a560-d1b7bfafdd6f", "label": "Station W (Montréal, café)" }, | |
| 23 | − { "key": "ff376cab-30e2-4cce-9999-ed755583c7f5", "label": "Chocolats Favoris (chaîne, dessert)" }, | |
| 24 | − { "key": "c5548a05-1776-49ac-9c9f-2ac8c998eed4", "label": "Le Super Qualité (Montréal, indien)" }, | |
| 25 | − { "key": "d84801b5-66f5-44b3-b0dc-52c0365f571c", "label": "Ryu Sushi (Montréal)" }, | |
| 26 | − { "key": "87c690ed-be3c-4a5d-b58b-676235faa9c1", "label": "Panda Boba-T (chaîne, bubble tea)" }, | |
| 27 | − { "key": "970d8e90-55ca-4655-9631-a8f9af08f4b7", "label": "Chez Ashton (23 succ., poutine)" }, | |
| 28 | − { "key": "e473d88d-b405-41ee-9ee8-2aa628eee604", "label": "Thaïzone (26 succ. QC)" }, | |
| 29 | − { "key": "f89d74e1-c2ab-4ace-8bce-39069900f5b4", "label": "Normandin (44 succ., familial)" }, | |
| 30 | − { "key": "e15b2545-2f2a-4e5f-baa6-00ab849cfa16", "label": "Valentine (76 succ. QC, casse-croûte)" }, | |
| 31 | − { "key": "657f8ce3-d288-4245-aecf-7ca017364b8e", "label": "Tutti Frutti (9 succ. QC, déjeuner)" }, | |
| 32 | − { "key": "00f8d52b-7aaf-4bcf-8722-6d538aad5a9a", "label": "Pacini (20 succ. QC, italien)" }, | |
| 33 | − { "key": "d679a49f-0a25-4694-82f2-56073a8b8790", "label": "Scores (25 succ., rôtisserie)" }, | |
| 34 | − { "key": "1696ecd3-d4ea-4224-9f98-9299ebcb339c", "label": "Mandy's (9 succ. QC, salades)" }, | |
| 35 | − { "key": "dce3e009-8038-47bc-8d5e-f336458e573f", "label": "Sushi Shop (141 succ. QC)" }, | |
| 36 | − { "key": "a0dac11c-e43c-4ae2-a37c-22d8542872e0", "label": "Ben & Florentine (54 succ. QC, déjeuner)" }, | |
| 37 | − { "key": "f0ac233f-df24-47a8-9e3a-a64520738ff2", "label": "Boustan (65 succ. QC, libanais)" }, | |
| 38 | − { "key": "45c52d27-bf8c-4995-9c4e-a0a2754101fb", "label": "Pizzéria NO.900 (32 succ. QC)" }, | |
| 39 | − { "key": "e3ca524e-57bb-4d8e-ad2f-53a35731c802", "label": "Sushi Taxi (30 succ. QC)" }, | |
| 40 | − { "key": "031978fa-09a0-493b-86d9-65aca8b3bef0", "label": "Cochon Dingue (7 succ., Québec)" }, | |
| 41 | − { "key": "d6a20802-625c-4970-920b-1b82c6867e9d", "label": "Salvatoré (3 succ. QC, pizza)" }, | |
| 42 | − { "key": "a23e2d27-6871-4d2f-be29-610426c251a2", "label": "La Belle Province — Montréal (2 succ.)" }, | |
| 43 | − { "key": "48c08050-9f8a-4ab9-a8a2-d58c9602df56", "label": "Madisons (2 succ. QC)" }, | |
| 44 | − { "key": "fd9eb8cb-4536-4bb6-bb19-6fb3be67808f", "label": "Freddy Pizzeria (Sherbrooke)" }, | |
| 45 | − { "key": "e8015de3-09ac-4cd2-a18d-0820dbe356ed", "label": "(intégration retirée chez UEAT)", "status": "inactif" }, | |
| 46 | − { "key": "dfde098b-ee0f-46a3-b1ad-09ec2acb54b9", "label": "(intégration retirée chez UEAT)", "status": "inactif" } | |
| 17 | + { | |
| 18 | + "key": "4268100d-6470-40b9-a755-e11342c91d6d", | |
| 19 | + "label": "Poulet Rouge (chaîne)" | |
| 20 | + }, | |
| 21 | + { | |
| 22 | + "key": "01d1c391-59cf-45a3-b051-7b2f9a623858", | |
| 23 | + "label": "Fairmont Le Château Frontenac (Québec)" | |
| 24 | + }, | |
| 25 | + { | |
| 26 | + "key": "b7282ce3-95cc-43ff-80b5-0cca8a7e4778", | |
| 27 | + "label": "Allô mon Coco (chaîne, déjeuner-brunch)" | |
| 28 | + }, | |
| 29 | + { | |
| 30 | + "key": "b1c2c004-bcff-4eb0-b3b9-d3bc5c8799a3", | |
| 31 | + "label": "Restaurant Impérial (Brossard, Pointe-Claire)" | |
| 32 | + }, | |
| 33 | + { | |
| 34 | + "key": "2f51961a-f378-4812-a31a-a0b45da3f87e", | |
| 35 | + "label": "Pizza Mia (Laval)" | |
| 36 | + }, | |
| 37 | + { | |
| 38 | + "key": "1b432c57-98d6-4681-a560-d1b7bfafdd6f", | |
| 39 | + "label": "Station W (Montréal, café)" | |
| 40 | + }, | |
| 41 | + { | |
| 42 | + "key": "ff376cab-30e2-4cce-9999-ed755583c7f5", | |
| 43 | + "label": "Chocolats Favoris (chaîne, dessert)" | |
| 44 | + }, | |
| 45 | + { | |
| 46 | + "key": "c5548a05-1776-49ac-9c9f-2ac8c998eed4", | |
| 47 | + "label": "Le Super Qualité (Montréal, indien)" | |
| 48 | + }, | |
| 49 | + { | |
| 50 | + "key": "d84801b5-66f5-44b3-b0dc-52c0365f571c", | |
| 51 | + "label": "Ryu Sushi (Montréal)" | |
| 52 | + }, | |
| 53 | + { | |
| 54 | + "key": "87c690ed-be3c-4a5d-b58b-676235faa9c1", | |
| 55 | + "label": "Panda Boba-T (chaîne, bubble tea)" | |
| 56 | + }, | |
| 57 | + { | |
| 58 | + "key": "970d8e90-55ca-4655-9631-a8f9af08f4b7", | |
| 59 | + "label": "Chez Ashton (23 succ., poutine)" | |
| 60 | + }, | |
| 61 | + { | |
| 62 | + "key": "e473d88d-b405-41ee-9ee8-2aa628eee604", | |
| 63 | + "label": "Thaïzone (26 succ. QC)" | |
| 64 | + }, | |
| 65 | + { | |
| 66 | + "key": "f89d74e1-c2ab-4ace-8bce-39069900f5b4", | |
| 67 | + "label": "Normandin (44 succ., familial)" | |
| 68 | + }, | |
| 69 | + { | |
| 70 | + "key": "e15b2545-2f2a-4e5f-baa6-00ab849cfa16", | |
| 71 | + "label": "Valentine (76 succ. QC, casse-croûte)" | |
| 72 | + }, | |
| 73 | + { | |
| 74 | + "key": "657f8ce3-d288-4245-aecf-7ca017364b8e", | |
| 75 | + "label": "Tutti Frutti (9 succ. QC, déjeuner)" | |
| 76 | + }, | |
| 77 | + { | |
| 78 | + "key": "00f8d52b-7aaf-4bcf-8722-6d538aad5a9a", | |
| 79 | + "label": "Pacini (20 succ. QC, italien)" | |
| 80 | + }, | |
| 81 | + { | |
| 82 | + "key": "d679a49f-0a25-4694-82f2-56073a8b8790", | |
| 83 | + "label": "Scores (25 succ., rôtisserie)" | |
| 84 | + }, | |
| 85 | + { | |
| 86 | + "key": "1696ecd3-d4ea-4224-9f98-9299ebcb339c", | |
| 87 | + "label": "Mandy's (9 succ. QC, salades)" | |
| 88 | + }, | |
| 89 | + { | |
| 90 | + "key": "dce3e009-8038-47bc-8d5e-f336458e573f", | |
| 91 | + "label": "Sushi Shop (141 succ. QC)" | |
| 92 | + }, | |
| 93 | + { | |
| 94 | + "key": "a0dac11c-e43c-4ae2-a37c-22d8542872e0", | |
| 95 | + "label": "Ben & Florentine (54 succ. QC, déjeuner)" | |
| 96 | + }, | |
| 97 | + { | |
| 98 | + "key": "f0ac233f-df24-47a8-9e3a-a64520738ff2", | |
| 99 | + "label": "Boustan (65 succ. QC, libanais)" | |
| 100 | + }, | |
| 101 | + { | |
| 102 | + "key": "45c52d27-bf8c-4995-9c4e-a0a2754101fb", | |
| 103 | + "label": "Pizzéria NO.900 (32 succ. QC)" | |
| 104 | + }, | |
| 105 | + { | |
| 106 | + "key": "e3ca524e-57bb-4d8e-ad2f-53a35731c802", | |
| 107 | + "label": "Sushi Taxi (30 succ. QC)" | |
| 108 | + }, | |
| 109 | + { | |
| 110 | + "key": "031978fa-09a0-493b-86d9-65aca8b3bef0", | |
| 111 | + "label": "Cochon Dingue (7 succ., Québec)" | |
| 112 | + }, | |
| 113 | + { | |
| 114 | + "key": "d6a20802-625c-4970-920b-1b82c6867e9d", | |
| 115 | + "label": "Salvatoré (3 succ. QC, pizza)" | |
| 116 | + }, | |
| 117 | + { | |
| 118 | + "key": "a23e2d27-6871-4d2f-be29-610426c251a2", | |
| 119 | + "label": "La Belle Province — Montréal (2 succ.)" | |
| 120 | + }, | |
| 121 | + { | |
| 122 | + "key": "48c08050-9f8a-4ab9-a8a2-d58c9602df56", | |
| 123 | + "label": "Madisons (2 succ. QC)" | |
| 124 | + }, | |
| 125 | + { | |
| 126 | + "key": "fd9eb8cb-4536-4bb6-bb19-6fb3be67808f", | |
| 127 | + "label": "Freddy Pizzeria (Sherbrooke)" | |
| 128 | + }, | |
| 129 | + { | |
| 130 | + "key": "e8015de3-09ac-4cd2-a18d-0820dbe356ed", | |
| 131 | + "label": "(intégration retirée chez UEAT)", | |
| 132 | + "status": "inactif" | |
| 133 | + }, | |
| 134 | + { | |
| 135 | + "key": "dfde098b-ee0f-46a3-b1ad-09ec2acb54b9", | |
| 136 | + "label": "(intégration retirée chez UEAT)", | |
| 137 | + "status": "inactif" | |
| 138 | + }, | |
| 139 | + { | |
| 140 | + "key": "e4c3af4f-c5ff-4fe7-a8e9-a8750ae532a8", | |
| 141 | + "label": "Morena Epicerie Traiteur (découvert via morena-food.com)" | |
| 142 | + }, | |
| 143 | + { | |
| 144 | + "key": "66ee81af-c10a-49b6-9faa-3e3b485a548e", | |
| 145 | + "label": "India Rosa (découvert via indiarosa.com)" | |
| 146 | + }, | |
| 147 | + { | |
| 148 | + "key": "b46660d2-389f-42d5-b161-d31b48ad80bf", | |
| 149 | + "label": "Restaurant Au Vieux Duluth (découvert via auvieuxduluth.com)" | |
| 150 | + }, | |
| 151 | + { | |
| 152 | + "key": "d6843f2a-5809-4d43-be8e-392e65a4a91c", | |
| 153 | + "label": "Aux Vivres (découvert via auxvivres.com)" | |
| 154 | + }, | |
| 155 | + { | |
| 156 | + "key": "c164671b-95fe-4f6f-8944-67eac68ad8c0", | |
| 157 | + "label": "Casa Grecque (découvert via casagrecque.ca)" | |
| 158 | + }, | |
| 159 | + { | |
| 160 | + "key": "36be39f4-3bfb-4d76-ab9b-bbb1985883ea", | |
| 161 | + "label": "Pizza Welldun (découvert via welldunpizza.ca)" | |
| 162 | + }, | |
| 163 | + { | |
| 164 | + "key": "e39d77ff-8183-4d75-9211-ed356dc857ea", | |
| 165 | + "label": "Bottega (découvert via bottega.ca)" | |
| 166 | + }, | |
| 167 | + { | |
| 168 | + "key": "55a3f719-d59a-4487-a0e2-b7aad293b247", | |
| 169 | + "label": "Sandhu (découvert via restaurantsandhu.ca)" | |
| 170 | + }, | |
| 171 | + { | |
| 172 | + "key": "627749b8-894b-4f13-a68b-c90b2ffd1702", | |
| 173 | + "label": "Rockaberry (découvert via rockaberry.ca)" | |
| 174 | + }, | |
| 175 | + { | |
| 176 | + "key": "4442c3e2-66b3-41ac-8cbf-a00a2a420320", | |
| 177 | + "label": "Maiko Sushi (découvert via maiko-sushi.com)" | |
| 178 | + }, | |
| 179 | + { | |
| 180 | + "key": "fc3bf368-dfcc-4a0b-8eb7-93aaeef67bcf", | |
| 181 | + "label": "Juliette & Chocolat (découvert via julietteetchocolat.com)" | |
| 182 | + }, | |
| 183 | + { | |
| 184 | + "key": "52f4a11d-75c9-4710-8f9c-15997aabf7bd", | |
| 185 | + "label": "Poivre Noir (découvert via poivrenoir.com)" | |
| 186 | + }, | |
| 187 | + { | |
| 188 | + "key": "1d596d77-a96f-46da-8192-785cd3766c58", | |
| 189 | + "label": "Pizzeria 67 (découvert via pizzeria67.com)" | |
| 190 | + }, | |
| 191 | + { | |
| 192 | + "key": "b0cbbf13-6f88-46a3-ad14-0c3bc85069f7", | |
| 193 | + "label": "Bistro-Bar le Brise-Bise (découvert via brisebise.ca)" | |
| 194 | + }, | |
| 195 | + { | |
| 196 | + "key": "77df4865-59bb-435e-818c-e743a4ca2751", | |
| 197 | + "label": "Pizza Fina (découvert via pizzafina.ca)" | |
| 198 | + }, | |
| 199 | + { | |
| 200 | + "key": "94652e5f-d4cc-47ac-81a3-ad9b1e31b826", | |
| 201 | + "label": "Sesame (découvert via sesamerestaurant.ca)" | |
| 202 | + }, | |
| 203 | + { | |
| 204 | + "key": "5f664929-18f2-4b2f-955d-b9f542312180", | |
| 205 | + "label": "Venice (découvert via venicemtl.com)" | |
| 206 | + }, | |
| 207 | + { | |
| 208 | + "key": "d25f3505-1acc-4422-a5c0-b83940b3f576", | |
| 209 | + "label": "Onigiri Shop (découvert via onigirishop.com)" | |
| 210 | + }, | |
| 211 | + { | |
| 212 | + "key": "ac77a7cf-e62b-40be-964b-497666c07c06", | |
| 213 | + "label": "Escondite (découvert via escondite.ca)" | |
| 214 | + }, | |
| 215 | + { | |
| 216 | + "key": "e4de92c1-12d0-49db-bb59-7baf0202faf8", | |
| 217 | + "label": "Nora Gray (découvert via noragray.com)" | |
| 218 | + } | |
| 47 | 219 | ] |
| 48 | 220 | }, |
| 49 | 221 | { |
@@ -86,4 +258,4 @@ | ||
| 86 | 258 | "status": "à faire" |
| 87 | 259 | } |
| 88 | 260 | ] |
| 89 | −} | |
| 261 | +} | |
| \ No newline at end of file | ||
modified
restoka/connectors/osm.py
+86 −4
@@ -31,8 +31,12 @@ QC_AREA = 3600061549 | ||
| 31 | 31 | MIRRORS = [ |
| 32 | 32 | "https://overpass.kumi.systems/api/interpreter", |
| 33 | 33 | "https://overpass-api.de/api/interpreter", |
| 34 | − "https://overpass.osm.ch/api/interpreter", | |
| 34 | + "https://overpass.private.coffee/api/interpreter", | |
| 35 | + "https://overpass.osm.jp/api/interpreter", | |
| 35 | 36 | ] |
| 37 | +# NB : overpass.osm.ch RETIRÉ des miroirs — c'est un extrait SUISSE seulement : | |
| 38 | +# il répond 200 avec 0 élément pour l'aire Québec, ce qui empoisonnait les | |
| 39 | +# runs (incident des runs #2/#3 : des types entiers « réussis » à 0 POI). | |
| 36 | 40 | |
| 37 | 41 | # (clé, valeur, type d'établissement Resto·Ka) |
| 38 | 42 | TARGETS = [ |
@@ -68,14 +72,51 @@ _CUISINE_MAP = { | ||
| 68 | 72 | "shawarma": "libanais-moyen-orient", "falafel": "libanais-moyen-orient", |
| 69 | 73 | "tapas": "mediterraneen", "portuguese": "mediterraneen", |
| 70 | 74 | "spanish": "mediterraneen", "american": "burgers", "diner": "dejeuner-brunch", |
| 75 | + # valeurs OSM courantes au QC, non couvertes par la v1 | |
| 76 | + "belgian": "francais", "bistro": "francais", "brasserie": "francais", | |
| 77 | + "turkish": "libanais-moyen-orient", "persian": "libanais-moyen-orient", | |
| 78 | + "iranian": "libanais-moyen-orient", "syrian": "libanais-moyen-orient", | |
| 79 | + "afghan": "libanais-moyen-orient", "pita": "libanais-moyen-orient", | |
| 80 | + "moroccan": "mediterraneen", "tunisian": "mediterraneen", | |
| 81 | + "gyro": "grec", "pakistani": "indien", "bangladeshi": "indien", | |
| 82 | + "sri_lankan": "indien", "nepalese": "indien", | |
| 83 | + "cantonese": "chinois", "szechuan": "chinois", "sichuan": "chinois", | |
| 84 | + "hong_kong": "chinois", "taiwanese": "chinois", "dim_sum": "chinois", | |
| 85 | + "dumpling": "chinois", "noodle": "chinois", "noodles": "chinois", | |
| 86 | + "malaysian": "thai", "indonesian": "thai", "filipino": "thai", | |
| 87 | + "teriyaki": "sushi-japonais", "izakaya": "sushi-japonais", | |
| 88 | + "peruvian": "mexicain", "brazilian": "mexicain", "colombian": "mexicain", | |
| 89 | + "salvadoran": "mexicain", "latin_american": "mexicain", | |
| 90 | + "argentinian": "bbq-grillades", "steak": "bbq-grillades", | |
| 91 | + "wings": "poulet", "fried_chicken": "poulet", | |
| 92 | + "hotdog": "fast-food", "hot_dog": "fast-food", "deli": "fast-food", | |
| 93 | + "bagel": "dejeuner-brunch", "pancake": "dejeuner-brunch", | |
| 94 | + "waffle": "cafe-dessert", "cake": "cafe-dessert", | |
| 95 | + "chocolate": "cafe-dessert", "tea": "cafe-dessert", | |
| 96 | + "juice": "cafe-dessert", "coffee": "cafe-dessert", | |
| 97 | + "pasta": "italien", "oysters": "fruits-de-mer", "fish": "fruits-de-mer", | |
| 98 | + "smoked_meat": "quebecois", "sugar_shack": "quebecois", | |
| 71 | 99 | } |
| 72 | 100 | |
| 101 | +# diet:*=yes|only|limited -> options alimentaires (§6) ; « limited » = offre | |
| 102 | +# partielle, on l'affiche quand même (mieux vaut signaler l'option qui existe) | |
| 73 | 103 | _DIET_TAGS = { |
| 74 | 104 | "diet:vegan": "vegan", "diet:vegetarian": "vegetarien", |
| 75 | 105 | "diet:gluten_free": "sans-gluten", "diet:halal": "halal", |
| 76 | 106 | "diet:kosher": "casher", |
| 77 | 107 | } |
| 78 | 108 | |
| 109 | +# payment:* -> étiquettes de service (modes de paiement acceptés) | |
| 110 | +_PAYMENT_TAGS = { | |
| 111 | + "payment:cash": "paiement-comptant", | |
| 112 | + "payment:credit_cards": "paiement-credit", | |
| 113 | + "payment:visa": "paiement-credit", | |
| 114 | + "payment:mastercard": "paiement-credit", | |
| 115 | + "payment:american_express": "paiement-credit", | |
| 116 | + "payment:debit_cards": "paiement-debit", | |
| 117 | + "payment:interac": "paiement-debit", | |
| 118 | +} | |
| 119 | + | |
| 79 | 120 | |
| 80 | 121 | class OsmConnector(BaseConnector): |
| 81 | 122 | source_id = "osm" |
@@ -95,7 +136,22 @@ class OsmConnector(BaseConnector): | ||
| 95 | 136 | for mirror in MIRRORS: |
| 96 | 137 | try: |
| 97 | 138 | resp = self.post(mirror, data={"data": query}) |
| 98 | − return resp.json().get("elements") or [] | |
| 139 | + data = resp.json() | |
| 140 | + # Overpass peut répondre 200 avec des résultats TRONQUÉS et un | |
| 141 | + # « remark » (timeout/mémoire) — traiter comme un échec de | |
| 142 | + # miroir plutôt que d'avaler une liste partielle (incident du | |
| 143 | + # run #2 : 6 213 POI au lieu de 13 531). | |
| 144 | + remark = (data.get("remark") or "").lower() | |
| 145 | + if "timed out" in remark or "error" in remark: | |
| 146 | + raise RuntimeError(f"réponse partielle : {remark[:200]}") | |
| 147 | + elements = data.get("elements") or [] | |
| 148 | + # 0 élément pour une requête à l'échelle de la PROVINCE est | |
| 149 | + # toujours implausible (chaque type compte des centaines de | |
| 150 | + # POI au QC) : miroir sans les données (extrait régional…) | |
| 151 | + if not elements: | |
| 152 | + raise RuntimeError("0 élément — miroir sans données " | |
| 153 | + "pour l'aire Québec") | |
| 154 | + return elements | |
| 99 | 155 | except Exception as exc: # miroir surchargé -> suivant |
| 100 | 156 | last_exc = exc |
| 101 | 157 | print(f"[resto-ka] osm: {mirror} a échoué ({exc}), miroir suivant", |
@@ -129,6 +185,12 @@ class OsmConnector(BaseConnector): | ||
| 129 | 185 | name = (tags.get("name") or "").strip() |
| 130 | 186 | if not name: |
| 131 | 187 | return None # un POI sans nom est inutilisable |
| 188 | + # marque/chaîne (brand=*) -> colonne chain ; branch=* = nom de la | |
| 189 | + # succursale, suffixé au nom (même convention que UEAT « Marque — Succ. ») | |
| 190 | + chain = (tags.get("brand") or "").strip() or None | |
| 191 | + branch = (tags.get("branch") or "").strip() | |
| 192 | + if branch and branch.lower() not in name.lower(): | |
| 193 | + name = f"{name} — {branch}" | |
| 132 | 194 | lat = el.get("lat") or (el.get("center") or {}).get("lat") |
| 133 | 195 | lng = el.get("lon") or (el.get("center") or {}).get("lon") |
| 134 | 196 | civic = (tags.get("addr:housenumber") or "").strip() |
@@ -140,6 +202,14 @@ class OsmConnector(BaseConnector): | ||
| 140 | 202 | services.append("delivery") |
| 141 | 203 | if tags.get("takeaway") != "only": |
| 142 | 204 | services.append("salle") |
| 205 | + if tags.get("wheelchair") in ("yes", "designated", "limited"): | |
| 206 | + services.append("accessible") | |
| 207 | + for t, svc in _PAYMENT_TAGS.items(): | |
| 208 | + if tags.get(t) == "yes" and svc not in services: | |
| 209 | + services.append(svc) | |
| 210 | + capacity = (tags.get("capacity") or "").strip() | |
| 211 | + if capacity.isdigit(): | |
| 212 | + services.append(f"capacite-{capacity}") | |
| 143 | 213 | hours = {} |
| 144 | 214 | if tags.get("opening_hours"): |
| 145 | 215 | hours["osm"] = tags["opening_hours"] |
@@ -149,6 +219,7 @@ class OsmConnector(BaseConnector): | ||
| 149 | 219 | external_id=osm_id, |
| 150 | 220 | name=name, |
| 151 | 221 | url=f"https://www.openstreetmap.org/{osm_id}", |
| 222 | + chain=chain, | |
| 152 | 223 | cuisines=self._cuisines(tags, name, value), |
| 153 | 224 | establishment_type=etype, |
| 154 | 225 | address=f"{civic} {street}".strip(), |
@@ -160,15 +231,26 @@ class OsmConnector(BaseConnector): | ||
| 160 | 231 | hours=hours, |
| 161 | 232 | services=services, |
| 162 | 233 | dietary_options=[d for t, d in _DIET_TAGS.items() |
| 163 | − if tags.get(t) in ("yes", "only")], | |
| 234 | + if tags.get(t) in ("yes", "only", "limited")], | |
| 164 | 235 | languages=["fr"], |
| 165 | 236 | menu=None, # découverte : pas de menu ici |
| 166 | 237 | ) |
| 167 | 238 | |
| 168 | 239 | def fetch(self) -> list[Restaurant]: |
| 169 | 240 | out: dict[str, Restaurant] = {} |
| 241 | + # run partiel : si UN type d'établissement échoue sur tous les miroirs, | |
| 242 | + # on continue avec les autres mais on signale le run comme incomplet — | |
| 243 | + # le pipeline suspendra alors les retraits (pas de fausses fermetures). | |
| 244 | + self.partial_run = False | |
| 170 | 245 | for key, value, etype in TARGETS: |
| 171 | − elements = self._overpass(key, value) | |
| 246 | + try: | |
| 247 | + elements = self._overpass(key, value) | |
| 248 | + except Exception as exc: | |
| 249 | + self.partial_run = True | |
| 250 | + print(f"[resto-ka] osm: {key}={value} ÉCHEC sur tous les " | |
| 251 | + f"miroirs ({exc}) — run marqué partiel, retraits " | |
| 252 | + "suspendus", file=sys.stderr) | |
| 253 | + continue | |
| 172 | 254 | kept = 0 |
| 173 | 255 | for el in elements: |
| 174 | 256 | resto = self._to_restaurant(el, etype, value) |
modified
restoka/connectors/ueat.py
+8 −2
@@ -56,8 +56,9 @@ _Q_FRANCHISES = """ | ||
| 56 | 56 | query getFranchises($orderType: OrderTypeEnum!) { |
| 57 | 57 | restaurant { name url logo } |
| 58 | 58 | franchises(modeType: $orderType) { |
| 59 | − companyProfileId code name currency countryCode | |
| 60 | − address { civicNumber streetName postalCode city province } | |
| 59 | + companyProfileId code name currency countryCode phoneNumber | |
| 60 | + address { civicNumber streetName postalCode city province | |
| 61 | + latitude longitude } | |
| 61 | 62 | } |
| 62 | 63 | } |
| 63 | 64 | """ |
@@ -342,6 +343,11 @@ class UeatConnector(BaseConnector): | ||
| 342 | 343 | address=f"{civic} {street}".strip(), |
| 343 | 344 | city=(addr.get("city") or "").strip(), |
| 344 | 345 | postal_code=(addr.get("postalCode") or "").strip(), |
| 346 | + # GPS de première partie (API) : prime sur le géocodage | |
| 347 | + # Nominatim (l'UPDATE fait COALESCE(:lat, lat) — un null ne | |
| 348 | + # peut pas écraser des coordonnées déjà géocodées) | |
| 349 | + lat=addr.get("latitude"), lng=addr.get("longitude"), | |
| 350 | + phone=(fr.get("phoneNumber") or "").strip(), | |
| 345 | 351 | website=(brand.get("url") or "").strip(), |
| 346 | 352 | services=["takeout"], |
| 347 | 353 | languages=["fr", "en"], |
modified
restoka/db.py
+23 −1
@@ -29,6 +29,14 @@ DRIFT_RATIO = 0.25 | ||
| 29 | 29 | DRIFT_MIN_BASE = 4 |
| 30 | 30 | DRIFT_HISTORY = 5 |
| 31 | 31 | |
| 32 | +# Garde « bootstrap » (source trop jeune pour une médiane fiable) : dès qu'il | |
| 33 | +# existe AU MOINS un run ok, si le run courant retourne moins de | |
| 34 | +# BOOTSTRAP_RATIO × le MAXIMUM historique, on suspend les retraits. Protège | |
| 35 | +# contre un run partiel (miroirs Overpass en 504…) qui, avec <3 runs | |
| 36 | +# d'historique, passait sous le radar de la garde médiane (incident du run | |
| 37 | +# OSM #2 : 6 213 trouvés vs 13 531 -> 7 354 fausses quasi-fermetures). | |
| 38 | +BOOTSTRAP_RATIO = 0.60 | |
| 39 | + | |
| 32 | 40 | # Dérive menu : si le nombre d'items d'un resto chute sous MENU_DROP_RATIO × |
| 33 | 41 | # le compte précédent (précédent >= MENU_DROP_MIN items), on conserve l'ancien |
| 34 | 42 | # menu et on consigne l'alerte (connecteur probablement cassé — CLAUDE.md §18). |
@@ -255,6 +263,14 @@ def _drift_alert(con: sqlite3.Connection, source: str, found: int) -> str | None | ||
| 255 | 263 | hist = con.execute( |
| 256 | 264 | "SELECT found FROM sync_log WHERE source=? AND ok=1" |
| 257 | 265 | " ORDER BY ts DESC LIMIT ?", (source, DRIFT_HISTORY)).fetchall() |
| 266 | + if not hist: | |
| 267 | + return None | |
| 268 | + # Garde bootstrap : active dès le 2e run, MÊME sans 3 runs d'historique. | |
| 269 | + max_found = max(r["found"] for r in hist) | |
| 270 | + if max_found >= DRIFT_MIN_BASE and found < BOOTSTRAP_RATIO * max_found: | |
| 271 | + return (f"dérive: {found} resto(s) trouvé(s) contre un maximum " | |
| 272 | + f"historique de {max_found} — retraits suspendus, " | |
| 273 | + "vérifier le connecteur") | |
| 258 | 274 | if len(hist) < 3: |
| 259 | 275 | return None |
| 260 | 276 | med_found = statistics.median(r["found"] for r in hist) |
@@ -265,7 +281,8 @@ def _drift_alert(con: sqlite3.Connection, source: str, found: int) -> str | None | ||
| 265 | 281 | |
| 266 | 282 | |
| 267 | 283 | def sync_source(con: sqlite3.Connection, source: str, |
| 268 | − restaurants: list[Restaurant]) -> dict: | |
| 284 | + restaurants: list[Restaurant], | |
| 285 | + partial: bool = False) -> dict: | |
| 269 | 286 | """Synchronise les restaurants d'une source. |
| 270 | 287 | |
| 271 | 288 | - nouveau resto -> insertion (+ menu, + historique de prix initial) |
@@ -273,6 +290,8 @@ def sync_source(con: sqlite3.Connection, source: str, | ||
| 273 | 290 | - resto disparu -> miss_count += 1, puis status=temporarily_closed |
| 274 | 291 | après MISS_GRACE exécutions (délai de grâce, jamais de suppression) |
| 275 | 292 | - dérive détectée -> alerte consignée, retraits suspendus |
| 293 | + - `partial=True` -> le connecteur signale un run incomplet (un | |
| 294 | + sous-ensemble de requêtes a échoué) : retraits suspendus d'office | |
| 276 | 295 | """ |
| 277 | 296 | now = time.time() |
| 278 | 297 | added = updated = 0 |
@@ -291,6 +310,9 @@ def sync_source(con: sqlite3.Connection, source: str, | ||
| 291 | 310 | items_sans_prix += 1 |
| 292 | 311 | |
| 293 | 312 | alert = _drift_alert(con, source, n) |
| 313 | + if partial and not alert: | |
| 314 | + alert = ("run partiel signalé par le connecteur — " | |
| 315 | + "retraits suspendus") | |
| 294 | 316 | |
| 295 | 317 | for resto in restaurants: |
| 296 | 318 | seen_uids.add(resto.uid) |
modified
restoka/ingest.py
+7 −2
@@ -29,7 +29,8 @@ def run(sources: list[str] | None = None) -> list[dict]: | ||
| 29 | 29 | t0 = time.time() |
| 30 | 30 | print(f"[resto-ka] sync {sid} ...") |
| 31 | 31 | try: |
| 32 | − restos = cls().fetch() | |
| 32 | + connector = cls() | |
| 33 | + restos = connector.fetch() | |
| 33 | 34 | finalized, dropped = [], 0 |
| 34 | 35 | for r in restos: |
| 35 | 36 | try: |
@@ -37,7 +38,11 @@ def run(sources: list[str] | None = None) -> list[dict]: | ||
| 37 | 38 | except Exception as exc: # une fiche malformée ne bloque pas la source |
| 38 | 39 | dropped += 1 |
| 39 | 40 | print(f"[resto-ka] fiche rejetée : {exc}", file=sys.stderr) |
| 40 | − stats = db.sync_source(con, sid, finalized) | |
| 41 | + # run partiel signalé par le connecteur (ex. OSM : un type | |
| 42 | + # d'établissement a échoué sur tous les miroirs) -> retraits suspendus | |
| 43 | + stats = db.sync_source(con, sid, finalized, | |
| 44 | + partial=getattr(connector, "partial_run", | |
| 45 | + False)) | |
| 41 | 46 | stats["seconds"] = round(time.time() - t0, 1) |
| 42 | 47 | if dropped: |
| 43 | 48 | stats["dropped"] = dropped |
| 44 | 49 | |