| |
1 | +# ----------------------------------------------------------------------------- |
| |
2 | +# Lou-Ka — Agrégateur de logements à louer (province de Québec) |
| |
3 | +# Auteur : Simon-Pierre Boucher — contact@spboucher.ai |
| |
4 | +# tal.py : historique au Tribunal administratif du logement (TAL / ex-RDL) |
| |
5 | +# |
| |
6 | +# Les décisions du TAL sont diffusées gratuitement par SOQUIJ |
| |
7 | +# (citoyens.soquij.qc.ca, tribunal no 35, plein texte depuis ~2009). Le site |
| |
8 | +# est protégé par une détection de bot (byscuit) : la RECHERCHE exige un |
| |
9 | +# rendu JS + IP résidentielle (Scrapfly, asp + render_js + session), mais la |
| |
10 | +# page d'une DÉCISION se charge ensuite sans rendu (même session = moins |
| |
11 | +# cher). Le moteur ne supporte pas les phrases entre guillemets : les mots |
| |
12 | +# sont combinés en ET implicite, d'où la stratégie |
| |
13 | +# recherche « <no civique> <nom de rue sans le type> » |
| |
14 | +# puis VÉRIFICATION de l'adresse exacte dans le texte intégral. |
| |
15 | +# |
| |
16 | +# Tout est mis en cache dans data/tal.db (base séparée : jamais de verrou |
| |
17 | +# sur louka.db) par adresse normalisée (no civique + rue + ville) : |
| |
18 | +# · tal_lookup : une ligne par adresse vérifiée (ou en file d'attente) ; |
| |
19 | +# · tal_decisions : décisions confirmées à l'adresse, classées par mots |
| |
20 | +# clés (non-paiement, résiliation, expulsion, reprise…) |
| |
21 | +# et par partie demanderesse (locateur vs locataire). |
| |
22 | +# |
| |
23 | +# Usage : python run.py tal [n] [budget_s] # précalcul par lots |
| |
24 | +# Lecture : lookup(address, city) — bloc « Historique TAL » de la fiche |
| |
25 | +# (/api/tal) ; une adresse jamais vérifiée est mise en file |
| |
26 | +# prioritaire et traitée au prochain passage du watch. |
| |
27 | +# ----------------------------------------------------------------------------- |
| |
28 | +from __future__ import annotations |
| |
29 | + |
| |
30 | +import html |
| |
31 | +import json |
| |
32 | +import os |
| |
33 | +import re |
| |
34 | +import sqlite3 |
| |
35 | +import threading |
| |
36 | +import time |
| |
37 | +import unicodedata |
| |
38 | +import urllib.parse |
| |
39 | +import urllib.request |
| |
40 | +from concurrent.futures import ThreadPoolExecutor, as_completed |
| |
41 | +from pathlib import Path |
| |
42 | + |
| |
43 | +DATA_DIR = Path(__file__).resolve().parent.parent / "data" |
| |
44 | +DB_PATH = DATA_DIR / "tal.db" |
| |
45 | +LOUKA_DB = DATA_DIR / "louka.db" |
| |
46 | + |
| |
47 | +SCRAPFLY_API = "https://api.scrapfly.io/scrape" |
| |
48 | +SOQUIJ_BASE = "https://citoyens.soquij.qc.ca" |
| |
49 | +TRIBUNAL_ID = "35" # Tribunal administratif du logement / RDL |
| |
50 | + |
| |
51 | +REFRESH_DAYS = 180 # revérifier une adresse après ~6 mois |
| |
52 | +ERROR_RETRY_DAYS = 7 # réessayer plus vite après une erreur |
| |
53 | +MAX_DECISIONS = 12 # décisions lues au maximum par adresse |
| |
54 | +WORKERS = 4 |
| |
55 | + |
| |
56 | +_SCHEMA = """ |
| |
57 | +CREATE TABLE IF NOT EXISTS tal_lookup ( |
| |
58 | + addr_key TEXT PRIMARY KEY, |
| |
59 | + civic TEXT, street TEXT, city TEXT, query TEXT, |
| |
60 | + status TEXT, -- queued | ok | error |
| |
61 | + found INTEGER, -- résultats bruts SOQUIJ (avant vérification) |
| |
62 | + matched INTEGER, -- décisions confirmées à l'adresse |
| |
63 | + priority INTEGER DEFAULT 0, -- 1 = demandé par une fiche (file rapide) |
| |
64 | + checked_at TEXT, |
| |
65 | + error TEXT |
| |
66 | +); |
| |
67 | +CREATE TABLE IF NOT EXISTS tal_decisions ( |
| |
68 | + addr_key TEXT, |
| |
69 | + decision_id TEXT, -- ID hexadécimal SOQUIJ |
| |
70 | + citation TEXT, -- ex. « 2026 QCTAL 21509 » |
| |
71 | + parties TEXT, |
| |
72 | + date TEXT, |
| |
73 | + demandeur TEXT, -- locateur | locataire | NULL (indéterminé) |
| |
74 | + tags TEXT, -- JSON : ["non-paiement", "résiliation de bail", …] |
| |
75 | + verdict TEXT, -- accueillie | rejetée | NULL |
| |
76 | + url TEXT, |
| |
77 | + PRIMARY KEY (addr_key, decision_id) |
| |
78 | +); |
| |
79 | +""" |
| |
80 | + |
| |
81 | +# --------------------------------------------------------------------------- |
| |
82 | +# Normalisation d'adresse |
| |
83 | +# --------------------------------------------------------------------------- |
| |
84 | + |
| |
85 | +_TYPES_VOIE = { |
| |
86 | + "rue", "avenue", "av", "ave", "boulevard", "boul", "blvd", "bd", "chemin", |
| |
87 | + "ch", "place", "pl", "cote", "allee", "montee", "carre", "impasse", |
| |
88 | + "terrasse", "tsse", "croissant", "crois", "rang", "route", "rte", |
| |
89 | + "promenade", "prom", "voie", "square", "cours", "sentier", "parc", |
| |
90 | + # types anglais (annonces montréalaises) |
| |
91 | + "st", "street", "rd", "road", "dr", "drive", "cres", "crescent", |
| |
92 | + "ct", "court", "lane", "ln", "hwy", "way", |
| |
93 | +} |
| |
94 | +_STOPWORDS = {"de", "du", "des", "la", "le", "les", "l", "d", "a", "au", "aux", |
| |
95 | + "app", "apt", "unite", "suite", "bureau", "local"} |
| |
96 | + |
| |
97 | + |
| |
98 | +def _fold(s: str) -> str: |
| |
99 | + """Minuscules sans accents (comparaisons et clés).""" |
| |
100 | + s = unicodedata.normalize("NFD", s) |
| |
101 | + return "".join(c for c in s if not unicodedata.combining(c)).lower() |
| |
102 | + |
| |
103 | + |
| |
104 | +def parse_address(address: str | None, city: str | None = None) -> dict | None: |
| |
105 | + """« 1573, rue Pierre-Corneille, Québec, Québec, G2E4W9 » -> |
| |
106 | + civic + rue + mots significatifs pour la recherche SOQUIJ.""" |
| |
107 | + if not address: |
| |
108 | + return None |
| |
109 | + parts = [p.strip() for p in address.split(",") if p.strip()] |
| |
110 | + if not parts: |
| |
111 | + return None |
| |
112 | + m = re.match(r"^(\d+[a-zA-Z]?)\b[\s,]*(.*)$", parts[0]) |
| |
113 | + if not m: |
| |
114 | + return None |
| |
115 | + civic = m.group(1) |
| |
116 | + street = m.group(2).strip() |
| |
117 | + if not street and len(parts) > 1: # « 1573 » puis « rue X » à part |
| |
118 | + street = parts[1] |
| |
119 | + if not street: |
| |
120 | + return None |
| |
121 | + ville = (city or "").strip() |
| |
122 | + if not ville and len(parts) > 2: |
| |
123 | + for p in parts[1:]: # sauter « Suite 1106 », « app. 4 »… |
| |
124 | + if not re.match(r"^(app|apt|suite|unit|unité|bureau|local|#|\d)", |
| |
125 | + p, re.I): |
| |
126 | + ville = p |
| |
127 | + break |
| |
128 | + |
| |
129 | + toks = [t for t in re.split(r"[^\w-]+", street) if t] |
| |
130 | + core = [t for t in toks |
| |
131 | + if _fold(t).rstrip(".") not in _TYPES_VOIE |
| |
132 | + and _fold(t) not in _STOPWORDS and len(t) >= 2 |
| |
133 | + and not re.fullmatch(r"\d+[a-zA-Z]?", t)] # no d'unité (« 16F ») |
| |
134 | + if not core: # ex. « 1re Avenue » : tout garder |
| |
135 | + core = toks |
| |
136 | + if not core: |
| |
137 | + return None |
| |
138 | + key = f"{civic}|{_fold(' '.join(core))}|{_fold(ville)}" |
| |
139 | + return {"civic": civic, "street": street, "city": ville, |
| |
140 | + "core": core, "query": f"{civic} {' '.join(core)}", "key": key} |
| |
141 | + |
| |
142 | + |
| |
143 | +# --------------------------------------------------------------------------- |
| |
144 | +# Accès SOQUIJ via Scrapfly |
| |
145 | +# --------------------------------------------------------------------------- |
| |
146 | + |
| |
147 | +def _scrapfly(url: str, *, render: bool, session: str) -> str: |
| |
148 | + key = os.environ.get("SCRAPFLY_KEY") |
| |
149 | + if not key: |
| |
150 | + raise RuntimeError("SCRAPFLY_KEY absent de l'environnement") |
| |
151 | + params = {"key": key, "url": url, "asp": "true", "country": "ca", |
| |
152 | + "proxy_pool": "public_residential_pool", "session": session} |
| |
153 | + if render: |
| |
154 | + params["render_js"] = "true" |
| |
155 | + params["rendering_wait"] = "5000" |
| |
156 | + u = SCRAPFLY_API + "?" + urllib.parse.urlencode(params) |
| |
157 | + data = None |
| |
158 | + for attempt in range(3): |
| |
159 | + try: |
| |
160 | + with urllib.request.urlopen(u, timeout=170) as resp: |
| |
161 | + data = json.load(resp) |
| |
162 | + break |
| |
163 | + except urllib.error.HTTPError as exc: |
| |
164 | + # 429 = limite de concurrence Scrapfly : attendre puis réessayer |
| |
165 | + if exc.code == 429 and attempt < 2: |
| |
166 | + time.sleep(8 * (attempt + 1)) |
| |
167 | + continue |
| |
168 | + raise |
| |
169 | + res = (data or {}).get("result") or {} |
| |
170 | + if res.get("status_code") != 200: |
| |
171 | + raise RuntimeError(f"SOQUIJ HTTP {res.get('status_code')}") |
| |
172 | + return res.get("content") or "" |
| |
173 | + |
| |
174 | + |
| |
175 | +def _to_text(html_doc: str) -> str: |
| |
176 | + """HTML -> texte plat, entités décodées, espaces normalisés.""" |
| |
177 | + doc = re.sub(r"<script.*?</script>|<style.*?</style>", " ", |
| |
178 | + html_doc, flags=re.S | re.I) |
| |
179 | + doc = re.sub(r"<[^>]+>", " ", doc) |
| |
180 | + doc = html.unescape(doc).replace("\xa0", " ") |
| |
181 | + return re.sub(r"\s+", " ", doc) |
| |
182 | + |
| |
183 | + |
| |
184 | +def _search(query: str, session: str) -> list[dict]: |
| |
185 | + """Recherche plein texte TAL ; -> [{id, parties, date}] (max 200).""" |
| |
186 | + recher = f"{TRIBUNAL_ID}_{query}_0___date" |
| |
187 | + target = SOQUIJ_BASE + "/index.php?" + urllib.parse.urlencode( |
| |
188 | + {"type": "listemc", "recher": recher}, |
| |
189 | + encoding="iso-8859-1", errors="replace") |
| |
190 | + content = _scrapfly(target, render=True, session=session) |
| |
191 | + text = _to_text(content) |
| |
192 | + if "Décisions trouvées" not in text and "Aucun résultat" not in text: |
| |
193 | + raise RuntimeError("page de résultats SOQUIJ non reconnue (anti-bot ?)") |
| |
194 | + rows = [] |
| |
195 | + for m in re.finditer( |
| |
196 | + r'href="(/php/decision\.php\?ID=([0-9A-Fa-f]+))"[^>]*>(.*?)</a>' |
| |
197 | + r'.*?class="tb-date"[^>]*>\s*([\d-]{8,10})', content, re.S): |
| |
198 | + rows.append({"id": m.group(2).upper(), |
| |
199 | + "parties": _to_text(m.group(3)).strip(), |
| |
200 | + "date": m.group(4), |
| |
201 | + "url": SOQUIJ_BASE + m.group(1)}) |
| |
202 | + return rows |
| |
203 | + |
| |
204 | + |
| |
205 | +_RE_CITATION = re.compile(r"\b(\d{4}\s+QC(?:TAL|RDL)\s+\d+)\b") |
| |
206 | + |
| |
207 | +_TAGS = [ |
| |
208 | + ("non-paiement de loyer", ("non-paiement", "loyer impaye", |
| |
209 | + "recouvrement du loyer", "ne paie pas le loyer", |
| |
210 | + "loyers impayes")), |
| |
211 | + ("résiliation de bail", ("resiliation",)), |
| |
212 | + ("expulsion", ("expulsion",)), |
| |
213 | + ("éviction", ("eviction",)), |
| |
214 | + ("reprise du logement", ("reprise du logement", "reprise de logement")), |
| |
215 | + ("fixation de loyer", ("fixation du loyer", "fixation de loyer")), |
| |
216 | + ("insalubrité", ("insalubr",)), |
| |
217 | + ("harcèlement", ("harcel",)), |
| |
218 | +] |
| |
219 | + |
| |
220 | + |
| |
221 | +def _classify(folded: str) -> tuple[str | None, list[str], str | None]: |
| |
222 | + """Texte plié -> (partie demanderesse, tags, verdict sommaire).""" |
| |
223 | + demandeur = None |
| |
224 | + m = re.search(r"\b(locateurs?|locatrices?|locataires?)\b[^a-z]{0,40}" |
| |
225 | + r"parties?\s+demanderesses?", folded) |
| |
226 | + if m: |
| |
227 | + demandeur = "locataire" if m.group(1).startswith("locata") else "locateur" |
| |
228 | + tags = [tag for tag, needles in _TAGS |
| |
229 | + if any(n in folded for n in needles)] |
| |
230 | + verdict = None |
| |
231 | + if re.search(r"\baccueille\b", folded): |
| |
232 | + verdict = "accueillie" |
| |
233 | + elif re.search(r"\brejette la demande\b", folded): |
| |
234 | + verdict = "rejetée" |
| |
235 | + return demandeur, tags, verdict |
| |
236 | + |
| |
237 | + |
| |
238 | +def _address_in_text(folded: str, civic: str, core: list[str]) -> bool: |
| |
239 | + """Le no civique suivi de près par le nom de la rue ? (adresse confirmée)""" |
| |
240 | + name = _fold(" ".join(core)) |
| |
241 | + pat = rf"\b{re.escape(civic.lower())}\b.{{0,80}}?{re.escape(name)}" |
| |
242 | + if re.search(pat, folded): |
| |
243 | + return True |
| |
244 | + # tolérance : dernier mot du nom seulement (ex. « st-jacques ») |
| |
245 | + last = _fold(core[-1]) |
| |
246 | + return bool(len(last) >= 4 and |
| |
247 | + re.search(rf"\b{re.escape(civic.lower())}\b.{{0,60}}?{re.escape(last)}", folded)) |
| |
248 | + |
| |
249 | + |
| |
250 | +def _check_address(parsed: dict, session: str) -> dict: |
| |
251 | + """Recherche + vérification pour UNE adresse. -> résultat à persister.""" |
| |
252 | + rows = _search(parsed["query"], session) |
| |
253 | + decisions = [] |
| |
254 | + for r in rows[:MAX_DECISIONS]: |
| |
255 | + try: |
| |
256 | + page = _scrapfly(r["url"], render=False, session=session) |
| |
257 | + except Exception: |
| |
258 | + continue |
| |
259 | + text = _to_text(page) |
| |
260 | + folded = _fold(text) |
| |
261 | + if not _address_in_text(folded, parsed["civic"], parsed["core"]): |
| |
262 | + continue |
| |
263 | + m = _RE_CITATION.search(text) |
| |
264 | + demandeur, tags, verdict = _classify(folded) |
| |
265 | + decisions.append({ |
| |
266 | + "decision_id": r["id"], |
| |
267 | + "citation": re.sub(r"\s+", " ", m.group(1)) if m else None, |
| |
268 | + "parties": r["parties"], "date": r["date"], |
| |
269 | + "demandeur": demandeur, "tags": tags, "verdict": verdict, |
| |
270 | + "url": r["url"]}) |
| |
271 | + return {"found": len(rows), "decisions": decisions} |
| |
272 | + |
| |
273 | + |
| |
274 | +# --------------------------------------------------------------------------- |
| |
275 | +# Persistance |
| |
276 | +# --------------------------------------------------------------------------- |
| |
277 | + |
| |
278 | +_lock = threading.Lock() |
| |
279 | + |
| |
280 | + |
| |
281 | +def _connect(ro: bool = False) -> sqlite3.Connection: |
| |
282 | + if ro: |
| |
283 | + con = sqlite3.connect(f"file:{DB_PATH}?mode=ro", uri=True, timeout=15) |
| |
284 | + else: |
| |
285 | + DATA_DIR.mkdir(parents=True, exist_ok=True) |
| |
286 | + con = sqlite3.connect(DB_PATH, timeout=30) |
| |
287 | + con.executescript(_SCHEMA) |
| |
288 | + con.row_factory = sqlite3.Row |
| |
289 | + return con |
| |
290 | + |
| |
291 | + |
| |
292 | +def _save(con: sqlite3.Connection, parsed: dict, status: str, |
| |
293 | + found: int = 0, decisions: list[dict] | None = None, |
| |
294 | + error: str | None = None) -> None: |
| |
295 | + now = time.strftime("%Y-%m-%d %H:%M:%S") |
| |
296 | + with con: |
| |
297 | + con.execute( |
| |
298 | + "INSERT OR REPLACE INTO tal_lookup " |
| |
299 | + "(addr_key, civic, street, city, query, status, found, matched," |
| |
300 | + " priority, checked_at, error) VALUES (?,?,?,?,?,?,?,?,0,?,?)", |
| |
301 | + (parsed["key"], parsed["civic"], parsed["street"], parsed["city"], |
| |
302 | + parsed["query"], status, found, len(decisions or []), now, error)) |
| |
303 | + con.execute("DELETE FROM tal_decisions WHERE addr_key=?", |
| |
304 | + (parsed["key"],)) |
| |
305 | + for d in decisions or []: |
| |
306 | + con.execute( |
| |
307 | + "INSERT OR REPLACE INTO tal_decisions VALUES (?,?,?,?,?,?,?,?,?)", |
| |
308 | + (parsed["key"], d["decision_id"], d["citation"], d["parties"], |
| |
309 | + d["date"], d["demandeur"], json.dumps(d["tags"], |
| |
310 | + ensure_ascii=False), d["verdict"], d["url"])) |
| |
311 | + |
| |
312 | + |
| |
313 | +# --------------------------------------------------------------------------- |
| |
314 | +# API de lecture (fiche / web.py) |
| |
315 | +# --------------------------------------------------------------------------- |
| |
316 | + |
| |
317 | +def lookup(address: str | None, city: str | None = None) -> dict: |
| |
318 | + """Historique TAL d'une adresse — lecture du cache ; une adresse inconnue |
| |
319 | + est mise en file prioritaire (traitée au prochain passage du watch).""" |
| |
320 | + parsed = parse_address(address, city) |
| |
321 | + if parsed is None: |
| |
322 | + return {"status": "na"} |
| |
323 | + con = _connect() |
| |
324 | + try: |
| |
325 | + row = con.execute("SELECT * FROM tal_lookup WHERE addr_key=?", |
| |
326 | + (parsed["key"],)).fetchone() |
| |
327 | + if row is None: |
| |
328 | + with _lock, con: |
| |
329 | + con.execute( |
| |
330 | + "INSERT OR IGNORE INTO tal_lookup " |
| |
331 | + "(addr_key, civic, street, city, query, status, priority) " |
| |
332 | + "VALUES (?,?,?,?,?,'queued',1)", |
| |
333 | + (parsed["key"], parsed["civic"], parsed["street"], |
| |
334 | + parsed["city"], parsed["query"])) |
| |
335 | + return {"status": "pending"} |
| |
336 | + if row["status"] != "ok": |
| |
337 | + return {"status": "pending" if row["status"] == "queued" |
| |
338 | + else "error"} |
| |
339 | + decs = [dict(r) for r in con.execute( |
| |
340 | + "SELECT * FROM tal_decisions WHERE addr_key=? ORDER BY date DESC", |
| |
341 | + (parsed["key"],)).fetchall()] |
| |
342 | + finally: |
| |
343 | + con.close() |
| |
344 | + items = [{"date": d["date"], "citation": d["citation"], |
| |
345 | + "demandeur": d["demandeur"], "tags": json.loads(d["tags"] or "[]"), |
| |
346 | + "verdict": d["verdict"], "url": d["url"]} for d in decs] |
| |
347 | + contre = sum(1 for d in items if d["demandeur"] == "locateur") |
| |
348 | + evict = sum(1 for d in items if d["demandeur"] == "locateur" |
| |
349 | + and set(d["tags"]) & |
| |
350 | + {"expulsion", "éviction", "reprise du logement", |
| |
351 | + "résiliation de bail", "non-paiement de loyer"}) |
| |
352 | + return {"status": "ok", "checked_at": row["checked_at"], |
| |
353 | + "n": len(items), "contre_locataire": contre, "eviction": evict, |
| |
354 | + "last_date": items[0]["date"] if items else None, |
| |
355 | + "decisions": items} |
| |
356 | + |
| |
357 | + |
| |
358 | +# --------------------------------------------------------------------------- |
| |
359 | +# Précalcul par lots (boucle watch / run.py tal) |
| |
360 | +# --------------------------------------------------------------------------- |
| |
361 | + |
| |
362 | +def _candidates(con: sqlite3.Connection, limit: int) -> list[dict]: |
| |
363 | + """File prioritaire, puis adresses jamais vérifiées (annonces récentes |
| |
364 | + d'abord), puis revérifications périmées.""" |
| |
365 | + out: list[dict] = [] |
| |
366 | + seen: set[str] = set() |
| |
367 | + |
| |
368 | + for r in con.execute( |
| |
369 | + "SELECT * FROM tal_lookup WHERE status='queued' " |
| |
370 | + "ORDER BY priority DESC LIMIT ?", (limit,)): |
| |
371 | + p = parse_address(f"{r['civic']} {r['street']}", r["city"]) |
| |
372 | + if p and p["key"] not in seen: |
| |
373 | + seen.add(p["key"]) |
| |
374 | + out.append(p) |
| |
375 | + if len(out) >= limit: |
| |
376 | + return out[:limit] |
| |
377 | + |
| |
378 | + known = {r["addr_key"]: r for r in con.execute( |
| |
379 | + "SELECT addr_key, status, checked_at FROM tal_lookup")} |
| |
380 | + stale_ok = time.strftime("%Y-%m-%d %H:%M:%S", |
| |
381 | + time.localtime(time.time() - REFRESH_DAYS * 86400)) |
| |
382 | + stale_err = time.strftime("%Y-%m-%d %H:%M:%S", |
| |
383 | + time.localtime(time.time() - ERROR_RETRY_DAYS * 86400)) |
| |
384 | + try: |
| |
385 | + lk = sqlite3.connect(f"file:{LOUKA_DB}?mode=ro", uri=True, timeout=15) |
| |
386 | + lk.row_factory = sqlite3.Row |
| |
387 | + rows = lk.execute( |
| |
388 | + "SELECT DISTINCT address, city FROM listings " |
| |
389 | + "WHERE active=1 AND published=1 AND address IS NOT NULL " |
| |
390 | + "AND address <> '' ORDER BY rowid DESC").fetchall() |
| |
391 | + lk.close() |
| |
392 | + except Exception: |
| |
393 | + rows = [] |
| |
394 | + for r in rows: |
| |
395 | + if len(out) >= limit: |
| |
396 | + break |
| |
397 | + p = parse_address(r["address"], r["city"]) |
| |
398 | + if p is None or p["key"] in seen: |
| |
399 | + continue |
| |
400 | + k = known.get(p["key"]) |
| |
401 | + if k is not None: |
| |
402 | + if k["status"] == "ok" and (k["checked_at"] or "") > stale_ok: |
| |
403 | + continue |
| |
404 | + if k["status"] == "error" and (k["checked_at"] or "") > stale_err: |
| |
405 | + continue |
| |
406 | + if k["status"] == "queued": |
| |
407 | + continue # déjà compté plus haut |
| |
408 | + seen.add(p["key"]) |
| |
409 | + out.append(p) |
| |
410 | + return out |
| |
411 | + |
| |
412 | + |
| |
413 | +def precompute(limit: int = 60, budget: int = 600) -> dict: |
| |
414 | + """Vérifie jusqu'à `limit` adresses (budget en secondes, WORKERS fils).""" |
| |
415 | + t0 = time.time() |
| |
416 | + con = _connect() |
| |
417 | + cands = _candidates(con, limit) |
| |
418 | + if not cands: |
| |
419 | + con.close() |
| |
420 | + return {"checked": 0, "matched": 0, "todo": 0} |
| |
421 | + checked = matched = errors = 0 |
| |
422 | + with ThreadPoolExecutor(max_workers=WORKERS) as pool: |
| |
423 | + futs = {} |
| |
424 | + for i, p in enumerate(cands): |
| |
425 | + if time.time() - t0 > budget: |
| |
426 | + break |
| |
427 | + futs[pool.submit(_check_address, p, |
| |
428 | + f"louka-tal-{i % WORKERS}")] = p |
| |
429 | + for fut in as_completed(futs): |
| |
430 | + p = futs[fut] |
| |
431 | + try: |
| |
432 | + res = fut.result() |
| |
433 | + _save(con, p, "ok", res["found"], res["decisions"]) |
| |
434 | + checked += 1 |
| |
435 | + matched += len(res["decisions"]) |
| |
436 | + if res["decisions"]: |
| |
437 | + print(f"[tal] {p['civic']} {p['street']} ({p['city']}) : " |
| |
438 | + f"{len(res['decisions'])} décision(s) confirmée(s)") |
| |
439 | + except Exception as exc: |
| |
440 | + _save(con, p, "error", error=str(exc)[:300]) |
| |
441 | + errors += 1 |
| |
442 | + con.close() |
| |
443 | + stats = {"checked": checked, "matched": matched, "errors": errors, |
| |
444 | + "seconds": round(time.time() - t0, 1)} |
| |
445 | + print(f"[tal] {stats}") |
| |
446 | + return stats |