SPB Git
spboucher.ai

spb/lou-ka

Public

Lou·Ka — tous les logements à louer du Québec, un seul endroit.

www.lou-ka.com
webappreal-estatequebecrental

[ka2] snapshot pré-mission immobilia

Simon-Pierre Boucher committed (Aug 24, 2026) parent 6ff906e Browse files

12 changed files +1,697 −1

12 changed files
Load all 12 files Unified Split
added .env.example +29 −0
@@ -0,0 +1,29 @@
1+# -----------------------------------------------------------------------------
2+# Lou-Ka — variables d'environnement (modèle — copier vers .env, NE PAS COMMITTER .env)
3+# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
4+# -----------------------------------------------------------------------------
5+
6+# Connecteurs / scraping
7+FIRECRAWL_API_KEY=
8+SCRAPFLY_KEY=
9+APIFY_TOKEN=
10+OXYLABS_PROXY_USER=
11+OXYLABS_PROXY_PASS=
12+BRIGHTDATA_API_KEY=
13+BRIGHTDATA_ZONE=
14+
15+# Session / auth
16+SESSION_SECRET=
17+LOUKA_BASE_URL=https://www.lou-ka.com
18+GOOGLE_CLIENT_ID=
19+GOOGLE_CLIENT_SECRET=
20+KA_SSO_SECRET=
21+KA_HUB_URL=https://www.groupe-ka.com
22+
23+# Hydro-Québec (estimation électricité — solveur reCAPTCHA)
24+HQ_CAPTCHA_KEY=
25+HQ_CAPTCHA_PROVIDER=2captcha
26+
27+# SerpApi — résolution Google Maps + avis des gestionnaires (louka/managers.py)
28+# Server-side SEULEMENT : jamais exposée au frontend ni écrite dans les logs.
29+SERPAPI_API_KEY=
added louka/building.py +244 −0
@@ -0,0 +1,244 @@
1+# -----------------------------------------------------------------------------
2+# Lou-Ka — Agrégateur de logements à louer (province de Québec)
3+# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
4+# building.py : « Passeport de l'immeuble » — entité persistante regroupant les
5+# annonces (actives ET historiques) d'un même bâtiment.
6+#
7+# Clé d'immeuble : la clé d'adresse civique de dedup._parse_address
8+# (« civique|rue|ville », préfixe « adr: ») — la même logique éprouvée que la
9+# déduplication inter-sources ; repli « geo:coord_key » (~11 m) quand l'adresse
10+# n'a pas de numéro civique fiable mais que l'annonce est géocodée.
11+#
12+# Tout indicateur est calculé UNIQUEMENT sur ce que Lou-Ka observe réellement
13+# (annonces collectées par les crawls). Un indicateur sans échantillon
14+# suffisant est omis ou marqué « donnees_insuffisantes » — jamais inventé.
15+# -----------------------------------------------------------------------------
16+from __future__ import annotations
17+
18+import json
19+import statistics
20+import time
21+
22+from . import db
23+from .dedup import CLASSIFIEDS, PORTALS, _parse_address
24+
25+VERSION = "immeuble-1.0"
26+
27+# seuils du score de rotation (annonces des 12 derniers mois / unités estimées)
28+ROTATION = [(0.5, "faible"), (1.0, "normale"), (2.0, "élevée")]
29+ROTATION_MAX = "très élevée"
30+MIN_OBS_DAYS = 180 # fenêtre d'observation minimale pour classer la rotation
31+MIN_ADS_ROTATION = 3 # nb d'annonces minimal pour classer la rotation
32+MIN_ADS_PRESSURE = 3 # nb minimal par fenêtre pour la pression sur les loyers
33+
34+
35+def building_key(address: str | None, city: str | None,
36+ coord_key: str | None) -> tuple[str | None, str | None]:
37+ """(clé d'immeuble, numéro d'unité) — adresse d'abord, géo en repli."""
38+ akey, unit = _parse_address(address or "", city or "")
39+ if akey:
40+ return f"adr:{akey}", unit
41+ if coord_key:
42+ return f"geo:{coord_key}", unit
43+ return None, unit
44+
45+
46+def _median(vals: list[float]) -> float | None:
47+ vals = [v for v in vals if v]
48+ return round(statistics.median(vals)) if vals else None
49+
50+
51+def _stats_for(members: list[dict], now: float) -> dict:
52+ """Statistiques d'un immeuble à partir de ses annonces (obs. Lou-Ka)."""
53+ d30, d90, d365, d730 = (now - 30 * 86400, now - 90 * 86400,
54+ now - 365 * 86400, now - 730 * 86400)
55+ actives = [m for m in members if m["active"]]
56+ first_obs = min(m["first_seen"] for m in members if m["first_seen"])
57+ last_obs = max(m["last_seen"] for m in members if m["last_seen"])
58+ obs_days = max(1, round((now - first_obs) / 86400))
59+
60+ units = {m["unit"] for m in members if m["unit"]}
61+ # unités estimées : numéros d'app distincts observés, plancher = annonces
62+ # actives en ce moment (elles existent forcément toutes en même temps)
63+ unites_estimees = max(len(units), len(actives), 1)
64+
65+ n_12m = sum(1 for m in members if (m["first_seen"] or 0) >= d365)
66+
67+ # loyers médians (annonces actives)
68+ med = _median([m["price"] for m in actives])
69+ par_cc: dict[str, int] = {}
70+ by_cc: dict[str, list[float]] = {}
71+ for m in actives:
72+ if m["price"] and m["bedrooms"] is not None:
73+ by_cc.setdefault(str(int(m["bedrooms"])), []).append(m["price"])
74+ for cc, vals in sorted(by_cc.items()):
75+ v = _median(vals)
76+ if v and len(vals) >= 2:
77+ par_cc[cc] = v
78+ vals_pi2 = [m["price"] / m["area"] for m in actives
79+ if m["price"] and m["area"] and m["area"] > 100]
80+ pi2 = round(statistics.median(vals_pi2), 2) if vals_pi2 else None
81+
82+ # pression sur les loyers : médiane des PREMIERS prix observés des annonces
83+ # apparues dans les 12 derniers mois vs celles des 12 mois précédents
84+ win_new = [m["first_price"] for m in members
85+ if m["first_price"] and (m["first_seen"] or 0) >= d365]
86+ win_old = [m["first_price"] for m in members
87+ if m["first_price"] and d730 <= (m["first_seen"] or 0) < d365]
88+ pression = None
89+ if len(win_new) >= MIN_ADS_PRESSURE and len(win_old) >= MIN_ADS_PRESSURE:
90+ m_new, m_old = statistics.median(win_new), statistics.median(win_old)
91+ if m_old > 0:
92+ pression = {
93+ "variation_12m": round((m_new - m_old) / m_old, 4),
94+ "mediane_12m": round(m_new), "mediane_12_24m": round(m_old),
95+ "n_12m": len(win_new), "n_12_24m": len(win_old),
96+ "statut": "calculated",
97+ }
98+
99+ # score de rotation : annonces des 12 derniers mois / unités estimées
100+ rotation: dict = {"statut": "donnees_insuffisantes"}
101+ if obs_days >= MIN_OBS_DAYS and len(members) >= MIN_ADS_ROTATION:
102+ ratio = n_12m / unites_estimees
103+ classe = ROTATION_MAX
104+ for seuil, nom in ROTATION:
105+ if ratio < seuil:
106+ classe = nom
107+ break
108+ rotation = {
109+ "classe": classe, "ratio": round(ratio, 2),
110+ "annonces_12m": n_12m, "unites_estimees": unites_estimees,
111+ "observation_jours": obs_days, "statut": "calculated",
112+ "methode": ("annonces observées les 12 derniers mois ÷ unités "
113+ "estimées (numéros d'app distincts, plancher = annonces "
114+ "actives simultanées). Mesure uniquement ce que Lou-Ka "
115+ "observe — pas un taux de roulement démographique."),
116+ }
117+
118+ sources: dict[str, int] = {}
119+ for m in members:
120+ sources[m["source"]] = sources.get(m["source"], 0) + 1
121+ gestionnaires = sorted(s for s in sources
122+ if s not in PORTALS and s not in CLASSIFIEDS)
123+
124+ return {
125+ "annonces_total": len(members),
126+ "annonces_actives": len(actives),
127+ "annonces_30j": sum(1 for m in members if (m["first_seen"] or 0) >= d30),
128+ "annonces_90j": sum(1 for m in members if (m["first_seen"] or 0) >= d90),
129+ "annonces_12m": n_12m,
130+ "unites_identifiees": len(units),
131+ "unites_estimees": unites_estimees,
132+ "loyer_median": med,
133+ "loyer_median_par_cc": par_cc or None,
134+ "pi2_median": pi2,
135+ "pression_loyers": pression,
136+ "rotation": rotation,
137+ "gestionnaires": gestionnaires,
138+ "sources": sources,
139+ "premiere_observation": first_obs,
140+ "derniere_observation": last_obs,
141+ "unites_actives": sorted(
142+ [{"uid": m["uid"], "unit_type": m["unit_type"],
143+ "price": m["price"], "bedrooms": m["bedrooms"]}
144+ for m in actives if not m["dup_of"]],
145+ key=lambda x: (x["price"] is None, x["price"] or 0))[:24],
146+ }
147+
148+
149+def rollup(con=None) -> dict:
150+ """Recalcule building_key de toutes les annonces + la table buildings.
151+
152+ Idempotent, recalcul complet (même philosophie que dedup.run) : les
153+ annonces INACTIVES participent — l'historique de l'immeuble est justement
154+ ce qui fait la valeur du passeport.
155+ """
156+ own = con is None
157+ if own:
158+ con = db.connect()
159+ now = time.time()
160+ rows = con.execute(
161+ "SELECT uid, source, address, city, coord_key, unit_type, bedrooms,"
162+ " price, area_sqft, first_seen, last_seen, active, dup_of, building_key"
163+ " FROM listings").fetchall()
164+
165+ # premier prix observé par annonce (pression sur les loyers)
166+ firstp = {r["uid"]: r["p"] for r in con.execute(
167+ "SELECT uid, (SELECT price FROM price_log p2 WHERE p2.uid=p1.uid"
168+ " AND p2.price IS NOT NULL ORDER BY ts LIMIT 1) p"
169+ " FROM (SELECT DISTINCT uid FROM price_log) p1")}
170+
171+ groups: dict[str, list[dict]] = {}
172+ key_updates: list[tuple[str | None, str]] = []
173+ for r in rows:
174+ bkey, unit = building_key(r["address"], r["city"], r["coord_key"])
175+ if bkey != r["building_key"]:
176+ key_updates.append((bkey, r["uid"]))
177+ if not bkey:
178+ continue
179+ groups.setdefault(bkey, []).append({
180+ "uid": r["uid"], "source": r["source"], "unit": unit,
181+ "unit_type": r["unit_type"], "bedrooms": r["bedrooms"],
182+ "price": r["price"], "area": r["area_sqft"],
183+ "first_seen": r["first_seen"], "last_seen": r["last_seen"],
184+ "active": r["active"], "dup_of": r["dup_of"],
185+ "first_price": firstp.get(r["uid"]) or r["price"],
186+ "address": r["address"], "city": r["city"],
187+ })
188+ if key_updates:
189+ con.executemany("UPDATE listings SET building_key=? WHERE uid=?",
190+ key_updates)
191+
192+ upserts = []
193+ for bkey, members in groups.items():
194+ # doublons inter-sources : un même logement publié sur 3 plateformes ne
195+ # compte qu'une fois dans les statistiques de l'immeuble
196+ uniq = [m for m in members if not m["dup_of"]]
197+ if not uniq:
198+ continue
199+ stats = _stats_for(uniq, now)
200+ # adresse d'affichage : la plus fréquente parmi les membres actifs
201+ addrs = [m["address"] for m in uniq if m["address"]]
202+ addr = max(set(addrs), key=addrs.count) if addrs else None
203+ cities = [m["city"] for m in uniq if m["city"]]
204+ city = max(set(cities), key=cities.count) if cities else None
205+ # position : coord de n'importe quel membre géocodé (via coord_key)
206+ upserts.append((bkey, addr, city,
207+ json.dumps(stats, ensure_ascii=False), VERSION, now))
208+ con.executemany(
209+ "INSERT INTO buildings (bkey, address, city, stats, version, computed_at)"
210+ " VALUES (?,?,?,?,?,?)"
211+ " ON CONFLICT(bkey) DO UPDATE SET address=excluded.address,"
212+ " city=excluded.city, stats=excluded.stats, version=excluded.version,"
213+ " computed_at=excluded.computed_at", upserts)
214+ con.commit()
215+ out = {"buildings": len(upserts), "listings_scanned": len(rows),
216+ "keys_updated": len(key_updates)}
217+ if own:
218+ con.close()
219+ return out
220+
221+
222+def fiche(bkey: str, con=None) -> dict | None:
223+ """Passeport d'un immeuble pour l'API (stats précalculées par rollup)."""
224+ own = con is None
225+ if own:
226+ con = db.connect()
227+ row = con.execute(
228+ "SELECT bkey, address, city, stats, version, computed_at FROM buildings"
229+ " WHERE bkey=?", (bkey,)).fetchone()
230+ if own and row is None:
231+ con.close()
232+ return None
233+ if row is None:
234+ return None
235+ out = {"bkey": row["bkey"], "address": row["address"], "city": row["city"],
236+ "version": row["version"], "computed_at": row["computed_at"],
237+ **json.loads(row["stats"] or "{}")}
238+ if own:
239+ con.close()
240+ return out
241+
242+
243+if __name__ == "__main__":
244+ print(rollup())
added louka/coutreel.py +196 −0
@@ -0,0 +1,196 @@
1+# -----------------------------------------------------------------------------
2+# Lou-Ka — Agrégateur de logements à louer (province de Québec)
3+# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
4+# coutreel.py : « Coût réel mensuel Lou-Ka » — loyer + frais non inclus,
5+# ligne par ligne, chaque poste étiqueté :
6+# included : explicitement inclus dans le bail (details.inclusions)
7+# observed : montant affiché par la source (ex. prix du stationnement)
8+# estimated : estimation sourcée (Hydro-Québec via hydro.py, médiane Internet)
9+# unknown : ni inclus ni estimable — dit explicitement, jamais inventé
10+#
11+# + prix au pi² avec percentile RÉELLEMENT calculé sur les comparables
12+# (même ville + même nombre de chambres, et rayon ~2 km si géocodée).
13+# -----------------------------------------------------------------------------
14+from __future__ import annotations
15+
16+import json
17+
18+from . import db
19+
20+# Internet résidentiel : médiane des forfaits de base au Québec (ordre de
21+# grandeur public CRTC/fournisseurs). Estimation générique, PAS une donnée
22+# de l'adresse — toujours présentée comme telle.
23+INTERNET_ESTIME = 60.0
24+
25+VERSION = "cout-reel-1.0"
26+
27+
28+def _pctile(values: list[float], x: float) -> int:
29+ below = sum(1 for v in values if v < x)
30+ return round(100 * below / len(values))
31+
32+
33+def pi2(con, l: dict) -> dict | None:
34+ """Prix au pi² + percentile parmi les comparables (calcul réel)."""
35+ price, area = l.get("price"), l.get("area_sqft")
36+ if not price or not area or area < 100:
37+ return None
38+ val = price / area
39+ out: dict = {"valeur": round(val, 2), "statut": "calculated"}
40+
41+ base = ("SELECT price, area_sqft, lat, lng FROM listings"
42+ " WHERE active=1 AND published=1 AND dup_of IS NULL"
43+ " AND price IS NOT NULL AND area_sqft > 100 AND uid<>?")
44+ args: list = [l["uid"]]
45+ if l.get("city"):
46+ base += " AND city=?"; args.append(l["city"])
47+ if l.get("bedrooms") is not None:
48+ base += " AND bedrooms=?"; args.append(l["bedrooms"])
49+ elif l.get("unit_type"):
50+ base += " AND unit_type=?"; args.append(l["unit_type"])
51+ rows = con.execute(base, args).fetchall()
52+ ville = [r["price"] / r["area_sqft"] for r in rows]
53+ if len(ville) >= 12:
54+ out["percentile_ville"] = _pctile(ville, val)
55+ out["n_ville"] = len(ville)
56+ out["portee_ville"] = (f"{l.get('city')}, "
57+ + (f"{int(l['bedrooms'])} ch."
58+ if l.get("bedrooms") is not None
59+ else l.get("unit_type") or "tous types"))
60+ lat, lng = l.get("lat"), l.get("lng")
61+ if lat is not None and lng is not None:
62+ secteur = [r["price"] / r["area_sqft"] for r in rows
63+ if r["lat"] is not None
64+ and abs(r["lat"] - lat) < 0.018 and abs(r["lng"] - lng) < 0.025]
65+ if len(secteur) >= 12:
66+ out["percentile_secteur"] = _pctile(secteur, val)
67+ out["n_secteur"] = len(secteur)
68+ if "percentile_ville" not in out and "percentile_secteur" not in out:
69+ out["percentile_note"] = "comparables insuffisants pour un percentile"
70+ return out
71+
72+
73+def compute(uid: str, con=None) -> dict | None:
74+ """Coût réel mensuel d'une annonce — module « Coût réel Lou-Ka »."""
75+ own = con is None
76+ if own:
77+ con = db.connect()
78+ try:
79+ row = con.execute(
80+ "SELECT uid, price, area_sqft, city, bedrooms, unit_type, lat, lng,"
81+ " details, address FROM listings WHERE uid=?", (uid,)).fetchone()
82+ if row is None:
83+ return None
84+ l = dict(row)
85+ try:
86+ details = json.loads(l.get("details") or "{}") or {}
87+ except (ValueError, TypeError):
88+ details = {}
89+ inc = details.get("inclusions") or {}
90+ price = l.get("price")
91+
92+ lignes: list[dict] = []
93+ lignes.append({"poste": "Loyer affiché", "statut": "observed",
94+ "montant": price, "source": "annonce"})
95+
96+ # --- électricité + chauffage (souvent le même compteur au Québec) ----
97+ hydro_est = None
98+ if not inc.get("electricity"):
99+ from . import hydro
100+ h = hydro.estimate(uid=uid, adresse=l.get("address"),
101+ lat=l.get("lat"), lng=l.get("lng"), solve=False)
102+ if h.get("disponible") and h.get("cout_mensuel"):
103+ hydro_est = round(float(h["cout_mensuel"]))
104+ if inc.get("electricity"):
105+ lignes.append({"poste": "Électricité", "statut": "included",
106+ "montant": 0, "source": "annonce (incluse)"})
107+ elif inc.get("electricity") is False:
108+ lignes.append({"poste": "Électricité",
109+ "statut": "estimated" if hydro_est else "unknown",
110+ "montant": hydro_est,
111+ "source": ("outil public d'estimation Hydro-Québec"
112+ if hydro_est else None),
113+ "note": None if hydro_est else
114+ "explicitement à la charge du locataire, "
115+ "montant non estimable pour cette adresse"})
116+ else:
117+ lignes.append({"poste": "Électricité",
118+ "statut": "estimated" if hydro_est else "unknown",
119+ "montant": hydro_est,
120+ "source": ("outil public d'estimation Hydro-Québec"
121+ if hydro_est else None),
122+ "note": None if hydro_est else
123+ "l'annonce ne précise pas si elle est incluse"})
124+
125+ if inc.get("heating"):
126+ lignes.append({"poste": "Chauffage", "statut": "included",
127+ "montant": 0, "source": "annonce (inclus)"})
128+ elif hydro_est:
129+ lignes.append({"poste": "Chauffage", "statut": "estimated",
130+ "montant": None,
131+ "note": "chauffage électrique répandu au Québec — "
132+ "déjà compté dans l'estimation d'électricité"})
133+ else:
134+ lignes.append({"poste": "Chauffage", "statut": "unknown",
135+ "montant": None,
136+ "note": "ni inclus ni estimable avec les données "
137+ "disponibles"})
138+
139+ if inc.get("hot_water"):
140+ lignes.append({"poste": "Eau chaude", "statut": "included",
141+ "montant": 0, "source": "annonce (incluse)"})
142+ else:
143+ lignes.append({"poste": "Eau chaude", "statut": "unknown",
144+ "montant": None,
145+ "note": "l'annonce ne précise pas si elle est incluse"})
146+
147+ if inc.get("internet"):
148+ lignes.append({"poste": "Internet", "statut": "included",
149+ "montant": 0, "source": "annonce (inclus)"})
150+ else:
151+ lignes.append({"poste": "Internet", "statut": "estimated",
152+ "montant": INTERNET_ESTIME,
153+ "source": "médiane des forfaits résidentiels de base "
154+ "au Québec (estimation générique)"})
155+
156+ parking = details.get("parking") or {}
157+ if parking.get("available"):
158+ if parking.get("included"):
159+ lignes.append({"poste": "Stationnement", "statut": "included",
160+ "montant": 0, "source": "annonce (inclus)"})
161+ elif parking.get("price"):
162+ lignes.append({"poste": "Stationnement", "statut": "observed",
163+ "montant": float(parking["price"]),
164+ "source": "annonce (prix affiché)"})
165+ else:
166+ lignes.append({"poste": "Stationnement", "statut": "unknown",
167+ "montant": None,
168+ "note": "disponible, prix non précisé par la source"})
169+
170+ # --- totaux : jamais de montant inventé ------------------------------
171+ total = None
172+ if price:
173+ total = float(price) + sum(
174+ (li["montant"] or 0) for li in lignes[1:]
175+ if li["statut"] in ("observed", "estimated")
176+ and li["montant"] is not None)
177+ inconnues = [li["poste"] for li in lignes if li["statut"] == "unknown"]
178+
179+ out = {
180+ "uid": uid, "version": VERSION,
181+ "loyer": price,
182+ "lignes": lignes,
183+ "total_estime": round(total) if total else None,
184+ "annuel_estime": round(total * 12) if total else None,
185+ "postes_inconnus": inconnues,
186+ "pi2": pi2(con, l),
187+ "methode": ("loyer affiché + frais non inclus connus (observés) ou "
188+ "estimés avec source ; les postes inconnus sont listés "
189+ "tels quels, jamais chiffrés arbitrairement"),
190+ }
191+ if total and l.get("area_sqft") and l["area_sqft"] > 100:
192+ out["total_pi2"] = round(total / l["area_sqft"], 2)
193+ return out
194+ finally:
195+ if own:
196+ con.close()
modified louka/db.py +140 −1
@@ -174,6 +174,80 @@ CREATE TABLE IF NOT EXISTS source_profiles (
174 logo_url TEXT, 174 logo_url TEXT,
175 updated_at REAL 175 updated_at REAL
176 ); 176 );
177+
178+CREATE TABLE IF NOT EXISTS listing_events (
179+ id INTEGER PRIMARY KEY AUTOINCREMENT,
180+ uid TEXT NOT NULL, -- listings.uid
181+ ts REAL NOT NULL,
182+ event TEXT NOT NULL, -- description | superficie | dispo |
183+ -- inclusions | photos | disparition |
184+ -- reapparition (prix -> price_log)
185+ old TEXT, -- valeur avant (JSON compact)
186+ new TEXT -- valeur après (JSON compact)
187+);
188+CREATE INDEX IF NOT EXISTS idx_levents_uid ON listing_events(uid, ts);
189+
190+CREATE TABLE IF NOT EXISTS buildings (
191+ bkey TEXT PRIMARY KEY, -- clé d'adresse « civique|rue|ville »
192+ -- (dedup._parse_address) ou « geo:lat,lng »
193+ address TEXT, -- adresse d'affichage (la plus fréquente)
194+ city TEXT,
195+ lat REAL, lng REAL,
196+ stats TEXT, -- JSON : unités, annonces, loyers,
197+ -- rotation, pression, gestionnaires…
198+ version TEXT, -- version du calcul (building.py)
199+ computed_at REAL
200+);
201+
202+CREATE TABLE IF NOT EXISTS recycled_cache (
203+ uid TEXT PRIMARY KEY, -- listings.uid
204+ matches TEXT, -- JSON : correspondances probables + signaux
205+ computed_at REAL
206+);
207+
208+CREATE TABLE IF NOT EXISTS managers (
209+ id INTEGER PRIMARY KEY AUTOINCREMENT,
210+ source_id TEXT UNIQUE, -- id de la source (data/sources.json)
211+ name TEXT, -- nom canonique
212+ aliases TEXT, -- JSON : variantes de nom
213+ website TEXT,
214+ phone TEXT,
215+ -- résolution Google Maps (SerpApi) — voir louka/managers.py
216+ gmaps_place_id TEXT,
217+ gmaps_data_id TEXT,
218+ gmaps_name TEXT,
219+ gmaps_address TEXT,
220+ gmaps_rating REAL,
221+ gmaps_reviews INTEGER, -- nombre d'avis annoncé par Google
222+ match_confidence REAL, -- 0..1 (association refusée si trop faible)
223+ match_signals TEXT, -- JSON : signaux ayant fondé l'association
224+ resolved_at REAL,
225+ resolve_failed INTEGER DEFAULT 0,
226+ review_stats TEXT, -- JSON : distribution, tendance, thèmes
227+ last_synced_at REAL
228+);
229+
230+CREATE TABLE IF NOT EXISTS manager_reviews (
231+ id INTEGER PRIMARY KEY AUTOINCREMENT,
232+ manager_id INTEGER NOT NULL,
233+ external_review_id TEXT, -- id Google (déduplication)
234+ source TEXT DEFAULT 'google_maps',
235+ rating REAL,
236+ text TEXT,
237+ published_at TEXT, -- ISO si dérivable
238+ relative_date_raw TEXT, -- « il y a 3 mois » (texte source)
239+ author_name TEXT,
240+ author_review_count INTEGER,
241+ owner_response TEXT,
242+ owner_response_date TEXT,
243+ source_url TEXT,
244+ fetched_at REAL,
245+ content_hash TEXT, -- déduplication de secours
246+ analysis TEXT -- JSON : topics/sentiment/severity (managers.py)
247+);
248+CREATE INDEX IF NOT EXISTS idx_mreviews_mgr ON manager_reviews(manager_id);
249+CREATE UNIQUE INDEX IF NOT EXISTS idx_mreviews_ext
250+ ON manager_reviews(manager_id, external_review_id);
177 """ 251 """
178 252
179 # Colonnes ajoutées après la v1 — migration automatique des bases existantes. 253 # Colonnes ajoutées après la v1 — migration automatique des bases existantes.
@@ -198,6 +272,7 @@ _MIGRATIONS = {
198 "images_ok": "TEXT", # JSON : galerie nettoyée (imgcheck.py) 272 "images_ok": "TEXT", # JSON : galerie nettoyée (imgcheck.py)
199 "img_audit": "TEXT", # JSON : traçabilité du contrôle images 273 "img_audit": "TEXT", # JSON : traçabilité du contrôle images
200 "coord_key": "TEXT", # clé immeuble « lat,lng » 4 déc. (jointure kascores) 274 "coord_key": "TEXT", # clé immeuble « lat,lng » 4 déc. (jointure kascores)
275+ "building_key": "TEXT", # clé immeuble par adresse (buildings.bkey)
201 }, 276 },
202 "sync_log": { 277 "sync_log": {
203 "stats": "TEXT", 278 "stats": "TEXT",
@@ -235,6 +310,8 @@ def connect() -> sqlite3.Connection:
235 " ON listings(active, published)") 310 " ON listings(active, published)")
236 con.execute("CREATE INDEX IF NOT EXISTS idx_listings_coord" 311 con.execute("CREATE INDEX IF NOT EXISTS idx_listings_coord"
237 " ON listings(coord_key)") 312 " ON listings(coord_key)")
313+ con.execute("CREATE INDEX IF NOT EXISTS idx_listings_bkey"
314+ " ON listings(building_key)")
238 con.commit() 315 con.commit()
239 # WAL : lectures (web) et écritures (sync, imgcheck) concurrentes sans 316 # WAL : lectures (web) et écritures (sync, imgcheck) concurrentes sans
240 # verrou global ; busy_timeout évite les « database is locked » ponctuels. 317 # verrou global ; busy_timeout évite les « database is locked » ponctuels.
@@ -276,6 +353,54 @@ def _drift_alert(con: sqlite3.Connection, source: str, found: int,
276 return None 353 return None
277 354
278 355
356+def _j(v) -> str:
357+ return json.dumps(v, ensure_ascii=False)
358+
359+
360+def _log_events(con: sqlite3.Connection, uid: str, ts: float,
361+ old: sqlite3.Row, new: dict) -> None:
362+ """Journalise les changements observés d'une annonce (listing_events).
363+
364+ Le prix a déjà son historique dédié (price_log) — ici : description,
365+ superficie, disponibilité, inclusions, photos. Valeurs compactes (pas de
366+ texte intégral) : l'objectif est la timeline de la fiche, pas l'archive.
367+ """
368+ ev: list[tuple[str, str | None, str | None]] = []
369+ old_desc = (old["description"] or "").strip()
370+ new_desc = (new.get("description") or "").strip()
371+ if old_desc != new_desc and (old_desc or new_desc):
372+ ev.append(("description", _j({"caracteres": len(old_desc)}),
373+ _j({"caracteres": len(new_desc)})))
374+ if (old["area_sqft"] or None) != (new.get("area_sqft") or None):
375+ ev.append(("superficie", _j(old["area_sqft"]), _j(new.get("area_sqft"))))
376+ if (old["availability_date"] or None) != (new.get("availability_date") or None):
377+ ev.append(("dispo", _j(old["availability_date"]),
378+ _j(new.get("availability_date"))))
379+ try:
380+ inc_old = (json.loads(old["details"] or "{}") or {}).get("inclusions") or {}
381+ inc_new = (json.loads(new.get("details") or "{}") or {}).get("inclusions") or {}
382+ except (ValueError, TypeError):
383+ inc_old = inc_new = {}
384+ if inc_old != inc_new:
385+ keys = set(inc_old) | set(inc_new)
386+ diff_old = {k: inc_old.get(k) for k in keys if inc_old.get(k) != inc_new.get(k)}
387+ diff_new = {k: inc_new.get(k) for k in keys if inc_old.get(k) != inc_new.get(k)}
388+ if diff_old or diff_new:
389+ ev.append(("inclusions", _j(diff_old), _j(diff_new)))
390+ try:
391+ imgs_old = json.loads(old["images"] or "[]")
392+ imgs_new = json.loads(new.get("images") or "[]")
393+ except (ValueError, TypeError):
394+ imgs_old = imgs_new = []
395+ if set(imgs_old) != set(imgs_new):
396+ ev.append(("photos", _j({"n": len(imgs_old)}), _j({"n": len(imgs_new)})))
397+ if ev:
398+ con.executemany(
399+ "INSERT INTO listing_events (uid, ts, event, old, new)"
400+ " VALUES (?,?,?,?,?)",
401+ [(uid, ts, e, o, n) for e, o, n in ev])
402+
403+
279 def sync_source(con: sqlite3.Connection, source: str, 404 def sync_source(con: sqlite3.Connection, source: str,
280 listings: list[Listing]) -> dict: 405 listings: list[Listing]) -> dict:
281 """Synchronise les annonces d'une source. 406 """Synchronise les annonces d'une source.
@@ -301,7 +426,8 @@ def sync_source(con: sqlite3.Connection, source: str,
301 seen_uids.add(lst.uid) 426 seen_uids.add(lst.uid)
302 h = lst.content_hash() 427 h = lst.content_hash()
303 row = con.execute( 428 row = con.execute(
304− "SELECT content_hash, price, images FROM listings WHERE uid=?", 429+ "SELECT content_hash, price, images, description, area_sqft,"
430+ " availability_date, details, active FROM listings WHERE uid=?",
305 (lst.uid,)).fetchone() 431 (lst.uid,)).fetchone()
306 # score de complétude + seuil de publication (quarantaine sous le seuil) 432 # score de complétude + seuil de publication (quarantaine sous le seuil)
307 from .quality import evaluate 433 from .quality import evaluate
@@ -382,8 +508,18 @@ def sync_source(con: sqlite3.Connection, source: str,
382 if lst.price != row["price"]: # changement de prix -> historique 508 if lst.price != row["price"]: # changement de prix -> historique
383 con.execute("INSERT INTO price_log (uid, ts, price) VALUES (?,?,?)", 509 con.execute("INSERT INTO price_log (uid, ts, price) VALUES (?,?,?)",
384 (lst.uid, now, lst.price)) 510 (lst.uid, now, lst.price))
511+ # timeline de la fiche : changements observés (hors prix)
512+ _log_events(con, lst.uid, now, row, params)
513+ if not row["active"]:
514+ con.execute("INSERT INTO listing_events (uid, ts, event, old, new)"
515+ " VALUES (?,?,?,?,?)",
516+ (lst.uid, now, "reapparition", None, None))
385 updated += 1 517 updated += 1
386 else: 518 else:
519+ if not row["active"]:
520+ con.execute("INSERT INTO listing_events (uid, ts, event, old, new)"
521+ " VALUES (?,?,?,?,?)",
522+ (lst.uid, now, "reapparition", None, None))
387 con.execute( 523 con.execute(
388 "UPDATE listings SET last_seen=?, miss_count=0, active=1 WHERE uid=?", 524 "UPDATE listings SET last_seen=?, miss_count=0, active=1 WHERE uid=?",
389 (now, lst.uid)) 525 (now, lst.uid))
@@ -402,6 +538,9 @@ def sync_source(con: sqlite3.Connection, source: str,
402 con.execute( 538 con.execute(
403 "UPDATE listings SET active=0, miss_count=?, updated_at=?" 539 "UPDATE listings SET active=0, miss_count=?, updated_at=?"
404 " WHERE uid=?", (r["miss_count"] + 1, now, r["uid"])) 540 " WHERE uid=?", (r["miss_count"] + 1, now, r["uid"]))
541+ con.execute("INSERT INTO listing_events (uid, ts, event, old, new)"
542+ " VALUES (?,?,?,?,?)",
543+ (r["uid"], now, "disparition", None, None))
405 removed += 1 544 removed += 1
406 else: 545 else:
407 con.execute("UPDATE listings SET miss_count=miss_count+1 WHERE uid=?", 546 con.execute("UPDATE listings SET miss_count=miss_count+1 WHERE uid=?",
added louka/history.py +95 −0
@@ -0,0 +1,95 @@
1+# -----------------------------------------------------------------------------
2+# Lou-Ka — Agrégateur de logements à louer (province de Québec)
3+# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
4+# history.py : « Historique Lou-Ka » d'une annonce — timeline construite à
5+# partir des observations réelles des crawls :
6+# - price_log (depuis la v1) : chaque changement de prix observé ;
7+# - listing_events (db.py/_log_events) : description, superficie, dispo,
8+# inclusions, photos, disparition/réapparition.
9+# Chaque élément est étiqueté `observed` : rien n'est estimé ni inventé.
10+# -----------------------------------------------------------------------------
11+from __future__ import annotations
12+
13+import json
14+import time
15+
16+from . import db
17+
18+
19+def timeline(uid: str, con=None) -> dict | None:
20+ """Historique complet d'une annonce (fiche + API).
21+
22+ Retourne : première/dernière observation, jours en ligne, prix initial et
23+ courant, variation, nombre de modifications et la timeline fusionnée
24+ (prix + événements), du plus récent au plus ancien.
25+ """
26+ own = con is None
27+ if own:
28+ con = db.connect()
29+ row = con.execute(
30+ "SELECT uid, price, first_seen, last_seen, active FROM listings"
31+ " WHERE uid=?", (uid,)).fetchone()
32+ if row is None:
33+ if own:
34+ con.close()
35+ return None
36+
37+ prices = con.execute(
38+ "SELECT ts, price FROM price_log WHERE uid=? ORDER BY ts", (uid,)).fetchall()
39+ events = con.execute(
40+ "SELECT ts, event, old, new FROM listing_events WHERE uid=? ORDER BY ts",
41+ (uid,)).fetchall()
42+
43+ items: list[dict] = []
44+ prev_price = None
45+ for p in prices:
46+ it = {"ts": p["ts"], "type": "prix", "prix": p["price"],
47+ "statut": "observed"}
48+ if prev_price is not None and p["price"] is not None:
49+ it["prix_avant"] = prev_price
50+ items.append(it)
51+ if p["price"] is not None:
52+ prev_price = p["price"]
53+ for e in events:
54+ def _load(v):
55+ try:
56+ return json.loads(v) if v is not None else None
57+ except (ValueError, TypeError):
58+ return None
59+ items.append({"ts": e["ts"], "type": e["event"],
60+ "avant": _load(e["old"]), "apres": _load(e["new"]),
61+ "statut": "observed"})
62+ items.sort(key=lambda x: x["ts"], reverse=True)
63+
64+ first_price = next((p["price"] for p in prices if p["price"] is not None), None)
65+ cur_price = row["price"]
66+ variation = None
67+ if first_price and cur_price and first_price > 0:
68+ variation = round((cur_price - first_price) / first_price, 4)
69+
70+ now = time.time()
71+ fin = row["last_seen"] if not row["active"] else now
72+ jours = max(0, round((fin - (row["first_seen"] or fin)) / 86400))
73+
74+ # modifications = éléments de timeline hors point de départ du prix
75+ n_modif = max(0, len([p for p in prices]) - 1) + len(
76+ [e for e in events if e["event"] not in ("disparition", "reapparition")])
77+
78+ out = {
79+ "uid": uid,
80+ "premiere_observation": row["first_seen"],
81+ "derniere_observation": row["last_seen"],
82+ "active": bool(row["active"]),
83+ "jours_en_ligne": jours,
84+ "prix_initial": first_price,
85+ "prix_actuel": cur_price,
86+ "variation": variation,
87+ "modifications": n_modif,
88+ "timeline": items[:60],
89+ "methode": ("Observations directes des synchronisations Lou-Ka "
90+ "(prix, description, superficie, disponibilité, inclusions, "
91+ "photos, retraits/retours). Aucune estimation."),
92+ }
93+ if own:
94+ con.close()
95+ return out
added louka/hiver.py +108 −0
@@ -0,0 +1,108 @@
1+# -----------------------------------------------------------------------------
2+# Lou-Ka — Agrégateur de logements à louer (province de Québec)
3+# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
4+# hiver.py : « Vie quotidienne en hiver » — indicateur 0-100 pensé pour le
5+# Québec : peut-on vivre son quotidien À PIED quand il fait -20 °C
6+# et que les trottoirs sont enneigés?
7+#
8+# Calcul déterministe et explicable à partir des données DÉJÀ collectées
9+# (poi_cache — plus proche commodité par catégorie, OpenStreetMap) :
10+# épicerie, pharmacie, arrêt de bus/métro, dépanneur. Chaque sous-score est
11+# une décroissance linéaire de la distance de marche (vol d'oiseau × 1,3).
12+# Pas de données de pente/exposition disponibles -> non pris en compte, et la
13+# méthodologie l'assume. Sans données POI : aucun score (jamais inventé).
14+# -----------------------------------------------------------------------------
15+from __future__ import annotations
16+
17+import json
18+
19+from . import db
20+
21+VERSION = "hiver-1.0"
22+DETOUR = 1.3 # vol d'oiseau -> distance de marche (même facteur que POI)
23+VITESSE_M_MIN = 80.0 # 4,8 km/h
24+
25+# (catégorie, libellé, poids, distance pleine note (m), distance note nulle (m))
26+CRITERES = [
27+ ("epicerie", "Épicerie", 0.35, 300, 1500),
28+ ("pharmacie", "Pharmacie", 0.20, 300, 1500),
29+ ("bus", "Arrêt de bus", 0.25, 150, 900),
30+ ("metro", "Métro", 0.05, 400, 1800),
31+ ("depanneur", "Dépanneur", 0.15, 200, 1000),
32+]
33+
34+CLASSES = [(80, "très pratique"), (60, "pratique"), (40, "exigeant")]
35+CLASSE_MIN = "difficile"
36+
37+
38+def _sous_score(dist_m: float, plein: float, nul: float) -> float:
39+ d = dist_m * DETOUR
40+ if d <= plein:
41+ return 100.0
42+ if d >= nul:
43+ return 0.0
44+ return 100.0 * (nul - d) / (nul - plein)
45+
46+
47+def score(lat: float | None, lng: float | None, con=None) -> dict | None:
48+ """Score hiver de l'immeuble (via poi_cache) — None si données absentes."""
49+ if lat is None or lng is None:
50+ return None
51+ own = con is None
52+ if own:
53+ con = db.connect()
54+ try:
55+ key = f"{round(lat, 4)},{round(lng, 4)}"
56+ row = con.execute("SELECT pois FROM poi_cache WHERE coord_key=?",
57+ (key,)).fetchone()
58+ if row is None:
59+ return None
60+ try:
61+ pois = {p["cat"]: p for p in json.loads(row["pois"] or "[]")}
62+ except (ValueError, TypeError):
63+ return None
64+ if not pois:
65+ return None
66+
67+ total = 0.0
68+ poids_total = 0.0
69+ detail = []
70+ for cat, label, poids, plein, nul in CRITERES:
71+ p = pois.get(cat)
72+ if p is None:
73+ # métro absent = normal hors Montréal : critère simplement omis
74+ if cat == "metro":
75+ continue
76+ s = 0.0
77+ item = {"critere": label, "score": 0, "distance_m": None,
78+ "note": "aucun repéré à distance de marche"}
79+ else:
80+ s = _sous_score(p["dist_m"], plein, nul)
81+ item = {"critere": label, "score": round(s),
82+ "distance_m": p["dist_m"],
83+ "minutes": max(1, round(p["dist_m"] * DETOUR
84+ / VITESSE_M_MIN)),
85+ "nom": p.get("name")}
86+ total += s * poids
87+ poids_total += poids
88+ detail.append(item)
89+ if poids_total == 0:
90+ return None
91+ note = round(total / poids_total)
92+ classe = CLASSE_MIN
93+ for seuil, nom in CLASSES:
94+ if note >= seuil:
95+ classe = nom
96+ break
97+ return {
98+ "score": note, "classe": classe, "detail": detail,
99+ "statut": "calculated", "version": VERSION,
100+ "methode": ("commodités essentielles à distance de marche "
101+ "(épicerie, pharmacie, bus/métro, dépanneur — "
102+ "OpenStreetMap), décroissance linéaire de la distance "
103+ "de marche estimée (vol d'oiseau × 1,3). Pente et "
104+ "exposition non disponibles : non prises en compte."),
105+ }
106+ finally:
107+ if own:
108+ con.close()
modified louka/ingest.py +14 −0
@@ -124,6 +124,12 @@ def watch(interval_seconds: int = 3600) -> None:
124 print(f"[lou-ka] dedup: {stats}") 124 print(f"[lou-ka] dedup: {stats}")
125 except Exception as exc: 125 except Exception as exc:
126 print(f"[lou-ka] dedup: erreur non bloquante: {exc}", file=sys.stderr) 126 print(f"[lou-ka] dedup: erreur non bloquante: {exc}", file=sys.stderr)
127+ try: # passeport des immeubles — après dedup (utilise dup_of + adresse)
128+ from . import building
129+ stats_b = building.rollup()
130+ print(f"[lou-ka] immeubles: {stats_b}")
131+ except Exception as exc:
132+ print(f"[lou-ka] immeubles: erreur non bloquante: {exc}", file=sys.stderr)
127 try: # juste valeur locative (fair value) — après dedup/geocode/quality 133 try: # juste valeur locative (fair value) — après dedup/geocode/quality
128 from . import fairvalue 134 from . import fairvalue
129 fairvalue.compute_all() 135 fairvalue.compute_all()
@@ -148,6 +154,14 @@ def watch(interval_seconds: int = 3600) -> None:
148 tal.precompute(limit=60, budget=600) 154 tal.precompute(limit=60, budget=600)
149 except Exception as exc: 155 except Exception as exc:
150 print(f"[lou-ka] tal: erreur non bloquante: {exc}", file=sys.stderr) 156 print(f"[lou-ka] tal: erreur non bloquante: {exc}", file=sys.stderr)
157+ try: # gestionnaires : résolution Google Maps + avis (SerpApi,
158+ # serveur seulement, budget borné ; sauté sans SERPAPI_API_KEY)
159+ from . import managers
160+ stats_m = managers.precompute(budget_s=120)
161+ print(f"[lou-ka] gestionnaires: {stats_m}")
162+ except Exception as exc:
163+ print(f"[lou-ka] gestionnaires: erreur non bloquante: {exc}",
164+ file=sys.stderr)
151 print(f"[lou-ka] prochaine synchronisation dans {interval_seconds}s") 165 print(f"[lou-ka] prochaine synchronisation dans {interval_seconds}s")
152 time.sleep(interval_seconds) 166 time.sleep(interval_seconds)
153 167
added louka/managers.py +529 −0
@@ -0,0 +1,529 @@
1+# -----------------------------------------------------------------------------
2+# Lou-Ka — Agrégateur de logements à louer (province de Québec)
3+# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
4+# managers.py : « Qui gère ce logement? » — fiche du gestionnaire/propriétaire.
5+#
6+# 1) Amorçage : les gestionnaires sont les sources DIRECTES de Lou-Ka
7+# (data/sources.json, hors portails et petites annonces) qui ont au moins
8+# une annonce collectée. Aucun gestionnaire n'est inventé.
9+# 2) Résolution Google Maps (SerpApi, serveur seulement) : la fiche Google
10+# n'est associée QUE si la confiance est suffisante (similarité de nom,
11+# domaine du site web, ville) — jamais « le premier résultat » aveuglément.
12+# Signaux et confiance stockés (match_confidence, match_signals).
13+# 3) Avis Google : synchronisation paginée (next_page_token, hl=fr, plus
14+# récents d'abord), dédupliqués par (manager_id, review_id Google) avec
15+# repli sur un hash de contenu. Les avis vivent dans NOTRE base : les pages
16+# ne déclenchent jamais d'appel SerpApi.
17+# 4) Analyse : statistiques réelles (distribution, récence, tendance — pas
18+# seulement la moyenne Google) + thèmes par lexique français, sentiment
19+# dérivé de la note, sévérité et confiance. Statut « inferred » assumé.
20+# -----------------------------------------------------------------------------
21+from __future__ import annotations
22+
23+import difflib
24+import hashlib
25+import json
26+import re
27+import time
28+import unicodedata
29+from datetime import datetime, timezone
30+
31+from . import db, serpapi_client
32+from .dedup import CLASSIFIEDS, PORTALS
33+
34+VERSION = "gestionnaire-1.0"
35+SEUIL_CONFIANCE = 0.60 # sous ce seuil : fiche Google NON associée
36+MAX_PAGES = 10 # garde-fou pagination avis (par synchronisation)
37+SYNC_TTL = 7 * 86400 # re-synchroniser les avis au plus aux 7 jours
38+RESOLVE_RETRY = 30 * 86400 # re-tenter une résolution échouée après 30 jours
39+
40+# suffixes/génériques retirés pour comparer les noms d'entreprises
41+_LEGAL = re.compile(
42+ r"\b(inc|ltee|ltée|ltd|senc|s\.e\.n\.c|enr|corp|corporation|groupe|group|"
43+ r"gestion|gestions|immobilier|immobiliere|immobilière|immeubles?|"
44+ r"appartements?|apartments?|properties|proprietes|propriétés|residences?|"
45+ r"résidences?|habitations?|siege|social|bureau|head|office|"
46+ r"les|le|la|de|du|des|et|and)\b")
47+
48+# types Google d'une fiche d'ENTREPRISE (vs fiche d'un immeuble précis)
49+_CORP_TYPES = re.compile(
50+ r"agence immobili|gestion immobili|property management|real estate|"
51+ r"societe de gestion|immobilier", re.I)
52+
53+
54+def _norm_name(s: str | None) -> str:
55+ s = unicodedata.normalize("NFKD", (s or "").lower())
56+ s = "".join(c for c in s if not unicodedata.combining(c))
57+ s = re.sub(r"[^a-z0-9 ]+", " ", s)
58+ s = _LEGAL.sub(" ", s)
59+ return " ".join(s.split())
60+
61+
62+def _domain(url: str | None) -> str | None:
63+ if not url:
64+ return None
65+ m = re.search(r"^(?:https?://)?(?:www\.)?([^/:?#]+)", url.strip().lower())
66+ return m.group(1) if m else None
67+
68+
69+# --- 1) amorçage depuis sources.json ----------------------------------------
70+
71+def seed(con=None) -> dict:
72+ """Crée/actualise un gestionnaire par source directe ayant des annonces."""
73+ own = con is None
74+ if own:
75+ con = db.connect()
76+ try:
77+ sources = json.loads((db.DB_PATH.parent / "sources.json")
78+ .read_text("utf-8"))["sources"]
79+ with_ads = {r["source"] for r in con.execute(
80+ "SELECT DISTINCT source FROM listings")}
81+ n = 0
82+ for s in sources:
83+ sid = s.get("id")
84+ if not sid or sid in PORTALS or sid in CLASSIFIEDS:
85+ continue
86+ if sid not in with_ads and s.get("connector") not in with_ads:
87+ continue
88+ con.execute(
89+ "INSERT INTO managers (source_id, name, website, aliases)"
90+ " VALUES (?,?,?,?)"
91+ " ON CONFLICT(source_id) DO UPDATE SET name=excluded.name,"
92+ " website=excluded.website",
93+ (sid, s.get("name"), s.get("url"),
94+ json.dumps({"region": s.get("region"),
95+ "connector": s.get("connector")},
96+ ensure_ascii=False)))
97+ n += 1
98+ con.commit()
99+ return {"managers": n}
100+ finally:
101+ if own:
102+ con.close()
103+
104+
105+# --- 2) résolution Google Maps (jamais « le premier résultat ») --------------
106+
107+def _score_candidate(mgr: dict, cand: dict) -> tuple[float, dict]:
108+ signals: dict = {}
109+ sim = difflib.SequenceMatcher(
110+ None, _norm_name(mgr["name"]), _norm_name(cand.get("title"))).ratio()
111+ signals["similarite_nom"] = round(sim, 3)
112+ conf = 0.6 * sim
113+ d_mgr, d_cand = _domain(mgr.get("website")), _domain(cand.get("website"))
114+ if d_mgr and d_cand and d_mgr == d_cand:
115+ conf += 0.3
116+ signals["meme_domaine_web"] = d_mgr
117+ region = None
118+ try:
119+ region = (json.loads(mgr.get("aliases") or "{}") or {}).get("region")
120+ except (ValueError, TypeError):
121+ pass
122+ addr = (cand.get("address") or "").lower()
123+ if region and _norm_name(region) and _norm_name(region) in _norm_name(addr):
124+ conf += 0.1
125+ signals["ville_correspondante"] = region
126+ if _CORP_TYPES.search(cand.get("type") or ""):
127+ signals["fiche_entreprise"] = cand.get("type")
128+ signals["type_fiche"] = cand.get("type")
129+ signals["nombre_avis"] = cand.get("reviews")
130+ return min(1.0, conf), signals
131+
132+
133+def resolve(manager_id: int, con=None) -> dict:
134+ """Associe (ou refuse d'associer) la fiche Google Maps du gestionnaire."""
135+ own = con is None
136+ if own:
137+ con = db.connect()
138+ try:
139+ mgr = con.execute("SELECT * FROM managers WHERE id=?",
140+ (manager_id,)).fetchone()
141+ if mgr is None:
142+ return {"ok": False, "raison": "gestionnaire inconnu"}
143+ mgr = dict(mgr)
144+ region = ""
145+ try:
146+ region = (json.loads(mgr.get("aliases") or "{}") or {}).get(
147+ "region") or ""
148+ except (ValueError, TypeError):
149+ pass
150+ q = f"{mgr['name']} {region} Québec".strip()
151+ data = serpapi_client.google_maps(q)
152+ cands = data.get("local_results") or []
153+ if not cands and data.get("place_results"):
154+ cands = [data["place_results"]]
155+
156+ scored = [( *_score_candidate(mgr, c), c) for c in cands[:8]]
157+ best, best_conf, best_sig = None, 0.0, {}
158+ if scored:
159+ top = max(s[0] for s in scored)
160+ # une chaîne a souvent une fiche Google PAR IMMEUBLE en plus du
161+ # siège : parmi les candidats plausibles (confiance proche du
162+ # meilleur ET au-dessus du seuil), préférer la fiche d'entreprise
163+ # puis la plus représentative (le plus d'avis)
164+ near = [s for s in scored
165+ if s[0] >= max(SEUIL_CONFIANCE, top - 0.15)]
166+ if near:
167+ best_conf, best_sig, best = max(
168+ near, key=lambda s: (bool(_CORP_TYPES.search(
169+ s[2].get("type") or "")),
170+ s[2].get("reviews") or 0, s[0]))
171+ else:
172+ best_conf, best_sig, best = max(scored, key=lambda s: s[0])
173+ now = time.time()
174+ if best is None or best_conf < SEUIL_CONFIANCE:
175+ con.execute(
176+ "UPDATE managers SET resolve_failed=1, resolved_at=?,"
177+ " match_confidence=?, match_signals=? WHERE id=?",
178+ (now, round(best_conf, 3) if best else None,
179+ json.dumps({"refus": "confiance insuffisante",
180+ "candidats": len(cands), **best_sig},
181+ ensure_ascii=False), manager_id))
182+ con.commit()
183+ return {"ok": False, "raison": "confiance insuffisante",
184+ "confiance": round(best_conf, 3)}
185+ con.execute(
186+ "UPDATE managers SET gmaps_place_id=?, gmaps_data_id=?,"
187+ " gmaps_name=?, gmaps_address=?, gmaps_rating=?, gmaps_reviews=?,"
188+ " match_confidence=?, match_signals=?, resolved_at=?,"
189+ " resolve_failed=0, phone=COALESCE(phone, ?) WHERE id=?",
190+ (best.get("place_id"), best.get("data_id"), best.get("title"),
191+ best.get("address"), best.get("rating"), best.get("reviews"),
192+ round(best_conf, 3), json.dumps(best_sig, ensure_ascii=False),
193+ now, best.get("phone"), manager_id))
194+ con.commit()
195+ return {"ok": True, "gmaps_name": best.get("title"),
196+ "confiance": round(best_conf, 3)}
197+ finally:
198+ if own:
199+ con.close()
200+
201+
202+# --- 3) synchronisation des avis (paginée, dédupliquée, en base) -------------
203+
204+def _content_hash(author: str | None, rating, text: str | None) -> str:
205+ raw = f"{author or ''}|{rating}|{(text or '')[:200]}"
206+ return hashlib.sha1(raw.encode("utf-8")).hexdigest()
207+
208+
209+def sync_reviews(manager_id: int, con=None, max_pages: int = MAX_PAGES) -> dict:
210+ """Rapatrie les avis Google (les plus récents d'abord) dans NOTRE base."""
211+ own = con is None
212+ if own:
213+ con = db.connect()
214+ try:
215+ mgr = con.execute("SELECT * FROM managers WHERE id=?",
216+ (manager_id,)).fetchone()
217+ if mgr is None or not mgr["gmaps_data_id"]:
218+ return {"ok": False, "raison": "fiche Google non associée"}
219+ known = {r["external_review_id"] for r in con.execute(
220+ "SELECT external_review_id FROM manager_reviews WHERE manager_id=?",
221+ (manager_id,))}
222+ known_hash = {r["content_hash"] for r in con.execute(
223+ "SELECT content_hash FROM manager_reviews WHERE manager_id=?"
224+ " AND external_review_id IS NULL", (manager_id,))}
225+
226+ token, new, pages = None, 0, 0
227+ while pages < max_pages:
228+ data = serpapi_client.google_maps_reviews(
229+ mgr["gmaps_data_id"], next_page_token=token)
230+ reviews = data.get("reviews") or []
231+ pages += 1
232+ page_new = 0
233+ now = time.time()
234+ for rv in reviews:
235+ ext = rv.get("review_id")
236+ text = rv.get("snippet") or rv.get("extracted_snippet",
237+ {}).get("original")
238+ user = rv.get("user") or {}
239+ chash = _content_hash(user.get("name"), rv.get("rating"), text)
240+ if (ext and ext in known) or (not ext and chash in known_hash):
241+ continue
242+ resp = rv.get("response") or {}
243+ analysis = _analyze_review(rv.get("rating"), text)
244+ con.execute(
245+ "INSERT INTO manager_reviews (manager_id,"
246+ " external_review_id, source, rating, text, published_at,"
247+ " relative_date_raw, author_name, author_review_count,"
248+ " owner_response, owner_response_date, source_url,"
249+ " fetched_at, content_hash, analysis) VALUES"
250+ " (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
251+ " ON CONFLICT(manager_id, external_review_id) DO NOTHING",
252+ (manager_id, ext, "google_maps", rv.get("rating"), text,
253+ rv.get("iso_date"), rv.get("date"), user.get("name"),
254+ user.get("reviews"), resp.get("snippet"),
255+ resp.get("iso_date") or resp.get("date"), rv.get("link"),
256+ now, chash, json.dumps(analysis, ensure_ascii=False)))
257+ if ext:
258+ known.add(ext)
259+ else:
260+ known_hash.add(chash)
261+ page_new += 1
262+ new += page_new
263+ token = (data.get("serpapi_pagination") or {}).get(
264+ "next_page_token")
265+ # tri « plus récents d'abord » : une page entièrement connue
266+ # signifie qu'on a rejoint l'historique déjà synchronisé
267+ if not token or (reviews and page_new == 0):
268+ break
269+ stats = _aggregate(con, manager_id)
270+ con.execute("UPDATE managers SET review_stats=?, last_synced_at=?"
271+ " WHERE id=?",
272+ (json.dumps(stats, ensure_ascii=False), time.time(),
273+ manager_id))
274+ con.commit()
275+ return {"ok": True, "nouveaux": new, "pages": pages,
276+ "total": stats.get("n")}
277+ finally:
278+ if own:
279+ con.close()
280+
281+
282+# --- 4) analyse : thèmes (lexique fr), sentiment, sévérité, agrégats ---------
283+
284+TOPICS = {
285+ "entretien": ["entretien", "entreten", "maintenance", "neglige", "négligé",
286+ "delabre", "délabré", "moisissure"],
287+ "réparations": ["reparation", "réparation", "reparer", "réparer", "bris",
288+ "brise", "brisé", "fuite", "degat", "dégât", "plomberie"],
289+ "communication": ["communication", "repond", "répond", "reponse",
290+ "réponse", "rappel", "courriel", "joindre",
291+ "injoignable", "appel", "message"],
292+ "personnel": ["personnel", "employe", "employé", "concierge", "equipe",
293+ "équipe", "gerant", "gérant", "agent", "proprietaire",
294+ "propriétaire"],
295+ "propreté": ["propre", "proprete", "propreté", "sale", "salete", "saleté",
296+ "malpropre"],
297+ "chauffage": ["chauffage", "chauffe", "chauffé", "froid", "temperature",
298+ "température"],
299+ "eau chaude": ["eau chaude"],
300+ "vermine": ["punaise", "coquerelle", "souris", "rat ", "rats", "insecte",
301+ "vermine", "fourmi", "extermina"],
302+ "bruit": ["bruit", "bruyant", "tapage", "insonoris"],
303+ "sécurité": ["securite", "sécurité", "securitaire", "sécuritaire", "vol",
304+ "intrusion", "serrure", "camera", "caméra"],
305+ "ascenseur": ["ascenseur"],
306+ "stationnement": ["stationnement", "parking", "deneigement",
307+ "déneigement"],
308+ "dépôt et remboursement": ["depot", "dépôt", "rembours", "caution"],
309+ "augmentation de loyer": ["augmentation", "hausse"],
310+ "bail et location": ["bail", "signature", "visite", "location"],
311+ "service client": ["service", "clientele", "clientèle", "professionnel",
312+ "courtois", "arrogant", "impoli", "respect"],
313+}
314+_GRAVES = {"vermine", "sécurité", "chauffage", "eau chaude"}
315+
316+
317+def _analyze_review(rating, text: str | None) -> dict:
318+ t = (text or "").lower()
319+ topics = sorted({name for name, kws in TOPICS.items()
320+ if any(k in t for k in kws)})
321+ if rating is None:
322+ sentiment = "inconnu"
323+ elif rating <= 2:
324+ sentiment = "négatif"
325+ elif rating >= 4:
326+ sentiment = "positif"
327+ else:
328+ sentiment = "neutre"
329+ severity = "faible"
330+ if sentiment == "négatif":
331+ severity = "élevée" if any(x in _GRAVES for x in topics) else "moyenne"
332+ if len(t) >= 80 and topics:
333+ confidence = 0.7
334+ elif t:
335+ confidence = 0.4
336+ else:
337+ confidence = 0.2
338+ return {"topics": topics, "sentiment": sentiment, "severite": severity,
339+ "confiance": confidence, "statut": "inferred",
340+ "methode": "lexique de thèmes fr + sentiment dérivé de la note"}
341+
342+
343+def _aggregate(con, manager_id: int) -> dict:
344+ rows = con.execute(
345+ "SELECT rating, published_at, owner_response, analysis"
346+ " FROM manager_reviews WHERE manager_id=?", (manager_id,)).fetchall()
347+ rated = [r for r in rows if r["rating"] is not None]
348+ out: dict = {"n": len(rows), "version": VERSION, "statut": "calculated",
349+ "methode": ("statistiques calculées sur les avis Google "
350+ "synchronisés en base (pas seulement la moyenne "
351+ "affichée par Google) — l'échantillon peut être "
352+ "plus petit que le total annoncé par Google, il "
353+ "s'accumule à chaque synchronisation ; thèmes "
354+ "par lexique — inférence, pas une lecture "
355+ "humaine")}
356+ if not rated:
357+ return out
358+ out["moyenne"] = round(sum(r["rating"] for r in rated) / len(rated), 2)
359+ dist = {str(i): 0 for i in range(1, 6)}
360+ for r in rated:
361+ k = str(int(round(r["rating"])))
362+ if k in dist:
363+ dist[k] += 1
364+ out["distribution"] = dist
365+ out["pct_negatif"] = round(100 * (dist["1"] + dist["2"]) / len(rated))
366+ out["pct_positif"] = round(100 * (dist["4"] + dist["5"]) / len(rated))
367+ out["avec_reponse_proprietaire"] = sum(
368+ 1 for r in rows if r["owner_response"])
369+
370+ cutoff = datetime.now(timezone.utc).timestamp() - 365 * 86400
371+ rec, old = [], []
372+ for r in rated:
373+ ts = _iso_ts(r["published_at"])
374+ (rec if ts and ts >= cutoff else old).append(r["rating"])
375+ if rec:
376+ out["moyenne_12m"] = round(sum(rec) / len(rec), 2)
377+ out["n_12m"] = len(rec)
378+ if len(rec) >= 5 and len(old) >= 5:
379+ delta = out["moyenne_12m"] - sum(old) / len(old)
380+ out["tendance"] = ("en amélioration" if delta >= 0.3 else
381+ "en dégradation" if delta <= -0.3 else "stable")
382+ out["tendance_delta"] = round(delta, 2)
383+
384+ themes: dict[str, dict] = {}
385+ for r in rows:
386+ try:
387+ a = json.loads(r["analysis"] or "{}")
388+ except (ValueError, TypeError):
389+ continue
390+ for tp in a.get("topics") or []:
391+ d = themes.setdefault(tp, {"mentions": 0, "negatif": 0,
392+ "positif": 0})
393+ d["mentions"] += 1
394+ if a.get("sentiment") == "négatif":
395+ d["negatif"] += 1
396+ elif a.get("sentiment") == "positif":
397+ d["positif"] += 1
398+ out["themes"] = dict(sorted(themes.items(),
399+ key=lambda kv: -kv[1]["mentions"]))
400+ out["plaintes_frequentes"] = [
401+ k for k, v in sorted(themes.items(), key=lambda kv: -kv[1]["negatif"])
402+ if v["negatif"] >= 2][:5]
403+ return out
404+
405+
406+def _iso_ts(iso: str | None) -> float | None:
407+ if not iso:
408+ return None
409+ try:
410+ return datetime.fromisoformat(iso.replace("Z", "+00:00")).timestamp()
411+ except ValueError:
412+ return None
413+
414+
415+# --- fiche API + boucle de fond ----------------------------------------------
416+
417+def fiche(source_id: str, con=None) -> dict | None:
418+ """Fiche gestionnaire pour l'API — tout vient de NOTRE base (0 SerpApi)."""
419+ own = con is None
420+ if own:
421+ con = db.connect()
422+ try:
423+ mgr = con.execute("SELECT * FROM managers WHERE source_id=?",
424+ (source_id,)).fetchone()
425+ if mgr is None:
426+ return None
427+ m = dict(mgr)
428+ n_active = con.execute(
429+ "SELECT COUNT(*) n FROM listings WHERE source=? AND active=1"
430+ " AND published=1 AND dup_of IS NULL", (source_id,)).fetchone()["n"]
431+ out = {
432+ "source_id": m["source_id"], "nom": m["name"],
433+ "site_web": m["website"], "telephone": m["phone"],
434+ "annonces_actives": n_active, "version": VERSION,
435+ }
436+ if m["gmaps_place_id"] and not m["resolve_failed"]:
437+ out["google_maps"] = {
438+ "nom": m["gmaps_name"], "adresse": m["gmaps_address"],
439+ "note": m["gmaps_rating"], "nombre_avis": m["gmaps_reviews"],
440+ "place_id": m["gmaps_place_id"],
441+ "confiance_association": m["match_confidence"],
442+ "signaux": json.loads(m["match_signals"] or "{}"),
443+ "statut": "inferred",
444+ "methode": ("fiche associée par similarité de nom, domaine "
445+ "web et ville — association refusée sous "
446+ f"{SEUIL_CONFIANCE:.2f} de confiance"),
447+ }
448+ try:
449+ out["avis"] = json.loads(m["review_stats"] or "null")
450+ except (ValueError, TypeError):
451+ out["avis"] = None
452+ out["avis_recents"] = [
453+ {"note": r["rating"], "texte": r["text"],
454+ "date": r["published_at"] or r["relative_date_raw"],
455+ "auteur": r["author_name"],
456+ "reponse_proprietaire": bool(r["owner_response"]),
457+ "analyse": json.loads(r["analysis"] or "{}"),
458+ "statut": "observed"}
459+ for r in con.execute(
460+ "SELECT rating, text, published_at, relative_date_raw,"
461+ " author_name, owner_response, analysis"
462+ " FROM manager_reviews WHERE manager_id=?"
463+ " ORDER BY published_at DESC, fetched_at DESC LIMIT 10",
464+ (m["id"],))]
465+ out["derniere_synchro"] = m["last_synced_at"]
466+ elif m["resolved_at"]:
467+ out["google_maps"] = {
468+ "statut": "non_associe",
469+ "note_methode": ("aucune fiche Google Maps n'a pu être "
470+ "associée avec une confiance suffisante — "
471+ "Lou-Ka préfère ne rien afficher plutôt que "
472+ "d'attribuer les avis d'une autre entreprise"),
473+ }
474+ return out
475+ finally:
476+ if own:
477+ con.close()
478+
479+
480+def precompute(budget_s: float = 120.0, con=None) -> dict:
481+ """Boucle de fond : amorce, résout et synchronise dans un budget temps."""
482+ if not serpapi_client.available():
483+ return {"skipped": "SERPAPI_API_KEY absente"}
484+ own = con is None
485+ if own:
486+ con = db.connect()
487+ t0 = time.time()
488+ resolved = synced = errors = 0
489+ try:
490+ seed(con)
491+ now = time.time()
492+ # 1) résolutions manquantes (ou échecs anciens à retenter)
493+ for r in con.execute(
494+ "SELECT id FROM managers WHERE resolved_at IS NULL"
495+ " OR (resolve_failed=1 AND resolved_at < ?)"
496+ " ORDER BY resolved_at IS NOT NULL, id",
497+ (now - RESOLVE_RETRY,)).fetchall():
498+ if time.time() - t0 > budget_s:
499+ break
500+ try:
501+ if resolve(r["id"], con).get("ok"):
502+ resolved += 1
503+ except serpapi_client.SerpApiError:
504+ errors += 1
505+ break # panne API : ne pas insister ce tour-ci
506+ # 2) avis périmés (les jamais-synchronisés d'abord)
507+ for r in con.execute(
508+ "SELECT id FROM managers WHERE gmaps_data_id IS NOT NULL"
509+ " AND resolve_failed=0 AND (last_synced_at IS NULL"
510+ " OR last_synced_at < ?)"
511+ " ORDER BY last_synced_at IS NOT NULL, last_synced_at",
512+ (now - SYNC_TTL,)).fetchall():
513+ if time.time() - t0 > budget_s:
514+ break
515+ try:
516+ if sync_reviews(r["id"], con).get("ok"):
517+ synced += 1
518+ except serpapi_client.SerpApiError:
519+ errors += 1
520+ break
521+ return {"resolved": resolved, "synced": synced, "errors": errors,
522+ "elapsed_s": round(time.time() - t0, 1)}
523+ finally:
524+ if own:
525+ con.close()
526+
527+
528+if __name__ == "__main__":
529+ print(precompute())
added louka/recycled.py +185 −0
@@ -0,0 +1,185 @@
1+# -----------------------------------------------------------------------------
2+# Lou-Ka — Agrégateur de logements à louer (province de Québec)
3+# Auteur : Simon-Pierre Boucher — contact@spboucher.ai
4+# recycled.py : détection des annonces RECYCLÉES — le même logement republié
5+# (souvent plus cher) après un passage antérieur sur le marché.
6+#
7+# Méthode : les candidates sont les annonces INACTIVES du même immeuble
8+# (building_key — même clé d'adresse que la déduplication). Chaque paire reçoit
9+# un score de similarité 0–100 fondé sur des signaux indépendants :
10+# numéro d'unité identique, type d'unité, chambres, superficie (±3 %),
11+# photos identiques (hash SHA1 du contrôle d'images), texte de description
12+# (similarité de Jaccard sur les mots).
13+# Seules les correspondances ≥ SEUIL_AFFICHAGE sont retournées, avec les
14+# signaux qui les fondent (inférence explicable, jamais de fusion automatique).
15+# Cache : table recycled_cache (TTL 7 jours — recalcul léger, borné à
16+# l'immeuble).
17+# -----------------------------------------------------------------------------
18+from __future__ import annotations
19+
20+import json
21+import re
22+import time
23+
24+from . import db
25+
26+TTL = 7 * 86400
27+SEUIL_AFFICHAGE = 70 # score minimal pour présenter une correspondance
28+MAX_CANDIDATES = 200 # garde-fou (tours à très fort volume)
29+
30+_WORD = re.compile(r"[a-zà-ÿ0-9]{3,}")
31+
32+
33+def _tokens(text: str | None) -> set[str]:
34+ return set(_WORD.findall((text or "").lower()))
35+
36+
37+def _jaccard(a: set[str], b: set[str]) -> float:
38+ if not a or not b:
39+ return 0.0
40+ return len(a & b) / len(a | b)
41+
42+
43+def _sha1_set(con, images_json: str | None) -> set[str]:
44+ try:
45+ urls = json.loads(images_json or "[]")[:20]
46+ except (ValueError, TypeError):
47+ return set()
48+ if not urls:
49+ return set()
50+ marks = ",".join("?" * len(urls))
51+ return {r["sha1"] for r in con.execute(
52+ f"SELECT sha1 FROM image_checks WHERE url IN ({marks})"
53+ " AND sha1 IS NOT NULL", urls)}
54+
55+
56+def _unit_of(address: str | None, city: str | None) -> str | None:
57+ from .dedup import _parse_address
58+ return _parse_address(address or "", city or "")[1]
59+
60+
61+def _score(con, cur: dict, cand: dict) -> tuple[int, list[str]]:
62+ """Score 0–100 + signaux lisibles justifiant la correspondance."""
63+ score = 0
64+ signals: list[str] = []
65+ if cur["unit"] and cand["unit"]:
66+ if cur["unit"] == cand["unit"]:
67+ score += 35
68+ signals.append(f"même numéro d'unité ({cur['unit']})")
69+ else:
70+ return 0, [] # unités connues et différentes : pas le même
71+ if cur["unit_type"] and cand["unit_type"]:
72+ if cur["unit_type"] == cand["unit_type"]:
73+ score += 10
74+ signals.append(f"même type ({cur['unit_type']})")
75+ else:
76+ return 0, []
77+ if cur["bedrooms"] is not None and cand["bedrooms"] is not None:
78+ if cur["bedrooms"] == cand["bedrooms"]:
79+ score += 8
80+ signals.append("même nombre de chambres")
81+ else:
82+ return 0, []
83+ if cur["area"] and cand["area"]:
84+ if abs(cur["area"] - cand["area"]) <= 0.03 * max(cur["area"], cand["area"]):
85+ score += 15
86+ signals.append(f"même superficie (≈{round(cand['area'])} pi²)")
87+ else:
88+ score -= 10
89+ # photos identiques (hash du contenu, pas l'URL)
90+ sh_cur = _sha1_set(con, cur["images"])
91+ sh_cand = _sha1_set(con, cand["images"])
92+ if sh_cur and sh_cand:
93+ overlap = len(sh_cur & sh_cand) / min(len(sh_cur), len(sh_cand))
94+ if overlap >= 0.5:
95+ score += 30
96+ signals.append(f"{len(sh_cur & sh_cand)} photo(s) identique(s)")
97+ elif overlap > 0:
98+ score += 12
99+ signals.append("photos partiellement identiques")
100+ # texte de description
101+ jac = _jaccard(_tokens(cur["description"]), _tokens(cand["description"]))
102+ if jac >= 0.7:
103+ score += 20
104+ signals.append("description quasi identique")
105+ elif jac >= 0.45:
106+ score += 10
107+ signals.append("description très similaire")
108+ # même source + même identifiant externe = republication certaine
109+ if cur["source"] == cand["source"]:
110+ score += 5
111+ return min(100, score), signals
112+
113+
114+def lookup(uid: str, con=None) -> dict:
115+ """Correspondances probables (annonces antérieures du même logement)."""
116+ own = con is None
117+ if own:
118+ con = db.connect()
119+ try:
120+ cached = con.execute(
121+ "SELECT matches, computed_at FROM recycled_cache WHERE uid=?",
122+ (uid,)).fetchone()
123+ if cached and time.time() - cached["computed_at"] < TTL:
124+ return json.loads(cached["matches"])
125+
126+ cur = con.execute(
127+ "SELECT uid, source, address, city, unit_type, bedrooms, price,"
128+ " area_sqft AS area, description, images, first_seen, building_key,"
129+ " dup_of FROM listings WHERE uid=?", (uid,)).fetchone()
130+ out: dict = {"matches": [], "statut": "inferred",
131+ "methode": ("similarité multi-signaux (unité, type, "
132+ "chambres, superficie, photos par hash, "
133+ "texte) parmi les annonces antérieures du "
134+ "même immeuble")}
135+ if cur is None or not cur["building_key"]:
136+ _store(con, uid, out)
137+ return out
138+ cur_d = dict(cur)
139+ cur_d["unit"] = _unit_of(cur["address"], cur["city"])
140+
141+ # exclure le groupe de doublons inter-sources courant (même annonce)
142+ dup_group = {uid, cur["dup_of"] or ""}
143+ for r in con.execute("SELECT uid FROM listings WHERE dup_of=?", (uid,)):
144+ dup_group.add(r["uid"])
145+
146+ cands = con.execute(
147+ "SELECT uid, source, address, city, unit_type, bedrooms, price,"
148+ " area_sqft AS area, description, images, first_seen, last_seen"
149+ " FROM listings WHERE building_key=? AND active=0 AND uid<>?"
150+ " ORDER BY last_seen DESC LIMIT ?",
151+ (cur["building_key"], uid, MAX_CANDIDATES)).fetchall()
152+ matches = []
153+ for c in cands:
154+ if c["uid"] in dup_group:
155+ continue
156+ # une annonce antérieure = terminée avant l'apparition de l'actuelle
157+ if (c["last_seen"] or 0) > (cur["first_seen"] or 0) + 7 * 86400:
158+ continue
159+ cd = dict(c)
160+ cd["unit"] = _unit_of(c["address"], c["city"])
161+ score, signals = _score(con, cur_d, cd)
162+ if score >= SEUIL_AFFICHAGE:
163+ matches.append({
164+ "uid": c["uid"], "source": c["source"],
165+ "prix": c["price"], "unit_type": c["unit_type"],
166+ "derniere_observation": c["last_seen"],
167+ "premiere_observation": c["first_seen"],
168+ "confiance": score, "signaux": signals,
169+ })
170+ matches.sort(key=lambda m: -m["confiance"])
171+ out["matches"] = matches[:5]
172+ _store(con, uid, out)
173+ return out
174+ finally:
175+ if own:
176+ con.close()
177+
178+
179+def _store(con, uid: str, out: dict) -> None:
180+ con.execute(
181+ "INSERT INTO recycled_cache (uid, matches, computed_at) VALUES (?,?,?)"
182+ " ON CONFLICT(uid) DO UPDATE SET matches=excluded.matches,"
183+ " computed_at=excluded.computed_at",
184+ (uid, json.dumps(out, ensure_ascii=False), time.time()))
185+ con.commit()
added louka/serpapi_client.py +86 −0
Large diff (86 lines) is not loaded by default to keep the page fast.Load diff
modified louka/web.py +59 −0
Large diff (71 lines) is not loaded by default to keep the page fast.Load diff
modified run.py +12 −0
Large diff (25 lines) is not loaded by default to keep the page fast.Load diff
Menu

Theme
↑↓ navigate ↵ open esc close / or ⌘K anywhere