Python 67%
TypeScript 18.2%
CSS 14.4%
1# -----------------------------------------------------------------------------2# Immo-Ka — Agrégateur de maisons à vendre (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# quality.py : contrôle qualité des annonces — score de complétude, contrôles de5# cohérence immobiliers, seuil de publication (quarantaine sous le seuil).6#7# `refresh(con)` recalcule pour toutes les annonces actives :8# - quality_score : complétude 0-100 (pondération des champs décisifs)9# - quality_issues : JSON (anomalies détectées, dures ou informatives)10# - published : 1 = affichable sur le site, 0 = quarantaine11# et les champs dérivés dans details : prix_pi2 (prix/superficie) et12# transaction (vente | location, détectée du libellé de prix).13#14# Règle de publication : prix plausible + ville + type de bien + au moins une15# image + contenu exploitable (description ou caractéristiques). Une annonce16# sous le seuil reste en base (re-synchronisée/enrichie aux prochains cycles)17# mais n'est pas affichée — elle sort de quarantaine dès qu'elle est complétée.18# -----------------------------------------------------------------------------19from __future__ import annotations2021import json22import os23import re24import sqlite325import time2627from .normalize import strip_accents2829# bornes de plausibilité (marché québécois)30PRICE_SALE_MIN, PRICE_SALE_MAX = 20_000, 80_000_00031PRICE_RENT_MIN, PRICE_RENT_MAX = 300, 25_00032AREA_MIN, AREA_MAX = 120, 25_000 # superficie habitable (pi²)33LOT_MAX = 200_000_000 # terrain (pi²) — grandes terres34YEAR_MIN = 16003536_RENT_RE = re.compile(r"/\s*mois|par mois|/\s*mth|/\s*month|\bmensuel|\ba louer\b|"37 r"\blouer\b|\blocation\b|\blease\b|\bfor rent\b")383940def _transaction(price, price_label: str, title: str) -> str:41 """vente | location — détectée du libellé (jamais confondre loyer et prix)."""42 key = strip_accents(f"{price_label} {title}".lower())43 if _RENT_RE.search(key):44 return "location"45 if price is not None and price < PRICE_RENT_MAX:46 # montant de loyer sans mot-clé : trop bas pour une vente au Québec47 return "location"48 return "vente"495051def assess(row: dict) -> tuple[int, list[str], int, dict]:52 """(score 0-100, anomalies, publiable 0/1, champs dérivés) pour une annonce.5354 `row` : dict aux clés des colonnes listings (features/details/images55 peuvent être des chaînes JSON ou déjà décodés)."""5657 def _json(v, default):58 if isinstance(v, (list, dict)):59 return v60 try:61 return json.loads(v) if v else default62 except (TypeError, ValueError):63 return default6465 images = _json(row.get("images"), [])66 features = _json(row.get("features"), [])67 details = _json(row.get("details"), {})68 desc = (row.get("description") or "").strip()69 price = row.get("price")70 issues: list[str] = []71 derived: dict = {}7273 tx = _transaction(price, row.get("price_label") or "", row.get("title") or "")74 derived["transaction"] = tx7576 # -- cohérence : prix plausible pour le type de transaction ----------------77 price_ok = price is not None and price > 078 if price_ok:79 lo, hi = ((PRICE_RENT_MIN, PRICE_RENT_MAX) if tx == "location"80 else (PRICE_SALE_MIN, PRICE_SALE_MAX))81 if not (lo <= price <= hi):82 issues.append(f"prix_hors_bornes:{price:.0f}$ ({tx})")83 price_ok = False8485 # -- cohérence : superficies plausibles -------------------------------------86 area = row.get("area_sqft")87 area_ok = area is not None and AREA_MIN <= area <= AREA_MAX88 if area is not None and not area_ok:89 issues.append(f"superficie_improbable:{area:.0f}pi2")90 lot = row.get("lot_sqft")91 if lot is not None and not (0 < lot <= LOT_MAX):92 issues.append(f"terrain_improbable:{lot:.0f}pi2")9394 # -- cohérence : pièces / chambres ------------------------------------------95 beds, baths = row.get("bedrooms"), row.get("bathrooms")96 if beds is not None and not (0 <= beds <= 30):97 issues.append(f"chambres_improbables:{beds}")98 beds = None99 if baths is not None and not (0 <= baths <= 20):100 issues.append(f"sdb_improbables:{baths}")101 baths = None102 ptype = (row.get("property_type") or "").strip()103 if beds and ptype == "Condo" and beds > 8:104 issues.append(f"chambres_vs_type:{beds}ch_condo")105106 # -- cohérence : année de construction ---------------------------------------107 year = row.get("year_built")108 current_year = time.gmtime().tm_year109 if year is not None and not (YEAR_MIN <= year <= current_year + 3):110 issues.append(f"annee_invalide:{year}")111112 # -- champ dérivé : prix au pi² (et m²) ---------------------------------------113 if price_ok and area_ok and tx == "vente":114 ppsf = price / area115 derived["prix_pi2"] = round(ppsf)116 derived["prix_m2"] = round(ppsf * 10.7639)117 if not (30 <= ppsf <= 3500):118 issues.append(f"prix_pi2_extreme:{ppsf:.0f}")119120 # -- score de complétude (0-100) ----------------------------------------------121 n_img = len(images)122 has_contact = bool(row.get("broker_name") or row.get("broker_phone"))123 score = 0124 score += 15 if price_ok else 0125 score += 8 if (row.get("city") or "").strip() else 0126 score += 7 if (row.get("address") or "").strip() else 0127 score += 8 if ptype else 0128 score += 10 if n_img >= 1 else 0129 score += 5 if n_img >= 8 else 0130 score += 12 if len(desc) >= 300 else 8 if len(desc) >= 80 else 4 if desc else 0131 score += 6 if beds is not None else 0132 score += 5 if baths is not None else 0133 score += 8 if area_ok else 0134 score += 4 if year is not None else 0135 score += 7 if row.get("lat") is not None else 0136 score += 3 if has_contact else 0137 score += 2 if lot is not None else 0138139 # -- seuil de publication ---------------------------------------------------140 # (une annonce SANS image reste publiable : le frontend applique l'image de141 # secours par type de bien ; le drapeau sans_image la marque à re-vérifier)142 if n_img < 1:143 issues.append("sans_image")144 # vendue/louée à la source : archivée (plus jamais affichée en résultats)145 statut = (row.get("status") or "").strip().lower()146 vendue = statut in ("vendu", "vendue", "loue", "louee", "loué", "louée",147 "sold", "rented", "retire", "retiré")148 if vendue:149 issues.append(f"statut:{statut}")150 # House-Ka : les cartes DDF (liste) n'ont ni description ni type — la151 # fiche s'enrichit au fil des passes de détail. Publication dès que le152 # prix est plausible et la ville connue ; type/description comptent dans153 # le score seulement.154 publishable = (155 not vendue156 and price_ok157 and bool((row.get("city") or "").strip())158 )159 if not publishable:160 why = []161 if not price_ok:162 why.append("prix")163 if not (row.get("city") or "").strip():164 why.append("ville")165 issues.append("quarantaine:" + "+".join(why))166167 return score, issues, int(publishable), derived168169170def refresh(con: sqlite3.Connection, sources: list[str] | None = None) -> dict:171 """Recalcule score/anomalies/publication pour les annonces actives.172173 Appelé après chaque synchronisation (ingest.run) — quelques secondes pour174 ~80 k lignes. Retourne un résumé {actives, publiees, quarantaine}."""175 _ensure_columns(con)176 sql = ("SELECT uid, source, title, price, price_label, city, address,"177 " property_type,"178 " bedrooms, bathrooms, area_sqft, lot_sqft, year_built, lat, status,"179 " broker_name, broker_phone, description, features, details, images,"180 " quality_score, quality_issues, published"181 " FROM listings WHERE active=1")182 args: list = []183 if sources:184 sql += f" AND source IN ({','.join('?' * len(sources))})"185 args = list(sources)186 updates = []187 n = pub = 0188 for r in con.execute(sql, args):189 row = dict(r)190 score, issues, publishable, derived = assess(row)191 n += 1192 pub += publishable193 details = {}194 try:195 details = json.loads(row.get("details") or "{}")196 except ValueError:197 pass198 changed_details = any(details.get(k) != v for k, v in derived.items())199 issues_json = json.dumps(issues, ensure_ascii=False) if issues else None200 if (score != row.get("quality_score") or publishable != row.get("published")201 or issues_json != row.get("quality_issues") or changed_details):202 details.update(derived)203 updates.append((score, issues_json, publishable,204 json.dumps(details, ensure_ascii=False), row["uid"]))205 if updates:206 con.executemany(207 "UPDATE listings SET quality_score=?, quality_issues=?, published=?,"208 " details=? WHERE uid=?", updates)209 con.commit()210 return {"actives": n, "publiees": pub, "quarantaine": n - pub,211 "recalculees": len(updates)}212213214def _ensure_columns(con: sqlite3.Connection) -> None:215 cols = {r["name"] for r in con.execute("PRAGMA table_info(listings)")}216 if "quality_score" not in cols:217 con.execute("ALTER TABLE listings ADD COLUMN quality_score INTEGER")218 if "quality_issues" not in cols:219 con.execute("ALTER TABLE listings ADD COLUMN quality_issues TEXT")220 if "published" not in cols:221 # 1 par défaut : la 1re passe refresh() met la vraie valeur partout222 con.execute("ALTER TABLE listings ADD COLUMN published INTEGER DEFAULT 1")223 con.execute("CREATE INDEX IF NOT EXISTS idx_listings_published"224 " ON listings(published)")225 con.commit()226227228def summary(con: sqlite3.Connection) -> dict:229 """Statistiques qualité pour /api/stats : complétude, quarantaine, anomalies."""230 _ensure_columns(con)231 row = con.execute(232 "SELECT COUNT(*) actives, SUM(published) publiees,"233 " ROUND(AVG(quality_score),1) completude_moyenne"234 " FROM listings WHERE active=1 AND dup_hidden=0").fetchone()235 per_source = [dict(r) for r in con.execute(236 "SELECT source, COUNT(*) n, SUM(published) publiees,"237 " ROUND(AVG(quality_score),1) completude,"238 " SUM(CASE WHEN quality_issues IS NOT NULL THEN 1 ELSE 0 END) anomalies"239 " FROM listings WHERE active=1 AND dup_hidden=0"240 " GROUP BY source ORDER BY n DESC")]241 anomalies: dict[str, int] = {}242 for r in con.execute(243 "SELECT quality_issues FROM listings WHERE active=1 AND dup_hidden=0"244 " AND quality_issues IS NOT NULL"):245 try:246 for issue in json.loads(r["quality_issues"]):247 anomalies[issue.split(":")[0]] = anomalies.get(issue.split(":")[0], 0) + 1248 except ValueError:249 continue250 d = dict(row)251 d["quarantaine"] = (d.get("actives") or 0) - (d.get("publiees") or 0)252 d["anomalies"] = dict(sorted(anomalies.items(), key=lambda kv: -kv[1]))253 d["par_source"] = per_source254 return d255