spb/rent-ka

1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# recycled.py : détection des annonces RECYCLÉES — le même logement republié5#               (souvent plus cher) après un passage antérieur sur le marché.6#7# Méthode : les candidates sont les annonces INACTIVES du même immeuble8# (building_key — même clé d'adresse que la déduplication). Chaque paire reçoit9# un score de similarité 0–100 fondé sur des signaux indépendants :10#   numéro d'unité identique, type d'unité, chambres, superficie (±3 %),11#   photos identiques (hash SHA1 du contrôle d'images), texte de description12#   (similarité de Jaccard sur les mots).13# Seules les correspondances ≥ SEUIL_AFFICHAGE sont retournées, avec les14# signaux qui les fondent (inférence explicable, jamais de fusion automatique).15# Cache : table recycled_cache (TTL 7 jours — recalcul léger, borné à16# l'immeuble).17# -----------------------------------------------------------------------------18from __future__ import annotations1920import json21import re22import time2324from . import db2526TTL = 7 * 8640027SEUIL_AFFICHAGE = 70     # score minimal pour présenter une correspondance28MAX_CANDIDATES = 200     # garde-fou (tours à très fort volume)2930_WORD = re.compile(r"[a-zà-ÿ0-9]{3,}")313233def _tokens(text: str | None) -> set[str]:34    return set(_WORD.findall((text or "").lower()))353637def _jaccard(a: set[str], b: set[str]) -> float:38    if not a or not b:39        return 0.040    return len(a & b) / len(a | b)414243def _sha1_set(con, images_json: str | None) -> set[str]:44    try:45        urls = json.loads(images_json or "[]")[:20]46    except (ValueError, TypeError):47        return set()48    if not urls:49        return set()50    marks = ",".join("?" * len(urls))51    return {r["sha1"] for r in con.execute(52        f"SELECT sha1 FROM image_checks WHERE url IN ({marks})"53        " AND sha1 IS NOT NULL", urls)}545556def _unit_of(address: str | None, city: str | None) -> str | None:57    from .dedup import _parse_address58    return _parse_address(address or "", city or "")[1]596061def _score(con, cur: dict, cand: dict) -> tuple[int, list[str]]:62    """Score 0–100 + signaux lisibles justifiant la correspondance."""63    score = 064    signals: list[str] = []65    if cur["unit"] and cand["unit"]:66        if cur["unit"] == cand["unit"]:67            score += 3568            signals.append(f"même numéro d'unité ({cur['unit']})")69        else:70            return 0, []          # unités connues et différentes : pas le même71    if cur["unit_type"] and cand["unit_type"]:72        if cur["unit_type"] == cand["unit_type"]:73            score += 1074            signals.append(f"même type ({cur['unit_type']})")75        else:76            return 0, []77    if cur["bedrooms"] is not None and cand["bedrooms"] is not None:78        if cur["bedrooms"] == cand["bedrooms"]:79            score += 880            signals.append("même nombre de chambres")81        else:82            return 0, []83    if cur["area"] and cand["area"]:84        if abs(cur["area"] - cand["area"]) <= 0.03 * max(cur["area"], cand["area"]):85            score += 1586            signals.append(f"même superficie (≈{round(cand['area'])} pi²)")87        else:88            score -= 1089    # photos identiques (hash du contenu, pas l'URL)90    sh_cur = _sha1_set(con, cur["images"])91    sh_cand = _sha1_set(con, cand["images"])92    if sh_cur and sh_cand:93        overlap = len(sh_cur & sh_cand) / min(len(sh_cur), len(sh_cand))94        if overlap >= 0.5:95            score += 3096            signals.append(f"{len(sh_cur & sh_cand)} photo(s) identique(s)")97        elif overlap > 0:98            score += 1299            signals.append("photos partiellement identiques")100    # texte de description101    jac = _jaccard(_tokens(cur["description"]), _tokens(cand["description"]))102    if jac >= 0.7:103        score += 20104        signals.append("description quasi identique")105    elif jac >= 0.45:106        score += 10107        signals.append("description très similaire")108    # même source + même identifiant externe = republication certaine109    if cur["source"] == cand["source"]:110        score += 5111    return min(100, score), signals112113114def lookup(uid: str, con=None) -> dict:115    """Correspondances probables (annonces antérieures du même logement)."""116    own = con is None117    if own:118        con = db.connect()119    try:120        cached = con.execute(121            "SELECT matches, computed_at FROM recycled_cache WHERE uid=?",122            (uid,)).fetchone()123        if cached and time.time() - cached["computed_at"] < TTL:124            return json.loads(cached["matches"])125126        cur = con.execute(127            "SELECT uid, source, address, city, unit_type, bedrooms, price,"128            " area_sqft AS area, description, images, first_seen, building_key,"129            " dup_of FROM listings WHERE uid=?", (uid,)).fetchone()130        out: dict = {"matches": [], "statut": "inferred",131                     "methode": ("multi-signal similarity (unit, type, "132                                 "bedrooms, area, photo hashes, text) among "133                                 "earlier listings of the same building")}134        if cur is None or not cur["building_key"]:135            _store(con, uid, out)136            return out137        cur_d = dict(cur)138        cur_d["unit"] = _unit_of(cur["address"], cur["city"])139140        # exclure le groupe de doublons inter-sources courant (même annonce)141        dup_group = {uid, cur["dup_of"] or ""}142        for r in con.execute("SELECT uid FROM listings WHERE dup_of=?", (uid,)):143            dup_group.add(r["uid"])144145        cands = con.execute(146            "SELECT uid, source, address, city, unit_type, bedrooms, price,"147            " area_sqft AS area, description, images, first_seen, last_seen"148            " FROM listings WHERE building_key=? AND active=0 AND uid<>?"149            " ORDER BY last_seen DESC LIMIT ?",150            (cur["building_key"], uid, MAX_CANDIDATES)).fetchall()151        matches = []152        for c in cands:153            if c["uid"] in dup_group:154                continue155            # une annonce antérieure = terminée avant l'apparition de l'actuelle156            if (c["last_seen"] or 0) > (cur["first_seen"] or 0) + 7 * 86400:157                continue158            cd = dict(c)159            cd["unit"] = _unit_of(c["address"], c["city"])160            score, signals = _score(con, cur_d, cd)161            if score >= SEUIL_AFFICHAGE:162                matches.append({163                    "uid": c["uid"], "source": c["source"],164                    "prix": c["price"], "unit_type": c["unit_type"],165                    "derniere_observation": c["last_seen"],166                    "premiere_observation": c["first_seen"],167                    "confiance": score, "signaux": signals,168                })169        matches.sort(key=lambda m: -m["confiance"])170        out["matches"] = matches[:5]171        _store(con, uid, out)172        return out173    finally:174        if own:175            con.close()176177178def _store(con, uid: str, out: dict) -> None:179    con.execute(180        "INSERT INTO recycled_cache (uid, matches, computed_at) VALUES (?,?,?)"181        " ON CONFLICT(uid) DO UPDATE SET matches=excluded.matches,"182        " computed_at=excluded.computed_at",183        (uid, json.dumps(out, ensure_ascii=False), time.time()))184    con.commit()185