Python 68.8%
TypeScript 18.6%
CSS 8.7%
JavaScript 3.3%
HTML 0.6%
1# -----------------------------------------------------------------------------2# Rent-Ka — Rental listings aggregator (Canada, outside Québec)3# Author: Simon-Pierre Boucher — contact@spboucher.ai4# schema.py : standardized data model (Listing) + central enrichment5# -----------------------------------------------------------------------------6"""Standard listing schema and field normalization.78Every connector, whatever its source site, must produce `Listing` objects9that conform to this schema. `finalize()` then applies the common10normalization layer (rentka/normalize.py): ISO dates, sqft areas, canonical11unit types, prices, ENGLISH display labels — connectors can stay simple and12fill raw fields (French labels from fr-ca templates included).13"""14from __future__ import annotations1516import hashlib17import json18from dataclasses import dataclass, field, asdict1920from .normalize import ( # ré-exportés pour les connecteurs existants21 bedrooms_from_unit_type,22 clean_address,23 clean_city,24 clean_sector,25 coerce_count,26 extract_details,27 merge_details,28 normalize_unit_type,29 parse_area_sqft,30 parse_availability_date,31 parse_bathrooms,32 parse_bedrooms,33 parse_price,34 price_is_from,35 strip_accents,36)3738__all__ = [39 "Listing", "infer_city", "normalize_unit_type", "parse_price",40 "parse_availability_date", "parse_area_sqft", "clean_address",41 "strip_accents",42]434445@dataclass46class Listing:47 """Standardized Rent-Ka listing."""4849 source: str # source id (see data/sources.json)50 external_id: str # identifier at the source51 url: str # listing page at the source52 title: str = "" # e.g. "200 Bay Street — 2 bedrooms"53 address: str = "" # civic address54 sector: str = "" # neighbourhood/borough (e.g. The Annex)55 city: str = "" # Toronto, Calgary, Halifax…56 province: str = "ON" # ON | BC | AB | SK | MB | NB | NS | PE | NL | YT | NT | NU57 unit_type: str = "" # Studio, n bedroom(s), Loft, Condo, House, Room…58 bedrooms: float | None = None # closed bedrooms59 bathrooms: float | None = None # bathrooms (1.5 = extra powder room)60 price: float | None = None # monthly rent ($ CAD), lowest when "from"61 price_label: str = "" # display text (e.g. "From $799/month")62 availability: str = "" # display text (e.g. "Available now")63 availability_date: str | None = None # ISO "2026-07-01", "now", or None64 area_sqft: float | None = None # area in sq ft65 pets: str | None = None # "yes" | "no" | "conditions" | None66 furnished: bool | None = None # furnished (None = unknown)67 description: str = ""68 digest: dict | None = None # structured description (rentka/textmine.py)69 amenities: list[str] = field(default_factory=list) # source text, for display70 details: dict = field(default_factory=dict) # structured fields (JSON)71 images: list[str] = field(default_factory=list) # absolute URLs72 lat: float | None = None73 lng: float | None = None7475 @property76 def uid(self) -> str:77 return f"{self.source}:{self.external_id}"7879 def content_hash(self) -> str:80 """Hash du contenu pour la détection de changements (pseudo-webhook)."""81 payload = asdict(self)82 blob = json.dumps(payload, sort_keys=True, ensure_ascii=False)83 return hashlib.sha256(blob.encode("utf-8")).hexdigest()8485 def finalize(self) -> "Listing":86 """Applique la normalisation commune. Appelé par le pipeline d'ingestion.8788 Idempotent ; ne remplace jamais une valeur explicite du connecteur.89 """90 import html as _html91 from .normalize import translate_label_en92 self.title = _html.unescape(self.title).strip()93 self.address = clean_address(_html.unescape(self.address))94 self.sector = clean_sector(self.sector)95 self.city = clean_city(self.city)96 self.unit_type = normalize_unit_type(self.unit_type)9798 # English display labels (central FR -> EN funnel — many connectors99 # read fr-ca page templates; unknown strings pass through untouched)100 self.amenities = [translate_label_en(a) for a in self.amenities]101 self.availability = translate_label_en(self.availability)102 self.price_label = translate_label_en(self.price_label)103 if self.pets in ("oui", "non"):104 self.pets = "yes" if self.pets == "oui" else "no"105106 if self.price is None:107 self.price = parse_price(self.price_label)108 if self.availability_date is None:109 self.availability_date = parse_availability_date(self.availability)110 if self.area_sqft is None:111 for texte in (self.description, " ".join(self.amenities), self.title):112 self.area_sqft = parse_area_sqft(texte)113 if self.area_sqft is not None:114 break115116 # source-provided coordinates: reject any point outside the covered117 # territory (swapped lat/lng, 0/0, typos) — the geocoder takes over118 # from the address rather than showing a pin in Kazakhstan.119 if self.lat is not None and self.lng is not None:120 prov = (self.province or "ON").upper()121 lo_lat, hi_lat, lo_lng, hi_lng = PROVINCE_BBOX.get(122 prov, CANADA_BBOX)123 ok = lo_lat <= self.lat <= hi_lat and lo_lng <= self.lng <= hi_lng124 if not ok:125 self.lat = self.lng = None126127 derived = extract_details(self.amenities, self.description, self.title)128 if self.price_label and price_is_from(self.price_label):129 derived["price_from"] = True130 self.details = merge_details(self.details, derived)131132 # chambres / salles de bain : valeur explicite du connecteur d'abord,133 # puis champs structurés de details, puis type d'unité (n½ -> n-2),134 # puis extraction texte — jamais de valeur inventée (None = inconnu)135 if self.bedrooms is None:136 for k in ("bedrooms", "Chambres", "Chambre(s)"):137 self.bedrooms = coerce_count(self.details.get(k))138 if self.bedrooms is not None:139 break140 if self.bedrooms is None:141 self.bedrooms = bedrooms_from_unit_type(self.unit_type)142 if self.bedrooms is None:143 self.bedrooms = parse_bedrooms(self.title, " | ".join(self.amenities),144 self.description)145 if self.bathrooms is None:146 for k in ("bathrooms", "Salles de bain", "Salle de bain",147 "Salle(s) de bain"):148 self.bathrooms = coerce_count(self.details.get(k))149 if self.bathrooms is not None:150 break151 if self.bathrooms is None:152 self.bathrooms = parse_bathrooms(" | ".join(self.amenities),153 self.description)154155 # description structurée (nettoyage + extraction + sections)156 if self.digest is None and self.description:157 try:158 from .textmine import analyser159 self.digest = analyser(self.description, price=self.price,160 sector=self.sector, city=self.city)161 except ImportError:162 pass # module absent : la fiche affichera le texte brut163164 if self.pets is None:165 self.pets = self.details.get("pets")166 else:167 self.details["pets"] = self.pets168 if self.furnished is None:169 furn = self.details.get("furnished")170 self.furnished = furn if isinstance(furn, bool) else None171 else:172 self.details["furnished"] = self.furnished173 return self174175176# ---------------------------------------------------------------------------177# Geography — coordinate sanity boxes per province (Canada, outside Québec)178# ---------------------------------------------------------------------------179180# (min_lat, max_lat, min_lng, max_lng)181CANADA_BBOX = (41.6, 83.2, -141.0, -52.5)182PROVINCE_BBOX = {183 "BC": (48.2, 60.0, -139.1, -114.0),184 "AB": (48.9, 60.0, -120.0, -109.9),185 "SK": (48.9, 60.0, -110.1, -101.3),186 "MB": (48.9, 60.0, -102.1, -88.9),187 "ON": (41.6, 56.9, -95.3, -74.3),188 "NB": (44.5, 48.1, -69.1, -63.7),189 "NS": (43.3, 47.1, -66.5, -59.6),190 "PE": (45.9, 47.1, -64.5, -61.9),191 "NL": (46.5, 60.5, -67.9, -52.5),192 "YT": (60.0, 69.7, -141.1, -123.7),193 "NT": (60.0, 78.9, -136.5, -101.9),194 "NU": (60.0, 83.2, -120.7, -61.0),195 # legacy rows / stray input — Québec box kept only for bbox sanity196 "QC": (44.5, 63.0, -80.0, -56.0),197}198199200def infer_city(sector: str, default: str = "") -> str:201 """Legacy helper (Québec-era): now a pass-through to the default."""202 return default203