# ----------------------------------------------------------------------------- # Lou-Ka — Agrégateur de logements à louer (province de Québec) # Auteur : Simon-Pierre Boucher — contact@spboucher.ai # connectors/headway.py : connecteur La Corporation Headway (headwayltee.com) # Site vitrine Wix (rendu serveur) sans liste d'unités individuelles : # une annonce par complexe immobilier (Place Prévert à Vanier, Place # Versant Nord / Place l'Heureux / Domaine Versant Nord à Ste-Foy, # Complexe Renaissance à Charlesbourg, Thibault et Curé-Pelletier à Lévis). # Le Domaine Anjou (Montréal) est exclu (hors Québec/Lévis). # ----------------------------------------------------------------------------- from __future__ import annotations import html as htmllib import re import unicodedata from ..schema import Listing, normalize_unit_type from .base import BaseConnector BASE = "https://www.headwayltee.com" # (url de page, [(nom du complexe, secteur, ville), ...]) PAGES: list[tuple[str, list[tuple[str, str, str]]]] = [ (f"{BASE}/logements-a-louer/appartement-quebec", [("Place Prévert", "Vanier", "Québec")]), (f"{BASE}/ste-foy", [("Place Versant Nord", "Sainte-Foy", "Québec"), ("Place l'Heureux", "Sainte-Foy", "Québec"), ("Domaine Versant Nord", "Sainte-Foy", "Québec")]), (f"{BASE}/logements-a-louer/appartement-charlesbourg", [("Complexe Renaissance", "Charlesbourg", "Québec")]), (f"{BASE}/logements-a-louer/levis", [("Thibault", "Lévis", "Lévis"), ("Curé-Pelletier", "Lévis", "Lévis")]), ] def _slug(s: str) -> str: s = unicodedata.normalize("NFD", s.lower()) s = "".join(c for c in s if unicodedata.category(c) != "Mn") return re.sub(r"[^a-z0-9]+", "-", s).strip("-") def _wix_image(url: str) -> str: """URL wixstatic pleine résolution (sans les transformations /v1/fill/...).""" return url.split("/v1/")[0] class HeadwayConnector(BaseConnector): source_id = "headway" request_delay = 0.6 def fetch(self) -> list[Listing]: listings: list[Listing] = [] for url, complexes in PAGES: try: html = self.get(url).text except Exception: continue try: listings.extend(self._parse_page(url, html, complexes)) except Exception: continue return listings # -- une page (1 à 3 complexes) ---------------------------------------------- def _parse_page(self, url: str, html: str, complexes: list[tuple[str, str, str]]) -> list[Listing]: # segments de texte visibles, avec leur position dans le HTML # (les segments contenant « { » sont du CSS/JS inliné par Wix : ignorés) segments = [(m.start(), htmllib.unescape(m.group(1)).replace("\xa0", " ").strip()) for m in re.finditer(r">([^<>]{2,400})<", html)] segments = [(p, t) for p, t in segments if t and "{" not in t and "}" not in t and not t.startswith(("var ", "window.", "/*", "//"))] # images : balises Nom du complexe ... (wixstatic) img_tags = [(m.start(), m.group(0)) for m in re.finditer(r"]+>", html)] # occurrences exactes des noms de complexes (titres de sections) name_slugs = {_slug(n): n for n, _, _ in complexes} name_events: list[tuple[int, str]] = [] # (pos, nom) for p, t in segments: if _slug(t) in name_slugs: name_events.append((p, name_slugs[_slug(t)])) # blocs "Adresse :" (libellé seul) -> associés au titre précédent le plus # proche ; les pages à complexe unique prennent le premier bloc trouvé. # nom -> (adresse, types, téléphone du bureau de location) info_blocks: dict[str, tuple[str, list[str], str]] = {} for i, (p, t) in enumerate(segments): if not re.match(r"^Adresse\s*:?\s*$", t): continue parts: list[str] = [] unit_types: list[str] = [] phone = "" after_phone_label = False for _, t2 in segments[i + 1:i + 40]: if re.search(r"Heures d'ouverture", t2, re.I): break if re.match(r"^Num[ée]ro de t[ée]l[ée]phone", t2, re.I): after_phone_label = True continue if after_phone_label and not phone: pm = re.search(r"\b(\d{3}-\d{3}-\d{4})\b", t2) if pm: phone = pm.group(1) continue if re.fullmatch(r"\d\s*(?:½|1/2)(?:\s*pi[èe]ces?)?|\d\s*pi[èe]ces", t2): nt = normalize_unit_type(t2) or t2 if nt not in unit_types: unit_types.append(nt) elif len(parts) < 3 and not re.search( r"Composition|Logements disponibles|sous-sol|Étage", t2, re.I): parts.append(t2) address = ", ".join(parts).strip(" ,") if "saint-sacrement" in address.lower(): # siège social, pas un immeuble continue owner = None for np, n in name_events: if np < p: owner = n if owner is None and len(complexes) == 1: owner = complexes[0][0] if owner and owner not in info_blocks: info_blocks[owner] = (address, unit_types, phone) # position de la section détaillée de chaque complexe (dernière # occurrence exacte du nom, sinon premier segment qui le contient) section_pos: dict[str, int | None] = {} for name, _, _ in complexes: name_flat = _slug(name) name_pos = None for p, n in name_events: if n == name: name_pos = p if name_pos is None: for p, t in segments: if name_flat in _slug(t): name_pos = p break section_pos[name] = name_pos results = [] for name, sector, city in complexes: name_flat = _slug(name) name_pos = section_pos[name] # fin de la section = début de la section détaillée suivante starts = sorted(p for p in section_pos.values() if p is not None and name_pos is not None and p > name_pos) section_end = starts[0] if starts else len(html) # images dont l'attribut alt commence par le nom du complexe images: list[str] = [] for _, tag in img_tags: alt_m = re.search(r'alt="([^"]*)"', tag) if not alt_m: continue alt = htmllib.unescape(alt_m.group(1)) if not _slug(alt).startswith(name_flat): continue src_m = re.search(r'src="(https://static\.wixstatic\.com/media/[^"]+)"', tag) if src_m: u = _wix_image(src_m.group(1)) if u not in images: images.append(u) # description = premier long paragraphe de la section du complexe description = "" if name_pos is not None: for p, t in segments: if name_pos < p < section_end and len(t) > 120 \ and not t.startswith("*") and "veuillez svp" not in t: description = t break address, unit_types, phone = info_blocks.get(name, ("", [], "")) # services disponibles (liste de la section du complexe) start = name_pos if name_pos is not None else 0 amenities = self._collect_list( segments, start, section_end, r"^services disponibles", r"^(à moins de|pourquoi|n'h[ée]sitez|si vous|acc[èe]s facile" r"|gr[âa]ce [àa]|press to zoom|contactez|\d/\d)") # commerces/services « à moins de 5 minutes à pied » -> description proximity = self._collect_list( segments, start, section_end, r"^([àa] moins de 5 minutes|acc[èe]s facile)", r"^(press to zoom|n'h[ée]sitez|gr[âa]ce [àa]|pourquoi" r"|contactez|la corporation|\d/\d)") if unit_types: comp = "Composition de l'immeuble : " + ", ".join(unit_types) + "." description = (description + " " + comp).strip() if description else comp if proximity: prox = "À moins de 5 minutes à pied : " + ", ".join(proximity) + "." description = (description + " " + prox).strip() if description else prox details: dict = {} if phone: details["contact"] = {"phone": phone} results.append(Listing( source=self.source_id, external_id=_slug(name), url=url, title=name, address=address, sector=sector, city=city, unit_type=unit_types[0] if len(unit_types) == 1 else "", availability="Sur demande (contacter l'agent de location)", description=description[:800], amenities=amenities, details=details, images=images[:20], )) return results # -- liste à puces après un libellé (bornée à la section du complexe) -------- @staticmethod def _collect_list(segments: list[tuple[int, str]], start: int, end: int, head_re: str, break_re: str, max_len: int = 110) -> list[str]: head_rx = re.compile(head_re, re.I) break_rx = re.compile(break_re, re.I) items: list[str] = [] for i, (p, t) in enumerate(segments): if not (start <= p < end) or not head_rx.match(t.lower()): continue for p2, t2 in segments[i + 1:i + 30]: if p2 >= end or break_rx.match(t2.lower()): break # phrases d'introduction / libellés : ignorés, pas des items if not (3 <= len(t2) <= max_len) or t2.endswith(":") \ or re.search(r"appelez-nous|t[ée]l[ée]phoner", t2, re.I): continue if t2 not in items: items.append(t2) break return items