spb/sorti-ka

1# -----------------------------------------------------------------------------2# Sorti-Ka — Agrégateur de sorties & événements (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/signelaval.py : connecteur Signé Laval (signelaval.com)5#   Source    : calendrier culturel officiel de Laval (Culture Laval) —6#               bibliothèques, spectacles, expositions, plein air (~400 fiches).7#   Extraction: sitemap public /sitemap/events/N.xml puis fiches rendues8#               serveur : JSON-LD Event complet (CMS Footlight / artsdata.ca)9#               avec dates ISO horodatées (fuseau -04:00), lieu, adresse,10#               ville, image IIIF, organisateur. Certains champs Footlight11#               sont des LISTES multilingues (["…", {"@language": "fr",12#               "@value": "…"}]) → _val privilégie la variante française13#               (bug corrigé à la validation live 2026-08-25).14#               GET directs, pas d'anti-bot.15#               INCRÉMENTAL avec cache disque (data/signelaval_cache.json) :16#               nouvelles fiches d'abord, re-visite roulante après 7 jours,17#               plafonné à SIGNELAVAL_MAX_FETCH fiches/sync.18#   Accès     : robots.txt « User-agent: * / Disallow: (vide) » + sitemap19#               public → scraping permis, identification honnête.20#   Prix      : offers = AggregateOffer sans montant sur les fiches vérifiées21#               → is_free inconnu (jamais inventé).22# -----------------------------------------------------------------------------23from __future__ import annotations2425import json26import re27import time2829from ..normalize import clean_text, parse_time30from ..schema import Event31from .base import BaseConnector3233BASE = "https://signelaval.com"34SITEMAP_INDEX = f"{BASE}/sitemap.xml"3536MAX_FETCH = int(__import__("os").environ.get("SIGNELAVAL_MAX_FETCH", "200"))37REFRESH_AFTER = 7 * 86400     # re-visite roulante d'une fiche (secondes)3839_LOC_RE = re.compile(r"<loc>([^<]+)</loc>")40_LD_RE = re.compile(r'<script type="application/ld\+json"[^>]*>(.*?)</script>',41                    re.S)424344def _val(x) -> str:45    """Valeur d'un champ JSON-LD Footlight : littéral, {"@value": …} ou LISTE46    multilingue (ex. ["Rythme latin", {"@language": "fr", "@value": "…"}]) —47    on privilégie la variante française, sinon la première non vide."""48    if isinstance(x, list):49        fallback = ""50        for item in x:51            if (isinstance(item, dict)52                    and str(item.get("@language", "")).startswith("fr")):53                v = _val(item)54                if v:55                    return v56            fallback = fallback or _val(item)57        return fallback58    if isinstance(x, dict):59        return str(x.get("@value") or "")60    return str(x or "")616263class SigneLavalConnector(BaseConnector):64    source_id = "signelaval"65    request_delay = 0.86667    # -- sitemap ----------------------------------------------------------68    def _fiches(self) -> list[str]:69        idx = self.get(SITEMAP_INDEX).text70        shards = [u for u in _LOC_RE.findall(idx) if "/sitemap/events/" in u]71        urls: list[str] = []72        for shard in shards:73            urls += [u for u in _LOC_RE.findall(self.get(shard).text)74                     if "/evenements/" in u]75        return urls7677    # -- fiche ------------------------------------------------------------78    def _parse_fiche(self, url: str, html: str) -> dict | None:79        """JSON-LD Event (Footlight) → ligne normalisable, ou None."""80        data = None81        for blob in _LD_RE.findall(html):82            try:83                d = json.loads(blob)84            except ValueError:85                continue86            if d.get("@type") == "Event":87                data = d88                break89        if not data:90            return None91        title = clean_text(_val(data.get("name")))92        if not title:93            return None94        start = _val(data.get("startDate"))95        end = _val(data.get("endDate"))96        loc = data.get("location") or {}97        addr = loc.get("address") or {}98        img = data.get("image") or {}99        image = (img.get("http://schema.org/url")100                 or img.get("thumbnailUrl") or "") if isinstance(img, dict) else ""101        org = data.get("organizer") or {}102        # /fr/evenements/<id>/<slug> → id stable103        parts = url.rstrip("/").split("/")104        ext = parts[-2] if len(parts) >= 2 else parts[-1]105        return {106            "external_id": ext, "url": url, "title": title,107            "description": clean_text(_val(data.get("description")))[:2000],108            "raw_categories": [title],109            "venue": clean_text(_val(loc.get("name"))),110            "address": clean_text(_val(addr.get("streetAddress"))),111            "city": clean_text(_val(addr.get("addressLocality"))) or "Laval",112            "postal_code": _val(addr.get("postalCode")),113            "start_date": start[:10] if start else None,114            "start_time": parse_time(start),115            "end_date": end[:10] if end else None,116            "end_time": parse_time(end),117            "status": ("cancelled"118                       if "cancel" in str(data.get("eventStatus", "")).lower()119                       else ""),120            "organizer": clean_text(_val(org.get("name"))),121            "website": data.get("url") or "",122            "image": image,123        }124125    # -- pipeline ---------------------------------------------------------126    def fetch(self) -> list[Event]:127        cache = self.load_cache()128        urls = self._fiches()129        current = set(urls)130        cache = {u: c for u, c in cache.items() if u in current}131132        now = time.time()133        candidates = sorted(134            (u for u in urls135             if u not in cache136             or now - cache[u].get("fetched_at", 0) > REFRESH_AFTER),137            key=lambda u: cache.get(u, {}).get("fetched_at", 0))[:MAX_FETCH]138139        def worker(url, session):140            return self._parse_fiche(url, session.get(url, timeout=20).text)141142        for url, row in self.fetch_many(candidates, worker, max_workers=4).items():143            cache[url] = {"fetched_at": now, "row": row}144        self.save_cache(cache)145146        events: list[Event] = []147        seen: set[str] = set()148        for entry in cache.values():149            row = entry.get("row")150            if not row or row["external_id"] in seen:151                continue152            seen.add(row["external_id"])153            events.append(Event(154                source=self.source_id,155                external_id=row["external_id"],156                url=row["url"], title=row["title"],157                description=row.get("description", ""),158                raw_categories=row.get("raw_categories") or [],159                venue=row.get("venue", ""),160                address=row.get("address", ""),161                city=row.get("city", ""),162                postal_code=row.get("postal_code", ""),163                start_date=row.get("start_date"),164                start_time=row.get("start_time"),165                end_date=row.get("end_date"),166                end_time=row.get("end_time"),167                status=row.get("status", ""),168                organizer=row.get("organizer", ""),169                website=row.get("website", ""),170                image=row.get("image", ""),171            ))172        return events173