spb/sorti-ka

1# -----------------------------------------------------------------------------2# Sorti-Ka — Agrégateur de sorties & événements (province de Québec)3# Auteur : Simon-Pierre Boucher — contact@spboucher.ai4# connectors/ticketmaster.py : connecteur Ticketmaster Canada (Québec) — SANS clé5#   Mode principal (sans clé) : scraping léger des pages découverte publiques6#     www.ticketmaster.ca/discover/<ville>?page=N (montreal, quebec-city,7#     gatineau). Chaque page embarque son état Next.js (__NEXT_DATA__) : la8#     requête serveur `cityEvents` y est déjà résolue avec 20 événements/page9#     (titre, date UTC + fuseau, salle géolocalisée lat/lng, adresse, code10#     postal, ville, URL billetterie, catégorie majeure, seatmap). Pagination11#     par ?page=N (vérifié 2026-08-18 : montreal total=1165, quebec-city=136).12#     Aucun prix publié dans ce flux → is_free/price_* inconnus (jamais13#     inventés). Artistes : patrons sûrs des titres (artists_from_title).14#     robots.txt : /discover/ non interdit (seuls /api/ismds/* le sont).15#     Anti-bot : GET direct d'abord ; si bloqué → Scrapfly ASP SANS render_js.16#     Plafonds : TM_MAX_EVENTS (défaut 360), TM_PAGES_PER_CITY (défaut 8).17#   Mode repli automatique (si TICKETMASTER_API_KEY apparaît dans .env) :18#     Discovery API officielle v2 — plus riche (priceRanges, attractions →19#     artistes, images 16:9). Les identifiants d'événements sont IDENTIQUES20#     dans les deux modes → aucun doublon lors d'un changement de mode.21# -----------------------------------------------------------------------------22from __future__ import annotations2324import json25import os2627from ..normalize import artists_from_title, utc_to_local28from ..schema import Event, normalize_status29from .base import BaseConnector3031# --- mode API officielle (avec clé) ------------------------------------------32API = "https://app.ticketmaster.com/discovery/v2/events.json"33PAGE_SIZE = 20034DEEP_PAGE_LIMIT = 1000        # l'API refuse page*size >= 1000 (pagination profonde)3536# --- mode scraping (sans clé) -------------------------------------------------37DISCOVER = "https://www.ticketmaster.ca/discover/{city}?page={page}"38CITIES = ("montreal", "quebec-city", "gatineau")   # villes découverte QC publiées39TM_MAX_EVENTS = int(os.environ.get("TM_MAX_EVENTS", "360"))40TM_PAGES_PER_CITY = int(os.environ.get("TM_PAGES_PER_CITY", "8"))41SCRAPE_PAGE_SIZE = 20         # taille fixe du flux cityEvents des pages découverte4243# catégories majeures Ticketmaster (segments Discovery) → libellés source44_CAT_NAMES = {45    "KZFzniwnSyZfZ7v7nJ": "Concerts",46    "KZFzniwnSyZfZ7v7na": "Arts & Theatre",47    "KZFzniwnSyZfZ7v7nE": "Sports",48    "KZFzniwnSyZfZ7v7n1": "Family",49    "KZFzniwnSyZfZ7v7n7": "Film",50}515253def _best_image(images: list[dict]) -> str:54    """Plus grande image 16:9 (sinon la plus large disponible)."""55    if not images:56        return ""57    ranked = sorted(images, key=lambda i: ((i.get("ratio") == "16_9"),58                                           i.get("width") or 0), reverse=True)59    return ranked[0].get("url") or ""606162class TicketmasterConnector(BaseConnector):63    source_id = "ticketmaster"64    request_delay = 1.0           # politesse scraping ; l'API tolère 5 req/s6566    def __init__(self) -> None:67        super().__init__()68        self._force_scrapfly = False69        self.session.headers["Accept-Language"] = "fr-CA,fr;q=0.9,en;q=0.5"7071    # ==========================================================================72    # Mode scraping SANS clé (pages découverte publiques)73    # ==========================================================================74    def _discover_html(self, url: str) -> str:75        """HTML d'une page découverte : GET direct, Scrapfly ASP en secours."""76        if not self._force_scrapfly:77            try:78                html = self.get(url).text79                if "__NEXT_DATA__" in html:80                    return html81            except Exception:82                pass83            if not os.environ.get("SCRAPFLY_API_KEY"):84                raise RuntimeError("GET direct bloqué et SCRAPFLY_API_KEY absent")85            self._force_scrapfly = True86            print("[sorti-ka] ticketmaster : GET direct bloqué → "87                  "Scrapfly ASP (sans render_js)")88        return self.get_rendered(url, render_js=False)8990    @staticmethod91    def _next_data(html: str) -> dict:92        i = html.find("__NEXT_DATA__")93        if i < 0:94            raise ValueError("__NEXT_DATA__ introuvable (anti-bot ?)")95        start = html.find(">", i) + 196        data, _ = json.JSONDecoder().raw_decode(html[start:].lstrip())97        return data9899    def _city_page(self, city: str, page: int) -> tuple[list[dict], int]:100        """Événements (hits) + total annoncé d'une page découverte."""101        data = self._next_data(self._discover_html(102            DISCOVER.format(city=city, page=page)))103        queries = (((data.get("props") or {}).get("pageProps") or {})104                   .get("initialReduxState") or {}).get("api", {}).get("queries", {})105        hits: list[dict] = []106        total = 0107        for key, q in queries.items():108            if key.startswith("cityEvents(") and f'"page":{page}' in key:109                d = q.get("data") or {}110                hits.extend(d.get("events") or [])111                total = max(total, int(d.get("total") or 0))112        return hits, total113114    def _scrape_hit_to_event(self, hit: dict) -> Event | None:115        ext_id = hit.get("id") or ""116        title = hit.get("title") or ""117        if not ext_id or not title:118            return None119        if hit.get("virtual"):120            return None121        # Phase 2 : un événement annulé est CONSERVÉ avec status=cancelled122        # (bandeau côté frontend + eventStatus schema.org) au lieu d'être jeté.123        status = "cancelled" if hit.get("cancelled") else ""124        v = hit.get("venue") or {}125        dates = hit.get("dates") or {}126        # startDate publié en UTC + fuseau → date/heure locales Québec ;127        # heure retenue seulement si la source l'affiche (showDateTime, pas TBA)128        start_date, start_time = utc_to_local(dates.get("startDate"))129        if hit.get("tba") or dates.get("dateDisplay") != "showDateTime":130            start_time = None131        cat_id = (hit.get("majorCategory") or {}).get("id") or ""132        venue_name = v.get("name") or ""133        return Event(134            source=self.source_id,135            external_id=ext_id,136            url=hit.get("url") or "",137            title=title,138            raw_categories=[_CAT_NAMES[cat_id]] if cat_id in _CAT_NAMES else [],139            venue=venue_name,140            address=v.get("addressLineOne") or "",141            city=v.get("city") or "",142            postal_code=v.get("code") or "",143            lat=v.get("latitude"),144            lng=v.get("longitude"),145            start_date=start_date,146            start_time=start_time,147            end_date=start_date,148            status=status,149            artists=artists_from_title(title, venue_name),150            # seul visuel publié dans ce flux : le plan de salle officiel151            image=hit.get("seatmapUrl") or "",152        )153154    def _fetch_scrape(self) -> list[Event]:155        by_id: dict[str, Event] = {}156        for city in CITIES:157            for page in range(TM_PAGES_PER_CITY):158                if len(by_id) >= TM_MAX_EVENTS:159                    break160                try:161                    hits, total = self._city_page(city, page)162                except Exception as exc:163                    print(f"[sorti-ka] ticketmaster : {city} page {page} "164                          f"ignorée : {exc}")165                    break166                if not hits:167                    break168                for hit in hits:169                    ev = self._scrape_hit_to_event(hit)170                    if ev is not None:171                        by_id.setdefault(ev.external_id, ev)172                if total and (page + 1) * SCRAPE_PAGE_SIZE >= total:173                    break174            if len(by_id) >= TM_MAX_EVENTS:175                break176        return list(by_id.values())177178    # ==========================================================================179    # Mode API officielle (repli automatique dès qu'une clé est fournie)180    # ==========================================================================181    def _page(self, key: str, page: int) -> dict:182        return self.get_json(API, params={183            "apikey": key, "countryCode": "CA", "stateCode": "QC",184            "locale": "fr-ca,en", "size": PAGE_SIZE, "page": page,185            "sort": "date,asc"})186187    def _hit_to_event(self, hit: dict) -> Event | None:188        ext_id = hit.get("id") or ""189        title = hit.get("name") or ""190        if not ext_id or not title:191            return None192        emb = hit.get("_embedded") or {}193        venues = emb.get("venues") or [{}]194        v = venues[0]195        loc = v.get("location") or {}196        dates = (hit.get("dates") or {}).get("start") or {}197        end_obj = (hit.get("dates") or {}).get("end") or {}198        end = end_obj.get("localDate")199        # statut officiel Discovery : onsale/offsale/cancelled/postponed/rescheduled200        status = normalize_status(201            ((hit.get("dates") or {}).get("status") or {}).get("code"))202        # classifications → libellés source (segment + genre + sous-genre)203        cats: list[str] = []204        for c in hit.get("classifications") or []:205            for k in ("segment", "genre", "subGenre"):206                name = (c.get(k) or {}).get("name") or ""207                if name and name.lower() != "undefined" and name not in cats:208                    cats.append(name)209        # fourchette de prix officielle (si publiée par la billetterie)210        price_min, price_label = None, ""211        ranges = [r for r in hit.get("priceRanges") or []212                  if r.get("min") is not None]213        if ranges:214            lo = min(float(r["min"]) for r in ranges)215            hi = max(float(r.get("max") or r["min"]) for r in ranges)216            price_min = lo if lo > 0 else None217            price_label = (f"{lo:g} $ à {hi:g} $ (Ticketmaster)" if hi > lo218                           else f"{lo:g} $ (Ticketmaster)")219        artists = [a.get("name") for a in emb.get("attractions") or []220                   if a.get("name")]221        return Event(222            source=self.source_id,223            external_id=ext_id,224            url=hit.get("url") or "",225            title=title,226            description=hit.get("info") or hit.get("pleaseNote") or "",227            raw_categories=cats,228            venue=v.get("name") or "",229            address=((v.get("address") or {}).get("line1") or ""),230            city=((v.get("city") or {}).get("name") or ""),231            postal_code=v.get("postalCode") or "",232            lat=loc.get("latitude"),233            lng=loc.get("longitude"),234            start_date=dates.get("localDate"),235            start_time=(None if dates.get("timeTBA") or dates.get("noSpecificTime")236                        else dates.get("localTime")),237            end_date=end or dates.get("localDate"),238            end_time=end_obj.get("localTime"),239            status=status,240            artists=artists,241            price_min=price_min,242            price_label=price_label,243            is_free=(False if price_min else None),244            organizer=((hit.get("promoter") or {}).get("name") or ""),245            image=_best_image(hit.get("images") or []),246        )247248    def _fetch_api(self, key: str) -> list[Event]:249        self.request_delay = 0.25     # quota officiel : 5 req/s250        events: list[Event] = []251        page, total_pages = 0, 1252        while page < total_pages and (page + 1) * PAGE_SIZE <= DEEP_PAGE_LIMIT:253            data = self._page(key, page)254            total_pages = int((data.get("page") or {}).get("totalPages") or 0)255            for hit in (data.get("_embedded") or {}).get("events") or []:256                ev = self._hit_to_event(hit)257                if ev is not None:258                    events.append(ev)259            page += 1260        return events261262    # ==========================================================================263    def fetch(self) -> list[Event]:264        key = os.environ.get("TICKETMASTER_API_KEY", "").strip()265        if key:266            return self._fetch_api(key)      # mode riche dès qu'une clé existe267        return self._fetch_scrape()          # mode sans clé (par défaut)268