Phase 2 — connecteurs : heure+description La Vitrine, statuts, end_time, offres, taxonomies
- lavitrine (plus grosse source, 5 087 à venir) : capte l HEURE de la slide (« 15:00 HNE », start_time 0 % → 51 % dès le 1er lot de 1 100 fiches) et la DESCRIPTION (accroche courte + début du texte À propos, desc≥200 0 % → 50 %) ; re-visite versionnée v2 du cache (montée progressive, ~400 fiches/sync). - eventbrite : end_time (98,8 %), full_description>JSON-LD>summary, organisateur via le JSON-LD de la fiche déjà visitée pour le prix (0 % → 67 % et montant), statut is_cancelled CONSERVÉ (bandeau) + urgency_signals sold out → soldout. Les séries (series_id) ne sont PAS déroulées : 0 série observée sur les pages découverte sondées. - atuvu : og:description enfin affectée à l événement (payée en crédits Scrapfly, elle était jetée) — SAUF le gabarit marketing « trouvez des billets… » constaté sur fiche réelle, jamais publié ; backfill au fil des re-scrapes lastmod, aucun crédit supplémentaire. - evenko : supports (premières parties) ajoutés aux artistes, door_time et tour_name en description (« Tournée : … · Portes : … »), representation_status → statut (soldout/cancelled). - bandsintown : offers[] type Tickets → lien billets direct + « sold out » → statut soldout. - brossard : taxonomie type-event résolue (1 requête WP REST, vraies catégories) ; description INTÉGRALE (fin de la coupe à 600 caractères) ; heure de fin de la bannière « de 15 h 00 à 16 h 00 » et du JSON-LD biblio. - longueuil : image captée dans l état Nuxt (mediaImages front_*_16_9/21_9) ; heure de fin des cartes <time>. - laval : préfixe « ANNULÉ - » du titre → statut cancelled, titre nettoyé. - ticketmaster (repli sans clé) : les annulés sont conservés avec statut ; mode API (si TICKETMASTER_API_KEY fournie) : dates.status.code → statut, end localTime → end_time. - montreal : heure de FIN du JSON-LD des fiches (endDate), backfill progressif sous le même plafond MTL_IMG_MAX. - sherbrooke : heure de fin des périodes UTC → locale. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
12 changed files +354 −58
modified
sortika/connectors/atuvu.py
+12 −0
@@ -70,6 +70,16 @@ class AtuvuConnector(BaseConnector): | ||
| 70 | 70 | if not title: |
| 71 | 71 | return [] |
| 72 | 72 | image = (_OG_RE["image"].search(html) or [None, ""])[1] |
| 73 | + # og:description : déjà téléchargée (crédits Scrapfly) — Phase 2, on | |
| 74 | + # l'affecte enfin à l'événement (elle était compilée mais jetée)… | |
| 75 | + # SAUF quand c'est le gabarit marketing du site (« trouvez des billets | |
| 76 | + # à tarif réduit… »), constaté sur fiche réelle : on ne publie pas de | |
| 77 | + # texte boilerplate comme description d'événement. Le backfill se fait | |
| 78 | + # au fil des re-scrapes lastmod (aucun crédit Scrapfly ajouté). | |
| 79 | + description = clean_text((_OG_RE["description"].search(html) | |
| 80 | + or [None, ""])[1]) | |
| 81 | + if "trouvez des billets" in description.lower(): | |
| 82 | + description = "" | |
| 73 | 83 | cats = [clean_text(c) for c in _CAT_RE.findall(html)][:2] |
| 74 | 84 | price_m = _PRICE_RE.search(html) |
| 75 | 85 | price = float(price_m.group(1).replace(",", ".")) if price_m else None |
@@ -83,6 +93,7 @@ class AtuvuConnector(BaseConnector): | ||
| 83 | 93 | "external_id": f"{slug}-{day}", |
| 84 | 94 | "start_time": parse_time(date_txt), # « … - 20h00 » de la fiche |
| 85 | 95 | "url": url, "title": title, "image": image, |
| 96 | + "description": description, | |
| 86 | 97 | "raw_categories": cats, "venue": clean_text(venue), |
| 87 | 98 | "city": clean_text(city or ""), "start_date": day, |
| 88 | 99 | "end_date": day, "price_min": price, |
@@ -116,6 +127,7 @@ class AtuvuConnector(BaseConnector): | ||
| 116 | 127 | source=self.source_id, |
| 117 | 128 | external_id=row["external_id"], |
| 118 | 129 | url=row["url"], title=row["title"], |
| 130 | + description=row.get("description", ""), | |
| 119 | 131 | raw_categories=row.get("raw_categories") or [], |
| 120 | 132 | venue=row.get("venue", ""), city=row.get("city", ""), |
| 121 | 133 | start_date=row.get("start_date"), |
modified
sortika/connectors/bandsintown.py
+13 −1
@@ -78,9 +78,18 @@ class BandsintownConnector(BaseConnector): | ||
| 78 | 78 | continue |
| 79 | 79 | dt = e.get("datetime") or "" # heure locale de la salle |
| 80 | 80 | art = e.get("artist") or {} |
| 81 | + # offres billetterie : lien billets direct + statut de vente | |
| 82 | + # (type Tickets ; status "available" / "sold out" chez la source) | |
| 83 | + ticket_url, offer_status = "", "" | |
| 84 | + for off in e.get("offers") or []: | |
| 85 | + if (off.get("type") or "").lower() == "tickets" and off.get("url"): | |
| 86 | + ticket_url = off["url"] | |
| 87 | + offer_status = (off.get("status") or "").lower() | |
| 88 | + break | |
| 81 | 89 | rows.append({ |
| 82 | 90 | "id": str(e.get("id") or ""), |
| 83 | − "url": e.get("url") or "", | |
| 91 | + "url": ticket_url or e.get("url") or "", | |
| 92 | + "offer_status": offer_status, | |
| 84 | 93 | "title": e.get("title") or "", |
| 85 | 94 | "description": e.get("description") or "", |
| 86 | 95 | "artist_name": art.get("name") or artist, |
@@ -144,6 +153,9 @@ class BandsintownConnector(BaseConnector): | ||
| 144 | 153 | start_time=(None if row.get("time") == "00:00" |
| 145 | 154 | else row.get("time")), |
| 146 | 155 | end_date=row.get("date"), |
| 156 | + # statut de vente de l'offre (« sold out » explicite) | |
| 157 | + status=("soldout" if "sold" in (row.get("offer_status") or "") | |
| 158 | + else ""), | |
| 147 | 159 | artists=row.get("lineup") or [row["artist_name"]], |
| 148 | 160 | # `free` vaut false par défaut chez Bandsintown : seul un |
| 149 | 161 | # true explicite est significatif (on n'invente rien) |
modified
sortika/connectors/brossard.py
+31 −5
@@ -38,10 +38,13 @@ _LD_RE = re.compile(r'<script type="application/ld\+json"[^>]*>(.*?)</script>', | ||
| 38 | 38 | re.S) |
| 39 | 39 | _OG_IMG_RE = re.compile(r'property="og:image" content="([^"]*)"') |
| 40 | 40 | # « Cet événement aura lieu le 6 décembre 2026, de 15 h 00 à 16 h 00 » |
| 41 | +# « … aura lieu le 6 décembre 2026, à 15 h 00 » | |
| 41 | 42 | # « … aura lieu du 3 juillet 2026 au 5 juillet 2026 » |
| 42 | 43 | _BANNER_RE = re.compile( |
| 43 | 44 | r"aura lieu\s+(?:le\s+(?P<jour>\d{1,2}(?:er)?\s+\S+\s+\d{4})" |
| 44 | − r"(?:,?\s*(?:de|à)\s+(?P<heure>\d{1,2}\s*h\s*(?:\d{2})?))?" | |
| 45 | + r"(?:,?\s*de\s+(?P<heure>\d{1,2}\s*h\s*(?:\d{2})?)" | |
| 46 | + r"\s*(?:à|-|–)\s*(?P<fin>\d{1,2}\s*h\s*(?:\d{2})?)" | |
| 47 | + r"|,?\s*à\s+(?P<heure2>\d{1,2}\s*h\s*(?:\d{2})?))?" | |
| 45 | 48 | r"|du\s+(?P<du>\d{1,2}(?:er)?\s+\S+\s+\d{4})\s+au\s+" |
| 46 | 49 | r"(?P<au>\d{1,2}(?:er)?\s+\S+\s+\d{4}))", re.I) |
| 47 | 50 | |
@@ -58,7 +61,7 @@ class BrossardConnector(BaseConnector): | ||
| 58 | 61 | batch = self.get_json(API, params={ |
| 59 | 62 | "per_page": PAGE_SIZE, "page": page, "orderby": "modified", |
| 60 | 63 | "order": "desc", |
| 61 | − "_fields": "id,link,title,content,modified"}) | |
| 64 | + "_fields": "id,link,title,content,modified,type-event"}) | |
| 62 | 65 | except Exception: |
| 63 | 66 | if page > 1: # « rest_post_invalid_page_number » = fin |
| 64 | 67 | break |
@@ -71,6 +74,20 @@ class BrossardConnector(BaseConnector): | ||
| 71 | 74 | page += 1 |
| 72 | 75 | return out |
| 73 | 76 | |
| 77 | + # -- taxonomie type-event : id → nom (1 requête, ~11 termes) --------------- | |
| 78 | + def _type_event_map(self) -> dict[int, str]: | |
| 79 | + """Vraies catégories WordPress (« Bibliothèque — adultes », | |
| 80 | + « Événements de la Ville »…) — remplace l'heuristique par titre.""" | |
| 81 | + try: | |
| 82 | + terms = self.get_json( | |
| 83 | + "https://brossard.ca/wp-json/wp/v2/type-event", | |
| 84 | + params={"per_page": 100, "_fields": "id,name"}) | |
| 85 | + return {int(t["id"]): clean_text(t.get("name") or "") | |
| 86 | + for t in terms if t.get("id")} | |
| 87 | + except Exception as exc: # taxonomie indisponible ≠ source cassée | |
| 88 | + print(f"[sorti-ka] brossard : taxonomie type-event ignorée : {exc}") | |
| 89 | + return {} | |
| 90 | + | |
| 74 | 91 | # -- fiche : dates/heure/lieu (bannière ville OU JSON-LD biblio) ---------- |
| 75 | 92 | def _parse_fiche(self, html: str) -> dict: |
| 76 | 93 | info: dict = {} |
@@ -89,6 +106,7 @@ class BrossardConnector(BaseConnector): | ||
| 89 | 106 | info["start_date"] = start[:10] or None |
| 90 | 107 | info["end_date"] = end[:10] or None |
| 91 | 108 | info["time"] = parse_time(start) |
| 109 | + info["end_time"] = parse_time(end) | |
| 92 | 110 | loc = ld.get("location") or [] |
| 93 | 111 | loc = loc[0] if isinstance(loc, list) and loc else loc |
| 94 | 112 | if isinstance(loc, dict): |
@@ -105,7 +123,9 @@ class BrossardConnector(BaseConnector): | ||
| 105 | 123 | if m.group("jour"): |
| 106 | 124 | info["start_date"] = parse_date_fr(m.group("jour")) |
| 107 | 125 | info["end_date"] = info["start_date"] |
| 108 | − info["time"] = parse_time(m.group("heure") or "") | |
| 126 | + info["time"] = parse_time(m.group("heure") | |
| 127 | + or m.group("heure2") or "") | |
| 128 | + info["end_time"] = parse_time(m.group("fin") or "") | |
| 109 | 129 | else: |
| 110 | 130 | info["start_date"] = parse_date_fr(m.group("du")) |
| 111 | 131 | info["end_date"] = parse_date_fr(m.group("au")) |
@@ -140,6 +160,7 @@ class BrossardConnector(BaseConnector): | ||
| 140 | 160 | records = [r for r in self._list() |
| 141 | 161 | if r.get("link") and (r.get("title") or {}).get("rendered")] |
| 142 | 162 | cache = self._enrich(records) |
| 163 | + type_names = self._type_event_map() | |
| 143 | 164 | events: list[Event] = [] |
| 144 | 165 | for rec in records: |
| 145 | 166 | info = cache.get(rec["link"]) or {} |
@@ -148,14 +169,18 @@ class BrossardConnector(BaseConnector): | ||
| 148 | 169 | start = info.get("start_date") or parse_date_fr(title) |
| 149 | 170 | if not start: |
| 150 | 171 | continue # fiche pas encore visitée et titre sans date |
| 172 | + cats = [type_names[t] for t in rec.get("type-event") or [] | |
| 173 | + if t in type_names] | |
| 151 | 174 | events.append(Event( |
| 152 | 175 | source=self.source_id, |
| 153 | 176 | external_id=str(rec["id"]), |
| 154 | 177 | url=rec["link"], |
| 155 | 178 | title=title, |
| 179 | + # description intégrale (la coupe à 600 caractères de la | |
| 180 | + # vague 1 amputait 90 % des fiches — Phase 2) | |
| 156 | 181 | description=clean_text( |
| 157 | − (rec.get("content") or {}).get("rendered") or "")[:600], | |
| 158 | − raw_categories=[title], | |
| 182 | + (rec.get("content") or {}).get("rendered") or ""), | |
| 183 | + raw_categories=cats or [title], | |
| 159 | 184 | venue=info.get("venue", ""), |
| 160 | 185 | address=info.get("address", ""), |
| 161 | 186 | postal_code=info.get("postal_code", ""), |
@@ -164,6 +189,7 @@ class BrossardConnector(BaseConnector): | ||
| 164 | 189 | start_date=start, |
| 165 | 190 | start_time=info.get("time"), |
| 166 | 191 | end_date=info.get("end_date") or start, |
| 192 | + end_time=info.get("end_time"), | |
| 167 | 193 | organizer="Ville de Brossard", |
| 168 | 194 | image=info.get("image", ""), |
| 169 | 195 | )) |
modified
sortika/connectors/evenko.py
+30 −5
@@ -22,14 +22,15 @@ import base64 | ||
| 22 | 22 | import json |
| 23 | 23 | from datetime import datetime, timezone |
| 24 | 24 | |
| 25 | −from ..schema import Event | |
| 25 | +from ..schema import Event, normalize_status | |
| 26 | 26 | from .base import BaseConnector |
| 27 | 27 | |
| 28 | 28 | API = "https://evenko.ca/api/search" |
| 29 | 29 | PAGE_SIZE = 100 |
| 30 | 30 | |
| 31 | −ATTRS = ["objectID", "title", "subtitle", "headliners", "venue", "show_date", | |
| 32 | − "show_time", "hide_show_time", "category", "genre", "description", | |
| 31 | +ATTRS = ["objectID", "title", "subtitle", "headliners", "supports", "venue", | |
| 32 | + "show_date", "show_time", "hide_show_time", "door_time", "tour_name", | |
| 33 | + "category", "genre", "description", | |
| 33 | 34 | "free", "age", "thumbnail", "status", "additional_information", |
| 34 | 35 | "presented_by", "event_tag", "promotional_tag"] |
| 35 | 36 | |
@@ -110,8 +111,31 @@ class EvenkoConnector(BaseConnector): | ||
| 110 | 111 | # explicitement de la cacher (hide_show_time) |
| 111 | 112 | start_time = (None if hit.get("hide_show_time") |
| 112 | 113 | else _unix_to_local_time(hit.get("show_time"))) |
| 113 | − # artistes : champ explicite `headliners` de la source | |
| 114 | + # artistes : têtes d'affiche (`headliners`) + premières | |
| 115 | + # parties (`supports`) — champs explicites de la source | |
| 114 | 116 | artists = [str(a) for a in hit.get("headliners") or [] if a] |
| 117 | + artists += [str(a) for a in hit.get("supports") or [] if a] | |
| 118 | + # statut billetterie : additional_information.representation_status | |
| 119 | + # (['buy_now'], ['sold_out'], ['cancelled']…) — jamais inventé | |
| 120 | + rep = (hit.get("additional_information") or {}) \ | |
| 121 | + .get("representation_status") or [] | |
| 122 | + status = "" | |
| 123 | + for token in (rep if isinstance(rep, list) else [rep]): | |
| 124 | + status = normalize_status(str(token)) | |
| 125 | + if status: | |
| 126 | + break | |
| 127 | + # description enrichie : tournée + heure des portes (factuels, | |
| 128 | + # publiés par la source ; aucun champ dédié au schéma) | |
| 129 | + description = hit.get("description") or "" | |
| 130 | + extras = [] | |
| 131 | + if hit.get("tour_name"): | |
| 132 | + extras.append(f"Tournée : {str(hit['tour_name']).strip()}") | |
| 133 | + door = _unix_to_local_time(hit.get("door_time")) | |
| 134 | + if door: | |
| 135 | + extras.append(f"Portes : {door.replace(':', ' h ')}") | |
| 136 | + if extras: | |
| 137 | + description = " — ".join( | |
| 138 | + p for p in (description, " · ".join(extras)) if p) | |
| 115 | 139 | # gratuité : champ `free` (aujourd'hui toujours null côté |
| 116 | 140 | # source, gardé au cas où il serait re-rempli) + étiquettes |
| 117 | 141 | # explicites « gratuit » (event_tag / promotional_tag). |
@@ -124,7 +148,7 @@ class EvenkoConnector(BaseConnector): | ||
| 124 | 148 | external_id=ext_id, |
| 125 | 149 | url=_ticket_url(hit) or (venue.get("website") or "https://evenko.ca/fr/calendrier"), |
| 126 | 150 | title=title, |
| 127 | − description=hit.get("description") or "", | |
| 151 | + description=description, | |
| 128 | 152 | raw_categories=[c for c in cats if c] or ["Concert"], |
| 129 | 153 | audience={"everyone": "Pour tous", "all_ages": "Pour tous", |
| 130 | 154 | "18+": "18 ans et plus", "16+": "16 ans et plus", |
@@ -134,6 +158,7 @@ class EvenkoConnector(BaseConnector): | ||
| 134 | 158 | start_date=day, |
| 135 | 159 | start_time=start_time, |
| 136 | 160 | end_date=day, |
| 161 | + status=status, | |
| 137 | 162 | artists=artists, |
| 138 | 163 | is_free=is_free, |
| 139 | 164 | organizer="evenko", |
modified
sortika/connectors/eventbrite.py
+54 −20
@@ -58,8 +58,10 @@ def _server_data(html: str) -> dict: | ||
| 58 | 58 | return data |
| 59 | 59 | |
| 60 | 60 | |
| 61 | −def _parse_offers(html: str) -> dict | None: | |
| 62 | − """AggregateOffer (lowPrice/highPrice/priceCurrency) du JSON-LD d'une fiche.""" | |
| 61 | +def _parse_detail(html: str) -> dict: | |
| 62 | + """JSON-LD d'une fiche : AggregateOffer (lowPrice/highPrice/priceCurrency) | |
| 63 | + + organisateur (organizer.name) + description complète — même requête.""" | |
| 64 | + price, organizer, description = None, "", "" | |
| 63 | 65 | for m in _LD_JSON_RE.finditer(html): |
| 64 | 66 | try: |
| 65 | 67 | d, _ = json.JSONDecoder().raw_decode(html[m.end():].lstrip()) |
@@ -68,8 +70,14 @@ def _parse_offers(html: str) -> dict | None: | ||
| 68 | 70 | for doc in (d if isinstance(d, list) else [d]): |
| 69 | 71 | if not isinstance(doc, dict): |
| 70 | 72 | continue |
| 73 | + org = doc.get("organizer") | |
| 74 | + if isinstance(org, dict) and org.get("name") and not organizer: | |
| 75 | + organizer = str(org["name"]) | |
| 76 | + desc = doc.get("description") | |
| 77 | + if isinstance(desc, str) and len(desc) > len(description): | |
| 78 | + description = desc[:2000] | |
| 71 | 79 | offers = doc.get("offers") |
| 72 | − if not offers: | |
| 80 | + if not offers or price is not None: | |
| 73 | 81 | continue |
| 74 | 82 | for off in (offers if isinstance(offers, list) else [offers]): |
| 75 | 83 | if not isinstance(off, dict) or off.get("@type") != "AggregateOffer": |
@@ -79,9 +87,10 @@ def _parse_offers(html: str) -> dict | None: | ||
| 79 | 87 | high = float(off.get("highPrice") or off.get("lowPrice")) |
| 80 | 88 | except (TypeError, ValueError): |
| 81 | 89 | continue |
| 82 | − return {"low": low, "high": high, | |
| 83 | − "cur": off.get("priceCurrency") or ""} | |
| 84 | − return None | |
| 90 | + price = {"low": low, "high": high, | |
| 91 | + "cur": off.get("priceCurrency") or ""} | |
| 92 | + break | |
| 93 | + return {"price": price, "organizer": organizer, "description": description} | |
| 85 | 94 | |
| 86 | 95 | |
| 87 | 96 | class EventbriteConnector(BaseConnector): |
@@ -110,17 +119,34 @@ class EventbriteConnector(BaseConnector): | ||
| 110 | 119 | return results |
| 111 | 120 | |
| 112 | 121 | # -- fiche → Event --------------------------------------------------------- |
| 113 | − def _result_to_event(self, r: dict, price: dict | None) -> Event | None: | |
| 122 | + def _result_to_event(self, r: dict, detail: dict | None) -> Event | None: | |
| 114 | 123 | ext_id = str(r.get("id") or r.get("eventbrite_event_id") or "") |
| 115 | 124 | title = r.get("name") or "" |
| 116 | 125 | if not ext_id or not title: |
| 117 | 126 | return None |
| 118 | − if r.get("is_online_event") or r.get("is_cancelled"): | |
| 127 | + if r.get("is_online_event"): | |
| 119 | 128 | return None |
| 120 | 129 | venue = r.get("primary_venue") or {} |
| 121 | 130 | addr = venue.get("address") or {} |
| 122 | 131 | if (addr.get("region") or "") != "QC": # filtre Québec strict |
| 123 | 132 | return None |
| 133 | + # statut billetterie (Phase 2) : annulé (is_cancelled — l'événement est | |
| 134 | + # conservé et affiché barré/bandeau) ; complet via urgency_signals | |
| 135 | + # (catégorie/message « sold out » explicite seulement, jamais déduit). | |
| 136 | + status = "cancelled" if r.get("is_cancelled") else "" | |
| 137 | + if not status: | |
| 138 | + us = r.get("urgency_signals") or {} | |
| 139 | + signals = " ".join([*(us.get("messages") or []), | |
| 140 | + *(us.get("categories") or [])]).lower() | |
| 141 | + if "sold" in signals and "out" in signals: | |
| 142 | + status = "soldout" | |
| 143 | + price = (detail or {}).get("price") | |
| 144 | + organizer = (detail or {}).get("organizer") or "" | |
| 145 | + # description : full_description (vide dans le flux de liste, constaté | |
| 146 | + # 2026-08-19) > JSON-LD de la fiche détail > summary (140 caractères) | |
| 147 | + description = (r.get("full_description") | |
| 148 | + or (detail or {}).get("description") | |
| 149 | + or r.get("summary") or "") | |
| 124 | 150 | cats: list[str] = [] |
| 125 | 151 | for t in r.get("tags") or []: |
| 126 | 152 | if (t.get("prefix") or "").startswith("Eventbrite"): |
@@ -148,7 +174,7 @@ class EventbriteConnector(BaseConnector): | ||
| 148 | 174 | external_id=ext_id, |
| 149 | 175 | url=r.get("url") or "", |
| 150 | 176 | title=title, |
| 151 | − description=r.get("summary") or "", | |
| 177 | + description=description, | |
| 152 | 178 | raw_categories=cats, |
| 153 | 179 | venue=venue.get("name") or "", |
| 154 | 180 | address=addr.get("address_1") or "", |
@@ -159,16 +185,20 @@ class EventbriteConnector(BaseConnector): | ||
| 159 | 185 | start_date=r.get("start_date"), |
| 160 | 186 | start_time=r.get("start_time"), |
| 161 | 187 | end_date=r.get("end_date"), |
| 188 | + end_time=r.get("end_time"), | |
| 189 | + status=status, | |
| 162 | 190 | is_free=is_free, |
| 163 | 191 | price_min=price_min, |
| 164 | 192 | price_label=label, |
| 193 | + organizer=organizer, | |
| 165 | 194 | image=img, |
| 166 | 195 | ) |
| 167 | 196 | |
| 168 | − # -- enrichissement prix (incrémental, plafonné) --------------------------- | |
| 169 | − def _enrich_prices(self, rows: list[dict]) -> dict: | |
| 170 | − """Cache {event_id: {fetched_at, price|None}} ; visite les fiches à | |
| 171 | − venir sans prix connu d'abord, re-visite roulante après 7 jours.""" | |
| 197 | + # -- enrichissement prix + organisateur (incrémental, plafonné) ------------ | |
| 198 | + def _enrich_details(self, rows: list[dict]) -> dict: | |
| 199 | + """Cache {event_id: {fetched_at, price|None, organizer}} ; fiches | |
| 200 | + jamais visitées d'abord, puis celles d'avant la Phase 2 (sans clé | |
| 201 | + organizer), puis re-visite roulante après 7 jours.""" | |
| 172 | 202 | cache = self.load_cache() |
| 173 | 203 | now = time.time() |
| 174 | 204 | current = {str(r.get("id")): r for r in rows if r.get("id")} |
@@ -178,18 +208,22 @@ class EventbriteConnector(BaseConnector): | ||
| 178 | 208 | (eid for eid, r in current.items() |
| 179 | 209 | if r.get("url") |
| 180 | 210 | and (eid not in cache |
| 211 | + or "organizer" not in cache[eid] | |
| 212 | + or "description" not in cache[eid] | |
| 181 | 213 | or now - cache[eid].get("fetched_at", 0) > REFRESH_AFTER)), |
| 182 | 214 | key=lambda eid: cache.get(eid, {}).get("fetched_at", 0))[:EB_DETAIL_MAX] |
| 183 | 215 | url_to_id = {current[eid]["url"]: eid for eid in candidates} |
| 184 | 216 | |
| 185 | 217 | def worker(url, session): |
| 186 | − return _parse_offers(session.get(url, timeout=20).text) | |
| 218 | + return _parse_detail(session.get(url, timeout=20).text) | |
| 187 | 219 | |
| 188 | − for url, price in self.fetch_many(list(url_to_id), worker, | |
| 189 | − max_workers=4).items(): | |
| 190 | − cache[url_to_id[url]] = {"fetched_at": now, "price": price} | |
| 220 | + for url, detail in self.fetch_many(list(url_to_id), worker, | |
| 221 | + max_workers=4).items(): | |
| 222 | + entry = {"fetched_at": now} | (detail or { | |
| 223 | + "price": None, "organizer": "", "description": ""}) | |
| 224 | + cache[url_to_id[url]] = entry | |
| 191 | 225 | self.save_cache(cache) |
| 192 | − return {eid: (entry or {}).get("price") for eid, entry in cache.items()} | |
| 226 | + return cache | |
| 193 | 227 | |
| 194 | 228 | # -- interface -------------------------------------------------------------- |
| 195 | 229 | def fetch(self) -> list[Event]: |
@@ -200,10 +234,10 @@ class EventbriteConnector(BaseConnector): | ||
| 200 | 234 | eid = str(r.get("id") or "") |
| 201 | 235 | if eid: |
| 202 | 236 | uniq.setdefault(eid, r) |
| 203 | − prices = self._enrich_prices(list(uniq.values())) | |
| 237 | + details = self._enrich_details(list(uniq.values())) | |
| 204 | 238 | events: list[Event] = [] |
| 205 | 239 | for eid, r in uniq.items(): |
| 206 | − ev = self._result_to_event(r, prices.get(eid)) | |
| 240 | + ev = self._result_to_event(r, details.get(eid)) | |
| 207 | 241 | if ev is not None: |
| 208 | 242 | events.append(ev) |
| 209 | 243 | return events |
modified
sortika/connectors/laval.py
+11 −0
@@ -10,10 +10,15 @@ | ||
| 10 | 10 | # ----------------------------------------------------------------------------- |
| 11 | 11 | from __future__ import annotations |
| 12 | 12 | |
| 13 | +import re | |
| 14 | + | |
| 13 | 15 | from ..normalize import utc_to_local |
| 14 | 16 | from ..schema import Event |
| 15 | 17 | from .base import BaseConnector |
| 16 | 18 | |
| 19 | +# la Ville publie l'annulation DANS le titre : « ANNULÉ - Rencontre avec … » | |
| 20 | +_CANCELLED_RE = re.compile(r"^\s*annul[eé]e?\s*[-–—:]\s*", re.I) | |
| 21 | + | |
| 17 | 22 | FEED = ("https://www.donneesquebec.ca/recherche/dataset/" |
| 18 | 23 | "df7add74-a741-4374-b3bd-22da8921c8b3/resource/" |
| 19 | 24 | "b51a25de-bd06-4247-87ba-2b1ea8228005/download/calendrier-activites.json") |
@@ -34,6 +39,11 @@ class LavalConnector(BaseConnector): | ||
| 34 | 39 | page = (rec.get("PageUrl") or "").strip() |
| 35 | 40 | if not title or not page: |
| 36 | 41 | continue |
| 42 | + # statut : préfixe « ANNULÉ - » du titre → cancelled (titre nettoyé) | |
| 43 | + status = "" | |
| 44 | + if _CANCELLED_RE.match(title): | |
| 45 | + status = "cancelled" | |
| 46 | + title = _CANCELLED_RE.sub("", title).strip() | |
| 37 | 47 | image = rec.get("ImageUrl") or "" |
| 38 | 48 | # une activité récurrente = une entrée par occurrence, même PageUrl |
| 39 | 49 | # → l'id externe inclut la date pour ne pas écraser les occurrences |
@@ -60,6 +70,7 @@ class LavalConnector(BaseConnector): | ||
| 60 | 70 | start_date=start_local or rec.get("EventDate"), |
| 61 | 71 | start_time=time_local, |
| 62 | 72 | end_date=end_local or rec.get("EndDate"), |
| 73 | + status=status, | |
| 63 | 74 | organizer="Ville de Laval", |
| 64 | 75 | image=(SITE + image) if image.startswith("/") else image, |
| 65 | 76 | )) |
modified
sortika/connectors/lavitrine.py
+34 −9
@@ -6,8 +6,10 @@ | ||
| 6 | 6 | # concerts (Grand Montréal + Québec + tournées). |
| 7 | 7 | # Extraction: sitemap.xml public (~2 000 fiches /fr/evenement/ + 185 |
| 8 | 8 | # /fr/exposition/, lastmod par URL) puis pages fiche rendues |
| 9 | −# serveur (h1, bloc .event_calendar : jour/mois/année, salle, | |
| 10 | −# ville ; og:image). GET directs — pas d'anti-bot. | |
| 9 | +# serveur (h1, bloc .event_calendar : jour/mois/année, HEURE | |
| 10 | +# (« 15:00 HNE » dans la slide), salle, ville ; og:image ; | |
| 11 | +# description = event_tabs_description-short + about-description). | |
| 12 | +# GET directs — pas d'anti-bot. | |
| 11 | 13 | # INCRÉMENTAL avec cache disque (data/lavitrine_cache.json) : |
| 12 | 14 | # nouvelles fiches d'abord, re-visite roulante après 7 jours, |
| 13 | 15 | # plafonné à LV_MAX_FETCH fiches/sync. |
@@ -44,13 +46,19 @@ _DATE_RE = re.compile( | ||
| 44 | 46 | r'<div class="event_calendar_slide-calender-date-text-small">([^<]+)</div>' |
| 45 | 47 | r'<div class="event_calendar_slide-calender-date-text-small">(\d{4})</div>') |
| 46 | 48 | _PLACE_RE = re.compile(r'class="text-style-2lines">([^<]*)</div><div>([^<]*)</div>') |
| 49 | +# l'heure de la représentation, dans la même slide, entre la date et la salle : | |
| 50 | +# <div class="text-size-small text-height-1-2">15:00 HNE</div> | |
| 51 | +_TIME_RE = re.compile(r'class="text-size-small[^"]*">\s*(\d{1,2}[:h]\d{2})') | |
| 52 | +# description de la fiche : accroche courte + début du texte long (onglet À propos) | |
| 53 | +_DESC_SHORT_RE = re.compile(r'event_tabs_description-short">(.*?)</div>', re.S) | |
| 54 | +_DESC_LONG_RE = re.compile(r'id="about-description">(.*?)</div>', re.S) | |
| 47 | 55 | |
| 48 | 56 | |
| 49 | 57 | _SLIDE_A_RE = re.compile( |
| 50 | 58 | r'<a[^>]*href="([^"]*)"[^>]*class="event_calendar_slide-content(.*?)</a>', re.S) |
| 51 | 59 | |
| 52 | 60 | |
| 53 | −def _slides(html: str) -> list[tuple[str, str, str, str, str, str, str]]: | |
| 61 | +def _slides(html: str) -> list[tuple[str, str, str, str, str, str, str, str | None]]: | |
| 54 | 62 | """Slides du calendrier : (id de fiche de la slide, url canonique, jour, |
| 55 | 63 | mois, année, salle, ville). |
| 56 | 64 | |
@@ -71,8 +79,10 @@ def _slides(html: str) -> list[tuple[str, str, str, str, str, str, str]]: | ||
| 71 | 79 | if not d: |
| 72 | 80 | continue |
| 73 | 81 | p = _PLACE_RE.search(block) |
| 82 | + t = _TIME_RE.search(block) | |
| 74 | 83 | out.append((slide_id, canon, d.group(1), d.group(2), d.group(3), |
| 75 | − p.group(1) if p else "", p.group(2) if p else "")) | |
| 84 | + p.group(1) if p else "", p.group(2) if p else "", | |
| 85 | + t.group(1).replace("h", ":") if t else None)) | |
| 76 | 86 | return out |
| 77 | 87 | |
| 78 | 88 | |
@@ -99,10 +109,17 @@ class LaVitrineConnector(BaseConnector): | ||
| 99 | 109 | return [] |
| 100 | 110 | img = _OG_IMG_RE.search(html) |
| 101 | 111 | image = img.group(1) if img else "" |
| 112 | + # description : accroche courte de la fiche + début du texte long | |
| 113 | + # (déjà dans le HTML téléchargé — Phase 2, gisement n° 1 de l'audit) | |
| 114 | + d_short = _DESC_SHORT_RE.search(html) | |
| 115 | + d_long = _DESC_LONG_RE.search(html) | |
| 116 | + description = " ".join(p for p in ( | |
| 117 | + clean_text(d_short.group(1)) if d_short else "", | |
| 118 | + clean_text(d_long.group(1)) if d_long else "") if p)[:2000] | |
| 102 | 119 | ext = url.rstrip("/").rsplit("/", 1)[-1] # id numérique stable |
| 103 | 120 | slug = url.rstrip("/").split("/")[-2] |
| 104 | 121 | rows = [] |
| 105 | − for slide_id, canon, day, month_txt, year, venue, city in _slides(html)[:60]: | |
| 122 | + for slide_id, canon, day, month_txt, year, venue, city, hhmm in _slides(html)[:60]: | |
| 106 | 123 | mo = _MONTHS.get(clean_text(month_txt).lower()[:4].rstrip(".")) \ |
| 107 | 124 | or _MONTHS.get(clean_text(month_txt).lower()[:3]) |
| 108 | 125 | if not mo: |
@@ -113,9 +130,11 @@ class LaVitrineConnector(BaseConnector): | ||
| 113 | 130 | # même date vue depuis plusieurs fiches → même uid, zéro doublon |
| 114 | 131 | "external_id": f"{slide_id or ext}-{iso}", |
| 115 | 132 | "url": canon or url, "title": title, "image": image, |
| 133 | + "description": description, | |
| 116 | 134 | "raw_categories": [title, slug.replace("-", " ")], |
| 117 | 135 | "venue": clean_text(venue or ""), "city": clean_text(city or ""), |
| 118 | 136 | "start_date": iso, "end_date": iso, |
| 137 | + "start_time": hhmm, # heure de la slide (« 15:00 HNE ») | |
| 119 | 138 | }) |
| 120 | 139 | return rows |
| 121 | 140 | |
@@ -126,11 +145,15 @@ class LaVitrineConnector(BaseConnector): | ||
| 126 | 145 | cache = {u: c for u, c in cache.items() if u in current} |
| 127 | 146 | |
| 128 | 147 | now = time.time() |
| 129 | − # nouvelles fiches d'abord, puis les plus anciennes visites (roulant) | |
| 148 | + # nouvelles fiches d'abord, puis celles parsées avant la Phase 2 | |
| 149 | + # (v < 2 : sans heure ni description — montée progressive), puis les | |
| 150 | + # plus anciennes visites (roulant) | |
| 130 | 151 | candidates = sorted( |
| 131 | 152 | (u for u in urls |
| 132 | − if u not in cache or now - cache[u].get("fetched_at", 0) > REFRESH_AFTER), | |
| 133 | − key=lambda u: cache.get(u, {}).get("fetched_at", 0))[:LV_MAX_FETCH] | |
| 153 | + if u not in cache or cache[u].get("v", 1) < 2 | |
| 154 | + or now - cache[u].get("fetched_at", 0) > REFRESH_AFTER), | |
| 155 | + key=lambda u: (cache.get(u, {}).get("v", 1) if u in cache else 0, | |
| 156 | + cache.get(u, {}).get("fetched_at", 0)))[:LV_MAX_FETCH] | |
| 134 | 157 | |
| 135 | 158 | def worker(url, session): |
| 136 | 159 | return self._parse_fiche(url, session.get(url, timeout=20).text) |
@@ -139,7 +162,7 @@ class LaVitrineConnector(BaseConnector): | ||
| 139 | 162 | if rows is None: # fiche cassée : on ne bloque pas la source |
| 140 | 163 | print(f"[sorti-ka] lavitrine : fiche ignorée {url}") |
| 141 | 164 | continue |
| 142 | − cache[url] = {"fetched_at": now, "rows": rows} | |
| 165 | + cache[url] = {"fetched_at": now, "rows": rows, "v": 2} | |
| 143 | 166 | self.save_cache(cache) |
| 144 | 167 | |
| 145 | 168 | events: list[Event] = [] |
@@ -153,9 +176,11 @@ class LaVitrineConnector(BaseConnector): | ||
| 153 | 176 | source=self.source_id, |
| 154 | 177 | external_id=row["external_id"], |
| 155 | 178 | url=row["url"], title=row["title"], |
| 179 | + description=row.get("description", ""), | |
| 156 | 180 | raw_categories=row.get("raw_categories") or [], |
| 157 | 181 | venue=row.get("venue", ""), city=row.get("city", ""), |
| 158 | 182 | start_date=row.get("start_date"), |
| 183 | + start_time=row.get("start_time"), | |
| 159 | 184 | end_date=row.get("end_date"), |
| 160 | 185 | image=row.get("image", ""), |
| 161 | 186 | )) |
modified
sortika/connectors/longueuil.py
+23 −1
@@ -42,6 +42,14 @@ _EXCERPT_RE = re.compile(r"event__excerpt[^>]*>\s*(.*?)\s*</p>", re.S) | ||
| 42 | 42 | _ADDR_RE = re.compile(r"address_line1:\"([^\"]*)\"") |
| 43 | 43 | _POSTAL_RE = re.compile(r"postal_code:\"([^\"]*)\"") |
| 44 | 44 | _LOCALITY_RE = re.compile(r"locality:\"([^\"]*)\"") |
| 45 | +# visuel de la fiche : URL mediaImages du CMS dans l'état Nuxt (styles | |
| 46 | +# front_medium_16_9 / front_large_21_9), échappée / | |
| 47 | +_IMG_RES = ( | |
| 48 | + re.compile(r"\"(https:\\u002F\\u002Fcms\.longueuil\.quebec\\u002Fsites" | |
| 49 | + r"[^\"]*?front_medium_16_9[^\"]*?)\""), | |
| 50 | + re.compile(r"\"(https:\\u002F\\u002Fcms\.longueuil\.quebec\\u002Fsites" | |
| 51 | + r"[^\"]*?front_large_21_9[^\"]*?)\""), | |
| 52 | +) | |
| 45 | 53 | |
| 46 | 54 | |
| 47 | 55 | def _js_str(raw: str) -> str: |
@@ -89,6 +97,7 @@ class LongueuilConnector(BaseConnector): | ||
| 89 | 97 | to_fetch = sorted( |
| 90 | 98 | (u for u in current |
| 91 | 99 | if u not in cache |
| 100 | + or "image" not in cache[u] # entrées d'avant la Phase 2 | |
| 92 | 101 | or now - cache[u].get("fetched_at", 0) > REFRESH_AFTER), |
| 93 | 102 | key=lambda u: cache.get(u, {}).get("fetched_at", 0))[:LGL_FICHE_MAX] |
| 94 | 103 | |
@@ -97,9 +106,16 @@ class LongueuilConnector(BaseConnector): | ||
| 97 | 106 | addr = _ADDR_RE.search(html) |
| 98 | 107 | postal = _POSTAL_RE.search(html) |
| 99 | 108 | loc = _LOCALITY_RE.search(html) |
| 109 | + image = "" | |
| 110 | + for img_re in _IMG_RES: | |
| 111 | + m = img_re.search(html) | |
| 112 | + if m: | |
| 113 | + image = _js_str(m.group(1)) | |
| 114 | + break | |
| 100 | 115 | return {"address": _js_str(addr.group(1)) if addr else "", |
| 101 | 116 | "postal": _js_str(postal.group(1)) if postal else "", |
| 102 | − "city": _js_str(loc.group(1)) if loc else ""} | |
| 117 | + "city": _js_str(loc.group(1)) if loc else "", | |
| 118 | + "image": image} | |
| 103 | 119 | |
| 104 | 120 | for url, info in self.fetch_many(to_fetch, worker, max_workers=4).items(): |
| 105 | 121 | if info is not None: |
@@ -111,6 +127,7 @@ class LongueuilConnector(BaseConnector): | ||
| 111 | 127 | r["address"] = info.get("address", "") |
| 112 | 128 | r["postal"] = info.get("postal", "") |
| 113 | 129 | r["city"] = info.get("city") or "Longueuil" |
| 130 | + r["image"] = info.get("image", "") | |
| 114 | 131 | |
| 115 | 132 | def fetch(self) -> list[Event]: |
| 116 | 133 | rows = self._cards(self.get(LIST_URL).text) |
@@ -126,6 +143,9 @@ class LongueuilConnector(BaseConnector): | ||
| 126 | 143 | start_time = parse_time(r["start"]) |
| 127 | 144 | if start_time == "00:00": # minuit = « toute la journée » |
| 128 | 145 | start_time = None |
| 146 | + end_time = parse_time(r["end"]) if r["end"] != r["start"] else None | |
| 147 | + if end_time == "00:00": | |
| 148 | + end_time = None | |
| 129 | 149 | events.append(Event( |
| 130 | 150 | source=self.source_id, |
| 131 | 151 | external_id=r["external_id"], |
@@ -140,5 +160,7 @@ class LongueuilConnector(BaseConnector): | ||
| 140 | 160 | start_date=start_date, |
| 141 | 161 | start_time=start_time, |
| 142 | 162 | end_date=end_date or start_date, |
| 163 | + end_time=end_time, | |
| 164 | + image=r.get("image", ""), | |
| 143 | 165 | )) |
| 144 | 166 | return events |
modified
sortika/connectors/montreal.py
+13 −7
@@ -26,9 +26,10 @@ PAGE = 1000 | ||
| 26 | 26 | IMG_MAX = int(os.environ.get("MTL_IMG_MAX", "300")) |
| 27 | 27 | |
| 28 | 28 | _OG_IMG_RE = re.compile(r'property="og:image" content="([^"]*)"') |
| 29 | −# heure locale de la fiche montreal.ca (JSON-LD Event, offset Québec inclus) : | |
| 30 | −# "startDate":"2026-10-14T19:30:00-04:00" → "19:30" | |
| 29 | +# heures locales de la fiche montreal.ca (JSON-LD Event, offset Québec inclus) : | |
| 30 | +# "startDate":"2026-10-14T19:30:00-04:00" → "19:30" ; idem "endDate" → heure de fin | |
| 31 | 31 | _LD_TIME_RE = re.compile(r'"startDate":"\d{4}-\d{2}-\d{2}T(\d{2}:\d{2}):') |
| 32 | +_LD_END_TIME_RE = re.compile(r'"endDate":"\d{4}-\d{2}-\d{2}T(\d{2}:\d{2}):') | |
| 32 | 33 | |
| 33 | 34 | # Valeurs génériques du champ `emplacement` = mode de l'événement, PAS un nom |
| 34 | 35 | # de lieu (vérifié sur le datastore complet 2026-08 : seules ces 3 premières |
@@ -64,11 +65,12 @@ class MontrealConnector(BaseConnector): | ||
| 64 | 65 | to_fetch = [] |
| 65 | 66 | for ev in events: |
| 66 | 67 | entry = cache.get(ev.url) |
| 67 | − if isinstance(entry, dict): # déjà au format vague 2 | |
| 68 | − continue | |
| 68 | + if isinstance(entry, dict) and "end_time" in entry: | |
| 69 | + continue # déjà au format Phase 2 | |
| 69 | 70 | last_day = (ev.end_date or ev.start_date or "")[:10] |
| 70 | 71 | if last_day >= today and len(to_fetch) < IMG_MAX: |
| 71 | − # None = jamais visité (prioritaire) ; str = hérité (backfill) | |
| 72 | + # None = jamais visité (prioritaire) ; str/dict vague 2 = | |
| 73 | + # hérité (backfill de l'heure de fin, priorité basse) | |
| 72 | 74 | to_fetch.append((entry is not None, ev.url)) |
| 73 | 75 | to_fetch = [u for _, u in sorted(to_fetch, key=lambda t: t[0])][:IMG_MAX] |
| 74 | 76 | |
@@ -76,11 +78,13 @@ class MontrealConnector(BaseConnector): | ||
| 76 | 78 | html = session.get(url, timeout=15).text |
| 77 | 79 | img = _OG_IMG_RE.search(html) |
| 78 | 80 | tm = _LD_TIME_RE.search(html) |
| 81 | + end_tm = _LD_END_TIME_RE.search(html) | |
| 79 | 82 | return {"image": img.group(1) if img else "", |
| 80 | − "time": tm.group(1) if tm else None} | |
| 83 | + "time": tm.group(1) if tm else None, | |
| 84 | + "end_time": end_tm.group(1) if end_tm else None} | |
| 81 | 85 | |
| 82 | 86 | for url, info in self.fetch_many(to_fetch, worker, max_workers=6).items(): |
| 83 | − cache[url] = info or {"image": "", "time": None} | |
| 87 | + cache[url] = info or {"image": "", "time": None, "end_time": None} | |
| 84 | 88 | self.save_cache(cache) |
| 85 | 89 | for ev in events: |
| 86 | 90 | entry = cache.get(ev.url) |
@@ -92,6 +96,8 @@ class MontrealConnector(BaseConnector): | ||
| 92 | 96 | ev.image = entry.get("image") or "" |
| 93 | 97 | if entry.get("time") and entry["time"] != "00:00": |
| 94 | 98 | ev.start_time = entry["time"] |
| 99 | + if entry.get("end_time") and entry["end_time"] != "00:00": | |
| 100 | + ev.end_time = entry["end_time"] | |
| 95 | 101 | |
| 96 | 102 | def fetch(self) -> list[Event]: |
| 97 | 103 | events: list[Event] = [] |
modified
sortika/connectors/sherbrooke.py
+2 −1
@@ -67,7 +67,7 @@ class SherbrookeConnector(BaseConnector): | ||
| 67 | 67 | # les périodes sont publiées en UTC (« …T04:00:00Z » = minuit |
| 68 | 68 | # local, placeholder « toute la journée ») → date + heure locales |
| 69 | 69 | start, start_time = utc_to_local(periods[0].get("dateDebut")) |
| 70 | − end, _ = utc_to_local(periods[-1].get("dateFin")) | |
| 70 | + end, end_time = utc_to_local(periods[-1].get("dateFin")) | |
| 71 | 71 | |
| 72 | 72 | cats, audience = [], [] |
| 73 | 73 | c = rec.get("categories") or {} |
@@ -100,6 +100,7 @@ class SherbrookeConnector(BaseConnector): | ||
| 100 | 100 | start_date=start or None, |
| 101 | 101 | start_time=start_time, |
| 102 | 102 | end_date=end or start or None, |
| 103 | + end_time=end_time, | |
| 103 | 104 | is_free=(not payant) if isinstance(payant, bool) else None, |
| 104 | 105 | organizer="Ville de Sherbrooke", |
| 105 | 106 | image=image, |
modified
sortika/connectors/ticketmaster.py
+13 −3
@@ -25,7 +25,7 @@ import json | ||
| 25 | 25 | import os |
| 26 | 26 | |
| 27 | 27 | from ..normalize import artists_from_title, utc_to_local |
| 28 | −from ..schema import Event | |
| 28 | +from ..schema import Event, normalize_status | |
| 29 | 29 | from .base import BaseConnector |
| 30 | 30 | |
| 31 | 31 | # --- mode API officielle (avec clé) ------------------------------------------ |
@@ -116,8 +116,11 @@ class TicketmasterConnector(BaseConnector): | ||
| 116 | 116 | title = hit.get("title") or "" |
| 117 | 117 | if not ext_id or not title: |
| 118 | 118 | return None |
| 119 | − if hit.get("cancelled") or hit.get("virtual"): | |
| 119 | + if hit.get("virtual"): | |
| 120 | 120 | return None |
| 121 | + # Phase 2 : un événement annulé est CONSERVÉ avec status=cancelled | |
| 122 | + # (bandeau côté frontend + eventStatus schema.org) au lieu d'être jeté. | |
| 123 | + status = "cancelled" if hit.get("cancelled") else "" | |
| 121 | 124 | v = hit.get("venue") or {} |
| 122 | 125 | dates = hit.get("dates") or {} |
| 123 | 126 | # startDate publié en UTC + fuseau → date/heure locales Québec ; |
@@ -142,6 +145,7 @@ class TicketmasterConnector(BaseConnector): | ||
| 142 | 145 | start_date=start_date, |
| 143 | 146 | start_time=start_time, |
| 144 | 147 | end_date=start_date, |
| 148 | + status=status, | |
| 145 | 149 | artists=artists_from_title(title, venue_name), |
| 146 | 150 | # seul visuel publié dans ce flux : le plan de salle officiel |
| 147 | 151 | image=hit.get("seatmapUrl") or "", |
@@ -190,7 +194,11 @@ class TicketmasterConnector(BaseConnector): | ||
| 190 | 194 | v = venues[0] |
| 191 | 195 | loc = v.get("location") or {} |
| 192 | 196 | dates = (hit.get("dates") or {}).get("start") or {} |
| 193 | − end = ((hit.get("dates") or {}).get("end") or {}).get("localDate") | |
| 197 | + end_obj = (hit.get("dates") or {}).get("end") or {} | |
| 198 | + end = end_obj.get("localDate") | |
| 199 | + # statut officiel Discovery : onsale/offsale/cancelled/postponed/rescheduled | |
| 200 | + status = normalize_status( | |
| 201 | + ((hit.get("dates") or {}).get("status") or {}).get("code")) | |
| 194 | 202 | # classifications → libellés source (segment + genre + sous-genre) |
| 195 | 203 | cats: list[str] = [] |
| 196 | 204 | for c in hit.get("classifications") or []: |
@@ -227,6 +235,8 @@ class TicketmasterConnector(BaseConnector): | ||
| 227 | 235 | start_time=(None if dates.get("timeTBA") or dates.get("noSpecificTime") |
| 228 | 236 | else dates.get("localTime")), |
| 229 | 237 | end_date=end or dates.get("localDate"), |
| 238 | + end_time=end_obj.get("localTime"), | |
| 239 | + status=status, | |
| 230 | 240 | artists=artists, |
| 231 | 241 | price_min=price_min, |
| 232 | 242 | price_label=price_label, |
modified
tests/test_connectors.py
+118 −6
@@ -46,8 +46,15 @@ def test_evenko_parses_fixture(monkeypatch): | ||
| 46 | 46 | assert "musique" in ev.categories or ev.categories |
| 47 | 47 | # vague 2 : heure locale du show_time + artistes du champ headliners |
| 48 | 48 | deep = next(e for e in events if "Deep Purple" in e.title) |
| 49 | − assert deep.artists == ["Deep Purple"] | |
| 49 | + # Phase 2 : les premières parties (`supports`) s'ajoutent aux artistes | |
| 50 | + assert deep.artists == ["Deep Purple", "Kansas", "Jefferson Starship"] | |
| 50 | 51 | assert deep.start_time and len(deep.start_time) == 5 # "HH:MM" |
| 52 | + # Phase 2 : tournée + heure des portes dans la description (factuels) | |
| 53 | + myles = next(e for e in events if "Myles Smith" in e.title) | |
| 54 | + assert "Tournée : My Mess, My Heart, My Life Tour" in myles.description | |
| 55 | + assert "Portes :" in myles.description | |
| 56 | + # representation_status ['buy_now'] = prévu → statut vide | |
| 57 | + assert all(e.status == "" for e in events) | |
| 51 | 58 | |
| 52 | 59 | |
| 53 | 60 | def test_atuvu_parses_fixture(): |
@@ -86,6 +93,12 @@ def test_lavitrine_parses_fixture(): | ||
| 86 | 93 | assert rows[0]["venue"] == "Théâtre Petit Champlain" |
| 87 | 94 | assert rows[0]["city"] == "Québec" |
| 88 | 95 | assert rows[0]["external_id"].startswith("23777-") |
| 96 | + # Phase 2 : l'heure de la slide (« 15:00 HNE ») est enfin captée | |
| 97 | + assert rows[0]["start_time"] == "15:00" | |
| 98 | + # Phase 2 : description = accroche courte + début du texte À propos | |
| 99 | + assert rows[0]["description"].startswith( | |
| 100 | + "DE CHARLES TRENET À STROMAE") | |
| 101 | + assert len(rows[0]["description"]) >= 200 | |
| 89 | 102 | |
| 90 | 103 | |
| 91 | 104 | def test_lepointdevente_parses_fixture(): |
@@ -197,22 +210,25 @@ def test_ticketmaster_without_key_scrapes_discover(monkeypatch): | ||
| 197 | 210 | ev = events[0] |
| 198 | 211 | assert ev.external_id == "310064A20EB06FDE" |
| 199 | 212 | assert ev.start_date == "2026-08-18" and ev.start_time == "19:00" # UTC→locale |
| 200 | − assert ev.venue == "Le Balcon X Terrasse" and ev.city == "Montreal" | |
| 213 | + # Phase 2 : « Montreal » (graphie source) → nom canonique MAMH | |
| 214 | + assert ev.venue == "Le Balcon X Terrasse" and ev.city == "Montréal" | |
| 201 | 215 | assert ev.lat == 45.505438 and ev.region == "Montréal" |
| 202 | 216 | assert ev.categories # Concerts → taxonomie canonique |
| 203 | 217 | assert ev.is_free is None and ev.price_min is None # jamais inventé |
| 204 | 218 | |
| 205 | 219 | |
| 206 | −def test_ticketmaster_scrape_skips_cancelled_and_tba_time(): | |
| 220 | +def test_ticketmaster_scrape_keeps_cancelled_with_status_and_tba_time(): | |
| 221 | + """Phase 2 : un événement annulé est CONSERVÉ avec status=cancelled | |
| 222 | + (bandeau frontend + eventStatus schema.org), plus jeté.""" | |
| 207 | 223 | from sortika.connectors.ticketmaster import TicketmasterConnector |
| 208 | 224 | c = TicketmasterConnector() |
| 209 | − assert c._scrape_hit_to_event({"id": "x", "title": "T", | |
| 210 | − "cancelled": True}) is None | |
| 225 | + ev = c._scrape_hit_to_event({"id": "x", "title": "T", "cancelled": True}) | |
| 226 | + assert ev is not None and ev.status == "cancelled" | |
| 211 | 227 | ev = c._scrape_hit_to_event({ |
| 212 | 228 | "id": "y", "title": "Sans heure annoncée", |
| 213 | 229 | "dates": {"dateDisplay": "showDateOnly", |
| 214 | 230 | "startDate": "2026-09-01T04:00:00Z"}, "venue": {}}) |
| 215 | − assert ev is not None and ev.start_time is None | |
| 231 | + assert ev is not None and ev.start_time is None and ev.status == "" | |
| 216 | 232 | |
| 217 | 233 | |
| 218 | 234 | def test_ticketmaster_parses_hit(): |
@@ -275,6 +291,100 @@ def test_bandsintown_filters_quebec(monkeypatch): | ||
| 275 | 291 | assert r["lineup"] == ["Deep Purple"] |
| 276 | 292 | |
| 277 | 293 | |
| 294 | +def test_atuvu_boilerplate_description_not_published(): | |
| 295 | + """La og:description gabarit (« trouvez des billets à tarif réduit… ») | |
| 296 | + n'est jamais publiée comme description d'événement.""" | |
| 297 | + from sortika.connectors.atuvu import AtuvuConnector | |
| 298 | + html = (FIXTURES / "atuvu_fiche.html").read_text(encoding="utf-8") | |
| 299 | + rows = AtuvuConnector()._parse_fiche("https://atuvu.ca/billets-spectacle/arkansas", html) | |
| 300 | + assert rows and rows[0]["description"] == "" | |
| 301 | + | |
| 302 | + | |
| 303 | +def test_laval_cancelled_prefix_becomes_status(monkeypatch): | |
| 304 | + """« ANNULÉ - Titre » → status=cancelled, titre nettoyé (Phase 2).""" | |
| 305 | + from sortika.connectors.laval import LavalConnector | |
| 306 | + c = LavalConnector() | |
| 307 | + rec = {"Title": "ANNULÉ - Rencontre avec Kevin Raphaël", | |
| 308 | + "PageUrl": "/activites/rencontre.aspx", | |
| 309 | + "EventDate": "2026-10-01T22:30:00", "EndDate": "2026-10-01T23:30:00"} | |
| 310 | + monkeypatch.setattr(c, "get_json", lambda *a, **kw: [rec]) | |
| 311 | + events = [e.finalize() for e in c.fetch()] | |
| 312 | + assert len(events) == 1 | |
| 313 | + assert events[0].status == "cancelled" | |
| 314 | + assert events[0].title == "Rencontre avec Kevin Raphaël" | |
| 315 | + | |
| 316 | + | |
| 317 | +def test_eventbrite_result_to_event_end_time_status_organizer(): | |
| 318 | + """Champs Phase 2 du payload __SERVER_DATA__ réel : end_time, statut | |
| 319 | + (is_cancelled conservé), full_description, organisateur du JSON-LD.""" | |
| 320 | + from sortika.connectors.eventbrite import EventbriteConnector | |
| 321 | + r = {"id": "123", "name": "Atelier de poterie", | |
| 322 | + "url": "https://www.eventbrite.ca/e/atelier-123", | |
| 323 | + "summary": "court", "full_description": "Une description longue " * 20, | |
| 324 | + "start_date": "2026-09-12", "start_time": "19:00", | |
| 325 | + "end_date": "2026-09-12", "end_time": "21:30", | |
| 326 | + "is_cancelled": True, "urgency_signals": {"messages": [], "categories": []}, | |
| 327 | + "primary_venue": {"name": "Studio X", | |
| 328 | + "address": {"region": "QC", "city": "Montréal", | |
| 329 | + "address_1": "1 rue Test", | |
| 330 | + "postal_code": "H2X 1Y6", | |
| 331 | + "latitude": 45.5, "longitude": -73.6}}} | |
| 332 | + ev = EventbriteConnector()._result_to_event( | |
| 333 | + r, {"price": None, "organizer": "Les Ateliers MTL", | |
| 334 | + "description": ""}).finalize() | |
| 335 | + assert ev.end_time == "21:30" | |
| 336 | + assert ev.status == "cancelled" # conservé, plus jeté | |
| 337 | + assert ev.organizer == "Les Ateliers MTL" | |
| 338 | + assert len(ev.description) >= 200 # full_description > summary | |
| 339 | + # summary court + description JSON-LD de la fiche → la plus riche gagne | |
| 340 | + r2 = dict(r, full_description="") | |
| 341 | + ev2 = EventbriteConnector()._result_to_event( | |
| 342 | + r2, {"price": None, "organizer": "", | |
| 343 | + "description": "Description détaillée de la fiche. " * 10}) | |
| 344 | + assert len(ev2.description) >= 200 | |
| 345 | + | |
| 346 | + | |
| 347 | +def test_eventbrite_parse_detail_offers_and_organizer(): | |
| 348 | + from sortika.connectors.eventbrite import _parse_detail | |
| 349 | + html = ('<script type="application/ld+json">{"@type":"Event",' | |
| 350 | + '"organizer":{"@type":"Organization","name":"Prod QC"},' | |
| 351 | + '"offers":[{"@type":"AggregateOffer","lowPrice":"25.00",' | |
| 352 | + '"highPrice":"45.00","priceCurrency":"CAD"}]}</script>') | |
| 353 | + d = _parse_detail(html) | |
| 354 | + assert d["organizer"] == "Prod QC" | |
| 355 | + assert d["price"] == {"low": 25.0, "high": 45.0, "cur": "CAD"} | |
| 356 | + assert d["description"] == "" # clé toujours présente | |
| 357 | + | |
| 358 | + | |
| 359 | +def test_bandsintown_offers_ticket_url_and_soldout(monkeypatch): | |
| 360 | + from sortika.connectors.bandsintown import BandsintownConnector | |
| 361 | + c = BandsintownConnector() | |
| 362 | + payload = [{"id": "9", "url": "https://www.bandsintown.com/e/9", | |
| 363 | + "datetime": "2026-09-02T20:00:00", "title": "", | |
| 364 | + "artist": {"name": "X"}, "lineup": ["X"], | |
| 365 | + "offers": [{"type": "Tickets", "status": "sold out", | |
| 366 | + "url": "https://billets.example/x"}], | |
| 367 | + "venue": {"name": "Salle Y", "city": "Québec", | |
| 368 | + "country": "Canada", "region": "QC"}}] | |
| 369 | + monkeypatch.setattr(c, "get_json", lambda *a, **kw: payload) | |
| 370 | + rows = c._artist_events("X") | |
| 371 | + assert rows[0]["url"] == "https://billets.example/x" | |
| 372 | + assert rows[0]["offer_status"] == "sold out" | |
| 373 | + | |
| 374 | + | |
| 375 | +def test_longueuil_image_from_nuxt_state(): | |
| 376 | + from sortika.connectors.longueuil import _IMG_RES, _js_str | |
| 377 | + state = ('x:{bg:"https:\\u002F\\u002Fcms.longueuil.quebec\\u002Fsites' | |
| 378 | + '\\u002Fdefault\\u002Ffiles\\u002Fstyles\\u002Ffront_medium_16_9' | |
| 379 | + '\\u002Fpublic\\u002Fmedias\\u002Fimages\\u002F2022-03' | |
| 380 | + '\\u002Fcvl231121-04h_0.jpg?itok=GYMG8rcf"}') | |
| 381 | + m = _IMG_RES[0].search(state) | |
| 382 | + assert m | |
| 383 | + url = _js_str(m.group(1)) | |
| 384 | + assert url.startswith("https://cms.longueuil.quebec/sites/") | |
| 385 | + assert "front_medium_16_9" in url and url.endswith("itok=GYMG8rcf") | |
| 386 | + | |
| 387 | + | |
| 278 | 388 | def test_brossard_parses_fiche_banner_and_jsonld(): |
| 279 | 389 | from sortika.connectors.brossard import BrossardConnector |
| 280 | 390 | c = BrossardConnector() |
@@ -284,6 +394,7 @@ def test_brossard_parses_fiche_banner_and_jsonld(): | ||
| 284 | 394 | '<meta property="og:image" content="https://brossard.ca/i.jpg">') |
| 285 | 395 | info = c._parse_fiche(ville) |
| 286 | 396 | assert info["start_date"] == "2026-12-06" and info["time"] == "15:00" |
| 397 | + assert info["end_time"] == "16:00" # Phase 2 : « de 15 h 00 à 16 h 00 » | |
| 287 | 398 | assert info["image"] == "https://brossard.ca/i.jpg" |
| 288 | 399 | # JSON-LD Event des fiches biblio.brossard.ca (structure réelle, 2026-08) |
| 289 | 400 | biblio = ('<script type="application/ld+json">[{"@context":"http://schema.org",' |
@@ -297,5 +408,6 @@ def test_brossard_parses_fiche_banner_and_jsonld(): | ||
| 297 | 408 | info = c._parse_fiche(biblio) |
| 298 | 409 | assert info["start_date"] == "2026-11-27" and info["time"] == "13:30" |
| 299 | 410 | assert info["end_date"] == "2026-11-27" |
| 411 | + assert info["end_time"] == "15:00" # Phase 2 : endDate JSON-LD | |
| 300 | 412 | assert info["venue"] == "Salle d'animation" |
| 301 | 413 | assert info["postal_code"] == "J4X 2A4" |
| 302 | 414 | |