\s*' r'([^<]*)\s*(?:

([^<]*)

)?', re.S) _META_ITEM_RE = re.compile(r'([^<]+)\s*' r'\s*(?:]*>)?([^<]+)', re.S) class QuoiFaireConnector(BaseConnector): source_id = "quoifaire" request_delay = 0.8 # -- listes (événements à venir seulement) -------------------------------- def _city_links(self, city: str) -> list[str]: links: list[str] = [] last = 1 page = 1 while page <= min(last, MAX_PAGES): url = f"{BASE}/{city}/evenements" + (f"?page={page}" if page > 1 else "") html = self.get(url).text batch = [u for u in _LINK_RE.findall(html) if u not in links] if page > 1 and not batch: break # fin réelle de la pagination links += batch pages = [int(p) for p in _PAGE_RE.findall(html)] if pages: last = max(last, max(pages)) page += 1 return links # -- fiche ----------------------------------------------------------------- def _parse_fiche(self, url: str, html: str) -> dict | None: h1 = _H1_RE.search(html) title = clean_text(h1.group(1)) if h1 else "" if not title: return None date_m = _DATE_RE.search(html) start, end = parse_date_range_fr(date_m.group(1) if date_m else None) if not start: return None # fiche sans date sûre : inutilisable price_m = _PRICE_RE.search(html) time_m = _TIME_RE.search(html) region_m = _REGION_RE.search(html) or _REGION2_RE.search(html) lieu_m = _LIEU_RE.search(html) img_m = _OG_IMG_RE.search(html) maps_m = _MAPS_RE.search(html) summary_m = _SUMMARY_RE.search(html) venue, address, city = "", "", "" if lieu_m: venue = clean_text(lieu_m.group(1)) address = clean_text(lieu_m.group(2) or "") # « 250 Boul. …, Lairet, La Cité-Limoilou, Québec, Capitale- # Nationale, G1L 5A7, Canada » → la ville précède la région adm. parts = [p.strip() for p in address.split(",")] if len(parts) >= 4: city = parts[-4] audience, category = "", "" for label, value in _META_ITEM_RE.findall(html): k, v = clean_text(label), clean_text(value) if k == "Auditoire" and v.lower() != "non spécifié": audience = v elif k == "Catégorie": category = v return { "title": title, "description": clean_text(summary_m.group(1)) if summary_m else "", "raw_categories": [c for c in (category, title) if c], "venue": venue, "address": address, "city": city, "tourist_region": clean_text(region_m.group(1)) if region_m else "", "lat": maps_m.group(1) if maps_m else None, "lng": maps_m.group(2) if maps_m else None, "start_date": start, "end_date": end, "start_time": parse_time(time_m.group(1)) if time_m else None, "price_label": clean_text(price_m.group(1)) if price_m else "", "audience": audience, "image": img_m.group(1) if img_m else "", } # -- pipeline --------------------------------------------------------------- def fetch(self) -> list[Event]: urls: list[str] = [] for city in CITIES: try: urls += [u for u in self._city_links(city) if u not in urls] except Exception as exc: # section 502 : on ne bloque pas la source print(f"[sorti-ka] quoifaire : section '{city}' ignorée : {exc}") cache = self.load_cache() current = set(urls) cache = {u: c for u, c in cache.items() if u in current} now = time.time() candidates = sorted( (u for u in urls if u not in cache or now - cache[u].get("fetched_at", 0) > REFRESH_AFTER), key=lambda u: cache.get(u, {}).get("fetched_at", 0))[:DETAIL_MAX] def worker(url, session): return self._parse_fiche(url, session.get(url, timeout=20).text) for url, row in self.fetch_many(candidates, worker, max_workers=4).items(): cache[url] = {"fetched_at": now, "row": row} self.save_cache(cache) events: list[Event] = [] seen: set[str] = set() for url, entry in cache.items(): row = entry.get("row") if not row: continue slug = url.rstrip("/").rsplit("/", 1)[-1] if slug in seen: # même fiche listée dans deux sections villes continue seen.add(slug) events.append(Event( source=self.source_id, external_id=slug, url=url, title=row["title"], description=row.get("description", ""), raw_categories=row.get("raw_categories") or [], audience=row.get("audience", ""), venue=row.get("venue", ""), address=row.get("address", ""), city=row.get("city", ""), tourist_region=row.get("tourist_region", ""), lat=row.get("lat"), lng=row.get("lng"), start_date=row.get("start_date"), start_time=row.get("start_time"), end_date=row.get("end_date"), price_label=row.get("price_label", ""), image=row.get("image", ""), )) return events