spb/crea-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   creaka/connectors/balados_rss.py4# Desc:   Connecteur ENRICHISSEMENT « balados-rss » — lecture directe des flux5#         RSS auto-déclarés des balados (AUCUNE clé requise). Palier 1 (§9).6#         Cap 150 flux/passage, re-visite 7 j, timeout court, fetch parallèle7#         poli (les flux vivent presque tous sur des hôtes différents).8# ==============================================================================9"""Enrichissement des balados par leur propre flux RSS.1011Le flux RSS d'un balado est publié PAR le créateur (auto-déclaré via Apple12Podcasts) : c'est la source la plus fraîche et la plus fiable sans clé.13On en extrait :1415- nb d'épisodes réels (``<item>``) + date du dernier épisode (``pubDate``)16  → détection dormant (>12 mois) bien plus juste que le releaseDate iTunes ;17- l'image du flux (``itunes:image``/``image/url``) → avatar si absent ;18- le lien officiel (``<link>``) → compte site-web (ou compte de plateforme19  si l'URL en est une), signal cross_link — le créateur le déclare lui-même ;20- la description du flux → bio si vide ;21- les catégories iTunes → niches §6.1.2223Substitut sans clé de l'API Podcast Index (connecteur `podcastindex`).24"""25from __future__ import annotations2627import re28from concurrent.futures import ThreadPoolExecutor, as_completed29from datetime import datetime, timedelta, timezone30from email.utils import parsedate_to_datetime31from xml.etree import ElementTree as ET3233import requests3435from ..identity import account36from ..normalize import map_niche, platform_from_url37from ..schema import Creator, now_iso38from .base import USER_AGENT, BaseConnector39from .balados_itunes import DORMANT_AFTER_DAYS, _GENRE_NICHE4041_ITUNES_NS = "{http://www.itunes.com/dtds/podcast-1.0.dtd}"42_MAX_BYTES = 8 * 1024 * 1024        # flux géants : on tronque (regex de repli)4344# hébergeurs de balados : leur page « show » n'est pas le site PERSONNEL du45# créateur → jamais transformée en compte site-web46_HOSTING_DOMAINS = (47    "anchor.fm", "podcasters.spotify.com", "feeds.feedburner.com",48    "buzzsprout.com", "podbean.com", "spreaker.com", "soundcloud.com",49    "audioboom.com", "transistor.fm", "simplecast.com", "libsyn.com",50    "megaphone.fm", "omny.fm", "acast.com", "captivate.fm", "podomatic.com",51    "rss.com", "pod.link", "podcastics.com", "ausha.co", "pippa.io",52    "audiomeans.fr", "wordpress.com", "blogspot.com", "squarespace.com",53    # portails PARTAGÉS entre plusieurs balados : un domaine commun ne doit54    # JAMAIS devenir un compte site-web (clé forte → risque de fusion §12.255    # entre deux créateurs différents)56    "baladoquebec.ca", "radio-canada.ca", "qub.ca", "telequebec.tv",57    "noovo.ca", "urbania.ca", "cogecomedia.com", "985fm.ca", "98.5fm.ca",58    # domaines des grandes plateformes : si platform_from_url ne reconnaît pas59    # le lien (page générique), ce n'est PAS un site personnel non plus60    "facebook.com", "instagram.com", "youtube.com", "tiktok.com", "x.com",61    "twitter.com", "spotify.com", "apple.com", "google.com", "linktr.ee",62    "beacons.ai", "patreon.com",63)646566def _parse_date(raw: str | None) -> datetime | None:67    """pubDate RFC-2822 (ou ISO en repli) → datetime UTC, sinon None."""68    if not raw:69        return None70    raw = raw.strip()71    try:72        dt = parsedate_to_datetime(raw)73    except (TypeError, ValueError):74        try:75            dt = datetime.fromisoformat(raw.replace("Z", "+00:00"))76        except ValueError:77            return None78    if dt.tzinfo is None:79        dt = dt.replace(tzinfo=timezone.utc)80    return dt.astimezone(timezone.utc)818283def parse_feed(raw: bytes) -> dict | None:84    """Flux RSS (bytes) → dict {episodes, last_episode, image, link,85    description, categories} ; None si illisible.8687    Voie principale : ElementTree. Repli (flux tronqué/malformé) : regex sur88    l'en-tête du canal + comptage des ``<item``.89    """90    try:91        root = ET.fromstring(raw)92    except ET.ParseError:93        return _parse_feed_regex(raw)94    channel = root.find("channel")95    if channel is None:                      # Atom ou format exotique96        return _parse_feed_regex(raw)97    items = channel.findall("item")98    last = None99    for it in items[:10]:                    # le plus récent est presque100        dt = _parse_date((it.findtext("pubDate") or "").strip())  # toujours en tête101        if dt and (last is None or dt > last):102            last = dt103    image = ""104    itunes_img = channel.find(f"{_ITUNES_NS}image")105    if itunes_img is not None:106        image = (itunes_img.get("href") or "").strip()107    if not image:108        image = (channel.findtext("image/url") or "").strip()109    cats: list[str] = []110    for cat in channel.iter(f"{_ITUNES_NS}category"):111        text = (cat.get("text") or "").strip()112        if text and text not in cats:113            cats.append(text)114    description = (channel.findtext("description")115                   or channel.findtext(f"{_ITUNES_NS}summary") or "").strip()116    # retirer un éventuel balisage HTML de la description117    description = re.sub(r"<[^>]+>", " ", description)118    description = re.sub(r"\s+", " ", description).strip()119    return {120        "episodes": len(items),121        "last_episode": last,122        "image": image,123        "link": (channel.findtext("link") or "").strip(),124        "description": description,125        "categories": cats,126    }127128129def _parse_feed_regex(raw: bytes) -> dict | None:130    """Repli tolérant : flux tronqué à _MAX_BYTES ou XML malformé."""131    text = raw.decode("utf-8", errors="replace")132    if "<channel" not in text and "<rss" not in text:133        return None134    head = text.split("<item", 1)[0]135    m_img = re.search(r'itunes:image[^>]*href="([^"]+)"', head)136    m_link = re.search(r"<link>\s*([^<\s]+)\s*</link>", head)137    m_desc = re.search(r"<description>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?</description>",138                       head, re.S)139    cats = re.findall(r'itunes:category[^>]*text="([^"]+)"', head)140    m_date = re.search(r"<pubDate>\s*(.*?)\s*</pubDate>", text)141    desc = re.sub(r"<[^>]+>", " ", m_desc.group(1)) if m_desc else ""142    return {143        "episodes": text.count("<item"),144        "last_episode": _parse_date(m_date.group(1)) if m_date else None,145        "image": m_img.group(1).strip() if m_img else "",146        "link": m_link.group(1).strip() if m_link else "",147        "description": re.sub(r"\s+", " ", desc).strip(),148        "categories": list(dict.fromkeys(cats)),149    }150151152def _needs_visit(metrics: dict, revisit_days: int) -> bool:153    checked = _parse_date(metrics.get("rss_checked"))154    if checked is None:155        return True156    return datetime.now(timezone.utc) - checked > timedelta(days=revisit_days)157158159def _site_domain(link: str) -> str:160    domain = link.split("://", 1)[-1].split("/", 1)[0].lower()161    return domain.removeprefix("www.")162163164class BaladosRssConnector(BaseConnector):165    source_id = "balados-rss"166    kind = "enrichment"167    request_delay = 0.2           # hôtes presque tous différents (poli quand même)168    timeout = 12                  # timeout court : un flux lent ne bloque pas le lot169    max_feeds = 150               # cap par passage (rotation complète en ~3 jours)170    revisit_days = 7              # cache : un flux déjà lu n'est relu qu'après 7 j171    workers = 6172173    def __init__(self) -> None:174        super().__init__()175        self.errors = 0176177    def _fetch_feed(self, url: str) -> dict | None:178        """Télécharge (borné à _MAX_BYTES) et parse un flux. None si échec."""179        resp = requests.get(url, timeout=self.timeout, stream=True,180                            headers={"User-Agent": USER_AGENT})181        resp.raise_for_status()182        chunks, size = [], 0183        for chunk in resp.iter_content(chunk_size=65536):184            chunks.append(chunk)185            size += len(chunk)186            if size >= _MAX_BYTES:187                break188        resp.close()189        return parse_feed(b"".join(chunks))190191    def enrich(self, creators: list[Creator]) -> list[Creator]:192        # candidats : comptes podcast avec feed_url, pas relus depuis 7 j ;193        # jamais lus d'abord (metrics.rss_checked absent)194        todo: list[tuple[Creator, object]] = []195        for cr in creators:196            for acc in cr.platforms:197                if acc.platform != "podcast" or not acc.metrics.get("feed_url"):198                    continue199                if _needs_visit(acc.metrics, self.revisit_days):200                    todo.append((cr, acc))201                break                          # un seul balado par fiche202        todo.sort(key=lambda t: bool(t[1].metrics.get("rss_checked")))203        todo = todo[:self.max_feeds]204205        results: dict[int, dict | None] = {}206        with ThreadPoolExecutor(max_workers=self.workers) as pool:207            futures = {pool.submit(self._fetch_feed,208                                   acc.metrics["feed_url"]): i209                       for i, (_, acc) in enumerate(todo)}210            for fut in as_completed(futures):211                try:212                    results[futures[fut]] = fut.result()213                except Exception:214                    self.errors += 1215                    results[futures[fut]] = None216217        enriched: list[Creator] = []218        now = datetime.now(timezone.utc)219        for i, (cr, acc) in enumerate(todo):220            feed = results.get(i)221            if feed is None:222                continue                       # échec réseau/parse : on réessaiera223            acc.metrics["rss_checked"] = now_iso()224            acc.last_checked = now_iso()225            if feed["episodes"]:226                acc.metrics["episodes"] = feed["episodes"]227            last = feed["last_episode"]228            if last is not None:229                acc.metrics["last_episode"] = last.strftime("%Y-%m-%dT%H:%M:%SZ")230                if now - last > timedelta(days=DORMANT_AFTER_DAYS):231                    acc.metrics["dormant"] = True232                    # dormant → inactive SEULEMENT si le balado est l'unique233                    # plateforme de la fiche (§13 ; réversible au retour)234                    if all(a.platform == "podcast" for a in cr.platforms):235                        cr.status = "inactive"236                else:237                    acc.metrics.pop("dormant", None)238            if feed["image"].startswith("http") and not cr.avatar_url:239                cr.avatar_url = feed["image"]240            if feed["description"] and len(cr.bio or "") < 40:241                cr.bio = feed["description"]242            niches = {map_niche(_GENRE_NICHE.get(c, c))243                      for c in feed["categories"]}244            niches.discard("autre")245            if niches:246                cr.niches = sorted(set(cr.niches or []) | niches)247            # <link> officiel auto-déclaré → compte (plateforme reconnue ou site-web)248            link = feed["link"]249            if link.startswith("http"):250                hit = platform_from_url(link)251                existing = {a.key for a in cr.platforms}252                if hit and f"{hit[0]}:{hit[1]}" not in existing:253                    cr.platforms.append(254                        account(hit[0], hit[1], "cross_link", url=link).finalize())255                elif not hit:256                    domain = _site_domain(link)257                    if (domain and "." in domain258                            and not any(h in link.lower() for h in _HOSTING_DOMAINS)259                            and f"site-web:{domain}" not in existing):260                        cr.platforms.append(261                            account("site-web", domain, "cross_link",262                                    url=link).finalize())263            enriched.append(cr)264        return enriched265