spb/crea-ka

1# ==============================================================================2# Author: Simon-Pierre Boucher <contact@spboucher.ai>3# File:   creaka/normalize.py4# Desc:   Normalisation commune — handles, URLs canoniques par plateforme,5#         niches, régions, langue, taille d'audience (voir CLAUDE.md §6, §7, §11)6# ==============================================================================7"""Couche de normalisation partagée par tous les connecteurs.89Chaque connecteur remplit des champs bruts ; `schema.Creator.finalize()`10appelle ces fonctions pour produire une fiche canonique. Idempotent.11"""12from __future__ import annotations1314import re15import unicodedata1617# --- taxonomies (CLAUDE.md §6) ------------------------------------------------18NICHES = {19    "humour", "mode", "beaute", "lifestyle", "famille-parentalite", "cuisine",20    "gaming", "tech", "sport-fitness", "plein-air", "voyage", "musique", "arts",21    "danse", "education", "finance-affaires", "sante-mieux-etre", "bouffe-resto",22    "actualite-opinion", "autre",23}2425PLATFORMS = {26    "instagram", "tiktok", "youtube", "twitch", "kick", "x", "facebook",27    "snapchat", "substack", "patreon", "onlyfans", "mym", "fansly",28    "linkedin", "threads", "spotify", "discord", "podcast", "site-web", "autre",29}3031CREATOR_TYPES = {32    "influenceur", "youtubeur", "streamer", "podcasteur", "createur-tiktok",33    "createur-ecrit", "musicien", "humoriste", "autre",34}3536LANGUAGES = {"fr", "en", "bilingue"}3738REGIONS = {39    "Bas-Saint-Laurent", "Saguenay–Lac-Saint-Jean", "Capitale-Nationale",40    "Mauricie", "Estrie", "Montréal", "Outaouais", "Abitibi-Témiscamingue",41    "Côte-Nord", "Nord-du-Québec", "Gaspésie–Îles-de-la-Madeleine",42    "Chaudière-Appalaches", "Laval", "Lanaudière", "Laurentides",43    "Montérégie", "Centre-du-Québec",44}4546# --- URLs canoniques par plateforme (CLAUDE.md §11) ---------------------------47# gabarit : handle SANS @ ; None = pas de forme canonique (garder l'URL source)48_URL_TEMPLATES: dict[str, str | None] = {49    "instagram": "https://www.instagram.com/{h}/",50    "tiktok": "https://www.tiktok.com/@{h}",51    "youtube": "https://www.youtube.com/@{h}",52    "twitch": "https://www.twitch.tv/{h}",53    "kick": "https://kick.com/{h}",54    "x": "https://x.com/{h}",55    "facebook": "https://www.facebook.com/{h}",56    "snapchat": "https://www.snapchat.com/add/{h}",57    "substack": "https://{h}.substack.com",58    "patreon": "https://www.patreon.com/{h}",59    "onlyfans": "https://onlyfans.com/{h}",60    "mym": "https://mym.fans/{h}",61    "fansly": "https://fansly.com/{h}",62    "linkedin": "https://www.linkedin.com/in/{h}",63    "threads": "https://www.threads.net/@{h}",64    # IDs Spotify et codes d'invitation Discord : sensibles à la casse →65    # pas de forme canonique reconstruite, on conserve l'URL source66    "spotify": None,67    "discord": None,68    "podcast": None,69    "site-web": None,70    "autre": None,71}7273# détection de plateforme à partir d'une URL (pages link-in-bio, bios, etc.)74_URL_PLATFORM_RES: list[tuple[str, re.Pattern]] = [75    ("instagram", re.compile(r"instagram\.com/([\w.\-]+)", re.I)),76    ("tiktok", re.compile(r"tiktok\.com/@([\w.\-]+)", re.I)),77    ("youtube", re.compile(r"youtube\.com/(?:@|c/|user/|channel/)([\w.\-]+)", re.I)),78    ("youtube", re.compile(r"youtu\.be/([\w.\-]+)", re.I)),79    ("twitch", re.compile(r"twitch\.tv/([\w.\-]+)", re.I)),80    ("kick", re.compile(r"kick\.com/([\w.\-]+)", re.I)),81    ("x", re.compile(r"(?:twitter|x)\.com/([\w.\-]+)", re.I)),82    ("facebook", re.compile(r"facebook\.com/([\w.\-]+)", re.I)),83    ("snapchat", re.compile(r"snapchat\.com/add/([\w.\-]+)", re.I)),84    ("substack", re.compile(r"https?://([\w\-]+)\.substack\.com", re.I)),85    ("patreon", re.compile(r"patreon\.com/([\w.\-]+)", re.I)),86    ("onlyfans", re.compile(r"onlyfans\.com/([\w.\-]+)", re.I)),87    ("mym", re.compile(r"mym\.fans/([\w.\-]+)", re.I)),88    ("fansly", re.compile(r"fansly\.com/([\w.\-]+)", re.I)),89    ("linkedin", re.compile(r"linkedin\.com/in/([\w.\-]+)", re.I)),90    ("threads", re.compile(r"threads\.net/@?([\w.\-]+)", re.I)),91    ("spotify", re.compile(92        r"open\.spotify\.com/(?:intl-[\w\-]+/)?(?:artist|show|user)/([A-Za-z0-9]+)",93        re.I)),94    ("discord", re.compile(r"discord(?:\.gg|(?:app)?\.com/invite)/([\w\-]+)", re.I)),95    # une page Apple Podcasts pointe le MÊME balado que balados-itunes96    # (handle = collectionId) → fusion naturelle des comptes podcast97    ("podcast", re.compile(r"podcasts\.apple\.com/\S*?/id(\d+)", re.I)),98]99100# chemins qui ne sont PAS des handles (pages génériques des plateformes)101_NOT_HANDLES = {102    "p", "reel", "reels", "stories", "share", "watch", "video", "videos",103    "playlist", "shorts", "live", "intent", "hashtag", "explore", "home",104    "profile.php", "groups", "pages", "events", "posts", "status", "search",105}106107108def strip_accents(text: str) -> str:109    """« Bédard » → « Bedard » (clés de comparaison, jamais pour l'affichage)."""110    return "".join(c for c in unicodedata.normalize("NFD", text)111                   if unicodedata.category(c) != "Mn")112113114def slugify(text: str) -> str:115    """Nom d'affichage → slug d'identifiant interne (ex. « Enola Bédard » → enola-bedard)."""116    s = strip_accents(text or "").lower()117    s = re.sub(r"[^a-z0-9]+", "-", s).strip("-")118    return s or "inconnu"119120121def clean_handle(raw: str | None) -> str:122    """Nettoie un handle brut : retire @, URL, espaces, barre finale ; minuscules.123124    Les handles des grandes plateformes sont insensibles à la casse — on125    normalise en minuscules pour la comparaison ET la forme canonique.126    """127    if not raw:128        return ""129    h = raw.strip()130    # si on a reçu une URL complète, garder le dernier segment utile131    if "://" in h or h.startswith("www."):132        h = h.rstrip("/").split("/")[-1]133    h = h.lstrip("@").split("?")[0].strip().strip("/")134    # ID de chaîne YouTube (UC…) : sensible à la casse — ne JAMAIS minusculiser135    if re.fullmatch(r"UC[0-9A-Za-z_-]{22}", h):136        return h137    return h.lower()138139140def normalize_platform(raw: str | None) -> str:141    """Nom de plateforme brut → valeur canonique de §6.2 (défaut : « autre »)."""142    if not raw:143        return "autre"144    p = strip_accents(raw).lower().strip()145    aliases = {146        "ig": "instagram", "insta": "instagram", "instagram": "instagram",147        "tiktok": "tiktok", "tik-tok": "tiktok", "tik tok": "tiktok",148        "youtube": "youtube", "yt": "youtube",149        "twitch": "twitch", "kick": "kick",150        "twitter": "x", "x": "x",151        "facebook": "facebook", "fb": "facebook",152        "snapchat": "snapchat", "snap": "snapchat",153        "substack": "substack", "patreon": "patreon", "onlyfans": "onlyfans",154        "of": "onlyfans", "mym": "mym", "fansly": "fansly",155        "linkedin": "linkedin", "threads": "threads",156        "spotify": "spotify", "discord": "discord",157        "podcast": "podcast", "balado": "podcast",158        "apple podcasts": "podcast", "apple-podcasts": "podcast",159        "site": "site-web", "site-web": "site-web", "website": "site-web",160        "web": "site-web", "blogue": "site-web", "blog": "site-web",161    }162    return aliases.get(p, p if p in PLATFORMS else "autre")163164165def canonical_url(platform: str, handle: str, fallback: str = "") -> str:166    """Forme canonique de l'URL d'un compte (CLAUDE.md §11)."""167    h = clean_handle(handle)168    # chaîne YouTube identifiée par son ID (pas de @handle connu)169    if platform == "youtube" and re.fullmatch(r"UC[0-9A-Za-z_-]{22}", h):170        return f"https://www.youtube.com/channel/{h}"171    tpl = _URL_TEMPLATES.get(platform)172    if tpl and h:173        return tpl.format(h=h)174    return fallback or ""175176177def platform_from_url(url: str) -> tuple[str, str] | None:178    """URL → (plateforme, handle) si reconnue, sinon None.179180    Utilisé par le connecteur link-in-bio : chaque lien d'une page Linktree181    est classé ici. Les chemins génériques (posts, vidéos) sont rejetés.182    """183    if not url:184        return None185    for platform, rx in _URL_PLATFORM_RES:186        m = rx.search(url)187        if m:188            handle = clean_handle(m.group(1))189            if not handle or handle in _NOT_HANDLES:190                return None191            return platform, handle192    return None193194195def clean_name(raw: str | None) -> str:196    """Nettoie un nom d'affichage (espaces, emojis décoratifs en bordure)."""197    if not raw:198        return ""199    name = re.sub(r"\s+", " ", raw).strip()200    # retirer les emojis/symboles en tête et en queue seulement (jamais au milieu :201    # certains noms de scène en contiennent volontairement)202    name = re.sub(r"^[\W_]+(?=\w)", "", name)203    name = re.sub(r"(?<=\w)[^\w).!?'’\"]+$", "", name)204    return name.strip()205206207def clean_bio(raw: str | None) -> str:208    """Bio publique : espaces normalisés, longueur bornée."""209    if not raw:210        return ""211    bio = re.sub(r"[ \t]+", " ", raw).strip()212    return bio[:1200]213214215def map_niche(raw: str | None) -> str:216    """Catégorie source → niche canonique (§6.1)."""217    if not raw:218        return "autre"219    c = strip_accents(raw).lower().strip()220    mapping = {221        "humour": "humour", "comedie": "humour", "comedy": "humour",222        "mode": "mode", "fashion": "mode",223        "beaute": "beaute", "beauty": "beaute", "makeup": "beaute",224        "lifestyle": "lifestyle", "tele-realite": "lifestyle",225        "famille": "famille-parentalite", "parentalite": "famille-parentalite",226        "famille-parentalite": "famille-parentalite", "maman": "famille-parentalite",227        "cuisine": "cuisine", "food": "bouffe-resto", "bouffe": "bouffe-resto",228        "bouffe-resto": "bouffe-resto", "resto": "bouffe-resto",229        "gaming": "gaming", "jeux video": "gaming", "jeux-video": "gaming",230        "tech": "tech", "technologie": "tech",231        "sport": "sport-fitness", "fitness": "sport-fitness",232        "sport-fitness": "sport-fitness", "entrainement": "sport-fitness",233        "plein air": "plein-air", "plein-air": "plein-air", "chasse": "plein-air",234        "peche": "plein-air", "outdoor": "plein-air",235        "voyage": "voyage", "travel": "voyage",236        "musique": "musique", "music": "musique",237        "arts": "arts", "art": "arts",238        "danse": "danse", "dance": "danse",239        "education": "education",240        "finance": "finance-affaires", "affaires": "finance-affaires",241        "finance-affaires": "finance-affaires", "entrepreneur": "finance-affaires",242        "business": "finance-affaires", "immobilier": "finance-affaires",243        "sante": "sante-mieux-etre", "mieux-etre": "sante-mieux-etre",244        "sante-mieux-etre": "sante-mieux-etre", "bien-etre": "sante-mieux-etre",245        "actualite": "actualite-opinion", "opinion": "actualite-opinion",246        "actualite-opinion": "actualite-opinion", "medias": "actualite-opinion",247        "podcast": "actualite-opinion",248    }249    return mapping.get(c, c if c in NICHES else "autre")250251252# localisation publique déclarée → région administrative (§7).253# ⚠️ On ne mappe QUE ce qui est sans ambiguïté ; « Québec » seul désigne la254# PROVINCE dans les listes médias → region None. Jamais d'inférence privée.255_CITY_REGION = {256    "montreal": "Montréal", "lachine": "Montréal", "verdun": "Montréal",257    "quebec (ville)": "Capitale-Nationale", "ville de quebec": "Capitale-Nationale",258    "charlevoix": "Capitale-Nationale",259    "laval": "Laval",260    "levis": "Chaudière-Appalaches", "beauce": "Chaudière-Appalaches",261    "gatineau": "Outaouais",262    "sherbrooke": "Estrie", "waterloo": "Estrie",  # Haute-Yamaska → Estrie (2021)263    "granby": "Estrie",264    "trois-rivieres": "Mauricie", "shawinigan": "Mauricie",265    "saguenay": "Saguenay–Lac-Saint-Jean", "chicoutimi": "Saguenay–Lac-Saint-Jean",266    "alma": "Saguenay–Lac-Saint-Jean",267    "rimouski": "Bas-Saint-Laurent", "riviere-du-loup": "Bas-Saint-Laurent",268    "gaspe": "Gaspésie–Îles-de-la-Madeleine",269    "rouyn-noranda": "Abitibi-Témiscamingue", "val-d'or": "Abitibi-Témiscamingue",270    "drummondville": "Centre-du-Québec", "victoriaville": "Centre-du-Québec",271    "longueuil": "Montérégie", "brossard": "Montérégie",272    "saint-jean-sur-richelieu": "Montérégie",273    "blainville": "Laurentides", "pointe-calumet": "Laurentides",274    "saint-jerome": "Laurentides", "mont-tremblant": "Laurentides",275    "terrebonne": "Lanaudière", "repentigny": "Lanaudière", "joliette": "Lanaudière",276    "sept-iles": "Côte-Nord", "baie-comeau": "Côte-Nord",277}278279280def map_region(location: str | None) -> tuple[str | None, str | None]:281    """Localisation publique (texte libre) → (région §7, ville) ou (None, None).282283    Prudence maximale : ne retourne une région que si la localisation déclarée284    est un lieu québécois reconnu sans ambiguïté. « Québec » seul = la province.285    """286    if not location:287        return None, None288    loc = strip_accents(location).lower().strip()289    # tronquer les précisions entre parenthèses SAUF « Québec (ville) »290    base = re.sub(r"\s*\(.*?\)\s*", " ", loc).strip().rstrip(",")291    if "quebec (ville)" in loc or base in ("ville de quebec",):292        return "Capitale-Nationale", "Québec"293    # retirer le suffixe province (« Blainville, Québec » → « blainville »)294    base = re.sub(r",?\s*(quebec|qc|canada)\s*$", "", base).strip().rstrip(",")295    if not base or base == "quebec":296        return None, None  # province seule → région inconnue297    for city, region in _CITY_REGION.items():298        if base == city or base.startswith(city + ",") or base.endswith(" " + city):299            return region, location.split(",")[0].split("(")[0].strip()300    if base in {strip_accents(r).lower() for r in REGIONS}:301        for r in REGIONS:302            if strip_accents(r).lower() == base:303                return r, None304    return None, None305306307def parse_count(raw) -> int | None:308    """« 1,2 M », « 12.3k », « 4 567 abonnés », 4567 → entier (None si inconnu)."""309    if raw is None:310        return None311    if isinstance(raw, (int, float)):312        return int(raw)313    if str(raw).strip().startswith("@"):314        return None  # un handle (« @nom5b ») n'est pas un compteur315    # frontières de mot obligatoires — sinon « tw5bu » se lirait « 5 milliards »316    m = re.search(r"(?<![\w@.])([\d][\d\s .,]*)\s*([kKmMbB])?(?![\w])", str(raw))317    if not m:318        return None319    num = m.group(1).replace(" ", "").replace(" ", "")320    # « 1.234.567 » / « 1,234,567 » = séparateurs de milliers ; « 1,2 »/« 1.2 » = décimal321    if num.count(",") > 1 or (num.count(",") == 1 and len(num.split(",")[1]) == 3322                              and not m.group(2)):323        num = num.replace(",", "")324    if num.count(".") > 1 or (num.count(".") == 1 and len(num.split(".")[1]) == 3325                              and not m.group(2)):326        num = num.replace(".", "")327    num = num.replace(",", ".")328    try:329        value = float(num)330    except ValueError:331        return None332    mult = {"k": 1e3, "m": 1e6, "b": 1e9}.get((m.group(2) or "").lower(), 1)333    return int(value * mult)334335336def audience_tier(followers: int | None) -> str:337    """Taille d'audience (§6.3), basée sur l'audience de la plateforme principale."""338    if not followers or followers <= 0:339        return "nano"340    if followers < 10_000:341        return "nano"342    if followers < 100_000:343        return "micro"344    if followers < 1_000_000:345        return "macro"346    return "mega"347348349def infer_creator_type(platforms: list[str], niche: str) -> str:350    """Type de créateur par défaut selon sa plateforme dominante et sa niche."""351    first = platforms[0] if platforms else ""352    if first == "youtube":353        return "youtubeur"354    if first in ("twitch", "kick"):355        return "streamer"356    if first == "tiktok":357        return "createur-tiktok"358    if first in ("substack", "site-web"):359        return "createur-ecrit"360    if first == "podcast":361        return "podcasteur"362    if niche == "humour":363        return "humoriste"364    if niche == "musique":365        return "musicien"366    return "influenceur"367368369def guess_language(*texts: str) -> str:370    """Langue de création (§6.4) à partir des textes publics. Défaut : fr (Québec)."""371    blob = strip_accents(" ".join(t for t in texts if t)).lower()372    if not blob:373        return "fr"374    if re.search(r"\b(bilingue|bilingual|fr/en|en/fr)\b", blob):375        return "bilingue"376    if re.search(r"\b(contenu|content)\s+(quotidien\s+)?(en\s+)?anglais\b", blob) \377            or re.search(r"\benglish(-|\s)?(only|content|speaking)\b", blob):378        return "en"379    return "fr"380