# ============================================================================== # Author: Simon-Pierre Boucher # File: creaka/normalize.py # Desc: Normalisation commune — handles, URLs canoniques par plateforme, # niches, régions, langue, taille d'audience (voir CLAUDE.md §6, §7, §11) # ============================================================================== """Couche de normalisation partagée par tous les connecteurs. Chaque connecteur remplit des champs bruts ; `schema.Creator.finalize()` appelle ces fonctions pour produire une fiche canonique. Idempotent. """ from __future__ import annotations import re import unicodedata # --- taxonomies (CLAUDE.md §6) ------------------------------------------------ NICHES = { "humour", "mode", "beaute", "lifestyle", "famille-parentalite", "cuisine", "gaming", "tech", "sport-fitness", "plein-air", "voyage", "musique", "arts", "danse", "education", "finance-affaires", "sante-mieux-etre", "bouffe-resto", "actualite-opinion", "autre", } PLATFORMS = { "instagram", "tiktok", "youtube", "twitch", "kick", "x", "facebook", "snapchat", "substack", "patreon", "onlyfans", "mym", "fansly", "linkedin", "threads", "spotify", "discord", "podcast", "site-web", "autre", } CREATOR_TYPES = { "influenceur", "youtubeur", "streamer", "podcasteur", "createur-tiktok", "createur-ecrit", "musicien", "humoriste", "autre", } LANGUAGES = {"fr", "en", "bilingue"} REGIONS = { "Bas-Saint-Laurent", "Saguenay–Lac-Saint-Jean", "Capitale-Nationale", "Mauricie", "Estrie", "Montréal", "Outaouais", "Abitibi-Témiscamingue", "Côte-Nord", "Nord-du-Québec", "Gaspésie–Îles-de-la-Madeleine", "Chaudière-Appalaches", "Laval", "Lanaudière", "Laurentides", "Montérégie", "Centre-du-Québec", } # --- URLs canoniques par plateforme (CLAUDE.md §11) --------------------------- # gabarit : handle SANS @ ; None = pas de forme canonique (garder l'URL source) _URL_TEMPLATES: dict[str, str | None] = { "instagram": "https://www.instagram.com/{h}/", "tiktok": "https://www.tiktok.com/@{h}", "youtube": "https://www.youtube.com/@{h}", "twitch": "https://www.twitch.tv/{h}", "kick": "https://kick.com/{h}", "x": "https://x.com/{h}", "facebook": "https://www.facebook.com/{h}", "snapchat": "https://www.snapchat.com/add/{h}", "substack": "https://{h}.substack.com", "patreon": "https://www.patreon.com/{h}", "onlyfans": "https://onlyfans.com/{h}", "mym": "https://mym.fans/{h}", "fansly": "https://fansly.com/{h}", "linkedin": "https://www.linkedin.com/in/{h}", "threads": "https://www.threads.net/@{h}", # IDs Spotify et codes d'invitation Discord : sensibles à la casse → # pas de forme canonique reconstruite, on conserve l'URL source "spotify": None, "discord": None, "podcast": None, "site-web": None, "autre": None, } # détection de plateforme à partir d'une URL (pages link-in-bio, bios, etc.) _URL_PLATFORM_RES: list[tuple[str, re.Pattern]] = [ ("instagram", re.compile(r"instagram\.com/([\w.\-]+)", re.I)), ("tiktok", re.compile(r"tiktok\.com/@([\w.\-]+)", re.I)), ("youtube", re.compile(r"youtube\.com/(?:@|c/|user/|channel/)([\w.\-]+)", re.I)), ("youtube", re.compile(r"youtu\.be/([\w.\-]+)", re.I)), ("twitch", re.compile(r"twitch\.tv/([\w.\-]+)", re.I)), ("kick", re.compile(r"kick\.com/([\w.\-]+)", re.I)), ("x", re.compile(r"(?:twitter|x)\.com/([\w.\-]+)", re.I)), ("facebook", re.compile(r"facebook\.com/([\w.\-]+)", re.I)), ("snapchat", re.compile(r"snapchat\.com/add/([\w.\-]+)", re.I)), ("substack", re.compile(r"https?://([\w\-]+)\.substack\.com", re.I)), ("patreon", re.compile(r"patreon\.com/([\w.\-]+)", re.I)), ("onlyfans", re.compile(r"onlyfans\.com/([\w.\-]+)", re.I)), ("mym", re.compile(r"mym\.fans/([\w.\-]+)", re.I)), ("fansly", re.compile(r"fansly\.com/([\w.\-]+)", re.I)), ("linkedin", re.compile(r"linkedin\.com/in/([\w.\-]+)", re.I)), ("threads", re.compile(r"threads\.net/@?([\w.\-]+)", re.I)), ("spotify", re.compile( r"open\.spotify\.com/(?:intl-[\w\-]+/)?(?:artist|show|user)/([A-Za-z0-9]+)", re.I)), ("discord", re.compile(r"discord(?:\.gg|(?:app)?\.com/invite)/([\w\-]+)", re.I)), # une page Apple Podcasts pointe le MÊME balado que balados-itunes # (handle = collectionId) → fusion naturelle des comptes podcast ("podcast", re.compile(r"podcasts\.apple\.com/\S*?/id(\d+)", re.I)), ] # chemins qui ne sont PAS des handles (pages génériques des plateformes) _NOT_HANDLES = { "p", "reel", "reels", "stories", "share", "watch", "video", "videos", "playlist", "shorts", "live", "intent", "hashtag", "explore", "home", "profile.php", "groups", "pages", "events", "posts", "status", "search", } def strip_accents(text: str) -> str: """« Bédard » → « Bedard » (clés de comparaison, jamais pour l'affichage).""" return "".join(c for c in unicodedata.normalize("NFD", text) if unicodedata.category(c) != "Mn") def slugify(text: str) -> str: """Nom d'affichage → slug d'identifiant interne (ex. « Enola Bédard » → enola-bedard).""" s = strip_accents(text or "").lower() s = re.sub(r"[^a-z0-9]+", "-", s).strip("-") return s or "inconnu" def clean_handle(raw: str | None) -> str: """Nettoie un handle brut : retire @, URL, espaces, barre finale ; minuscules. Les handles des grandes plateformes sont insensibles à la casse — on normalise en minuscules pour la comparaison ET la forme canonique. """ if not raw: return "" h = raw.strip() # si on a reçu une URL complète, garder le dernier segment utile if "://" in h or h.startswith("www."): h = h.rstrip("/").split("/")[-1] h = h.lstrip("@").split("?")[0].strip().strip("/") # ID de chaîne YouTube (UC…) : sensible à la casse — ne JAMAIS minusculiser if re.fullmatch(r"UC[0-9A-Za-z_-]{22}", h): return h return h.lower() def normalize_platform(raw: str | None) -> str: """Nom de plateforme brut → valeur canonique de §6.2 (défaut : « autre »).""" if not raw: return "autre" p = strip_accents(raw).lower().strip() aliases = { "ig": "instagram", "insta": "instagram", "instagram": "instagram", "tiktok": "tiktok", "tik-tok": "tiktok", "tik tok": "tiktok", "youtube": "youtube", "yt": "youtube", "twitch": "twitch", "kick": "kick", "twitter": "x", "x": "x", "facebook": "facebook", "fb": "facebook", "snapchat": "snapchat", "snap": "snapchat", "substack": "substack", "patreon": "patreon", "onlyfans": "onlyfans", "of": "onlyfans", "mym": "mym", "fansly": "fansly", "linkedin": "linkedin", "threads": "threads", "spotify": "spotify", "discord": "discord", "podcast": "podcast", "balado": "podcast", "apple podcasts": "podcast", "apple-podcasts": "podcast", "site": "site-web", "site-web": "site-web", "website": "site-web", "web": "site-web", "blogue": "site-web", "blog": "site-web", } return aliases.get(p, p if p in PLATFORMS else "autre") def canonical_url(platform: str, handle: str, fallback: str = "") -> str: """Forme canonique de l'URL d'un compte (CLAUDE.md §11).""" h = clean_handle(handle) # chaîne YouTube identifiée par son ID (pas de @handle connu) if platform == "youtube" and re.fullmatch(r"UC[0-9A-Za-z_-]{22}", h): return f"https://www.youtube.com/channel/{h}" tpl = _URL_TEMPLATES.get(platform) if tpl and h: return tpl.format(h=h) return fallback or "" def platform_from_url(url: str) -> tuple[str, str] | None: """URL → (plateforme, handle) si reconnue, sinon None. Utilisé par le connecteur link-in-bio : chaque lien d'une page Linktree est classé ici. Les chemins génériques (posts, vidéos) sont rejetés. """ if not url: return None for platform, rx in _URL_PLATFORM_RES: m = rx.search(url) if m: handle = clean_handle(m.group(1)) if not handle or handle in _NOT_HANDLES: return None return platform, handle return None def clean_name(raw: str | None) -> str: """Nettoie un nom d'affichage (espaces, emojis décoratifs en bordure).""" if not raw: return "" name = re.sub(r"\s+", " ", raw).strip() # retirer les emojis/symboles en tête et en queue seulement (jamais au milieu : # certains noms de scène en contiennent volontairement) name = re.sub(r"^[\W_]+(?=\w)", "", name) name = re.sub(r"(?<=\w)[^\w).!?'’\"]+$", "", name) return name.strip() def clean_bio(raw: str | None) -> str: """Bio publique : espaces normalisés, longueur bornée.""" if not raw: return "" bio = re.sub(r"[ \t]+", " ", raw).strip() return bio[:1200] def map_niche(raw: str | None) -> str: """Catégorie source → niche canonique (§6.1).""" if not raw: return "autre" c = strip_accents(raw).lower().strip() mapping = { "humour": "humour", "comedie": "humour", "comedy": "humour", "mode": "mode", "fashion": "mode", "beaute": "beaute", "beauty": "beaute", "makeup": "beaute", "lifestyle": "lifestyle", "tele-realite": "lifestyle", "famille": "famille-parentalite", "parentalite": "famille-parentalite", "famille-parentalite": "famille-parentalite", "maman": "famille-parentalite", "cuisine": "cuisine", "food": "bouffe-resto", "bouffe": "bouffe-resto", "bouffe-resto": "bouffe-resto", "resto": "bouffe-resto", "gaming": "gaming", "jeux video": "gaming", "jeux-video": "gaming", "tech": "tech", "technologie": "tech", "sport": "sport-fitness", "fitness": "sport-fitness", "sport-fitness": "sport-fitness", "entrainement": "sport-fitness", "plein air": "plein-air", "plein-air": "plein-air", "chasse": "plein-air", "peche": "plein-air", "outdoor": "plein-air", "voyage": "voyage", "travel": "voyage", "musique": "musique", "music": "musique", "arts": "arts", "art": "arts", "danse": "danse", "dance": "danse", "education": "education", "finance": "finance-affaires", "affaires": "finance-affaires", "finance-affaires": "finance-affaires", "entrepreneur": "finance-affaires", "business": "finance-affaires", "immobilier": "finance-affaires", "sante": "sante-mieux-etre", "mieux-etre": "sante-mieux-etre", "sante-mieux-etre": "sante-mieux-etre", "bien-etre": "sante-mieux-etre", "actualite": "actualite-opinion", "opinion": "actualite-opinion", "actualite-opinion": "actualite-opinion", "medias": "actualite-opinion", "podcast": "actualite-opinion", } return mapping.get(c, c if c in NICHES else "autre") # localisation publique déclarée → région administrative (§7). # ⚠️ On ne mappe QUE ce qui est sans ambiguïté ; « Québec » seul désigne la # PROVINCE dans les listes médias → region None. Jamais d'inférence privée. _CITY_REGION = { "montreal": "Montréal", "lachine": "Montréal", "verdun": "Montréal", "quebec (ville)": "Capitale-Nationale", "ville de quebec": "Capitale-Nationale", "charlevoix": "Capitale-Nationale", "laval": "Laval", "levis": "Chaudière-Appalaches", "beauce": "Chaudière-Appalaches", "gatineau": "Outaouais", "sherbrooke": "Estrie", "waterloo": "Estrie", # Haute-Yamaska → Estrie (2021) "granby": "Estrie", "trois-rivieres": "Mauricie", "shawinigan": "Mauricie", "saguenay": "Saguenay–Lac-Saint-Jean", "chicoutimi": "Saguenay–Lac-Saint-Jean", "alma": "Saguenay–Lac-Saint-Jean", "rimouski": "Bas-Saint-Laurent", "riviere-du-loup": "Bas-Saint-Laurent", "gaspe": "Gaspésie–Îles-de-la-Madeleine", "rouyn-noranda": "Abitibi-Témiscamingue", "val-d'or": "Abitibi-Témiscamingue", "drummondville": "Centre-du-Québec", "victoriaville": "Centre-du-Québec", "longueuil": "Montérégie", "brossard": "Montérégie", "saint-jean-sur-richelieu": "Montérégie", "blainville": "Laurentides", "pointe-calumet": "Laurentides", "saint-jerome": "Laurentides", "mont-tremblant": "Laurentides", "terrebonne": "Lanaudière", "repentigny": "Lanaudière", "joliette": "Lanaudière", "sept-iles": "Côte-Nord", "baie-comeau": "Côte-Nord", } def map_region(location: str | None) -> tuple[str | None, str | None]: """Localisation publique (texte libre) → (région §7, ville) ou (None, None). Prudence maximale : ne retourne une région que si la localisation déclarée est un lieu québécois reconnu sans ambiguïté. « Québec » seul = la province. """ if not location: return None, None loc = strip_accents(location).lower().strip() # tronquer les précisions entre parenthèses SAUF « Québec (ville) » base = re.sub(r"\s*\(.*?\)\s*", " ", loc).strip().rstrip(",") if "quebec (ville)" in loc or base in ("ville de quebec",): return "Capitale-Nationale", "Québec" # retirer le suffixe province (« Blainville, Québec » → « blainville ») base = re.sub(r",?\s*(quebec|qc|canada)\s*$", "", base).strip().rstrip(",") if not base or base == "quebec": return None, None # province seule → région inconnue for city, region in _CITY_REGION.items(): if base == city or base.startswith(city + ",") or base.endswith(" " + city): return region, location.split(",")[0].split("(")[0].strip() if base in {strip_accents(r).lower() for r in REGIONS}: for r in REGIONS: if strip_accents(r).lower() == base: return r, None return None, None def parse_count(raw) -> int | None: """« 1,2 M », « 12.3k », « 4 567 abonnés », 4567 → entier (None si inconnu).""" if raw is None: return None if isinstance(raw, (int, float)): return int(raw) if str(raw).strip().startswith("@"): return None # un handle (« @nom5b ») n'est pas un compteur # frontières de mot obligatoires — sinon « tw5bu » se lirait « 5 milliards » m = re.search(r"(? 1 or (num.count(",") == 1 and len(num.split(",")[1]) == 3 and not m.group(2)): num = num.replace(",", "") if num.count(".") > 1 or (num.count(".") == 1 and len(num.split(".")[1]) == 3 and not m.group(2)): num = num.replace(".", "") num = num.replace(",", ".") try: value = float(num) except ValueError: return None mult = {"k": 1e3, "m": 1e6, "b": 1e9}.get((m.group(2) or "").lower(), 1) return int(value * mult) def audience_tier(followers: int | None) -> str: """Taille d'audience (§6.3), basée sur l'audience de la plateforme principale.""" if not followers or followers <= 0: return "nano" if followers < 10_000: return "nano" if followers < 100_000: return "micro" if followers < 1_000_000: return "macro" return "mega" def infer_creator_type(platforms: list[str], niche: str) -> str: """Type de créateur par défaut selon sa plateforme dominante et sa niche.""" first = platforms[0] if platforms else "" if first == "youtube": return "youtubeur" if first in ("twitch", "kick"): return "streamer" if first == "tiktok": return "createur-tiktok" if first in ("substack", "site-web"): return "createur-ecrit" if first == "podcast": return "podcasteur" if niche == "humour": return "humoriste" if niche == "musique": return "musicien" return "influenceur" def guess_language(*texts: str) -> str: """Langue de création (§6.4) à partir des textes publics. Défaut : fr (Québec).""" blob = strip_accents(" ".join(t for t in texts if t)).lower() if not blob: return "fr" if re.search(r"\b(bilingue|bilingual|fr/en|en/fr)\b", blob): return "bilingue" if re.search(r"\b(contenu|content)\s+(quotidien\s+)?(en\s+)?anglais\b", blob) \ or re.search(r"\benglish(-|\s)?(only|content|speaking)\b", blob): return "en" return "fr"