spb/qc-influenceurs Public

1#!/usr/bin/env python32"""Fusionne les extractions (data/extracted/*.json) en un dataset unique dédupliqué.34Sorties : data/qc_influenceurs.json + data/qc_influenceurs.csv5"""6import csv7import glob8import json9import os10import re11import unicodedata1213BASE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))14EXTRACTED = os.path.join(BASE, "data", "extracted")15OUT_JSON = os.path.join(BASE, "data", "qc_influenceurs.json")16OUT_CSV = os.path.join(BASE, "data", "qc_influenceurs.csv")1718PLATFORMS = ["instagram", "tiktok", "youtube", "facebook", "x", "twitch"]19METRIC_KEYS = {20    "instagram": "instagram_followers",21    "tiktok": "tiktok_followers",22    "youtube": "youtube_subscribers",23    "facebook": "facebook_followers",24    "x": "x_followers",25    "twitch": "twitch_followers",26}272829# variantes connues du même créateur (clé normalisée -> clé canonique)30MANUAL_ALIASES = {31    "carl is cooking": "carl arsenault",32}3334CATEGORY_FIXES = {35    "tennis / sport": "sport",36    "art / miniatures": "autre",37    "docuséries / arts": "autre",38}394041def norm_name(name: str) -> str:42    s = re.sub(r"\(.*?\)", " ", name or "")  # "Nabil Lahrech (Aiekillu)" == "Nabil Lahrech"43    s = unicodedata.normalize("NFKD", s)44    s = "".join(c for c in s if not unicodedata.combining(c))45    s = re.sub(r"[^a-z0-9]+", " ", s.lower()).strip()46    return s474849def name_aliases(name: str):50    """Clés supplémentaires : contenu des parenthèses (pseudo) et nom complet."""51    keys = {norm_name(name)}52    for inner in re.findall(r"\((.*?)\)", name or ""):53        k = norm_name(inner)54        if k:55            keys.add(k)56    return keys575859def norm_handle(h):60    if not h:61        return None62    h = str(h).strip().lstrip("@").lower()63    h = re.sub(r"^https?://(www\.)?(instagram|tiktok|youtube|facebook|x|twitter|twitch)\.(com|tv)/(@)?", "", h)64    h = h.strip("/").split("/")[0].split("?")[0]65    return h or None666768def merge_record(dst, src):69    for p in PLATFORMS:70        h = norm_handle((src.get("handles") or {}).get(p))71        if h and not dst["handles"].get(p):72            dst["handles"][p] = h73    for mk in METRIC_KEYS.values():74        v = (src.get("metrics") or {}).get(mk)75        if isinstance(v, (int, float)) and v > 0:76            cur = dst["metrics"].get(mk)77            # garder la valeur la plus élevée (les listes datées sous-estiment)78            if not cur or v > cur:79                dst["metrics"][mk] = int(v)80    if src.get("category") and (not dst.get("category") or dst["category"] == "autre"):81        dst["category"] = src["category"]82    if src.get("location") and dst.get("location") in (None, "", "Québec"):83        dst["location"] = src["location"]84    note = (src.get("notes") or "").strip()85    if note and note not in dst["notes"]:86        dst["notes"] = (dst["notes"] + " | " + note).strip(" |")87    for s in src.get("sources") or []:88        if s not in dst["sources"]:89            dst["sources"].append(s)909192def main():93    records = []94    for f in sorted(glob.glob(os.path.join(EXTRACTED, "*.json"))):95        try:96            data = json.load(open(f))97        except Exception as e:98            print(f"ERREUR lecture {f}: {e}")99            continue100        if isinstance(data, dict):101            data = data.get("influencers") or data.get("data") or []102        print(f"{os.path.basename(f)}: {len(data)} records")103        records.extend(data)104105    by_key = {}106    handle_index = {}  # (platform, handle) -> key107    alias_index = {}   # alias normalisé -> key108    NON_QC = ("toronto", "vancouver", "calgary", "winnipeg", "scarborough",109              "ajax", "ontario", "terre-neuve", "alberta", "colombie-britannique")110    for r in records:111        if not r.get("name"):112            continue113        loc = (r.get("location") or "").lower()114        if any(x in loc for x in NON_QC):115            continue  # hors Québec (listes canadiennes mal filtrées)116        key = norm_name(r["name"])117        if not key:118            continue119        key = MANUAL_ALIASES.get(key, key)120        # match par handle ou alias (pseudo entre parenthèses) si le nom diffère121        matched = None122        for p in PLATFORMS:123            h = norm_handle((r.get("handles") or {}).get(p))124            if h and (p, h) in handle_index:125                matched = handle_index[(p, h)]126                break127        if not matched:128            for a in name_aliases(r["name"]):129                if a in alias_index:130                    matched = alias_index[a]131                    break132        key = matched or key133        if key not in by_key:134            by_key[key] = {135                "name": r["name"].strip(),136                "handles": {p: None for p in PLATFORMS},137                "metrics": {mk: None for mk in METRIC_KEYS.values()},138                "category": None,139                "location": None,140                "notes": "",141                "sources": [],142            }143        merge_record(by_key[key], r)144        for p in PLATFORMS:145            h = by_key[key]["handles"].get(p)146            if h:147                handle_index[(p, h)] = key148        for a in name_aliases(r["name"]) | name_aliases(by_key[key]["name"]):149            alias_index[a] = key150151    out = list(by_key.values())152    for r in out:153        vals = [v for v in r["metrics"].values() if v]154        r["total_followers"] = sum(vals) if vals else None155        r["platform_count"] = sum(1 for p in PLATFORMS if r["handles"].get(p) or r["metrics"].get(METRIC_KEYS[p]))156        t = r["total_followers"] or 0157        r["tier"] = "méga (1M+)" if t >= 1_000_000 else "macro (100K+)" if t >= 100_000 else "micro (10K+)" if t >= 10_000 else "nano/inconnu"158        r["category"] = CATEGORY_FIXES.get(r["category"] or "autre", r["category"] or "autre")159        r["location"] = r["location"] or "Québec"160161    out.sort(key=lambda r: -(r["total_followers"] or 0))162163    json.dump(out, open(OUT_JSON, "w"), ensure_ascii=False, indent=2)164165    with open(OUT_CSV, "w", newline="") as fh:166        w = csv.writer(fh)167        w.writerow(["nom", "categorie", "localisation", "tier", "total_followers",168                    "instagram_handle", "instagram_followers", "tiktok_handle", "tiktok_followers",169                    "youtube_handle", "youtube_subscribers", "facebook_handle", "facebook_followers",170                    "x_handle", "x_followers", "twitch_handle", "twitch_followers",171                    "nb_plateformes", "notes", "sources"])172        for r in out:173            w.writerow([r["name"], r["category"], r["location"], r["tier"], r["total_followers"] or "",174                        r["handles"]["instagram"] or "", r["metrics"]["instagram_followers"] or "",175                        r["handles"]["tiktok"] or "", r["metrics"]["tiktok_followers"] or "",176                        r["handles"]["youtube"] or "", r["metrics"]["youtube_subscribers"] or "",177                        r["handles"]["facebook"] or "", r["metrics"]["facebook_followers"] or "",178                        r["handles"]["x"] or "", r["metrics"]["x_followers"] or "",179                        r["handles"]["twitch"] or "", r["metrics"]["twitch_followers"] or "",180                        r["platform_count"], r["notes"], ";".join(r["sources"])])181182    print(f"\nTotal dédupliqué : {len(out)} influenceurs")183    print(f"Avec au moins une métrique : {sum(1 for r in out if r['total_followers'])}")184    print(f"Méga (1M+) : {sum(1 for r in out if r['tier'].startswith('méga'))}")185    print(f"Macro (100K+) : {sum(1 for r in out if r['tier'].startswith('macro'))}")186187188if __name__ == "__main__":189    main()190