spb/groupe-ka

1#!/usr/bin/env node2// Auteur : Simon-Pierre Boucher — contact@spboucher.ai3// KA ID v2.2 — Matrix factorization implicite (ALS, Hu-Koren-Volinsky 2008).4// Job batch quotidien (pm2 groupe-ka-kaid-mf, cron 03:12) : factorise la5// matrice membre × annonce de chaque univers à partir du journal6// d'interactions et des favoris (confiance c = 1 + α·r, r pondéré par type7// de signal et décroissance temporelle), puis écrit le top-20 de8// recommandations par membre dans `mf_recs`. Le endpoint /api/sso/prefs les9// fusionne dans le profil (`mf`) et le rerank des satellites les booste.10//11// PRÊT MÊME SANS VOLUME : sous les seuils (3 membres, 4 annonces, 612// interactions croisées par univers), l'univers est sauté proprement — le13// modèle s'activera de lui-même quand la base grandira.14//15// Vie privée : les membres ayant désactivé la personnalisation sont exclus16// de l'entraînement ET ne reçoivent aucune recommandation.17//18// Usage : node scripts/kaid-mf.mjs [--app <univers>] [--verbose]19import Database from "better-sqlite3";20import path from "node:path";2122const DB_PATH =23  process.env.KA_DB_PATH ?? path.join(process.cwd(), "data", "ka-id.db");24const ARGS = process.argv.slice(2);25const ONLY_APP = ARGS.includes("--app") ? ARGS[ARGS.indexOf("--app") + 1] : null;26const VERBOSE = ARGS.includes("--verbose");2728// hyperparamètres (implicit ALS)29const K_MAX = 16;        // facteurs latents (réduit automatiquement si petit)30const LAMBDA = 0.1;      // régularisation31const ALPHA = 40;        // échelle de confiance c = 1 + α·r_norm32const ITERS = 12;        // alternances33const TOP_N = 20;        // recommandations conservées par membre34// seuils d'activation par univers35const MIN_USERS = 3;36const MIN_ITEMS = 4;37const MIN_NNZ = 6;3839const HALF_LIFE_DAYS = 45;40const WEIGHTS = {41  favorite: 8, external_click: 5, detail_dwell: 4, share: 4,42  detail_view: 2.5, compare: 2.5, alert_open: 2, click: 1,43  map_marker_click: 1,44};45const NEGATIVE = new Set(["hide", "dismiss", "unfavorite"]);4647const db = new Database(DB_PATH);48db.pragma("journal_mode = WAL");49db.exec(`50  CREATE TABLE IF NOT EXISTS mf_recs (51    user_id INTEGER NOT NULL,52    app TEXT NOT NULL,53    item_id TEXT NOT NULL,54    score REAL NOT NULL,55    rank INTEGER NOT NULL,56    run_at TEXT NOT NULL DEFAULT (datetime('now')),57    PRIMARY KEY (user_id, app, rank)58  );59  CREATE INDEX IF NOT EXISTS mf_recs_user ON mf_recs(user_id, app);60`);6162const log = (...a) => console.log("[kaid-mf]", ...a);63const vlog = (...a) => VERBOSE && console.log("[kaid-mf]", ...a);6465function decay(createdAt) {66  const t = Date.parse(String(createdAt).replace(" ", "T") + "Z");67  if (Number.isNaN(t)) return 0.5;68  const days = Math.max(0, (Date.now() - t) / 86_400_000);69  return Math.pow(0.5, days / HALF_LIFE_DAYS);70}7172/** Interactions (membre, annonce) → confiance r, négatifs retirés. */73function buildInteractions(app) {74  const r = new Map(); // "uid|item" -> poids accumulé75  const neg = new Set();76  const key = (u, i) => `${u}|${i}`;7778  const events = db.prepare(79    `SELECT e.user_id, e.entity_id, e.event_type, e.created_at80     FROM user_events e JOIN users u ON u.id = e.user_id81     WHERE e.app = ? AND e.entity_id IS NOT NULL82       AND e.created_at > datetime('now', '-180 days')83       AND COALESCE(u.personalization, 1) = 184     ORDER BY e.id DESC LIMIT 200000`,85  ).all(app);86  for (const e of events) {87    if (NEGATIVE.has(e.event_type)) {88      neg.add(key(e.user_id, e.entity_id));89      continue;90    }91    const w = WEIGHTS[e.event_type];92    if (!w) continue;93    const k = key(e.user_id, e.entity_id);94    r.set(k, (r.get(k) ?? 0) + w * decay(e.created_at));95  }96  const favs = db.prepare(97    `SELECT f.user_id, f.item_id, f.created_at98     FROM favorites f JOIN users u ON u.id = f.user_id99     WHERE f.app = ? AND COALESCE(u.personalization, 1) = 1`,100  ).all(app);101  for (const f of favs) {102    const k = key(f.user_id, f.item_id);103    r.set(k, (r.get(k) ?? 0) + WEIGHTS.favorite * Math.max(0.35, decay(f.created_at)));104  }105  for (const k of neg) r.delete(k);106107  const hidden = new Map(); // user_id -> Set(item_id)108  for (const h of db.prepare(109    "SELECT user_id, item_id FROM hidden_items WHERE app = ?").all(app)) {110    if (!hidden.has(h.user_id)) hidden.set(h.user_id, new Set());111    hidden.get(h.user_id).add(h.item_id);112  }113  return { r, hidden };114}115116/** Résout A·x = b (A symétrique définie positive k×k) — Gauss pivot partiel. */117function solve(A, b, k) {118  const M = new Float64Array(k * (k + 1));119  for (let i = 0; i < k; i++) {120    for (let j = 0; j < k; j++) M[i * (k + 1) + j] = A[i * k + j];121    M[i * (k + 1) + k] = b[i];122  }123  for (let col = 0; col < k; col++) {124    let piv = col;125    for (let row = col + 1; row < k; row++)126      if (Math.abs(M[row * (k + 1) + col]) > Math.abs(M[piv * (k + 1) + col])) piv = row;127    if (piv !== col)128      for (let j = col; j <= k; j++) {129        const t = M[col * (k + 1) + j];130        M[col * (k + 1) + j] = M[piv * (k + 1) + j];131        M[piv * (k + 1) + j] = t;132      }133    const d = M[col * (k + 1) + col] || 1e-9;134    for (let row = col + 1; row < k; row++) {135      const f = M[row * (k + 1) + col] / d;136      if (!f) continue;137      for (let j = col; j <= k; j++) M[row * (k + 1) + j] -= f * M[col * (k + 1) + j];138    }139  }140  const x = new Float64Array(k);141  for (let i = k - 1; i >= 0; i--) {142    let s = M[i * (k + 1) + k];143    for (let j = i + 1; j < k; j++) s -= M[i * (k + 1) + j] * x[j];144    x[i] = s / (M[i * (k + 1) + i] || 1e-9);145  }146  return x;147}148149/** Un passage d'alternance : recalcule X (les « lignes ») à Y fixé. */150function alsStep(X, Y, rowsOf, nRows, k) {151  // YtY + λI (précalculé une fois par passage)152  const base = new Float64Array(k * k);153  const nY = Y.length / k;154  for (let i = 0; i < nY; i++)155    for (let a = 0; a < k; a++) {156      const ya = Y[i * k + a];157      if (!ya) continue;158      for (let b = 0; b < k; b++) base[a * k + b] += ya * Y[i * k + b];159    }160  for (let a = 0; a < k; a++) base[a * k + a] += LAMBDA;161162  const A = new Float64Array(k * k);163  const bvec = new Float64Array(k);164  for (let u = 0; u < nRows; u++) {165    A.set(base);166    bvec.fill(0);167    for (const [i, c] of rowsOf(u)) {168      const extra = ALPHA * c; // (c_ui − 1) avec c_ui = 1 + α·c169      for (let a = 0; a < k; a++) {170        const ya = Y[i * k + a];171        if (!ya) continue;172        bvec[a] += (1 + ALPHA * c) * ya;173        for (let b = 0; b < k; b++) A[a * k + b] += extra * ya * Y[i * k + b];174      }175    }176    X.set(solve(A, bvec, k), u * k);177  }178}179180function factorizeApp(app) {181  const { r, hidden } = buildInteractions(app);182  const users = new Map();183  const items = new Map();184  for (const key of r.keys()) {185    const [u, i] = key.split(/\|(.+)/s);186    if (!users.has(u)) users.set(u, users.size);187    if (!items.has(i)) items.set(i, items.size);188  }189  const nU = users.size, nI = items.size, nnz = r.size;190  if (nU < MIN_USERS || nI < MIN_ITEMS || nnz < MIN_NNZ) {191    log(`${app} : volume insuffisant (membres=${nU}, annonces=${nI}, ` +192        `interactions=${nnz}) — modèle en veille`);193    return { app, trained: false, users: nU, items: nI };194  }195  const k = Math.max(2, Math.min(K_MAX, Math.floor(Math.min(nU, nI) / 2)));196197  // normaliser r (le poids brut varie de 1 à ~40) → r/8 borné à 3198  const byUser = Array.from({ length: nU }, () => []);199  const byItem = Array.from({ length: nI }, () => []);200  for (const [key, w] of r) {201    const [u, i] = key.split(/\|(.+)/s);202    const uu = users.get(u), ii = items.get(i);203    const c = Math.min(3, w / 8);204    byUser[uu].push([ii, c]);205    byItem[ii].push([uu, c]);206  }207208  // init déterministe légère (hash) — reproductible d'un run à l'autre209  const X = new Float64Array(nU * k);210  const Y = new Float64Array(nI * k);211  let seed = 42;212  const rand = () => {213    seed = (seed * 1103515245 + 12345) & 0x7fffffff;214    return (seed / 0x7fffffff - 0.5) * 0.1;215  };216  for (let i = 0; i < X.length; i++) X[i] = rand();217  for (let i = 0; i < Y.length; i++) Y[i] = rand();218219  for (let it = 0; it < ITERS; it++) {220    alsStep(X, Y, (u) => byUser[u], nU, k);221    alsStep(Y, X, (i) => byItem[i], nI, k);222  }223224  // recommandations : items non vus, non masqués, score > 0, top-N225  const itemIds = [...items.keys()];226  const insert = db.prepare(227    `INSERT INTO mf_recs (user_id, app, item_id, score, rank)228     VALUES (?, ?, ?, ?, ?)`,229  );230  const clear = db.prepare("DELETE FROM mf_recs WHERE app = ?");231  let written = 0;232  const tx = db.transaction(() => {233    clear.run(app);234    for (const [uidStr, u] of users) {235      const userId = Number(uidStr);236      const seen = new Set(byUser[u].map(([i]) => i));237      const hid = hidden.get(userId) ?? new Set();238      const scores = [];239      for (let i = 0; i < nI; i++) {240        if (seen.has(i) || hid.has(itemIds[i])) continue;241        let s = 0;242        for (let a = 0; a < k; a++) s += X[u * k + a] * Y[i * k + a];243        if (s > 0.05) scores.push([s, i]);244      }245      scores.sort((a, b) => b[0] - a[0]);246      const top = scores.slice(0, TOP_N);247      const max = top.length ? top[0][0] : 1;248      top.forEach(([s, i], rank) => {249        insert.run(userId, app, itemIds[i], Math.round((s / max) * 100) / 100, rank + 1);250        written++;251      });252    }253  });254  tx();255  log(`${app} : entraîné (membres=${nU}, annonces=${nI}, nnz=${nnz}, k=${k}) ` +256      `→ ${written} recommandations`);257  return { app, trained: true, users: nU, items: nI, written };258}259260// ---------------------------------------------------------------- principal261const apps = ONLY_APP262  ? [ONLY_APP]263  : db.prepare(264      `SELECT DISTINCT app FROM (265         SELECT app FROM user_events UNION SELECT app FROM favorites)`,266    ).all().map((r0) => r0.app);267268log(`démarrage — base ${DB_PATH}, univers : ${apps.join(", ") || "(aucun)"}`);269let trained = 0;270for (const app of apps) {271  try {272    if (factorizeApp(app).trained) trained++;273  } catch (e) {274    log(`${app} : ERREUR — ${e.message}`);275  }276}277// purge des recommandations orphelines de plus de 7 jours278db.prepare("DELETE FROM mf_recs WHERE run_at < datetime('now', '-7 days')").run();279log(`terminé : ${trained}/${apps.length} univers entraînés`);280