SPB Git

Repository Public

moe-routing.tsx 8.1 KB 137 lines tsx

1import type { Locale } from "@/schema";2import { createRng, entropy, softmax, sum, topK } from "../core";3import { param, type SimDef } from "../types";4import { fmt, fmtInt, px } from "../ui/format";5import { Readouts } from "../ui/Readouts";67export interface MoeModel {8  experts: number;9  /** Effective k (the slider value capped at the number of experts). */10  k: number;11  temperature: number;12  tokens: string[];13  /** logits[t][e]: gate logit of expert e for token t. */14  logits: number[][];15  /** gate[t] = softmax(logits[t] / temperature): sums to 1 over the experts. */16  gate: number[][];17  /** selected[t]: the k experts with the largest gate probability, largest first. */18  selected: number[][];19  /** weights[t][e]: gate renormalised over the selected experts (0 for the others); sums to 1. */20  weights: number[][];21  /** kept[t]: gate probability mass on the selected experts, before renormalisation. */22  kept: number[];23  /** entropy[t]: entropy of the gate divided by ln N (1 = uniform, 0 = one expert). */24  entropy: number[];25  /** loadCum[t][e]: number of tokens 0…t routed to expert e. */26  loadCum: number[][];27  /** Largest final load divided by the mean load (1 = perfectly balanced). */28  imbalance: number;29}3031export function computeMoe(p: Readonly<Record<string, number>>, seed: number): MoeModel {32  const experts = Math.round(p.experts), k = Math.min(Math.round(p.topK), experts), nTok = Math.round(p.tokens), temperature = p.temperature;33  const rng = createRng(seed, "moe");34  // A fixed preference per expert (some experts are "popular") plus a token-specific term.35  const bias = Array.from({ length: experts }, () => rng.normal(0, 0.6));36  const logits: number[][] = [], gate: number[][] = [], selected: number[][] = [], weights: number[][] = [], kept: number[] = [], ent: number[] = [], loadCum: number[][] = [];37  const load = new Array<number>(experts).fill(0);38  for (let t = 0; t < nTok; t++) {39    const z = bias.map((b) => b + rng.normal());40    const g = softmax(z, temperature);41    const sel = topK(g, k);42    const mass = sum(sel.map((e) => g[e]));43    const w = new Array<number>(experts).fill(0);44    for (const e of sel) { w[e] = g[e] / mass; load[e] += 1; }45    logits.push(z); gate.push(g); selected.push(sel); weights.push(w); kept.push(mass); ent.push(entropy(g) / Math.log(experts)); loadCum.push([...load]);46  }47  const meanLoad = (nTok * k) / experts;48  return { experts, k, temperature, tokens: Array.from({ length: nTok }, (_, i) => `t${i + 1}`), logits, gate, selected, weights, kept, entropy: ent, loadCum, imbalance: Math.max(...load) / meanLoad };49}5051const TXT = {52  fr: { tokens: "Jetons", gate: "Porte : softmax(z / τ), puis top-k", experts: "Experts", load: "Charge cumulée : jetons routés vers chaque expert", token: "Jeton courant", sel: "Experts retenus", kept: "Masse de la porte conservée", ent: "Entropie de la porte (normalisée)", imb: "Déséquilibre final (max / moyenne)", aria: "Routage d'un jeton vers les k meilleurs experts" },53  en: { tokens: "Tokens", gate: "Gate: softmax(z / τ), then top-k", experts: "Experts", load: "Cumulative load: tokens routed to each expert", token: "Current token", sel: "Selected experts", kept: "Gate mass kept", ent: "Gate entropy (normalised)", imb: "Final imbalance (max / mean)", aria: "Routing of a token to its top-k experts" },54} as const;5556function MoeView({ model: m, locale, step }: { model: MoeModel; locale: Locale; step: number }) {57  const L = TXT[locale];58  const W = 560, H = 432, pad = 14;59  const t = Math.min(Math.max(0, step), m.tokens.length - 1);60  const nTok = m.tokens.length;61  const cw = Math.min(46, (W - 2 * pad) / nTok), chipW = cw - 6;62  const tokX = (i: number) => W / 2 + (i - (nTok - 1) / 2) * cw;63  const colW = (W - 2 * pad) / m.experts, boxW = Math.min(colW - 8, 88);64  const ex = (e: number) => pad + colW * (e + 0.5);65  const gateY = 86, gateH = 30, boxY = 178, boxH = 62, base = 400, maxBar = 104;66  const final = m.loadCum[nTok - 1];67  const maxLoad = Math.max(1, ...final);68  const sel = new Set(m.selected[t]);69  const load = m.loadCum[t];70  return (71    <>72      <svg viewBox={`0 0 ${W} ${H}`} role="img" aria-label={L.aria} className="sim-svg sim-moe">73        <text x={pad} y={16} className="sim-label">{L.tokens}</text>74        {m.tokens.map((tok, i) => (75          <g key={i} className="sim-moe-token" data-state={i === t ? "current" : i < t ? "done" : "todo"}>76            <rect x={px(tokX(i) - chipW / 2)} y={26} width={px(chipW)} height={30} rx={8} />77            <text x={px(tokX(i))} y={46} textAnchor="middle">{tok}</text>78          </g>79        ))}80        <path d={`M${px(tokX(t))} 56 C${px(tokX(t))} 74 ${W / 2} 68 ${W / 2} ${gateY}`} className="sim-moe-feed" />81        <rect x={W / 2 - 150} y={gateY} width={300} height={gateH} rx={15} className="sim-moe-gate" />82        <text x={W / 2} y={gateY + 20} textAnchor="middle" className="sim-moe-gate-text">{L.gate}</text>83        {m.gate[t].map((g, e) => (84          <path key={e} d={`M${W / 2} ${gateY + gateH} C${W / 2} ${gateY + gateH + 34} ${px(ex(e))} ${boxY - 34} ${px(ex(e))} ${boxY}`}85            className="sim-moe-link" data-on={sel.has(e)} strokeWidth={px(1 + 9 * g)} />86        ))}87        {m.gate[t].map((g, e) => (88          <g key={e} className="sim-moe-expert" data-on={sel.has(e)}>89            <rect x={px(ex(e) - boxW / 2)} y={boxY} width={px(boxW)} height={boxH} rx={10} />90            <text x={px(ex(e))} y={boxY + 18} textAnchor="middle" className="sim-moe-name">E{e + 1}</text>91            <text x={px(ex(e))} y={boxY + 35} textAnchor="middle" className="sim-moe-p">p {fmt(locale, g, 2, true)}</text>92            <text x={px(ex(e))} y={boxY + 52} textAnchor="middle" className="sim-moe-w">{sel.has(e) ? `w ${fmt(locale, m.weights[t][e], 2, true)}` : "—"}</text>93          </g>94        ))}95        <text x={pad} y={270} className="sim-label">{L.load}</text>96        <line x1={pad} x2={W - pad} y1={base} y2={base} className="sim-axis" />97        {load.map((n, e) => {98          const h = (maxBar * n) / maxLoad, bw = Math.min(boxW * 0.62, 44);99          return (100            <g key={e} className="sim-moe-load" data-on={sel.has(e)}>101              <rect x={px(ex(e) - bw / 2)} y={px(base - h)} width={px(bw)} height={px(h)} rx={3} />102              <text x={px(ex(e))} y={px(base - h - 6)} textAnchor="middle" className="sim-moe-count">{fmtInt(locale, n)}</text>103              <text x={px(ex(e))} y={base + 18} textAnchor="middle" className="sim-tick">E{e + 1}</text>104            </g>105          );106        })}107      </svg>108      <Readouts items={[109        { label: L.token, value: `${m.tokens[t]} (${fmtInt(locale, t + 1)} / ${fmtInt(locale, nTok)})` },110        { label: L.sel, value: m.selected[t].map((e) => `E${e + 1}`).join(", "), tone: "blue" },111        { label: L.kept, value: fmt(locale, m.kept[t], 3) },112        { label: L.ent, value: fmt(locale, m.entropy[t], 3) },113        { label: L.imb, value: fmt(locale, m.imbalance, 2) },114      ]} />115    </>116  );117}118119export const moeRouting: SimDef<MoeModel> = {120  kind: "moe-routing",121  title: { fr: "Mélange d'experts : routage top-k", en: "Mixture of Experts: top-k routing" },122  description: {123    fr: "Pour chaque jeton, la porte calcule un softmax sur $N$ experts, ne garde que les $k$ plus probables et renormalise leurs poids ; la sortie est la somme pondérée de ces $k$ experts. Simplification : les logits de la porte sont tirés au hasard (préférence fixe par expert + terme propre au jeton) au lieu d'être produits par un routeur appris.",124    en: "For each token the gate computes a softmax over $N$ experts, keeps only the $k$ most probable and renormalises their weights; the output is the weighted sum of those $k$ experts. Simplification: gate logits are random draws (a fixed preference per expert plus a token-specific term) rather than the output of a trained router.",125  },126  params: [127    param("experts", "Experts N", "Experts N", 2, 8, 1, 6),128    param("topK", "Experts actifs k", "Active experts k", 1, 8, 1, 2),129    param("tokens", "Jetons", "Tokens", 4, 12, 1, 8),130    param("temperature", "Température τ", "Temperature τ", 0.1, 3, 0.1, 1),131  ],132  stochastic: true,133  compute: (p, seed) => computeMoe(p, seed),134  steps: (m) => m.tokens.length,135  View: MoeView,136};137
Files main
Menu

Theme