Repository Public
1import type { Locale } from "@/schema";2import { createRng, entropy, softmax, sum, topK } from "../core";3import { param, type SimDef } from "../types";4import { fmt, fmtInt, px } from "../ui/format";5import { Readouts } from "../ui/Readouts";67export interface MoeModel {8 experts: number;9 /** Effective k (the slider value capped at the number of experts). */10 k: number;11 temperature: number;12 tokens: string[];13 /** logits[t][e]: gate logit of expert e for token t. */14 logits: number[][];15 /** gate[t] = softmax(logits[t] / temperature): sums to 1 over the experts. */16 gate: number[][];17 /** selected[t]: the k experts with the largest gate probability, largest first. */18 selected: number[][];19 /** weights[t][e]: gate renormalised over the selected experts (0 for the others); sums to 1. */20 weights: number[][];21 /** kept[t]: gate probability mass on the selected experts, before renormalisation. */22 kept: number[];23 /** entropy[t]: entropy of the gate divided by ln N (1 = uniform, 0 = one expert). */24 entropy: number[];25 /** loadCum[t][e]: number of tokens 0…t routed to expert e. */26 loadCum: number[][];27 /** Largest final load divided by the mean load (1 = perfectly balanced). */28 imbalance: number;29}3031export function computeMoe(p: Readonly<Record<string, number>>, seed: number): MoeModel {32 const experts = Math.round(p.experts), k = Math.min(Math.round(p.topK), experts), nTok = Math.round(p.tokens), temperature = p.temperature;33 const rng = createRng(seed, "moe");34 // A fixed preference per expert (some experts are "popular") plus a token-specific term.35 const bias = Array.from({ length: experts }, () => rng.normal(0, 0.6));36 const logits: number[][] = [], gate: number[][] = [], selected: number[][] = [], weights: number[][] = [], kept: number[] = [], ent: number[] = [], loadCum: number[][] = [];37 const load = new Array<number>(experts).fill(0);38 for (let t = 0; t < nTok; t++) {39 const z = bias.map((b) => b + rng.normal());40 const g = softmax(z, temperature);41 const sel = topK(g, k);42 const mass = sum(sel.map((e) => g[e]));43 const w = new Array<number>(experts).fill(0);44 for (const e of sel) { w[e] = g[e] / mass; load[e] += 1; }45 logits.push(z); gate.push(g); selected.push(sel); weights.push(w); kept.push(mass); ent.push(entropy(g) / Math.log(experts)); loadCum.push([...load]);46 }47 const meanLoad = (nTok * k) / experts;48 return { experts, k, temperature, tokens: Array.from({ length: nTok }, (_, i) => `t${i + 1}`), logits, gate, selected, weights, kept, entropy: ent, loadCum, imbalance: Math.max(...load) / meanLoad };49}5051const TXT = {52 fr: { tokens: "Jetons", gate: "Porte : softmax(z / τ), puis top-k", experts: "Experts", load: "Charge cumulée : jetons routés vers chaque expert", token: "Jeton courant", sel: "Experts retenus", kept: "Masse de la porte conservée", ent: "Entropie de la porte (normalisée)", imb: "Déséquilibre final (max / moyenne)", aria: "Routage d'un jeton vers les k meilleurs experts" },53 en: { tokens: "Tokens", gate: "Gate: softmax(z / τ), then top-k", experts: "Experts", load: "Cumulative load: tokens routed to each expert", token: "Current token", sel: "Selected experts", kept: "Gate mass kept", ent: "Gate entropy (normalised)", imb: "Final imbalance (max / mean)", aria: "Routing of a token to its top-k experts" },54} as const;5556function MoeView({ model: m, locale, step }: { model: MoeModel; locale: Locale; step: number }) {57 const L = TXT[locale];58 const W = 560, H = 432, pad = 14;59 const t = Math.min(Math.max(0, step), m.tokens.length - 1);60 const nTok = m.tokens.length;61 const cw = Math.min(46, (W - 2 * pad) / nTok), chipW = cw - 6;62 const tokX = (i: number) => W / 2 + (i - (nTok - 1) / 2) * cw;63 const colW = (W - 2 * pad) / m.experts, boxW = Math.min(colW - 8, 88);64 const ex = (e: number) => pad + colW * (e + 0.5);65 const gateY = 86, gateH = 30, boxY = 178, boxH = 62, base = 400, maxBar = 104;66 const final = m.loadCum[nTok - 1];67 const maxLoad = Math.max(1, ...final);68 const sel = new Set(m.selected[t]);69 const load = m.loadCum[t];70 return (71 <>72 <svg viewBox={`0 0 ${W} ${H}`} role="img" aria-label={L.aria} className="sim-svg sim-moe">73 <text x={pad} y={16} className="sim-label">{L.tokens}</text>74 {m.tokens.map((tok, i) => (75 <g key={i} className="sim-moe-token" data-state={i === t ? "current" : i < t ? "done" : "todo"}>76 <rect x={px(tokX(i) - chipW / 2)} y={26} width={px(chipW)} height={30} rx={8} />77 <text x={px(tokX(i))} y={46} textAnchor="middle">{tok}</text>78 </g>79 ))}80 <path d={`M${px(tokX(t))} 56 C${px(tokX(t))} 74 ${W / 2} 68 ${W / 2} ${gateY}`} className="sim-moe-feed" />81 <rect x={W / 2 - 150} y={gateY} width={300} height={gateH} rx={15} className="sim-moe-gate" />82 <text x={W / 2} y={gateY + 20} textAnchor="middle" className="sim-moe-gate-text">{L.gate}</text>83 {m.gate[t].map((g, e) => (84 <path key={e} d={`M${W / 2} ${gateY + gateH} C${W / 2} ${gateY + gateH + 34} ${px(ex(e))} ${boxY - 34} ${px(ex(e))} ${boxY}`}85 className="sim-moe-link" data-on={sel.has(e)} strokeWidth={px(1 + 9 * g)} />86 ))}87 {m.gate[t].map((g, e) => (88 <g key={e} className="sim-moe-expert" data-on={sel.has(e)}>89 <rect x={px(ex(e) - boxW / 2)} y={boxY} width={px(boxW)} height={boxH} rx={10} />90 <text x={px(ex(e))} y={boxY + 18} textAnchor="middle" className="sim-moe-name">E{e + 1}</text>91 <text x={px(ex(e))} y={boxY + 35} textAnchor="middle" className="sim-moe-p">p {fmt(locale, g, 2, true)}</text>92 <text x={px(ex(e))} y={boxY + 52} textAnchor="middle" className="sim-moe-w">{sel.has(e) ? `w ${fmt(locale, m.weights[t][e], 2, true)}` : "—"}</text>93 </g>94 ))}95 <text x={pad} y={270} className="sim-label">{L.load}</text>96 <line x1={pad} x2={W - pad} y1={base} y2={base} className="sim-axis" />97 {load.map((n, e) => {98 const h = (maxBar * n) / maxLoad, bw = Math.min(boxW * 0.62, 44);99 return (100 <g key={e} className="sim-moe-load" data-on={sel.has(e)}>101 <rect x={px(ex(e) - bw / 2)} y={px(base - h)} width={px(bw)} height={px(h)} rx={3} />102 <text x={px(ex(e))} y={px(base - h - 6)} textAnchor="middle" className="sim-moe-count">{fmtInt(locale, n)}</text>103 <text x={px(ex(e))} y={base + 18} textAnchor="middle" className="sim-tick">E{e + 1}</text>104 </g>105 );106 })}107 </svg>108 <Readouts items={[109 { label: L.token, value: `${m.tokens[t]} (${fmtInt(locale, t + 1)} / ${fmtInt(locale, nTok)})` },110 { label: L.sel, value: m.selected[t].map((e) => `E${e + 1}`).join(", "), tone: "blue" },111 { label: L.kept, value: fmt(locale, m.kept[t], 3) },112 { label: L.ent, value: fmt(locale, m.entropy[t], 3) },113 { label: L.imb, value: fmt(locale, m.imbalance, 2) },114 ]} />115 </>116 );117}118119export const moeRouting: SimDef<MoeModel> = {120 kind: "moe-routing",121 title: { fr: "Mélange d'experts : routage top-k", en: "Mixture of Experts: top-k routing" },122 description: {123 fr: "Pour chaque jeton, la porte calcule un softmax sur $N$ experts, ne garde que les $k$ plus probables et renormalise leurs poids ; la sortie est la somme pondérée de ces $k$ experts. Simplification : les logits de la porte sont tirés au hasard (préférence fixe par expert + terme propre au jeton) au lieu d'être produits par un routeur appris.",124 en: "For each token the gate computes a softmax over $N$ experts, keeps only the $k$ most probable and renormalises their weights; the output is the weighted sum of those $k$ experts. Simplification: gate logits are random draws (a fixed preference per expert plus a token-specific term) rather than the output of a trained router.",125 },126 params: [127 param("experts", "Experts N", "Experts N", 2, 8, 1, 6),128 param("topK", "Experts actifs k", "Active experts k", 1, 8, 1, 2),129 param("tokens", "Jetons", "Tokens", 4, 12, 1, 8),130 param("temperature", "Température τ", "Temperature τ", 0.1, 3, 0.1, 1),131 ],132 stochastic: true,133 compute: (p, seed) => computeMoe(p, seed),134 steps: (m) => m.tokens.length,135 View: MoeView,136};137