Repository Public
1import type { Locale } from "@/schema";2import { createRng, entropy, hashString, softmax } from "../core";3import { param, type SimConfig, type SimDef } from "../types";4import { fmt, px } from "../ui/format";5import { Readouts } from "../ui/Readouts";67export const MAX_ATTENTION_TOKENS = 10;8const DEFAULT_TOKENS = { fr: "le,chat,dort,sur,le,tapis", en: "the,cat,sleeps,on,the,mat" } as const;910export function attentionTokens(config: SimConfig): string[] {11 const raw = typeof config.tokens === "string" && config.tokens.trim() ? config.tokens : DEFAULT_TOKENS[config.locale === "en" ? "en" : "fr"];12 return raw.split(",").map((t) => t.trim()).filter(Boolean).slice(0, MAX_ATTENTION_TOKENS);13}1415export interface AttentionModel {16 tokens: string[];17 causal: boolean;18 /** weights[i][j]: attention of token i (query) to token j (key). Each row sums to 1. */19 weights: number[][];20 /** Entropy of each row divided by ln n (1 = uniform attention). */21 rowEntropy: number[];22 meanEntropy: number;23 /** Mean attention weight on the token itself. */24 selfWeight: number;25}2627export function computeAttention(p: Readonly<Record<string, number>>, seed: number, config: SimConfig): AttentionModel {28 const tokens = attentionTokens(config);29 const n = tokens.length, d = Math.round(p.dim), causal = config.causal === true;30 // Token embedding (same word → same vector) plus a smaller positional vector.31 const emb = tokens.map((tok, i) => {32 const tr = createRng(seed ^ hashString(tok.toLowerCase()), "att-token"), pr = createRng(seed + i, "att-position");33 return Array.from({ length: d }, () => tr.normal() + 0.5 * pr.normal());34 });35 const mr = createRng(seed, "att-projections");36 const mat = () => Array.from({ length: d }, () => Array.from({ length: d }, () => mr.normal(0, 1 / Math.sqrt(d))));37 const Wq = mat(), Wk = mat();38 const apply = (M: number[][], v: number[]) => M.map((row) => row.reduce((s, w, i) => s + w * v[i], 0));39 const Q = emb.map((e) => apply(Wq, e)), K = emb.map((e) => apply(Wk, e));40 const weights = Q.map((q, i) => softmax(K.map((k, j) => {41 if (causal && j > i) return -Infinity;42 const score = q.reduce((s, v, a) => s + v * k[a], 0) / Math.sqrt(d);43 return score - p.locality * Math.abs(i - j);44 }), p.temperature));45 const rowEntropy = weights.map((w) => (n > 1 ? entropy(w) / Math.log(n) : 0));46 return {47 tokens, causal, weights, rowEntropy,48 meanEntropy: rowEntropy.reduce((s, v) => s + v, 0) / n,49 selfWeight: weights.reduce((s, w, i) => s + w[i], 0) / n,50 };51}5253const TXT = {54 fr: { aria: "Carte des poids d'attention : une ligne par jeton requête, une colonne par jeton clé", axes: "requête ↓ clé →", ent: "Entropie moyenne des lignes (normalisée)", self: "Poids moyen sur soi-même", mask: "Masque causal", yes: "oui", no: "non" },55 en: { aria: "Attention weight map: one row per query token, one column per key token", axes: "query ↓ key →", ent: "Mean row entropy (normalised)", self: "Mean weight on itself", mask: "Causal mask", yes: "yes", no: "no" },56} as const;5758function AttentionView({ model: m, locale }: { model: AttentionModel; locale: Locale }) {59 const L = TXT[locale];60 const n = m.tokens.length, W = 560, left = 92, top = 34;61 const cell = Math.min(48, (W - left - 8) / n), H = top + n * cell + 8;62 const short = (s: string, max: number) => (s.length > max ? `${s.slice(0, Math.max(1, max - 1))}…` : s);63 const maxChars = Math.max(2, Math.floor(cell / 6.4));64 return (65 <>66 <svg viewBox={`0 0 ${W} ${px(H)}`} role="img" aria-label={L.aria} className="sim-svg sim-att">67 <text x={left - 8} y={22} textAnchor="end" className="sim-tick">{L.axes}</text>68 {m.tokens.map((t, j) => <text key={`c${j}`} x={px(left + (j + 0.5) * cell)} y={24} textAnchor="middle" className="sim-att-token">{short(t, maxChars)}</text>)}69 {m.tokens.map((t, i) => <text key={`r${i}`} x={left - 8} y={px(top + (i + 0.5) * cell + 4)} textAnchor="end" className="sim-att-token">{short(t, 11)}</text>)}70 {m.weights.map((row, i) => row.map((w, j) => (71 <g key={`${i}-${j}`}>72 <rect x={px(left + j * cell + 1)} y={px(top + i * cell + 1)} width={px(cell - 2)} height={px(cell - 2)} rx={4} className="sim-att-cell" fillOpacity={px(w)} data-masked={m.causal && j > i} />73 {cell >= 30 && !(m.causal && j > i) ? <text x={px(left + (j + 0.5) * cell)} y={px(top + (i + 0.5) * cell + 4)} textAnchor="middle" className="sim-att-value" data-strong={w > 0.5}>{fmt(locale, w, 2, true)}</text> : null}74 </g>75 )))}76 </svg>77 <Readouts items={[78 { label: L.ent, value: fmt(locale, m.meanEntropy, 3) },79 { label: L.self, value: fmt(locale, m.selfWeight, 3) },80 { label: L.mask, value: m.causal ? L.yes : L.no },81 ]} />82 </>83 );84}8586export const attention: SimDef<AttentionModel> = {87 kind: "attention",88 title: { fr: "Auto-attention à produit scalaire normalisé", en: "Scaled dot-product self-attention" },89 description: {90 fr: "Poids d'attention $\\operatorname{softmax}\\big((q_i^{\\top} k_j/\\sqrt{d} - \\lambda\\,|i-j|)/\\tau\\big)$ d'une seule tête. Les plongements (un vecteur par mot, plus un terme de position) et les projections $Q$ et $K$ sont tirés de la graine : la carte montre le mécanisme, pas ce qu'un modèle entraîné a appris. $\\lambda$ est un biais de position relative simplifié ; $\\tau = 1$ et $\\lambda = 0$ donnent la formule standard.",91 en: "Attention weights $\\operatorname{softmax}\\big((q_i^{\\top} k_j/\\sqrt{d} - \\lambda\\,|i-j|)/\\tau\\big)$ of a single head. Embeddings (one vector per word plus a positional term) and the $Q$ and $K$ projections are drawn from the seed: the map shows the mechanism, not what a trained model has learned. $\\lambda$ is a simplified relative-position bias; $\\tau = 1$ and $\\lambda = 0$ give the standard formula.",92 },93 params: [94 param("temperature", "Température τ", "Temperature τ", 0.1, 5, 0.1, 1),95 param("dim", "Dimension d", "Dimension d", 4, 32, 4, 16),96 param("locality", "Biais de proximité λ", "Locality bias λ", 0, 3, 0.1, 0),97 ],98 stochastic: true,99 configKeys: ["tokens", "causal"],100 validateConfig: (c) => {101 const out: string[] = [];102 if (c.tokens !== undefined) {103 if (typeof c.tokens !== "string") out.push("config.tokens: expected comma-separated text");104 else {105 const n = c.tokens.split(",").map((t) => t.trim()).filter(Boolean).length;106 if (n < 2) out.push("config.tokens: at least 2 comma-separated tokens are needed");107 if (n > MAX_ATTENTION_TOKENS) out.push(`config.tokens: at most ${MAX_ATTENTION_TOKENS} tokens (got ${n})`);108 }109 }110 if (c.causal !== undefined && typeof c.causal !== "boolean") out.push("config.causal: expected true or false");111 return out;112 },113 compute: computeAttention,114 View: AttentionView,115};116