SPB Git

Repository Public

attention.tsx 6.7 KB 116 lines tsx

1import type { Locale } from "@/schema";2import { createRng, entropy, hashString, softmax } from "../core";3import { param, type SimConfig, type SimDef } from "../types";4import { fmt, px } from "../ui/format";5import { Readouts } from "../ui/Readouts";67export const MAX_ATTENTION_TOKENS = 10;8const DEFAULT_TOKENS = { fr: "le,chat,dort,sur,le,tapis", en: "the,cat,sleeps,on,the,mat" } as const;910export function attentionTokens(config: SimConfig): string[] {11  const raw = typeof config.tokens === "string" && config.tokens.trim() ? config.tokens : DEFAULT_TOKENS[config.locale === "en" ? "en" : "fr"];12  return raw.split(",").map((t) => t.trim()).filter(Boolean).slice(0, MAX_ATTENTION_TOKENS);13}1415export interface AttentionModel {16  tokens: string[];17  causal: boolean;18  /** weights[i][j]: attention of token i (query) to token j (key). Each row sums to 1. */19  weights: number[][];20  /** Entropy of each row divided by ln n (1 = uniform attention). */21  rowEntropy: number[];22  meanEntropy: number;23  /** Mean attention weight on the token itself. */24  selfWeight: number;25}2627export function computeAttention(p: Readonly<Record<string, number>>, seed: number, config: SimConfig): AttentionModel {28  const tokens = attentionTokens(config);29  const n = tokens.length, d = Math.round(p.dim), causal = config.causal === true;30  // Token embedding (same word → same vector) plus a smaller positional vector.31  const emb = tokens.map((tok, i) => {32    const tr = createRng(seed ^ hashString(tok.toLowerCase()), "att-token"), pr = createRng(seed + i, "att-position");33    return Array.from({ length: d }, () => tr.normal() + 0.5 * pr.normal());34  });35  const mr = createRng(seed, "att-projections");36  const mat = () => Array.from({ length: d }, () => Array.from({ length: d }, () => mr.normal(0, 1 / Math.sqrt(d))));37  const Wq = mat(), Wk = mat();38  const apply = (M: number[][], v: number[]) => M.map((row) => row.reduce((s, w, i) => s + w * v[i], 0));39  const Q = emb.map((e) => apply(Wq, e)), K = emb.map((e) => apply(Wk, e));40  const weights = Q.map((q, i) => softmax(K.map((k, j) => {41    if (causal && j > i) return -Infinity;42    const score = q.reduce((s, v, a) => s + v * k[a], 0) / Math.sqrt(d);43    return score - p.locality * Math.abs(i - j);44  }), p.temperature));45  const rowEntropy = weights.map((w) => (n > 1 ? entropy(w) / Math.log(n) : 0));46  return {47    tokens, causal, weights, rowEntropy,48    meanEntropy: rowEntropy.reduce((s, v) => s + v, 0) / n,49    selfWeight: weights.reduce((s, w, i) => s + w[i], 0) / n,50  };51}5253const TXT = {54  fr: { aria: "Carte des poids d'attention : une ligne par jeton requête, une colonne par jeton clé", axes: "requête ↓  clé →", ent: "Entropie moyenne des lignes (normalisée)", self: "Poids moyen sur soi-même", mask: "Masque causal", yes: "oui", no: "non" },55  en: { aria: "Attention weight map: one row per query token, one column per key token", axes: "query ↓  key →", ent: "Mean row entropy (normalised)", self: "Mean weight on itself", mask: "Causal mask", yes: "yes", no: "no" },56} as const;5758function AttentionView({ model: m, locale }: { model: AttentionModel; locale: Locale }) {59  const L = TXT[locale];60  const n = m.tokens.length, W = 560, left = 92, top = 34;61  const cell = Math.min(48, (W - left - 8) / n), H = top + n * cell + 8;62  const short = (s: string, max: number) => (s.length > max ? `${s.slice(0, Math.max(1, max - 1))}…` : s);63  const maxChars = Math.max(2, Math.floor(cell / 6.4));64  return (65    <>66      <svg viewBox={`0 0 ${W} ${px(H)}`} role="img" aria-label={L.aria} className="sim-svg sim-att">67        <text x={left - 8} y={22} textAnchor="end" className="sim-tick">{L.axes}</text>68        {m.tokens.map((t, j) => <text key={`c${j}`} x={px(left + (j + 0.5) * cell)} y={24} textAnchor="middle" className="sim-att-token">{short(t, maxChars)}</text>)}69        {m.tokens.map((t, i) => <text key={`r${i}`} x={left - 8} y={px(top + (i + 0.5) * cell + 4)} textAnchor="end" className="sim-att-token">{short(t, 11)}</text>)}70        {m.weights.map((row, i) => row.map((w, j) => (71          <g key={`${i}-${j}`}>72            <rect x={px(left + j * cell + 1)} y={px(top + i * cell + 1)} width={px(cell - 2)} height={px(cell - 2)} rx={4} className="sim-att-cell" fillOpacity={px(w)} data-masked={m.causal && j > i} />73            {cell >= 30 && !(m.causal && j > i) ? <text x={px(left + (j + 0.5) * cell)} y={px(top + (i + 0.5) * cell + 4)} textAnchor="middle" className="sim-att-value" data-strong={w > 0.5}>{fmt(locale, w, 2, true)}</text> : null}74          </g>75        )))}76      </svg>77      <Readouts items={[78        { label: L.ent, value: fmt(locale, m.meanEntropy, 3) },79        { label: L.self, value: fmt(locale, m.selfWeight, 3) },80        { label: L.mask, value: m.causal ? L.yes : L.no },81      ]} />82    </>83  );84}8586export const attention: SimDef<AttentionModel> = {87  kind: "attention",88  title: { fr: "Auto-attention à produit scalaire normalisé", en: "Scaled dot-product self-attention" },89  description: {90    fr: "Poids d'attention $\\operatorname{softmax}\\big((q_i^{\\top} k_j/\\sqrt{d} - \\lambda\\,|i-j|)/\\tau\\big)$ d'une seule tête. Les plongements (un vecteur par mot, plus un terme de position) et les projections $Q$ et $K$ sont tirés de la graine : la carte montre le mécanisme, pas ce qu'un modèle entraîné a appris. $\\lambda$ est un biais de position relative simplifié ; $\\tau = 1$ et $\\lambda = 0$ donnent la formule standard.",91    en: "Attention weights $\\operatorname{softmax}\\big((q_i^{\\top} k_j/\\sqrt{d} - \\lambda\\,|i-j|)/\\tau\\big)$ of a single head. Embeddings (one vector per word plus a positional term) and the $Q$ and $K$ projections are drawn from the seed: the map shows the mechanism, not what a trained model has learned. $\\lambda$ is a simplified relative-position bias; $\\tau = 1$ and $\\lambda = 0$ give the standard formula.",92  },93  params: [94    param("temperature", "Température τ", "Temperature τ", 0.1, 5, 0.1, 1),95    param("dim", "Dimension d", "Dimension d", 4, 32, 4, 16),96    param("locality", "Biais de proximité λ", "Locality bias λ", 0, 3, 0.1, 0),97  ],98  stochastic: true,99  configKeys: ["tokens", "causal"],100  validateConfig: (c) => {101    const out: string[] = [];102    if (c.tokens !== undefined) {103      if (typeof c.tokens !== "string") out.push("config.tokens: expected comma-separated text");104      else {105        const n = c.tokens.split(",").map((t) => t.trim()).filter(Boolean).length;106        if (n < 2) out.push("config.tokens: at least 2 comma-separated tokens are needed");107        if (n > MAX_ATTENTION_TOKENS) out.push(`config.tokens: at most ${MAX_ATTENTION_TOKENS} tokens (got ${n})`);108      }109    }110    if (c.causal !== undefined && typeof c.causal !== "boolean") out.push("config.causal: expected true or false");111    return out;112  },113  compute: computeAttention,114  View: AttentionView,115};116
Files main
Menu

Theme