Repository Public
1/**2 * =============================================================================3 * SP BOUCHER | SOFTWARE & RESEARCH4 * ARTIFICIAL INTELLIGENCE • FINANCE • ENGINEERING5 * =============================================================================6 *7 * Project : Sheet8 * File : src/templates/ai/transformer-architecture.ts9 * Module : templates10 * Repository : https://git.spboucher.ai/sheet11 * Project Website : https://sheet.spboucher.ai12 * Created : 2026-11-1713 * Last Modified : 2026-11-1714 *15 * Author : Simon-Pierre Boucher16 * Position : Professor of Finance17 * Institution : Université du Québec en Outaouais (UQO)18 * Organization : SP Boucher19 * Website : https://spboucher.ai20 * Git : https://git.spboucher.ai21 *22 * Preferred Contact: contact@spboucher.ai23 * Development : dev@spboucher.ai24 * Support : support@spboucher.ai25 * Security/Admin : admin@spboucher.ai26 *27 * Copyright (c) 202628 * Simon-Pierre Boucher29 * All rights reserved where applicable. This file is governed by the30 * applicable project license, copyright notices and relevant law.31 * Existing third-party rights and notices remain unaffected.32 *33 * License : None specified (no license file in this repository)34 *35 * Do not embed credentials, private keys, access tokens,36 * certificates or confidential personal information.37 *38 * =============================================================================39 * END OF SP BOUCHER SOURCE HEADER40 * =============================================================================41 */4243import type { TemplateDef } from "../types";44import { T, pickSources } from "./shared";4546const r = String.raw;4748export const transformerArchitecture: TemplateDef = {49 id: "transformer-architecture",50 category: "ai",51 title: { fr: "Architecture du Transformer", en: "Transformer architecture" },52 description: {53 fr: "Encodeur, décodeur, auto-attention, encodage positionnel, connexions résiduelles et normalisation : le schéma complet, équations à l'appui.",54 en: "Encoder, decoder, self-attention, positional encoding, residual connections and normalisation: the full picture, with the equations.",55 },56 build: (locale) => {57 const t = T(locale);58 return {59 schemaVersion: 1,60 metadata: {61 title: t("L'architecture du Transformer", "The Transformer architecture"),62 slug: t("architecture-transformer", "transformer-architecture"),63 locale,64 summary: t(65 "Comment un Transformer transforme une suite de jetons en prédictions : l'auto-attention relie toutes les positions entre elles, puis chaque bloc affine la représentation.",66 "How a Transformer turns a sequence of tokens into predictions: self-attention connects every position to every other, and each block refines the representation.",67 ),68 topics: [t("IA", "AI"), t("Apprentissage profond", "Deep Learning")],69 tags: ["transformer", "attention", t("encodeur-décodeur", "encoder-decoder"), t("encodage positionnel", "positional encoding"), "LLM"],70 level: "intermediate",71 kind: "architecture",72 objectives: [73 t("Suivre le trajet d'un jeton à travers l'encodeur et le décodeur.", "Follow a token through the encoder and the decoder."),74 t("Écrire et interpréter l'équation de l'attention à produit scalaire normalisé.", "Write and interpret the scaled dot-product attention equation."),75 t("Expliquer pourquoi il faut un encodage positionnel, des connexions résiduelles et une normalisation.", "Explain why positional encoding, residual connections and normalisation are needed."),76 t("Distinguer les variantes encodeur seul, décodeur seul et encodeur-décodeur.", "Tell encoder-only, decoder-only and encoder–decoder variants apart."),77 ],78 example: true,79 },80 theme: { preset: "scientific-light", accent: "blue" },81 blocks: [82 { id: "h-title", type: "heading", level: 1, eyebrow: t("Architecture · Apprentissage profond", "Architecture · Deep Learning"), text: t("L'architecture du Transformer", "The Transformer architecture") },83 {84 id: "intro-lead", type: "paragraph", lead: true,85 text: t(86 "Le Transformer est une architecture de réseau de neurones qui traite une séquence **sans récurrence ni convolution** : chaque position consulte directement toutes les autres grâce à un mécanisme d'attention[^vaswani2017]. Cette idée, proposée en 2017 pour la traduction automatique, est aujourd'hui la base des grands modèles de langage.",87 "The Transformer is a neural network architecture that processes a sequence **without recurrence or convolution**: every position looks directly at every other one through an attention mechanism[^vaswani2017]. Proposed in 2017 for machine translation, the idea now underpins large language models.",88 ),89 },90 {91 id: "c-ideas", type: "cards", columns: 3,92 items: [93 { icon: "eye", tone: "blue", title: t("L'attention relie tout", "Attention connects everything"), text: t("Chaque jeton calcule une moyenne pondérée de tous les autres. Deux mots éloignés sont reliés en une seule opération.", "Each token computes a weighted average of all the others. Two distant words are linked in a single operation.") },94 { icon: "bolt", tone: "teal", title: t("Tout se calcule en parallèle", "Everything runs in parallel"), text: t("À l'entraînement, toutes les positions sont traitées en même temps, là où un réseau récurrent avance pas à pas.", "During training all positions are processed at once, where a recurrent network moves one step at a time.") },95 { icon: "layers", tone: "violet", title: t("Un même bloc, empilé", "One block, stacked"), text: t("Le modèle répète $N$ fois un bloc identique : attention, puis réseau à propagation avant, chacun entouré d'une connexion résiduelle et d'une normalisation.", "The model repeats the same block $N$ times: attention, then a feed-forward network, each wrapped in a residual connection and a normalisation.") },96 ],97 },9899 { id: "h-overview", type: "heading", level: 2, text: t("Vue d'ensemble : encodeur et décodeur", "The big picture: encoder and decoder") },100 {101 id: "d-arch", type: "diagram", span: 8, kind: "architecture", direction: "TB", routing: "orthogonal",102 title: t("Le Transformer d'origine (encodeur-décodeur)", "The original Transformer (encoder–decoder)"),103 caption: t(104 "Une couche d'encodeur et une couche de décodeur sont dessinées ; le modèle d'origine en empile $N = 6$ de chaque[^vaswani2017].",105 "One encoder layer and one decoder layer are drawn; the original model stacks $N = 6$ of each[^vaswani2017].",106 ),107 alt: t(108 "Schéma en deux colonnes. À gauche, l'encodeur : jetons source, plongement et encodage positionnel, auto-attention multi-tête, addition et normalisation, réseau à propagation avant, addition et normalisation. À droite, le décodeur : jetons cible décalés, plongement et encodage positionnel, auto-attention masquée, attention croisée qui reçoit la sortie de l'encodeur, réseau à propagation avant, puis couche linéaire et softmax qui donnent la probabilité du prochain jeton. Des flèches pointillées représentent les connexions résiduelles.",109 "Two-column diagram. On the left, the encoder: source tokens, embedding and positional encoding, multi-head self-attention, add and normalise, feed-forward network, add and normalise. On the right, the decoder: shifted target tokens, embedding and positional encoding, masked self-attention, cross-attention receiving the encoder output, feed-forward network, then a linear layer and a softmax giving the probability of the next token. Dashed arrows show the residual connections.",110 ),111 sourceIds: ["vaswani2017"],112 groups: [113 { id: "g-enc", label: t("Encodeur (× N)", "Encoder (× N)"), tone: "blue" },114 { id: "g-dec", label: t("Décodeur (× N)", "Decoder (× N)"), tone: "violet" },115 ],116 nodes: [117 { id: "n-src", label: t("Jetons source", "Source tokens"), shape: "pill", tone: "neutral", icon: "token" },118 { id: "n-emb-src", label: t("Plongement + position", "Embedding + position"), shape: "round", tone: "teal" },119 { id: "n-enc-attn", label: t("Auto-attention multi-tête", "Multi-head self-attention"), shape: "round", tone: "blue", icon: "eye", group: "g-enc" },120 { id: "n-enc-norm1", label: t("Addition + normalisation", "Add + norm"), shape: "rect", tone: "amber", group: "g-enc" },121 { id: "n-enc-ffn", label: t("Propagation avant", "Feed-forward"), sublabel: t("par position", "position-wise"), shape: "round", tone: "green", group: "g-enc" },122 { id: "n-enc-norm2", label: t("Addition + normalisation", "Add + norm"), shape: "rect", tone: "amber", group: "g-enc" },123 { id: "n-tgt", label: t("Jetons cible décalés", "Shifted target tokens"), shape: "pill", tone: "neutral", icon: "token" },124 { id: "n-emb-tgt", label: t("Plongement + position", "Embedding + position"), shape: "round", tone: "teal" },125 { id: "n-dec-mattn", label: t("Auto-attention masquée", "Masked self-attention"), shape: "round", tone: "violet", icon: "eye", group: "g-dec" },126 { id: "n-dec-norm1", label: t("Addition + normalisation", "Add + norm"), shape: "rect", tone: "amber", group: "g-dec" },127 { id: "n-dec-xattn", label: t("Attention croisée", "Cross-attention"), sublabel: t("Q : décodeur · K, V : encodeur", "Q: decoder · K, V: encoder"), shape: "round", tone: "pink", icon: "link", group: "g-dec" },128 { id: "n-dec-norm2", label: t("Addition + normalisation", "Add + norm"), shape: "rect", tone: "amber", group: "g-dec" },129 { id: "n-dec-ffn", label: t("Propagation avant", "Feed-forward"), sublabel: t("par position", "position-wise"), shape: "round", tone: "green", group: "g-dec" },130 { id: "n-dec-norm3", label: t("Addition + normalisation", "Add + norm"), shape: "rect", tone: "amber", group: "g-dec" },131 { id: "n-linear", label: t("Couche linéaire", "Linear layer"), sublabel: t("vers le vocabulaire", "to the vocabulary"), shape: "rect", tone: "neutral" },132 { id: "n-softmax", label: "Softmax", sublabel: t("probabilité du prochain jeton", "next-token probability"), shape: "pill", tone: "red", icon: "chart" },133 ],134 edges: [135 { id: "e-src-emb", from: "n-src", to: "n-emb-src" },136 { id: "e-emb-attn", from: "n-emb-src", to: "n-enc-attn" },137 { id: "e-attn-n1", from: "n-enc-attn", to: "n-enc-norm1" },138 { id: "e-res-enc1", from: "n-emb-src", to: "n-enc-norm1", style: "dashed", tone: "amber", label: t("résiduelle", "residual") },139 { id: "e-n1-ffn", from: "n-enc-norm1", to: "n-enc-ffn" },140 { id: "e-ffn-n2", from: "n-enc-ffn", to: "n-enc-norm2" },141 { id: "e-res-enc2", from: "n-enc-norm1", to: "n-enc-norm2", style: "dashed", tone: "amber" },142 { id: "e-tgt-emb", from: "n-tgt", to: "n-emb-tgt" },143 { id: "e-emb-mattn", from: "n-emb-tgt", to: "n-dec-mattn" },144 { id: "e-mattn-n1", from: "n-dec-mattn", to: "n-dec-norm1" },145 { id: "e-res-dec1", from: "n-emb-tgt", to: "n-dec-norm1", style: "dashed", tone: "amber" },146 { id: "e-n1-xattn", from: "n-dec-norm1", to: "n-dec-xattn", label: "Q" },147 { id: "e-memory", from: "n-enc-norm2", to: "n-dec-xattn", tone: "pink", label: t("K, V (mémoire)", "K, V (memory)") },148 { id: "e-xattn-n2", from: "n-dec-xattn", to: "n-dec-norm2" },149 { id: "e-res-dec2", from: "n-dec-norm1", to: "n-dec-norm2", style: "dashed", tone: "amber" },150 { id: "e-n2-ffn", from: "n-dec-norm2", to: "n-dec-ffn" },151 { id: "e-ffn-n3", from: "n-dec-ffn", to: "n-dec-norm3" },152 { id: "e-res-dec3", from: "n-dec-norm2", to: "n-dec-norm3", style: "dashed", tone: "amber" },153 { id: "e-n3-linear", from: "n-dec-norm3", to: "n-linear" },154 { id: "e-linear-softmax", from: "n-linear", to: "n-softmax" },155 ],156 legend: [157 { label: t("Flux des représentations", "Flow of representations"), tone: "neutral", shape: "arrow" },158 { label: t("Connexion résiduelle (l'entrée contourne la sous-couche)", "Residual connection (the input bypasses the sub-layer)"), tone: "amber", shape: "dashed" },159 { label: t("Sortie de l'encodeur lue par le décodeur", "Encoder output read by the decoder"), tone: "pink", shape: "arrow" },160 { label: t("Attention", "Attention"), tone: "blue", shape: "square" },161 { label: t("Réseau à propagation avant", "Feed-forward network"), tone: "green", shape: "square" },162 { label: t("Addition + normalisation", "Add + norm"), tone: "amber", shape: "square" },163 ],164 animation: {165 steps: [166 { title: t("1. Des jetons aux vecteurs", "1. From tokens to vectors"), text: t("Chaque jeton source devient un vecteur de dimension $d_{\\text{model}}$, auquel on ajoute un encodage de sa position.", "Each source token becomes a vector of dimension $d_{\\text{model}}$, to which an encoding of its position is added."), nodes: ["n-src", "n-emb-src"], edges: ["e-src-emb"] },167 { title: t("2. L'encodeur met en contexte", "2. The encoder builds context"), text: t("L'auto-attention laisse chaque position consulter toutes les autres ; la sortie est additionnée à l'entrée puis normalisée.", "Self-attention lets each position look at all the others; the output is added to the input and normalised."), nodes: ["n-enc-attn", "n-enc-norm1"], edges: ["e-emb-attn", "e-attn-n1", "e-res-enc1"] },168 { title: t("3. Transformation par position", "3. Position-wise transformation"), text: t("Le même petit réseau à deux couches est appliqué séparément à chaque position, là encore avec résiduelle et normalisation.", "The same small two-layer network is applied to each position separately, again with a residual and a normalisation."), nodes: ["n-enc-ffn", "n-enc-norm2"], edges: ["e-n1-ffn", "e-ffn-n2", "e-res-enc2"] },169 { title: t("4. Le décodeur lit ce qu'il a déjà produit", "4. The decoder reads what it has produced"), text: t("L'auto-attention est masquée : la position $i$ ne voit que les positions $\\le i$, ce qui préserve la génération de gauche à droite.", "Self-attention is masked: position $i$ only sees positions $\\le i$, which preserves left-to-right generation."), nodes: ["n-tgt", "n-emb-tgt", "n-dec-mattn", "n-dec-norm1"], edges: ["e-tgt-emb", "e-emb-mattn", "e-mattn-n1", "e-res-dec1"] },170 { title: t("5. Le décodeur consulte la source", "5. The decoder consults the source"), text: t("Dans l'attention croisée, les requêtes viennent du décodeur ; les clés et les valeurs viennent de la sortie de l'encodeur.", "In cross-attention the queries come from the decoder; the keys and values come from the encoder output."), nodes: ["n-enc-norm2", "n-dec-xattn", "n-dec-norm2"], edges: ["e-n1-xattn", "e-memory", "e-xattn-n2", "e-res-dec2"] },171 { title: t("6. Prédire le prochain jeton", "6. Predict the next token"), text: t("Après le réseau à propagation avant, une couche linéaire projette vers le vocabulaire et le softmax donne une distribution de probabilité.", "After the feed-forward network, a linear layer projects onto the vocabulary and the softmax yields a probability distribution."), nodes: ["n-dec-ffn", "n-dec-norm3", "n-linear", "n-softmax"], edges: ["e-n2-ffn", "e-ffn-n3", "e-res-dec3", "e-n3-linear", "e-linear-softmax"] },172 ],173 },174 },175 {176 id: "c-variants", type: "cards", span: 4, columns: 2,177 items: [178 { icon: "search", tone: "blue", badge: t("Encodeur seul", "Encoder-only"), title: t("Comprendre", "Understand"), text: t("Attention bidirectionnelle sur toute l'entrée. Adapté à la classification et à l'extraction ; BERT en est l'exemple type[^devlin2018].", "Bidirectional attention over the whole input. Suited to classification and extraction; BERT is the textbook example[^devlin2018].") },179 { icon: "sparkles", tone: "violet", badge: t("Décodeur seul", "Decoder-only"), title: t("Générer", "Generate"), text: t("Uniquement l'auto-attention masquée, sans attention croisée. C'est la forme des modèles de langage génératifs de la famille GPT[^brown2020].", "Masked self-attention only, no cross-attention. This is the shape of generative language models of the GPT family[^brown2020].") },180 { icon: "arrows", tone: "pink", badge: t("Encodeur-décodeur", "Encoder–decoder"), title: t("Transformer une séquence en une autre", "Map one sequence to another"), text: t("Le schéma d'origine, conçu pour la traduction : une séquence est lue, une autre est produite[^vaswani2017].", "The original design, built for translation: one sequence is read, another is produced[^vaswani2017].") },181 ],182 },183184 { id: "h-attention", type: "heading", level: 2, text: t("L'auto-attention, pièce maîtresse", "Self-attention, the centrepiece") },185 {186 id: "eq-attention", type: "equation", span: 6, label: t("Attention à produit scalaire normalisé", "Scaled dot-product attention"), derivation: "standard", sourceIds: ["vaswani2017"],187 latex: r`\mathrm{Attention}(Q, K, V) = \mathrm{softmax}\!\left(\frac{Q K^{\top}}{\sqrt{d_k}}\right) V`,188 explanation: t(189 "Chaque requête est comparée à toutes les clés par produit scalaire. Le softmax, appliqué ligne par ligne, convertit ces scores en poids positifs de somme 1 ; la sortie est la moyenne des valeurs pondérée par ces poids[^vaswani2017].",190 "Each query is compared with every key by a dot product. The softmax, applied row by row, turns these scores into positive weights that sum to 1; the output is the average of the values weighted by them[^vaswani2017].",191 ),192 variables: [193 { symbol: "Q", meaning: t("Matrice des requêtes, une ligne par position qui interroge ($n \\times d_k$)", "Query matrix, one row per position asking ($n \\times d_k$)") },194 { symbol: "K", meaning: t("Matrice des clés, une ligne par position consultée ($m \\times d_k$)", "Key matrix, one row per position being looked at ($m \\times d_k$)") },195 { symbol: "V", meaning: t("Matrice des valeurs, l'information effectivement transmise ($m \\times d_v$)", "Value matrix, the information actually passed on ($m \\times d_v$)") },196 { symbol: "d_k", meaning: t("Dimension des requêtes et des clés", "Dimension of the queries and keys") },197 { symbol: r`\mathrm{softmax}`, meaning: t("Fonction $z_i \\mapsto e^{z_i} / \\sum_j e^{z_j}$, appliquée à chaque ligne", "Function $z_i \\mapsto e^{z_i} / \\sum_j e^{z_j}$, applied to each row") },198 ],199 },200 {201 id: "eq-multihead", type: "equation", span: 6, label: t("Attention multi-tête", "Multi-head attention"), derivation: "standard", sourceIds: ["vaswani2017"],202 latex: r`\begin{aligned} \mathrm{MultiHead}(Q, K, V) &= \mathrm{Concat}(\mathrm{head}_1, \dots, \mathrm{head}_h)\, W^{O} \\ \mathrm{head}_i &= \mathrm{Attention}(Q W_i^{Q},\; K W_i^{K},\; V W_i^{V}) \end{aligned}`,203 explanation: t(204 "Plutôt qu'une seule attention, le modèle en calcule $h$ en parallèle dans des sous-espaces différents, puis concatène les résultats. Chaque tête peut ainsi se spécialiser dans un type de relation[^vaswani2017].",205 "Instead of a single attention, the model computes $h$ of them in parallel in different subspaces, then concatenates the results. Each head can thus specialise in one kind of relation[^vaswani2017].",206 ),207 variables: [208 { symbol: "h", meaning: t("Nombre de têtes d'attention", "Number of attention heads") },209 { symbol: r`W_i^{Q},\, W_i^{K},\, W_i^{V}`, meaning: t("Projections apprises de la tête $i$", "Learned projections of head $i$") },210 { symbol: r`W^{O}`, meaning: t("Projection de sortie appliquée après la concaténation", "Output projection applied after concatenation") },211 ],212 },213 {214 id: "sim-attention", type: "simulation", span: 8, kind: "attention", seed: 7,215 params: { temperature: 1, dim: 16, locality: 0 },216 config: { tokens: t("la,banque,au,bord,de,la,rivière", "the,bank,on,the,river,side"), causal: false },217 title: t("Explorer les poids d'attention", "Explore attention weights"),218 caption: t("Simulation à but pédagogique : les poids sont calculés sur des vecteurs jouets, pas par un modèle entraîné.", "Teaching simulation: the weights are computed on toy vectors, not by a trained model."),219 alt: t(220 "Simulation interactive montrant, pour une courte phrase, la matrice des poids d'attention : chaque ligne est un jeton qui interroge, chaque colonne un jeton consulté, et chaque ligne somme à 1.",221 "Interactive simulation showing, for a short sentence, the matrix of attention weights: each row is a querying token, each column a token being looked at, and every row sums to 1.",222 ),223 },224 {225 id: "co-scale", type: "callout", span: 4, tone: "note", title: t("Pourquoi diviser par $\\sqrt{d_k}$ ?", "Why divide by $\\sqrt{d_k}$?"),226 text: t(227 "Quand $d_k$ est grand, les produits scalaires prennent de grandes valeurs et le softmax sature : presque tout le poids va à une seule clé et les gradients deviennent très petits. La division par $\\sqrt{d_k}$ ramène les scores à une échelle où l'apprentissage reste possible[^vaswani2017].",228 "When $d_k$ is large, dot products grow large and the softmax saturates: almost all the weight goes to a single key and gradients become very small. Dividing by $\\sqrt{d_k}$ brings the scores back to a scale where learning remains possible[^vaswani2017].",229 ),230 },231 {232 id: "co-mask", type: "callout", tone: "definition", title: t("Masque causal", "Causal mask"),233 text: t(234 "Dans le décodeur, on ajoute $-\\infty$ aux scores des positions futures avant le softmax : leur poids devient exactement 0. La prédiction à la position $i$ ne dépend donc que des sorties connues aux positions inférieures à $i$[^vaswani2017].",235 "In the decoder, $-\\infty$ is added to the scores of future positions before the softmax: their weight becomes exactly 0. The prediction at position $i$ therefore depends only on the known outputs at positions below $i$[^vaswani2017].",236 ),237 },238239 { id: "h-position", type: "heading", level: 2, text: t("Donner un ordre aux jetons", "Giving tokens an order") },240 {241 id: "p-position", type: "paragraph", span: 6,242 text: t(243 "L'attention seule ne sait rien de l'ordre : permuter les jetons d'entrée permute simplement les sorties. Il faut donc **injecter la position** dans les vecteurs. L'article d'origine additionne au plongement un signal sinusoïdal de fréquence différente pour chaque dimension, et note que des plongements de position appris donnent des résultats presque identiques[^vaswani2017].",244 "Attention alone knows nothing about order: permuting the input tokens simply permutes the outputs. Position must therefore be **injected** into the vectors. The original paper adds to the embedding a sinusoidal signal with a different frequency for each dimension, and notes that learned position embeddings give nearly identical results[^vaswani2017].",245 ),246 },247 {248 id: "eq-position", type: "equation", span: 6, label: t("Encodage positionnel sinusoïdal", "Sinusoidal positional encoding"), derivation: "standard", sourceIds: ["vaswani2017"],249 latex: r`\begin{aligned} PE_{(pos,\,2i)} &= \sin\!\left(\frac{pos}{10000^{2i/d_{\text{model}}}}\right) \\ PE_{(pos,\,2i+1)} &= \cos\!\left(\frac{pos}{10000^{2i/d_{\text{model}}}}\right) \end{aligned}`,250 variables: [251 { symbol: "pos", meaning: t("Position du jeton dans la séquence (0, 1, 2, …)", "Position of the token in the sequence (0, 1, 2, …)") },252 { symbol: "i", meaning: t("Indice de la paire de dimensions, de 0 à $d_{\\text{model}}/2 - 1$", "Index of the pair of dimensions, from 0 to $d_{\\text{model}}/2 - 1$") },253 { symbol: r`d_{\text{model}}`, meaning: t("Dimension des vecteurs du modèle", "Dimension of the model's vectors") },254 ],255 },256257 { id: "h-block", type: "heading", level: 2, text: t("Ce qui rend l'empilement entraînable", "What makes the stack trainable") },258 {259 id: "eq-residual", type: "equation", span: 6, label: t("Résiduelle et normalisation", "Residual and normalisation"), derivation: "standard", sourceIds: ["vaswani2017", "he2016", "ba2016"],260 latex: r`y = \mathrm{LayerNorm}\big(x + \mathrm{Sublayer}(x)\big)`,261 explanation: t(262 "La connexion résiduelle[^he2016] laisse le signal et le gradient traverser le bloc sans obstacle ; la normalisation de couche[^ba2016] stabilise l'échelle des activations. Chaque sous-couche du Transformer d'origine est enveloppée ainsi[^vaswani2017].",263 "The residual connection[^he2016] lets the signal and the gradient cross the block unhindered; layer normalisation[^ba2016] stabilises the scale of the activations. Every sub-layer of the original Transformer is wrapped this way[^vaswani2017].",264 ),265 variables: [266 { symbol: "x", meaning: t("Entrée de la sous-couche", "Input of the sub-layer") },267 { symbol: r`\mathrm{Sublayer}`, meaning: t("Attention ou réseau à propagation avant", "Attention or feed-forward network") },268 { symbol: r`\mathrm{LayerNorm}`, meaning: t("Normalisation de chaque vecteur à moyenne 0 et variance 1, suivie d'un gain et d'un biais appris", "Normalisation of each vector to mean 0 and variance 1, followed by a learned gain and bias") },269 ],270 },271 {272 id: "eq-ffn", type: "equation", span: 6, label: t("Réseau à propagation avant", "Feed-forward network"), derivation: "standard", sourceIds: ["vaswani2017"],273 latex: r`\mathrm{FFN}(x) = \max(0,\; x W_1 + b_1)\, W_2 + b_2`,274 explanation: t(275 "Deux transformations linéaires séparées par une ReLU, appliquées à chaque position indépendamment. La couche intermédiaire est plus large que le modèle ($d_{ff} = 2048$ pour $d_{\\text{model}} = 512$ dans le modèle de base)[^vaswani2017].",276 "Two linear maps separated by a ReLU, applied to each position independently. The inner layer is wider than the model ($d_{ff} = 2048$ for $d_{\\text{model}} = 512$ in the base model)[^vaswani2017].",277 ),278 variables: [279 { symbol: r`W_1,\, b_1`, meaning: t("Poids et biais de la première couche ($d_{\\text{model}} \\to d_{ff}$)", "Weights and bias of the first layer ($d_{\\text{model}} \\to d_{ff}$)") },280 { symbol: r`W_2,\, b_2`, meaning: t("Poids et biais de la seconde couche ($d_{ff} \\to d_{\\text{model}}$)", "Weights and bias of the second layer ($d_{ff} \\to d_{\\text{model}}$)") },281 ],282 },283 {284 id: "tb-complexity", type: "table", span: 8, headerColumn: true, sourceIds: ["vaswani2017"],285 caption: t(286 "Comparaison par couche, pour une séquence de longueur $n$, une dimension $d$ et un noyau de taille $k$ (tableau 1 de l'article d'origine)[^vaswani2017].",287 "Per-layer comparison for a sequence of length $n$, dimension $d$ and kernel size $k$ (Table 1 of the original paper)[^vaswani2017].",288 ),289 columns: [290 { label: t("Type de couche", "Layer type") },291 { label: t("Complexité par couche", "Complexity per layer") },292 { label: t("Opérations séquentielles", "Sequential operations") },293 { label: t("Longueur maximale du chemin", "Maximum path length") },294 ],295 rows: [296 [t("Auto-attention", "Self-attention"), "$O(n^2 \\cdot d)$", "$O(1)$", "$O(1)$"],297 [t("Récurrente", "Recurrent"), "$O(n \\cdot d^2)$", "$O(n)$", "$O(n)$"],298 [t("Convolutive", "Convolutional"), "$O(k \\cdot n \\cdot d^2)$", "$O(1)$", "$O(\\log_k n)$"],299 ],300 },301 {302 id: "co-quadratic", type: "callout", span: 4, tone: "warning", title: t("Le prix : un coût quadratique", "The price: quadratic cost"),303 text: t(304 "Chaque position consulte toutes les autres : le calcul et la mémoire de l'attention croissent comme $n^2$. C'est ce qui limite la longueur du contexte, et ce qui motive le cache clés-valeurs à la génération.",305 "Every position looks at every other: the computation and memory of attention grow as $n^2$. This is what limits context length, and what motivates the key–value cache at generation time.",306 ),307 },308 {309 id: "tb-base", type: "table", headerColumn: true, sourceIds: ["vaswani2017"],310 caption: t("Hyperparamètres du modèle « de base » de l'article d'origine[^vaswani2017].", "Hyperparameters of the “base” model of the original paper[^vaswani2017]."),311 columns: [{ label: t("Paramètre", "Parameter") }, { label: t("Symbole", "Symbol"), align: "center" }, { label: t("Valeur", "Value"), align: "right" }],312 rows: [313 [t("Couches par pile", "Layers per stack"), "$N$", "6"],314 [t("Dimension du modèle", "Model dimension"), "$d_{\\text{model}}$", "512"],315 [t("Dimension interne du réseau à propagation avant", "Inner dimension of the feed-forward network"), "$d_{ff}$", "2048"],316 [t("Têtes d'attention", "Attention heads"), "$h$", "8"],317 [t("Dimension des clés et des valeurs par tête", "Key and value dimension per head"), "$d_k = d_v$", "64"],318 ],319 },320321 { id: "h-check", type: "heading", level: 2, text: t("Vérifier sa compréhension", "Check your understanding") },322 {323 id: "quiz-attention", type: "quiz", span: 6,324 question: t(325 "On retire l'encodage positionnel d'un encodeur Transformer, puis on inverse l'ordre des jetons d'entrée. Que deviennent les vecteurs de sortie ?",326 "The positional encoding is removed from a Transformer encoder, then the order of the input tokens is reversed. What happens to the output vectors?",327 ),328 choices: [329 { text: t("Ce sont les mêmes vecteurs, simplement dans l'ordre inverse.", "They are the same vectors, simply in reverse order."), correct: true, explanation: t("Exact : sans information de position, l'auto-attention et le réseau par position sont équivariants aux permutations.", "Correct: without position information, self-attention and the position-wise network are permutation-equivariant.") },330 { text: t("Ils changent complètement, car l'attention lit de gauche à droite.", "They change completely, because attention reads left to right."), correct: false, explanation: t("L'attention n'a pas de sens de lecture : elle compare toutes les paires de positions en une fois.", "Attention has no reading direction: it compares all pairs of positions at once.") },331 { text: t("Ils sont identiques et dans le même ordre qu'avant.", "They are identical and in the same order as before."), correct: false, explanation: t("Chaque sortie reste attachée à son jeton : l'ordre des sorties suit celui des entrées.", "Each output stays attached to its token: the order of the outputs follows that of the inputs.") },332 ],333 },334 {335 id: "acc-faq", type: "accordion", span: 6,336 items: [337 {338 title: t("Requête, clé, valeur : d'où viennent ces trois matrices ?", "Query, key, value: where do these three matrices come from?"),339 text: t(340 "Dans l'auto-attention, elles sont toutes trois des projections linéaires apprises de la **même** suite de vecteurs. Dans l'attention croisée, les requêtes viennent du décodeur, les clés et les valeurs de la sortie de l'encodeur.",341 "In self-attention all three are learned linear projections of the **same** sequence of vectors. In cross-attention the queries come from the decoder, the keys and values from the encoder output.",342 ),343 },344 {345 title: t("La normalisation se place-t-elle avant ou après la sous-couche ?", "Does normalisation go before or after the sub-layer?"),346 text: t(347 "L'article d'origine normalise **après** l'addition, comme dans l'équation ci-dessus[^vaswani2017]. Beaucoup d'implémentations récentes normalisent l'entrée de la sous-couche (variante dite *pre-norm*). Le schéma de cette fiche suit l'article d'origine.",348 "The original paper normalises **after** the addition, as in the equation above[^vaswani2017]. Many recent implementations normalise the input of the sub-layer instead (the *pre-norm* variant). The diagram in this sheet follows the original paper.",349 ),350 },351 {352 title: t("Erreur fréquente : « l'attention explique la décision du modèle »", "Common mistake: “attention explains the model's decision”"),353 text: t(354 "Les poids d'attention indiquent quelles positions ont été mélangées dans une tête donnée, à une couche donnée. Ils ne constituent pas, à eux seuls, une explication causale de la sortie : le réseau comporte de nombreuses têtes, des résiduelles et des couches non linéaires.",355 "Attention weights show which positions were mixed in a given head at a given layer. On their own they are not a causal explanation of the output: the network has many heads, residual paths and non-linear layers.",356 ),357 },358 {359 title: t("Un modèle décodeur seul a-t-il encore une attention croisée ?", "Does a decoder-only model still have cross-attention?"),360 text: t(361 "Non. Sans encodeur, il n'y a rien à consulter : chaque bloc se réduit à l'auto-attention masquée et au réseau à propagation avant. L'invite et le texte généré forment une seule et même séquence.",362 "No. With no encoder there is nothing to consult: each block reduces to masked self-attention and the feed-forward network. The prompt and the generated text form one single sequence.",363 ),364 },365 ],366 },367 { id: "refs", type: "references", title: t("Références", "References") },368 ],369 sources: pickSources("vaswani2017", "he2016", "ba2016", "devlin2018", "brown2020"),370 claims: [371 { id: "cl-no-recurrence", status: "sourced", sourceIds: ["vaswani2017"], blockId: "intro-lead", text: t("Le Transformer repose uniquement sur l'attention, sans récurrence ni convolution.", "The Transformer relies on attention alone, without recurrence or convolution.") },372 { id: "cl-attention-eq", status: "sourced", sourceIds: ["vaswani2017"], blockId: "eq-attention", text: t("L'attention s'écrit softmax(QKᵀ/√d_k)·V.", "Attention is written softmax(QKᵀ/√d_k)·V.") },373 { id: "cl-scale", status: "sourced", sourceIds: ["vaswani2017"], blockId: "co-scale", text: t("La division par √d_k évite que le softmax entre dans des régions à très petits gradients.", "Dividing by √d_k keeps the softmax out of regions with very small gradients.") },374 { id: "cl-residual", status: "sourced", sourceIds: ["vaswani2017", "he2016", "ba2016"], blockId: "eq-residual", text: t("Chaque sous-couche est entourée d'une connexion résiduelle suivie d'une normalisation de couche.", "Each sub-layer is wrapped in a residual connection followed by layer normalisation.") },375 { id: "cl-complexity", status: "sourced", sourceIds: ["vaswani2017"], blockId: "tb-complexity", text: t("Une couche d'auto-attention coûte O(n²·d) et relie deux positions quelconques par un chemin de longueur O(1).", "A self-attention layer costs O(n²·d) and connects any two positions by a path of length O(1).") },376 { id: "cl-permutation", status: "derived", sourceIds: ["vaswani2017"], blockId: "quiz-attention", text: t("Sans encodage positionnel, l'encodeur est équivariant aux permutations : cela découle de la forme de l'équation d'attention, qui ne dépend pas des indices de position.", "Without positional encoding the encoder is permutation-equivariant: this follows from the form of the attention equation, which does not depend on position indices.") },377 { id: "cl-base", status: "sourced", sourceIds: ["vaswani2017"], blockId: "tb-base", text: t("Le modèle de base utilise N = 6, d_model = 512, d_ff = 2048, h = 8 et d_k = d_v = 64.", "The base model uses N = 6, d_model = 512, d_ff = 2048, h = 8 and d_k = d_v = 64.") },378 ],379 };380 },381};382