spb/artificial-neural-networks-book Public

1% ============================================================================2%  Artificial Neural Networks — Methods, Equations and Graphical3%  Representations4%  Author  : Simon-Pierre Boucher — contact@spboucher.ai5%  Chapter 5 : Recurrent Networks — RNN, LSTM and GRU (chapters/05-rnn.tex)6% ============================================================================78\chapter{Recurrent Networks: RNN, LSTM and GRU}9\label{chap:rnn}1011Feedforward architectures map a fixed-size input to a fixed-size output;12they have no mechanism for processing sequences of arbitrary length, nor13any notion of order or memory. Recurrent neural networks (RNNs) remove this14limitation by maintaining a \emph{hidden state} that is updated at every15time step and acts as a compressed summary of everything the network has16seen so far. This chapter develops the vanilla recurrent network and its17training algorithm, backpropagation through time; analyzes why gradients18vanish or explode over long horizons; and presents the two gated19architectures — the long short-term memory (LSTM) of Hochreiter and20Schmidhuber \cite{hochreiter1997} and the gated recurrent unit (GRU) of Cho21et al.\ \cite{cho2014} — that made learning long-range dependencies22practical. We close with bidirectional networks and the encoder--decoder23paradigm for sequence-to-sequence learning.2425% ----------------------------------------------------------------------------26\section{The simple recurrent network}27\label{sec:rnn-simple}2829\begin{definition}[Recurrent neural network]30A recurrent neural network processes a sequence31$\vect{x}_1, \vect{x}_2, \dots, \vect{x}_T$, with32$\vect{x}_t \in \R^{d}$, by maintaining a hidden state33$\vect{h}_t \in \R^{n}$ computed from the current input and the previous34state, $\vect{h}_t = f(\vect{h}_{t-1}, \vect{x}_t; \theta)$, where the35parameters $\theta$ are \emph{shared across all time steps}.36\end{definition}3738The standard (Elman) form uses a $\tanh$ nonlinearity for the state update39and a linear read-out:40\begin{equation}41  \vect{h}_t42  = \tanh\!\left(\mat{W}_{h}\,\vect{h}_{t-1}43                 + \mat{W}_{x}\,\vect{x}_t + \vect{b}\right),44  \label{eq:rnn-hidden}45\end{equation}46\begin{equation}47  \vect{y}_t = \mat{W}_{y}\,\vect{h}_t + \vect{b}_y,48  \label{eq:rnn-output}49\end{equation}50with $\mat{W}_{x} \in \R^{n \times d}$,51$\mat{W}_{h} \in \R^{n \times n}$ and52$\mat{W}_{y} \in \R^{m \times n}$. For classification tasks the output53\eqref{eq:rnn-output} is typically passed through a softmax,54$\hat{\vect{y}}_t = \softmax(\mat{W}_{y}\vect{h}_t + \vect{b}_y)$.55Weight sharing across time is the defining structural property: the same56pair $(\mat{W}_h, \mat{W}_x)$ is applied at every step, so the RNN is a57discrete-time dynamical system whose parameters do not grow with the58sequence length. Figure~\ref{fig:rnn-unrolled} shows the two equivalent59views of this computation: the \emph{folded} form, a single cell with a60feedback loop, and the \emph{unfolded} form, a deep network with one layer61per time step and tied weights.6263\begin{figure}[htbp]64  \centering65  \begin{tikzpicture}66    % ---------- folded form ----------67    \node[ninput]  (fx) at (0,0)   {$\vect{x}_t$};68    \node[mem, minimum width=1.3cm] (fc) at (0,1.9) {$\vect{h}$};69    \node[noutput] (fy) at (0,3.8) {$\vect{y}_t$};70    \draw[fleche] (fx) -- (fc) node[midway,right,etiquette] {$\mat{W}_x$};71    \draw[fleche] (fc) -- (fy) node[midway,right,etiquette] {$\mat{W}_y$};72    \draw[fleche] (fc.east) .. controls +(1.5,1.0) and +(1.5,-1.0) ..73      (fc.east) node[pos=0.5, right=0.25cm, etiquette] {$\mat{W}_h$};74    % ---------- unfold symbol ----------75    \node at (3.35,1.9) {\Large $=$};76    \node[etiquette] at (3.35,2.5) {unfold};77    % ---------- unfolded form ----------78    \foreach \i/\lab in {1/{t-1}, 2/{t}, 3/{t+1}} {79      \node[ninput]  (x\i) at (3.0+\i*2.5, 0)   {$\vect{x}_{\lab}$};80      \node[mem, minimum width=1.3cm] (c\i) at (3.0+\i*2.5, 1.9)81        {$\vect{h}_{\lab}$};82      \node[noutput] (y\i) at (3.0+\i*2.5, 3.8) {$\vect{y}_{\lab}$};83      \draw[fleche] (x\i) -- (c\i);84      \draw[fleche] (c\i) -- (y\i);85    }86    \draw[fleche] (4.3,1.9) -- (c1.west)87      node[very near start, above, etiquette] {$\cdots$};88    \draw[fleche] (c1) -- (c2)89      node[midway, above, etiquette] {$\mat{W}_h$};90    \draw[fleche] (c2) -- (c3)91      node[midway, above, etiquette] {$\mat{W}_h$};92    \draw[fleche] (c3.east) -- (11.7,1.9)93      node[very near end, above, etiquette] {$\cdots$};94  \end{tikzpicture}95  \caption{The two equivalent views of a recurrent network. Left: folded96    form — a single cell whose hidden state $\vect{h}$ feeds back into97    itself through $\mat{W}_h$. Right: unfolded form — the same cell98    replicated over time steps $t-1$, $t$, $t+1$, with all replicas sharing99    the weights $(\mat{W}_x, \mat{W}_h, \mat{W}_y)$ of100    \eqref{eq:rnn-hidden}--\eqref{eq:rnn-output}.}101  \label{fig:rnn-unrolled}102\end{figure}103104% ----------------------------------------------------------------------------105\section{Backpropagation through time}106\label{sec:rnn-bptt}107108Training proceeds by \emph{unrolling} the recurrence into the feedforward109network of Figure~\ref{fig:rnn-unrolled} (right) and applying standard110backpropagation to the unrolled graph — hence the name backpropagation111through time (BPTT). For a sequence-level loss112$\Loss = \sum_{t=1}^{T} \Loss_t$, the gradient with respect to the113recurrent matrix accumulates contributions over all pairs of time steps:114\begin{equation}115  \frac{\partial \Loss}{\partial \mat{W}_h}116  = \sum_{t=1}^{T} \sum_{k=1}^{t}117    \frac{\partial \Loss_t}{\partial \vect{h}_t}118    \left( \prod_{i=k+1}^{t}119      \frac{\partial \vect{h}_i}{\partial \vect{h}_{i-1}} \right)120    \frac{\partial \vect{h}_k}{\partial \mat{W}_h}.121  \label{eq:rnn-loss-grad}122\end{equation}123The critical object in \eqref{eq:rnn-loss-grad} is the product of124Jacobians that transports the error signal from step $t$ back to step $k$.125Differentiating \eqref{eq:rnn-hidden}, each factor is126$\mat{W}_h\transp$ scaled by the local slope of the nonlinearity, so127\begin{equation}128  \frac{\partial \vect{h}_t}{\partial \vect{h}_k}129  = \prod_{i=k+1}^{t}130    \frac{\partial \vect{h}_i}{\partial \vect{h}_{i-1}}131  = \prod_{i=k+1}^{t}132    \operatorname{diag}\!\bigl(\tanh'(\vect{a}_i)\bigr)\, \mat{W}_h\transp,133  \label{eq:rnn-jacobian}134\end{equation}135where $\vect{a}_i = \mat{W}_h \vect{h}_{i-1} + \mat{W}_x \vect{x}_i +136\vect{b}$ is the pre-activation. Bounding each factor by its largest137singular value $\sigma_{\max}(\mat{W}_h)$ and using138$|\tanh'| \le \gamma = 1$ gives139\begin{equation}140  \left\lVert \frac{\partial \vect{h}_t}{\partial \vect{h}_k} \right\rVert141  \;\le\; \bigl(\gamma\, \sigma_{\max}(\mat{W}_h)\bigr)^{\,t-k}.142  \label{eq:rnn-jacobian-bound}143\end{equation}144145Equation \eqref{eq:rnn-jacobian-bound} exposes the fundamental pathology146of the simple RNN. If $\sigma_{\max}(\mat{W}_h) < 1/\gamma$, the bound147decays exponentially in the time lag $t-k$: gradients \emph{vanish}, and148the network cannot learn dependencies spanning more than a few dozen149steps. Conversely, if the spectral radius of $\mat{W}_h$ exceeds $1$ — a150necessary condition — the product can grow exponentially: gradients151\emph{explode}, producing loss spikes and numerical overflow. Both regimes152are generic; only a narrow band around unit gain propagates error signals153faithfully over long horizons.154155\begin{remark}[Gradient clipping]156Exploding gradients admit a simple remedy: rescale the gradient157$\vect{g} = \nabla_\theta \Loss$ whenever its norm exceeds a threshold158$\tau$,159\begin{equation}160  \vect{g} \;\leftarrow\;161  \begin{cases}162    \dfrac{\tau}{\lVert \vect{g} \rVert}\, \vect{g}163      & \text{if } \lVert \vect{g} \rVert > \tau, \\[2ex]164    \vect{g} & \text{otherwise.}165  \end{cases}166  \label{eq:rnn-clip}167\end{equation}168Vanishing gradients have no comparably simple fix; they are an169\emph{architectural} problem, and it is precisely this problem that the170gated cells of Sections~\ref{sec:rnn-lstm} and~\ref{sec:rnn-gru} solve.171\end{remark}172173Algorithm~\ref{alg:rnn-bptt} assembles the complete estimation procedure:174a forward sweep that stores all pre-activations and states, a backward175sweep that transports the error signal $\vect{\delta}_t$ from step $T$176down to step $1$ while accumulating the shared-weight gradients of177\eqref{eq:rnn-loss-grad}, followed by clipping \eqref{eq:rnn-clip} and a178gradient step.179180\begin{algorithm}[htbp]181  \caption{Backpropagation through time (BPTT) for the simple RNN}182  \label{alg:rnn-bptt}183  \begin{algorithmic}[1]184    \Require sequence $(\vect{x}_1, \dots, \vect{x}_T)$, losses $\Loss_t$,185      parameters $(\mat{W}_x, \mat{W}_h, \mat{W}_y, \vect{b}, \vect{b}_y)$,186      learning rate $\eta$, clipping threshold $\tau$187    \State $\vect{h}_0 \gets \vect{0}$188    \For{$t = 1, \dots, T$} \Comment{forward pass: store all189      $\vect{a}_t, \vect{h}_t$}190      \State $\vect{a}_t \gets \mat{W}_h \vect{h}_{t-1}191        + \mat{W}_x \vect{x}_t + \vect{b}$;\quad192        $\vect{h}_t \gets \tanh(\vect{a}_t)$193      \State $\vect{y}_t \gets \mat{W}_y \vect{h}_t + \vect{b}_y$194    \EndFor195    \State $\nabla_{\mat{W}_x}, \nabla_{\mat{W}_h}, \nabla_{\mat{W}_y},196      \nabla_{\vect{b}}, \nabla_{\vect{b}_y} \gets \vect{0}$;\quad197      $\vect{\delta} \gets \vect{0}$198    \For{$t = T, \dots, 1$} \Comment{backward pass:199      $\vect{\delta}$ carries $\partial\Loss/\partial\vect{h}_t$}200      \State $\vect{\delta} \gets201        \mat{W}_y\transp\, \nabla_{\vect{y}_t}\Loss_t + \vect{\delta}$202      \State $\vect{\delta}_a \gets \vect{\delta} \odot203        \bigl(\vect{1} - \tanh^2(\vect{a}_t)\bigr)$204      \State $\nabla_{\mat{W}_y} \gets \nabla_{\mat{W}_y}205        + \nabla_{\vect{y}_t}\Loss_t\, \vect{h}_t\transp$;\quad206        $\nabla_{\vect{b}_y} \gets \nabla_{\vect{b}_y}207        + \nabla_{\vect{y}_t}\Loss_t$208      \State $\nabla_{\mat{W}_h} \gets \nabla_{\mat{W}_h}209        + \vect{\delta}_a \vect{h}_{t-1}\transp$;\quad210        $\nabla_{\mat{W}_x} \gets \nabla_{\mat{W}_x}211        + \vect{\delta}_a \vect{x}_t\transp$;\quad212        $\nabla_{\vect{b}} \gets \nabla_{\vect{b}} + \vect{\delta}_a$213      \State $\vect{\delta} \gets \mat{W}_h\transp \vect{\delta}_a$214        \Comment{transport the error to step $t-1$}215    \EndFor216    \State clip each gradient by \eqref{eq:rnn-clip} with threshold $\tau$217    \State update each parameter $\theta \gets \theta - \eta\,218      \nabla_\theta$219  \end{algorithmic}220\end{algorithm}221222% ----------------------------------------------------------------------------223\section{Long short-term memory (LSTM)}224\label{sec:rnn-lstm}225226The long short-term memory network \cite{hochreiter1997} replaces the227purely multiplicative recurrence \eqref{eq:rnn-hidden} with an228\emph{additive} one. It introduces a second state vector, the \emph{cell229state} $\vect{c}_t$, which traverses time along a path modified only by230elementwise gating — the ``constant error carousel.'' Three learned gates,231each a sigmoid layer reading the current input $\vect{x}_t$ and the232previous hidden state $\vect{h}_{t-1}$, control what the cell forgets,233what it writes, and what it exposes:234\begin{align}235  \vect{f}_t &= \sigma\!\left(\mat{W}_f \vect{x}_t236      + \mat{U}_f \vect{h}_{t-1} + \vect{b}_f\right)237      && \text{(forget gate)}238  \label{eq:rnn-lstm-f} \\239  \vect{i}_t &= \sigma\!\left(\mat{W}_i \vect{x}_t240      + \mat{U}_i \vect{h}_{t-1} + \vect{b}_i\right)241      && \text{(input gate)}242  \label{eq:rnn-lstm-i} \\243  \vect{o}_t &= \sigma\!\left(\mat{W}_o \vect{x}_t244      + \mat{U}_o \vect{h}_{t-1} + \vect{b}_o\right)245      && \text{(output gate)}246  \label{eq:rnn-lstm-o} \\247  \tilde{\vect{c}}_t &= \tanh\!\left(\mat{W}_c \vect{x}_t248      + \mat{U}_c \vect{h}_{t-1} + \vect{b}_c\right)249      && \text{(candidate content)}250  \label{eq:rnn-lstm-ctilde} \\251  \vect{c}_t &= \vect{f}_t \odot \vect{c}_{t-1}252      + \vect{i}_t \odot \tilde{\vect{c}}_t253      && \text{(cell state update)}254  \label{eq:rnn-lstm-c} \\255  \vect{h}_t &= \vect{o}_t \odot \tanh(\vect{c}_t)256      && \text{(hidden state)}257  \label{eq:rnn-lstm-h}258\end{align}259Here $\sigma$ is the logistic sigmoid, so each gate takes values in260$(0,1)^n$ and acts as a soft, differentiable switch applied coordinatewise261through the Hadamard product $\odot$. The forget gate $\vect{f}_t$ decides262how much of the previous cell $\vect{c}_{t-1}$ to retain; the input gate263$\vect{i}_t$ decides how much of the new candidate $\tilde{\vect{c}}_t$ to264write; the output gate $\vect{o}_t$ decides how much of the (squashed)265cell to reveal in the hidden state. Figure~\ref{fig:rnn-lstm-cell} traces266these six equations through the cell.267268\begin{property}[Constant error carousel]269\label{prop:rnn-cec}270Along the cell-state path of \eqref{eq:rnn-lstm-c}, the Jacobian of the271recurrence is diagonal,272\begin{equation}273  \frac{\partial \vect{c}_t}{\partial \vect{c}_{t-1}}274  = \operatorname{diag}(\vect{f}_t)275  \quad (\text{holding the gates fixed}),276  \label{eq:rnn-lstm-cec}277\end{equation}278with entries in $(0,1)$ controlled by the forget gate rather than by279repeated multiplication with $\mat{W}_h$. When the network sets280$\vect{f}_t \approx \vect{1}$, error signals flow backward through281\eqref{eq:rnn-lstm-cec} essentially unattenuated over hundreds of steps,282in contrast with the exponential bound \eqref{eq:rnn-jacobian-bound} of283the simple RNN.284\end{property}285286\begin{remark}287A useful practical consequence of Property~\ref{prop:rnn-cec} is to288initialize the forget-gate bias $\vect{b}_f$ to a positive value (e.g.\289$1$ or $2$), so that $\vect{f}_t \approx \vect{1}$ at the start of290training and the network begins by \emph{remembering}, only later learning291what to forget.292\end{remark}293294\begin{figure}[htbp]295  \centering296  \begin{tikzpicture}297    % ================= cell state line (top, violet) =================298    \node[mem, minimum width=1.2cm] (cprev) at (0.2,5)  {$\vect{c}_{t-1}$};299    \node[op]  (multf) at (2.6,5) {$\odot$};300    \node[op]  (plus)  at (6.4,5) {$+$};301    \node[mem, minimum width=1.2cm] (cnew)  at (12.4,5) {$\vect{c}_t$};302    \draw[fleche, cmem!70!black, very thick] (cprev) -- (multf);303    \draw[fleche, cmem!70!black, very thick] (multf) -- (plus);304    \draw[fleche, cmem!70!black, very thick] (plus)  -- (cnew);305    \fill[cmem!70!black] (8.6,5) circle (1.6pt);306    % ================= gates (bottom row) =================307    \node[gate] (fgate) at (2.6,1.2)  {$\sigma$};308    \node[gate] (igate) at (4.6,1.2)  {$\sigma$};309    \node[gate] (cgate) at (6.4,1.2)  {$\tanh$};310    \node[gate] (ogate) at (10.4,1.2) {$\sigma$};311    % ================= interior op nodes =================312    \node[op] (multi) at (6.4,3.3)  {$\odot$};   % i_t (.) c~_t313    \node[op] (tanhc) at (8.6,3.3)  {$\tanh$};   % tanh(c_t)314    \node[op] (multo) at (10.4,2.2) {$\odot$};   % o_t (.) tanh(c_t)315    % ================= gate outputs =================316    \draw[fleche] (fgate) -- (multf)317      node[pos=0.55, right, etiquette] {$\vect{f}_t$};318    \draw[fleche, rounded corners=3pt] (igate.north) |- (multi.west)319      node[pos=0.25, right, etiquette] {$\vect{i}_t$};320    \draw[fleche] (cgate) -- (multi)321      node[pos=0.5, right, etiquette] {$\tilde{\vect{c}}_t$};322    \draw[fleche] (multi) -- (plus);323    \draw[fleche] (ogate) -- (multo)324      node[pos=0.5, right, etiquette] {$\vect{o}_t$};325    % ================= tanh branch from the state line =================326    \draw[fleche] (8.6,5) -- (tanhc);327    \draw[fleche, rounded corners=3pt] (tanhc.south) |- (multo.west);328    % ================= h_t output =================329    \node[mem, minimum width=1.2cm] (hnew) at (12.4,2.2) {$\vect{h}_t$};330    \draw[fleche] (multo) -- (hnew);331    \draw[fleche, rounded corners=3pt] (11.6,2.2) |- (12.4,0.6)332      node[pos=1.0, right, etiquette] {to $\vect{y}_t$};333    \fill[black!70] (11.6,2.2) circle (1.4pt);334    % ================= input trunk (bottom) =================335    \node[mem, minimum width=1.2cm] (hprev) at (-0.6,0) {$\vect{h}_{t-1}$};336    \node[ninput] (xt) at (1.2,-1.5) {$\vect{x}_t$};337    \draw[thick] (hprev.east) -- (10.4,0);338    \draw[fleche] (xt) -- (1.2,0);339    \foreach \x in {1.2, 2.6, 4.6, 6.4} \fill[black!70] (\x,0) circle (1.4pt);340    \draw[fleche] (2.6,0) -- (fgate.south);341    \draw[fleche] (4.6,0) -- (igate.south);342    \draw[fleche] (6.4,0) -- (cgate.south);343    \draw[fleche] (10.4,0) -- (ogate.south);344    % ================= cell frame =================345    \begin{scope}[on background layer]346      \node[draw=black!35, dashed, rounded corners=6pt, fill=black!2,347            fit={(1.7,-0.6) (11.7,5.75)}, inner sep=2pt] (frame) {};348    \end{scope}349    \node[etiquette, anchor=south west] at (1.75,5.85) {LSTM cell};350  \end{tikzpicture}351  \caption{The LSTM cell, tracing equations352    \eqref{eq:rnn-lstm-f}--\eqref{eq:rnn-lstm-h}. The cell state runs353    horizontally along the top (violet): it is first scaled by the forget354    gate ($\odot$ with $\vect{f}_t$), then incremented ($+$) with the355    gated candidate $\vect{i}_t \odot \tilde{\vect{c}}_t$. The three356    $\sigma$ gates and the $\tanh$ candidate layer (bottom, green) all357    read the shared input trunk carrying $\vect{h}_{t-1}$ and358    $\vect{x}_t$. The updated cell $\vect{c}_t$ is squashed by $\tanh$ and359    multiplied by the output gate $\vect{o}_t$ to produce the hidden state360    $\vect{h}_t$, which exits right and branches toward the output.}361  \label{fig:rnn-lstm-cell}362\end{figure}363364% ----------------------------------------------------------------------------365\section{Gated recurrent unit (GRU)}366\label{sec:rnn-gru}367368The gated recurrent unit \cite{cho2014} is a streamlined gated cell that369merges the LSTM's cell and hidden states into a single vector370$\vect{h}_t$ and uses only two gates — roughly $25\%$ fewer parameters371than the LSTM:372\begin{align}373  \vect{z}_t &= \sigma\!\left(\mat{W}_z \vect{x}_t374      + \mat{U}_z \vect{h}_{t-1} + \vect{b}_z\right)375      && \text{(update gate)}376  \label{eq:rnn-gru-z} \\377  \vect{r}_t &= \sigma\!\left(\mat{W}_r \vect{x}_t378      + \mat{U}_r \vect{h}_{t-1} + \vect{b}_r\right)379      && \text{(reset gate)}380  \label{eq:rnn-gru-r} \\381  \tilde{\vect{h}}_t &= \tanh\!\left(\mat{W}_h \vect{x}_t382      + \mat{U}_h (\vect{r}_t \odot \vect{h}_{t-1}) + \vect{b}_h\right)383      && \text{(candidate state)}384  \label{eq:rnn-gru-htilde} \\385  \vect{h}_t &= (\vect{1} - \vect{z}_t) \odot \vect{h}_{t-1}386      + \vect{z}_t \odot \tilde{\vect{h}}_t387      && \text{(interpolation)}388  \label{eq:rnn-gru-h}389\end{align}390The reset gate $\vect{r}_t$ controls how much of the past state391contributes to the candidate \eqref{eq:rnn-gru-htilde}: with392$\vect{r}_t \approx \vect{0}$ the unit ignores its history and behaves393like a freshly initialized network. The update gate $\vect{z}_t$ then394interpolates \eqref{eq:rnn-gru-h} between copying the old state and395writing the new candidate — the same leaky-integration principle as the396LSTM's forget/input pair, realized as an explicit convex combination.397Empirically, GRU and LSTM perform comparably across language and speech398benchmarks, with the LSTM slightly more robust on tasks requiring precise399counting; the GRU is often preferred when parameter economy or training400speed matters.401402% ----------------------------------------------------------------------------403\section{Bidirectional networks and sequence-to-sequence learning}404\label{sec:rnn-seq2seq}405406\paragraph{Bidirectional RNNs.}407The recurrences above are causal: $\vect{h}_t$ summarizes only408$\vect{x}_1, \dots, \vect{x}_t$. Many labeling tasks (tagging, speech409frames, contextual encoding) benefit from future context as well. A410bidirectional RNN runs two independent recurrent networks over the411sequence — one forward, one backward — and concatenates their states:412\begin{equation}413  \overrightarrow{\vect{h}}_t414    = f\!\left(\overrightarrow{\mat{W}} \vect{x}_t415        + \overrightarrow{\mat{U}}\, \overrightarrow{\vect{h}}_{t-1}\right),416  \qquad417  \overleftarrow{\vect{h}}_t418    = f\!\left(\overleftarrow{\mat{W}} \vect{x}_t419        + \overleftarrow{\mat{U}}\, \overleftarrow{\vect{h}}_{t+1}\right),420  \label{eq:rnn-bidir}421\end{equation}422\begin{equation}423  \vect{y}_t = g\!\left(\mat{V}\,424    [\,\overrightarrow{\vect{h}}_t \,;\, \overleftarrow{\vect{h}}_t\,]425    + \vect{b}\right),426  \label{eq:rnn-bidir-out}427\end{equation}428so each output sees both past and future. The price is that the full429sequence must be available in advance: bidirectional models suit offline430labeling, not streaming generation.431432\paragraph{Encoder--decoder (seq2seq).}433To map an input sequence to an output sequence of different length —434machine translation being the canonical example — the encoder--decoder435architecture \cite{cho2014} uses two recurrent networks. An436\emph{encoder} consumes the source $\vect{x}_1, \dots, \vect{x}_{T_x}$437and compresses it into a context vector $\vect{c}$ (typically its final438hidden state); a \emph{decoder} then generates the target439autoregressively, each token conditioned on the context and on all440previously generated tokens:441\begin{equation}442  p(\vect{y}_1, \dots, \vect{y}_{T'} \mid443    \vect{x}_1, \dots, \vect{x}_{T_x})444  = \prod_{t=1}^{T'}445    p\!\left(\vect{y}_t \mid \vect{y}_{<t},\, \vect{c}\right).446  \label{eq:rnn-seq2seq}447\end{equation}448Training maximizes the log-likelihood of \eqref{eq:rnn-seq2seq} with449\emph{teacher forcing} (feeding the ground-truth $\vect{y}_{t-1}$ as the450decoder input at step $t$); inference replaces it with the model's own451predictions, usually explored with beam search.452Figure~\ref{fig:rnn-seq2seq} sketches the architecture.453454\begin{figure}[htbp]455  \centering456  \begin{tikzpicture}457    % encoder458    \foreach \i in {1,2,3} {459      \node[ninput] (ex\i) at (\i*1.9-1.9, 0) {$\vect{x}_\i$};460      \node[blochidden, minimum width=1.3cm] (ec\i) at (\i*1.9-1.9, 1.6) {};461      \draw[fleche] (ex\i) -- (ec\i);462    }463    \draw[fleche] (ec1) -- (ec2);464    \draw[fleche] (ec2) -- (ec3);465    \node[etiquette] at (1.9, 2.35) {encoder};466    % context467    \node[mem, minimum width=1.1cm] (ctx) at (5.5, 1.6) {$\vect{c}$};468    \draw[fleche] (ec3) -- (ctx);469    % decoder470    \foreach \i in {1,2,3} {471      \node[blocoutput, minimum width=1.3cm] (dc\i) at (5.6+\i*1.9, 1.6) {};472      \node[noutput] (dy\i) at (5.6+\i*1.9, 3.2) {$\vect{y}_\i$};473      \draw[fleche] (dc\i) -- (dy\i);474    }475    \draw[fleche] (ctx) -- (dc1);476    \draw[fleche] (dc1) -- (dc2);477    \draw[fleche] (dc2) -- (dc3);478    \node[etiquette] at (9.4, 0.6) {decoder};479    % autoregressive feedback (dashed)480    \draw[flechep] (dy1.east) to[out=-30,in=120]481      node[pos=0.4, above=1pt, etiquette] {$\vect{y}_{t-1}$}482      ([xshift=-8pt]dc2.north);483    \draw[flechep] (dy2.east) to[out=-30,in=120]484      ([xshift=-8pt]dc3.north);485  \end{tikzpicture}486  \caption{Encoder--decoder (seq2seq) architecture. The encoder compresses487    the source sequence into a context vector $\vect{c}$; the decoder488    generates the target autoregressively following489    \eqref{eq:rnn-seq2seq}, each step receiving the previously emitted490    token (dashed arrows).}491  \label{fig:rnn-seq2seq}492\end{figure}493494\begin{remark}[The bottleneck that led to attention]495The fixed-size context vector $\vect{c}$ in \eqref{eq:rnn-seq2seq} is an496information bottleneck: translation quality degrades visibly on long497sentences, because an entire source sequence must be squeezed into a498single vector. Letting the decoder look back at \emph{all} encoder states499— attention — removes this bottleneck and is the subject of the next500chapter.501\end{remark}502