% ============================================================================
%  Artificial Neural Networks — Methods, Equations and Graphical
%  Representations
%  Author  : Simon-Pierre Boucher — contact@spboucher.ai
%  Chapter 13 : Emerging Architectures (chapters/13-emerging.tex)
% ============================================================================
\chapter{Emerging Architectures: Capsules, Neural ODEs, Memory-Augmented
  Networks and KANs}\label{chap:emerging}

The architectures of the preceding chapters share a common template: fixed
computational graphs whose edges carry scalar weights and whose nodes apply
fixed nonlinearities. This final chapter examines four families that each
break the template along a different axis. \emph{Capsule networks} replace
scalar neurons by vector-valued units routed by iterative agreement.
\emph{Neural ordinary differential equations} replace discrete layers by a
continuous-time flow whose depth is chosen by a numerical solver.
\emph{Memory-augmented networks} couple a neural controller to an external,
differentiable random-access memory. \emph{Kolmogorov--Arnold networks}
move the learnable nonlinearity from the nodes onto the edges themselves.
In every case we follow the discipline of this book: the defining
equations, the estimation algorithm, and a faithful graphical
representation.

% ============================================================================
\section{Capsule Networks and Routing by Agreement}
\label{sec:em-capsules}

Convolutional networks achieve tolerance to small translations through
pooling, but in doing so they discard the precise spatial relationships
between parts — a face detector built on max-pooling responds to a jumble
of eyes and mouth almost as strongly as to a correctly arranged face.
Capsule networks \cite{sabour2017} address this by making the network's
elementary unit a \emph{vector} rather than a scalar.

\begin{definition}[Capsule]\label{def:em-capsule}
A \emph{capsule} is a group of neurons whose activity vector
$\vect{u} \in \R^{d}$ encodes the instantiation parameters (pose,
deformation, texture) of an entity, and whose length
$\lVert\vect{u}\rVert \in [0,1)$ encodes the probability that the entity
is present in the input.
\end{definition}

\subsection{Prediction, coupling and squashing}

Let $\vect{u}_i$ be the output of capsule $i$ in layer $\ell$. Capsule $i$
casts a \emph{vote} for the pose of each capsule $j$ in layer $\ell+1$
through a learned transformation matrix $\mat{W}_{ij}$, and the votes are
aggregated with data-dependent \emph{coupling coefficients} $c_{ij}$:
\begin{align}
  \hat{\vect{u}}_{j|i} &= \mat{W}_{ij}\,\vect{u}_i,
  \label{eq:em-caps-pred}\\
  \vect{s}_j &= \sum_i c_{ij}\,\hat{\vect{u}}_{j|i}.
  \label{eq:em-caps-sum}
\end{align}
The couplings are a softmax over \emph{routing logits} $b_{ij}$,
initialized to zero, so that each lower capsule distributes a unit budget
of attention across the capsules of the next layer:
\begin{equation}
  c_{ij} \;=\; \frac{\exp(b_{ij})}{\sum_k \exp(b_{ik})}.
  \label{eq:em-caps-coupling}
\end{equation}
The total input $\vect{s}_j$ is converted to an output vector by the
\emph{squashing} nonlinearity, which preserves orientation while mapping
length into $[0,1)$ so it can act as a probability:
\begin{equation}
  \vect{v}_j \;=\;
  \frac{\lVert\vect{s}_j\rVert^2}{1+\lVert\vect{s}_j\rVert^2}\,
  \frac{\vect{s}_j}{\lVert\vect{s}_j\rVert}.
  \label{eq:em-caps-squash}
\end{equation}

\subsection{Dynamic routing as an estimation algorithm}

The logits are refined by \emph{routing by agreement}: a vote that aligns
with the emerging consensus $\vect{v}_j$ has its coupling reinforced,
\begin{equation}
  b_{ij} \;\leftarrow\; b_{ij} + \hat{\vect{u}}_{j|i} \cdot \vect{v}_j,
  \label{eq:em-caps-agree}
\end{equation}
which is a soft clustering in pose space: parts whose predicted poses
agree are assigned to the same whole. Algorithm~\ref{alg:em-routing}
summarizes the procedure of \cite{sabour2017}, typically run for $r=3$
iterations between consecutive capsule layers.

\begin{algorithm}[t]
\caption{Dynamic routing by agreement \cite{sabour2017}}
\label{alg:em-routing}
\begin{algorithmic}[1]
\Require prediction vectors $\hat{\vect{u}}_{j|i}$ from
         \eqref{eq:em-caps-pred}, number of iterations $r$
\Ensure  output capsules $\vect{v}_j$
\State $b_{ij} \gets 0$ for all $i, j$
\For{$t = 1, \dots, r$}
  \State $c_{ij} \gets \softmax_j(b_{ij})$
         \Comment{coupling coefficients, \eqref{eq:em-caps-coupling}}
  \State $\vect{s}_j \gets \sum_i c_{ij}\,\hat{\vect{u}}_{j|i}$
         \Comment{weighted vote aggregation, \eqref{eq:em-caps-sum}}
  \State $\vect{v}_j \gets
         \dfrac{\lVert\vect{s}_j\rVert^2}{1+\lVert\vect{s}_j\rVert^2}\,
         \dfrac{\vect{s}_j}{\lVert\vect{s}_j\rVert}$
         \Comment{squash, \eqref{eq:em-caps-squash}}
  \State $b_{ij} \gets b_{ij} + \hat{\vect{u}}_{j|i}\cdot\vect{v}_j$
         \Comment{agreement update, \eqref{eq:em-caps-agree}}
\EndFor
\State \Return $\vect{v}_j$
\end{algorithmic}
\end{algorithm}

Figure~\ref{fig:em-capsules} shows two capsule layers after routing has
converged: the width and opacity of each arrow is proportional to the
coupling coefficient $c_{ij}$, so the diagram makes visible the
part--whole assignment that \eqref{eq:em-caps-agree} computes.

\begin{figure}[t]
  \centering
  \begin{tikzpicture}[scale=1.0]
    % primary capsules
    \node[nhidden, minimum size=9mm] (u1) at (0, 1.7) {$\vect{u}_1$};
    \node[nhidden, minimum size=9mm] (u2) at (0, 0.0) {$\vect{u}_2$};
    \node[nhidden, minimum size=9mm] (u3) at (0,-1.7) {$\vect{u}_3$};
    % output capsules
    \node[noutput, minimum size=10mm] (v1) at (5.6, 0.9) {$\vect{v}_1$};
    \node[noutput, minimum size=10mm] (v2) at (5.6,-0.9) {$\vect{v}_2$};
    % couplings: widths proportional to c_ij
    \draw[fleche, line width=1.9pt, cinput!90!black]
      (u1) -- (v1)
      node[pos=0.40, above=1pt, sloped, font=\scriptsize, text=black]
      {$\hat{\vect{u}}_{1|1}$};
    \draw[fleche, line width=0.5pt, cinput!90!black, opacity=0.30]
      (u1) -- (v2);
    \draw[fleche, line width=1.5pt, cinput!90!black, opacity=0.85]
      (u2) -- (v1);
    \draw[fleche, line width=0.7pt, cinput!90!black, opacity=0.40]
      (u2) -- (v2);
    \draw[fleche, line width=0.5pt, cinput!90!black, opacity=0.30]
      (u3) -- (v1);
    \draw[fleche, line width=1.9pt, cinput!90!black]
      (u3) -- (v2);
    % annotations
    \node[etiquette, align=center] at (0, 2.85)
      {primary capsules\\(layer $\ell$)};
    \node[etiquette, align=center] at (5.6, 2.05)
      {output capsules\\(layer $\ell+1$)};
    \node[etiquette, align=left, anchor=west] at (6.55, 0.9)
      {$\lVert\vect{v}_1\rVert \approx 1$:\\entity present};
    \node[etiquette, align=left, anchor=west] at (6.55,-0.9)
      {$\lVert\vect{v}_2\rVert \approx 1$:\\entity present};
    \node[etiquette, align=center] at (2.8,-2.75)
      {arrow width $\propto$ coupling coefficient $c_{ij}$
       of \eqref{eq:em-caps-coupling}};
  \end{tikzpicture}
  \caption{Two capsule layers after routing by agreement
    (Algorithm~\ref{alg:em-routing}). Each primary capsule sends a
    prediction vector $\hat{\vect{u}}_{j|i} = \mat{W}_{ij}\vect{u}_i$ to
    every output capsule; iterating
    \eqref{eq:em-caps-coupling}--\eqref{eq:em-caps-agree} concentrates the
    couplings (arrow width) on the wholes whose pose the parts agree on:
    here $\vect{u}_1, \vect{u}_2$ are routed to $\vect{v}_1$ and
    $\vect{u}_3$ to $\vect{v}_2$.}
  \label{fig:em-capsules}
\end{figure}

\subsection{Margin loss}

Since capsule lengths are probabilities, classification uses a per-class
\emph{margin loss}: with $T_k = 1$ if class $k$ is present and $0$
otherwise,
\begin{equation}
  L_k \;=\; T_k \max\bigl(0,\, m^{+} - \lVert\vect{v}_k\rVert\bigr)^2
  \;+\; \lambda\,(1-T_k)\max\bigl(0,\,
  \lVert\vect{v}_k\rVert - m^{-}\bigr)^2,
  \label{eq:em-caps-margin}
\end{equation}
with $m^{+}=0.9$, $m^{-}=0.1$ and $\lambda = 0.5$ in \cite{sabour2017};
the total loss $\Loss = \sum_k L_k$ is complemented by a small
reconstruction penalty that regularizes the capsule poses.

\begin{remark}
Routing replaces pooling: instead of discarding position information, the
network performs an explicit, differentiable assignment of parts to
wholes. The price is computational — routing is an inner-loop iteration
per layer and per example — and capsule networks have not yet scaled
beyond mid-sized benchmarks, but the equivariance principle they embody
continues to influence architecture design.
\end{remark}

% ============================================================================
\section{Neural Ordinary Differential Equations}
\label{sec:em-node}

\subsection{From residual blocks to continuous depth}

A residual block computes
$\vect{h}_{t+1} = \vect{h}_t + f(\vect{h}_t, \theta_t)$. Reading the layer
index $t$ as time, this is exactly one step of the explicit Euler scheme
with unit step size. Neural ODEs \cite{chen2018} take the continuous
limit: the hidden state is defined as the solution of an initial value
problem,
\begin{equation}
  \frac{d\vect{h}(t)}{dt} = f\bigl(\vect{h}(t), t, \theta\bigr),
  \qquad
  \vect{h}(t_1) = \vect{h}(t_0)
  + \int_{t_0}^{t_1} f\bigl(\vect{h}(t), t, \theta\bigr)\,dt,
  \label{eq:em-node-dynamics}
\end{equation}
evaluated by a black-box solver
$\vect{h}(t_1) = \mathrm{ODESolve}(\vect{h}(t_0), f, t_0, t_1, \theta)$
such as an adaptive Runge--Kutta method.

\begin{remark}[ResNet as Euler discretization]\label{rem:em-euler}
With step size $\Delta t$, the Euler scheme applied to
\eqref{eq:em-node-dynamics} reads
\begin{equation}
  \vect{h}_{t+1} \;=\; \vect{h}_t
  + \Delta t\, f(\vect{h}_t, t, \theta),
  \label{eq:em-node-euler}
\end{equation}
which for $\Delta t = 1$ is precisely the residual block of a ResNet. A
neural ODE is thus a residual network whose number of layers — the number
of solver steps — is chosen adaptively at run time, trading accuracy
against compute without retraining.
\end{remark}

\subsection{The adjoint method}

Naively backpropagating through every internal solver step would store all
intermediate states. The \emph{adjoint sensitivity method} avoids this.
Define the adjoint state as the sensitivity of the loss to the hidden
state at each instant,
\begin{equation}
  \vect{a}(t) \;=\; \frac{\partial \Loss}{\partial \vect{h}(t)}.
  \label{eq:em-node-adjoint}
\end{equation}
The adjoint obeys its own linear ODE, integrated \emph{backwards} in time
from $\vect{a}(t_1) = \partial\Loss/\partial\vect{h}(t_1)$:
\begin{equation}
  \frac{d\vect{a}(t)}{dt}
  \;=\; -\,\vect{a}(t)\transp\,
  \frac{\partial f\bigl(\vect{h}(t), t, \theta\bigr)}{\partial \vect{h}},
  \label{eq:em-node-adjoint-ode}
\end{equation}
and the parameter gradient is obtained by a single quadrature along the
same backward pass:
\begin{equation}
  \frac{d\Loss}{d\theta}
  \;=\; -\int_{t_1}^{t_0} \vect{a}(t)\transp\,
  \frac{\partial f\bigl(\vect{h}(t), t, \theta\bigr)}{\partial \theta}
  \,dt.
  \label{eq:em-node-grad}
\end{equation}
In practice one concatenates the state, the adjoint and the accumulating
gradient into a single \emph{augmented state} and makes one backward
solver call:
\begin{equation}
  \frac{d}{dt}
  \begin{bmatrix} \vect{h}(t)\\[2pt] \vect{a}(t)\\[2pt]
    \dfrac{d\Loss}{d\theta}(t) \end{bmatrix}
  =
  \begin{bmatrix} f(\vect{h}, t, \theta)\\[2pt]
    -\vect{a}\transp\,\partial f/\partial\vect{h}\\[2pt]
    -\vect{a}\transp\,\partial f/\partial\theta \end{bmatrix},
  \label{eq:em-node-augmented}
\end{equation}
where the vector--Jacobian products are computed by ordinary reverse-mode
automatic differentiation of $f$ alone.
Algorithm~\ref{alg:em-adjoint} assembles the full gradient computation.

\begin{property}[Constant memory in depth]\label{prop:em-memory}
Training with
\eqref{eq:em-node-adjoint-ode}--\eqref{eq:em-node-augmented} requires
storing only the endpoint states, so the memory cost of a neural ODE is
$O(1)$ in the effective depth, whereas backpropagation through an
$L$-layer residual network stores $O(L)$ activations. The hidden
trajectory $\vect{h}(t)$ needed inside \eqref{eq:em-node-adjoint-ode} is
recovered on the fly by integrating \eqref{eq:em-node-dynamics}
backwards alongside the adjoint.
\end{property}

\begin{algorithm}[t]
\caption{Adjoint-method gradient for a neural ODE \cite{chen2018}}
\label{alg:em-adjoint}
\begin{algorithmic}[1]
\Require dynamics $f_\theta$, initial state $\vect{h}(t_0)$, loss $\Loss$
\State $\vect{h}(t_1) \gets
       \mathrm{ODESolve}\bigl(\vect{h}(t_0), f, t_0, t_1, \theta\bigr)$
       \Comment{forward solve of \eqref{eq:em-node-dynamics}}
\State $\vect{a}(t_1) \gets \partial\Loss/\partial\vect{h}(t_1)$
       \Comment{terminal adjoint, \eqref{eq:em-node-adjoint}}
\State $\vect{s}(t_1) \gets
       \bigl[\vect{h}(t_1),\ \vect{a}(t_1),\ \vect{0}\bigr]$
       \Comment{augmented state}
\State $\bigl[\vect{h}(t_0), \vect{a}(t_0), d\Loss/d\theta\bigr] \gets
       \mathrm{ODESolve}\bigl(\vect{s}(t_1),
       \text{dynamics \eqref{eq:em-node-augmented}}, t_1, t_0\bigr)$
       \Comment{backward solve}
\State \Return $d\Loss/d\theta$ \ and \
       $\vect{a}(t_0) = \partial\Loss/\partial\vect{h}(t_0)$
\end{algorithmic}
\end{algorithm}

Figure~\ref{fig:em-node} contrasts the two views of depth: the discrete
Euler/ResNet staircase of \eqref{eq:em-node-euler} against the smooth flow
of \eqref{eq:em-node-dynamics}, drawn over the vector field
$f(h) = 0.8\,h\,(1-h/3)$ that both are following.

\begin{figure}[t]
  \centering
  \begin{tikzpicture}
    \begin{axis}[
        width=0.82\textwidth, height=6.6cm,
        xlabel={$t$ (depth)}, ylabel={$h(t)$},
        xmin=-0.15, xmax=4.45, ymin=0, ymax=3.15,
        legend style={at={(0.97,0.06)}, anchor=south east,
                      font=\scriptsize, draw=black!30},
        tick label style={font=\scriptsize},
        label style={font=\small},
      ]
      % faint vector field of the dynamics f(h)=0.8 h (1-h/3)
      \foreach \t in {0.1,0.6,1.1,1.6,2.1,2.6,3.1,3.6,4.1}{
        \foreach \h in {0.35,0.75,1.15,1.55,1.95,2.35,2.75}{
          \edef\temp{\noexpand\draw[-{Stealth[length=1.0mm]},
            black!25, thin]
            (axis cs:\t,\h) --
            (axis cs:{\t+0.16},{\h+0.16*0.8*\h*(1-\h/3)});}
          \temp
        }
      }
      % continuous ODE trajectory (logistic solution)
      \addplot[cinput, very thick, smooth, domain=0:4.3, samples=80]
        {3/(1+5*exp(-0.8*x))};
      \addlegendentry{continuous flow, \eqref{eq:em-node-dynamics}}
      % Euler / ResNet steps, dt = 1
      \addplot[chidden!80!black, thick, dashed,
               mark=*, mark options={solid, fill=chidden}]
        coordinates {(0,0.5) (1,0.8333) (2,1.3148) (3,1.9056)
                     (4,2.4617)};
      \addlegendentry{Euler / ResNet steps, $\Delta t = 1$,
        \eqref{eq:em-node-euler}}
    \end{axis}
  \end{tikzpicture}
  \caption{A neural ODE as the continuous limit of a residual network.
    Grey arrows: the learned vector field $f(h,t,\theta)$. Blue: the
    exact trajectory of \eqref{eq:em-node-dynamics}. Orange: the Euler
    discretization \eqref{eq:em-node-euler} with $\Delta t = 1$, i.e.\ a
    four-block residual network following the same field with
    accumulating discretization error. An adaptive solver places as many
    evaluations as the requested tolerance demands.}
  \label{fig:em-node}
\end{figure}

% ============================================================================
\section{Memory-Augmented Networks: Neural Turing Machines}
\label{sec:em-ntm}

Recurrent networks store all their knowledge in a fixed-size hidden state.
Neural Turing Machines \cite{graves2014} decouple computation from
storage: a controller network (typically an LSTM) interacts with an
external memory matrix $\mat{M}_t \in \R^{N\times W}$ ($N$ locations of
width $W$) through read and write \emph{heads} that are differentiable
end-to-end, so the whole system is trained by ordinary gradient descent.

\subsection{Reading and writing}

Every interaction is mediated by a normalized attention weighting
$\vect{w}_t \in \R^N$, $\sum_i w_t(i) = 1$, over memory locations. Reading
returns the weighted average of the rows,
\begin{equation}
  \vect{r}_t \;=\; \sum_{i=1}^{N} w_t(i)\,\mat{M}_t(i),
  \label{eq:em-ntm-read}
\end{equation}
and writing decomposes, like an LSTM gate pair, into an erase followed by
an add, with $\vect{e}_t \in [0,1]^W$ and $\vect{a}_t \in \R^W$ emitted by
the controller:
\begin{align}
  \tilde{\mat{M}}_t(i) &= \mat{M}_{t-1}(i) \odot
    \bigl[\vect{1} - w_t(i)\,\vect{e}_t\bigr],
  \label{eq:em-ntm-erase}\\
  \mat{M}_t(i) &= \tilde{\mat{M}}_t(i) + w_t(i)\,\vect{a}_t.
  \label{eq:em-ntm-add}
\end{align}

\subsection{The differentiable addressing pipeline}

The weighting $\vect{w}_t$ is produced by a four-stage pipeline that
blends content-based and location-based addressing. First, the controller
emits a key $\vect{k}_t$ compared to every row by cosine similarity,
\begin{equation}
  K[\vect{u},\vect{v}] \;=\;
  \frac{\vect{u}\cdot\vect{v}}{\lVert\vect{u}\rVert\,
        \lVert\vect{v}\rVert},
  \label{eq:em-ntm-cosine}
\end{equation}
sharpened by a scalar $\beta_t > 0$ and normalized:
\begin{equation}
  w_t^{c}(i) \;=\;
  \frac{\exp\bigl(\beta_t\,K[\vect{k}_t, \mat{M}_t(i)]\bigr)}
       {\sum_j \exp\bigl(\beta_t\,K[\vect{k}_t, \mat{M}_t(j)]\bigr)}.
  \label{eq:em-ntm-content}
\end{equation}
The content weighting is then interpolated with the previous weighting by
a gate $g_t \in [0,1]$,
\begin{equation}
  \vect{w}_t^{g} \;=\; g_t\,\vect{w}_t^{c}
  + (1-g_t)\,\vect{w}_{t-1},
  \label{eq:em-ntm-interp}
\end{equation}
rotated by a circular convolution with a shift distribution $\vect{s}_t$
(location-based addressing, enabling ``move one slot to the right''),
\begin{equation}
  \tilde{w}_t(i) \;=\; \sum_{j=0}^{N-1} w_t^{g}(j)\,
  s_t\bigl((i-j) \bmod N\bigr),
  \label{eq:em-ntm-shift}
\end{equation}
and finally re-sharpened with $\gamma_t \geq 1$ to undo the blurring
introduced by the convolution:
\begin{equation}
  w_t(i) \;=\;
  \frac{\tilde{w}_t(i)^{\gamma_t}}{\sum_j \tilde{w}_t(j)^{\gamma_t}}.
  \label{eq:em-ntm-sharpen}
\end{equation}
Algorithm~\ref{alg:em-ntm} chains
\eqref{eq:em-ntm-cosine}--\eqref{eq:em-ntm-sharpen} with the memory update
\eqref{eq:em-ntm-erase}--\eqref{eq:em-ntm-add}; every operation is smooth
in its inputs, so gradients flow from the task loss into the controller,
the heads and the memory itself. The complete system is drawn in
Figure~\ref{fig:em-ntm}.

\begin{algorithm}[t]
\caption{One NTM head at timestep $t$ \cite{graves2014}}
\label{alg:em-ntm}
\begin{algorithmic}[1]
\Require controller outputs $\vect{k}_t, \beta_t, g_t, \vect{s}_t,
         \gamma_t$ (and $\vect{e}_t, \vect{a}_t$ for a write head),
         previous weighting $\vect{w}_{t-1}$, memory $\mat{M}_{t-1}$
\State $w_t^c(i) \gets \softmax_i\bigl(\beta_t\,
       K[\vect{k}_t,\mat{M}_{t-1}(i)]\bigr)$
       \Comment{content addressing,
       \eqref{eq:em-ntm-cosine}--\eqref{eq:em-ntm-content}}
\State $\vect{w}_t^{g} \gets g_t \vect{w}_t^{c}
       + (1-g_t)\,\vect{w}_{t-1}$
       \Comment{interpolation, \eqref{eq:em-ntm-interp}}
\State $\tilde{w}_t(i) \gets \sum_j w^{g}_t(j)\,s_t((i-j)\bmod N)$
       \Comment{circular shift, \eqref{eq:em-ntm-shift}}
\State $w_t(i) \gets \tilde{w}_t(i)^{\gamma_t} \big/
       \sum_j \tilde{w}_t(j)^{\gamma_t}$
       \Comment{sharpening, \eqref{eq:em-ntm-sharpen}}
\If{write head}
  \State apply erase then add,
         \eqref{eq:em-ntm-erase}--\eqref{eq:em-ntm-add}
\Else
  \State \Return read vector
         $\vect{r}_t = \sum_i w_t(i)\,\mat{M}_t(i)$,
         \eqref{eq:em-ntm-read}
\EndIf
\end{algorithmic}
\end{algorithm}

\begin{figure}[t]
  \centering
  \begin{tikzpicture}[scale=1.0]
    % controller
    \node[blochidden, minimum width=2.5cm, minimum height=1.5cm,
          align=center]
      (ctrl) at (0,0) {Controller\\(LSTM)};
    \draw[fleche] (-2.6,-0.45) node[left, font=\small] {$\vect{x}_t$}
      -- (-1.25,-0.45);
    \draw[fleche] (-1.25, 0.45)
      -- (-2.6, 0.45) node[left, font=\small] {$\vect{y}_t$};
    % heads
    \node[gate, minimum width=1.9cm] (wh) at (3.7, 1.5)
      {write head};
    \node[gate, minimum width=1.9cm] (rh) at (3.7,-1.5)
      {read head};
    \draw[fleche] (1.25, 0.45) -| (wh.south);
    \draw[fleche] (1.25,-0.45) -| (rh.north);
    \node[etiquette, anchor=south, align=center] at (1.85, 0.55)
      {$\vect{k}_t, \beta_t, g_t, \vect{s}_t,$\\
       $\gamma_t, \vect{e}_t, \vect{a}_t$};
    \node[etiquette, anchor=north] at (2.15,-0.55)
      {$\vect{k}_t, \beta_t, g_t, \vect{s}_t, \gamma_t$};
    % memory grid: 4 rows (locations N) x 5 columns (width W)
    \begin{scope}[shift={(6.3,-1.24)}]
      % highlighted row (addressed): row index 2 (third from bottom)
      \fill[cmem!35] (0, 1.24) rectangle (3.10, 1.86);
      \foreach \r in {0,1,2,3}{
        \foreach \c in {0,1,2,3,4}{
          \draw[black!60] (\c*0.62, \r*0.62)
            rectangle ++(0.62,0.62);
        }
      }
      \node[etiquette] at (1.55, 2.85)
        {memory $\mat{M}_t \in \R^{N\times W}$};
      \node[etiquette, anchor=west] at (3.22, 1.55) {$w_t$};
    \end{scope}
    % head <-> memory arrows (enter the addressed row's left edge)
    \draw[fleche, cgate!60!black] (wh.east)
      -- (5.9, 1.5) -- (5.9, 0.48) -- (6.28, 0.48);
    \draw[fleche, cgate!60!black] (rh.east)
      -- (5.9, -1.5) -- (5.9, 0.14) -- (6.28, 0.14);
    \node[etiquette, anchor=south, align=center] at (5.45, 1.58)
      {erase, add\\
       \eqref{eq:em-ntm-erase}--\eqref{eq:em-ntm-add}};
    \node[etiquette, anchor=north] at (5.42, -1.62)
      {address $w_t$};
    % read vector returned to controller (dashed)
    \draw[flechep, cmem!70!black]
      (rh.south) |- (0,-2.75)
      node[pos=0.75, above, etiquette] {read vector $\vect{r}_t$,
        \eqref{eq:em-ntm-read}}
      -- (0,-0.75);
  \end{tikzpicture}
  \caption{The Neural Turing Machine. The controller emits addressing
    parameters for each head; the write head modifies the memory by erase
    \eqref{eq:em-ntm-erase} then add \eqref{eq:em-ntm-add}, the read head
    returns the attention-weighted content of the addressed row (violet),
    and the read vector feeds back into the controller at the next step
    (dashed). Every operation is differentiable, so the system trains
    end-to-end by gradient descent.}
  \label{fig:em-ntm}
\end{figure}

\begin{remark}
The Differentiable Neural Computer refines the NTM with dynamic slot
allocation via usage vectors and a temporal link matrix that lets read
heads replay memory in the order it was written. Although superseded in
practice by Transformers — whose attention \emph{is} a form of
content-based addressing \eqref{eq:em-ntm-content} over an internal
memory of past tokens — the NTM/DNC line established the memory-augmented
paradigm that today's retrieval-augmented models inherit.
\end{remark}

% ============================================================================
\section{Kolmogorov--Arnold Networks}
\label{sec:em-kan}

\subsection{The representation theorem}

\begin{theorem}[Kolmogorov--Arnold superposition]\label{thm:em-kart}
Every continuous function $f : [0,1]^n \to \R$ can be written as
\begin{equation}
  f(x_1, \dots, x_n) \;=\;
  \sum_{q=1}^{2n+1} \Phi_q\!\left(
  \sum_{p=1}^{n} \phi_{q,p}(x_p)\right),
  \label{eq:em-kan-kart}
\end{equation}
where the $\Phi_q : \R \to \R$ and $\phi_{q,p} : [0,1] \to \R$ are
continuous \emph{univariate} functions.
\end{theorem}

Multivariate continuity thus reduces entirely to sums and compositions of
one-dimensional functions — the only truly multivariate operation in
\eqref{eq:em-kan-kart} is addition.

\subsection{KAN layers}

Kolmogorov--Arnold networks \cite{liu2024kan} turn this structure into an
architecture. Where an MLP layer computes
$\sigma(\mat{W}\vect{x} + \vect{b})$ — fixed nonlinearities on the nodes,
learnable scalars on the edges — a KAN layer places a \emph{learnable
univariate function on every edge} and reduces nodes to pure summation:
\begin{equation}
  x_{\ell+1,\,j} \;=\; \sum_{i=1}^{n_\ell}
  \phi_{\ell,j,i}\bigl(x_{\ell,\,i}\bigr),
  \label{eq:em-kan-layer}
\end{equation}
and a full network is a composition of such function matrices
$\Phi_\ell = \bigl(\phi_{\ell,j,i}\bigr)_{j,i}$:
\begin{equation}
  \mathrm{KAN}(\vect{x}) \;=\;
  \bigl(\Phi_{L-1} \circ \cdots \circ \Phi_1 \circ
  \Phi_0\bigr)(\vect{x}).
  \label{eq:em-kan-compose}
\end{equation}
Each edge function is parameterized as a B-spline plus a smooth residual
basis that keeps gradients well-behaved:
\begin{equation}
  \phi(x) \;=\; w_b\,\mathrm{silu}(x)
  + w_s \sum_{i} c_i\,B_i(x),
  \label{eq:em-kan-spline}
\end{equation}
with learnable spline coefficients $c_i$ over a grid of knots. The
authors of \cite{liu2024kan} stress that
\eqref{eq:em-kan-layer}--\eqref{eq:em-kan-compose} generalize the exact
depth-2, width-$(2n{+}1)$ form of Theorem~\ref{thm:em-kart} to arbitrary
depths and widths. Figure~\ref{fig:em-kan} draws a small KAN with the
learned univariate function displayed on each edge.

\begin{figure}[t]
  \centering
  \begin{tikzpicture}[scale=1.0]
    % nodes
    \node[ninput]  (x1) at (0, 1.0) {$x_1$};
    \node[ninput]  (x2) at (0,-1.0) {$x_2$};
    \node[op, minimum size=7mm] (m1) at (3.4, 1.8) {$+$};
    \node[op, minimum size=7mm] (m2) at (3.4, 0.0) {$+$};
    \node[op, minimum size=7mm] (m3) at (3.4,-1.8) {$+$};
    \node[noutput, minimum size=9mm] (y) at (6.8, 0.0) {$y$};
    % edges first (below boxes)
    \draw[fleche, black!55] (x1) -- (m1);
    \draw[fleche, black!55] (x1) -- (m2);
    \draw[fleche, black!55] (x1) -- (m3);
    \draw[fleche, black!55] (x2) -- (m1);
    \draw[fleche, black!55] (x2) -- (m2);
    \draw[fleche, black!55] (x2) -- (m3);
    \draw[fleche, black!55] (m1) -- (y);
    \draw[fleche, black!55] (m2) -- (y);
    \draw[fleche, black!55] (m3) -- (y);
    % mini function boxes on edges (drawn after edges to sit on top)
    % straight-ish edges: boxes at pos ~0.4 ; crossing edges: pos ~0.72
    \node[draw=black!60, fill=white, minimum width=8mm,
          minimum height=5.5mm, inner sep=0.5pt] at (1.36, 1.32) {};
    \draw[cinput, thick]
      plot[domain=-0.30:0.30, samples=19]
      ({1.36+\x}, {1.32+0.14*sin(600*\x)});
    \node[draw=black!60, fill=white, minimum width=8mm,
          minimum height=5.5mm, inner sep=0.5pt] at (1.36, 0.60) {};
    \draw[cinput, thick]
      plot[domain=-0.30:0.30, samples=19]
      ({1.36+\x}, {0.60+1.1*\x*\x-0.09});
    \node[draw=black!60, fill=white, minimum width=8mm,
          minimum height=5.5mm, inner sep=0.5pt] at (2.448,-1.016) {};
    \draw[cinput, thick]
      plot[domain=-0.30:0.30, samples=19]
      ({2.448+\x}, {-1.016+0.13*tanh(6*\x)});
    \node[draw=black!60, fill=white, minimum width=8mm,
          minimum height=5.5mm, inner sep=0.5pt] at (2.448, 1.016) {};
    \draw[cinput, thick]
      plot[domain=-0.30:0.30, samples=19]
      ({2.448+\x}, {1.016-0.13*tanh(6*\x)});
    \node[draw=black!60, fill=white, minimum width=8mm,
          minimum height=5.5mm, inner sep=0.5pt] at (1.36,-0.60) {};
    \draw[cinput, thick]
      plot[domain=-0.30:0.30, samples=19]
      ({1.36+\x}, {-0.60+0.14*sin(300*\x)});
    \node[draw=black!60, fill=white, minimum width=8mm,
          minimum height=5.5mm, inner sep=0.5pt] at (1.36,-1.32) {};
    \draw[cinput, thick]
      plot[domain=-0.30:0.30, samples=19]
      ({1.36+\x}, {-1.32-1.1*\x*\x+0.09});
    % layer 2 boxes
    \node[draw=black!60, fill=white, minimum width=8mm,
          minimum height=5.5mm, inner sep=0.5pt] at (5.1, 0.90) {};
    \draw[coutput, thick]
      plot[domain=-0.30:0.30, samples=19]
      ({5.1+\x}, {0.90+0.13*tanh(6*\x)});
    \node[draw=black!60, fill=white, minimum width=8mm,
          minimum height=5.5mm, inner sep=0.5pt] at (5.1, 0.0) {};
    \draw[coutput, thick]
      plot[domain=-0.30:0.30, samples=19]
      ({5.1+\x}, {0.0+0.14*sin(600*\x)});
    \node[draw=black!60, fill=white, minimum width=8mm,
          minimum height=5.5mm, inner sep=0.5pt] at (5.1,-0.90) {};
    \draw[coutput, thick]
      plot[domain=-0.30:0.30, samples=19]
      ({5.1+\x}, {-0.90+1.1*\x*\x-0.09});
    % annotations
    \node[etiquette, align=center] at (0,-2.6)
      {inputs};
    \node[etiquette, align=center] at (3.4,-2.6)
      {summation nodes \eqref{eq:em-kan-layer}};
    \node[etiquette, align=center] at (6.8,-2.6)
      {output};
    \node[etiquette, align=center] at (3.4, 2.75)
      {a learnable univariate $\phi_{\ell,j,i}$
       \eqref{eq:em-kan-spline} on \emph{every} edge};
  \end{tikzpicture}
  \caption{A small Kolmogorov--Arnold network with $n_0 = 2$ inputs,
    $n_1 = 3$ hidden summation nodes and one output. Each edge carries
    its own learnable univariate function (inset curves), parameterized
    as a B-spline \eqref{eq:em-kan-spline}; the nodes only add their
    incoming values, exactly as in the superposition
    \eqref{eq:em-kan-kart}.}
  \label{fig:em-kan}
\end{figure}

\subsection{KANs versus MLPs}

Table~\ref{tab:em-kan-mlp} summarizes the structural contrast. KANs trade
the hardware-friendliness of dense matrix multiplication for
interpretability: a trained spline can be plotted, pruned, and often
symbolically identified ($\sin$, $x^2$, $\exp$), which has made KANs
attractive for scientific and symbolic-regression tasks.

\begin{table}[t]
  \centering
  \caption{Multilayer perceptrons versus Kolmogorov--Arnold networks.}
  \label{tab:em-kan-mlp}
  \begin{tabular}{@{}lll@{}}
    \toprule
    & MLP & KAN \\
    \midrule
    Nonlinearity & fixed, on nodes & learnable, on edges \\
    Edge parameters & scalar weights $w_{ij}$ &
      spline functions $\phi_{j,i}$ \\
    Node operation & $\sigma(\mat{W}\vect{x}+\vect{b})$ &
      summation \eqref{eq:em-kan-layer} \\
    Theoretical anchor & universal approximation &
      superposition \eqref{eq:em-kan-kart} \\
    Strengths & fast dense algebra, scales &
      small-scale accuracy, interpretable \\
    Weaknesses & opaque parameters &
      slower training, unproven at scale \\
    \bottomrule
  \end{tabular}
\end{table}

\begin{remark}[Grid extension]\label{rem:em-grid}
The spline grid in \eqref{eq:em-kan-spline} can be refined during
training: a KAN first fitted on a coarse grid is re-projected onto a
finer one (a small least-squares problem per edge), increasing capacity
exactly where resolution is needed without restarting optimization — a
form of continuation in model space with no analogue in standard MLP
training.
\end{remark}

% ============================================================================
\section{Outlook}
\label{sec:em-outlook}

The four families of this chapter relax, in turn, each frozen ingredient
of the classical neural network: the scalar unit (capsules), the discrete
layer (neural ODEs), the fixed-size state (memory-augmented networks) and
the fixed nonlinearity (KANs). None has displaced the Transformer as the
dominant general-purpose architecture, but each has permanently enlarged
the design space — routing lives on in mixture-of-experts gating,
continuous-depth models in diffusion and flow-based generation, external
memory in retrieval-augmented systems, and learnable univariate bases in
scientific machine learning. The history of the field, from the
perceptron onward, suggests that ideas of this kind rarely disappear;
they wait for the scale, the hardware or the objective that lets them
matter.