% ============================================================================
%  Artificial Neural Networks — Methods, Equations and Graphical
%  Representations
%  Author  : Simon-Pierre Boucher — contact@spboucher.ai
%  Chapter 4 : Convolutional Neural Networks (chapters/04-cnn.tex)
% ============================================================================
\chapter{Convolutional Neural Networks}\label{chap:cnn}

Fully connected networks treat every input coordinate as unrelated to every
other: a pixel in the top-left corner of an image and its immediate
neighbour are, as far as the architecture is concerned, as distant as two
arbitrary pixels. Convolutional neural networks (CNNs) build the spatial
structure of the data directly into the architecture through three
mechanisms: \emph{local connectivity} (each unit sees only a small window
of the input), \emph{weight sharing} (the same filter is applied at every
position, so a feature detector learned in one place is available
everywhere), and \emph{hierarchical composition} (stacked layers assemble
edges into textures, textures into parts, parts into objects). This chapter
develops the convolution and pooling operations, assembles them into the
canonical convolutional architecture of LeCun et
al.~\cite{lecun1998}, analyses the residual connection of He et
al.~\cite{he2016} that made very deep networks trainable, and closes with
the notion of receptive field and a brief genealogy of modern
architectures.

% ----------------------------------------------------------------------------
\section{From the Visual Cortex to the Convolutional Layer}

The lineage of the CNN begins in neurophysiology. Recording from the cat's
visual cortex, Hubel and Wiesel identified \emph{simple cells}, which
respond to oriented edges at specific retinal positions, and \emph{complex
cells}, which respond to the same features with tolerance to small shifts.
Fukushima's \emph{Neocognitron} (1980) translated this two-cell hierarchy
into a network that alternates feature-extracting S-cells — local
receptive fields with shared weights, the conceptual ancestor of the
convolutional layer — with position-tolerant C-cells, the ancestor of
pooling. What the Neocognitron lacked was end-to-end supervised training:
that ingredient was supplied when backpropagation was applied to
weight-shared architectures, culminating in LeNet-5~\cite{lecun1998}, the
first widely deployed CNN, which read millions of bank cheques using the
canonical pattern
$[\text{conv} \to \text{pool}] \times N \to \text{dense} \to
\text{output}$ that this chapter formalizes.

% ----------------------------------------------------------------------------
\section{The Convolution Operation}

\subsection{Definition}

\begin{definition}[Discrete two-dimensional convolution]\label{def:cnn-conv}
Let $I$ be an input map and $K \in \R^{k_h \times k_w}$ a kernel (filter).
The discrete convolution of $I$ by $K$ is
\begin{equation}\label{eq:cnn-conv2d}
  S(i,j) \;=\; (I * K)(i,j)
  \;=\; \sum_{m=0}^{k_h-1} \sum_{n=0}^{k_w-1} I(i-m,\, j-n)\, K(m,n).
\end{equation}
\end{definition}

\begin{remark}[Convolution versus cross-correlation]\label{rem:cnn-crosscorr}
Deep learning frameworks implement \emph{cross-correlation} — the same sum
without flipping the kernel, i.e.\ with indices $I(i+m,\,j+n)$. Since the
kernel entries are learned, the two operations are equivalent up to a
re-parameterization of the weights, and the term ``convolution'' is used
for both. All equations below follow the cross-correlation convention.
\end{remark}

In a convolutional \emph{layer}, the input is a stack of $C_{\mathrm{in}}$
channels $\vect{x} \in \R^{C_{\mathrm{in}} \times H \times W}$, and each of
the $C_{\mathrm{out}}$ output channels is produced by its own
three-dimensional filter plus a bias. With stride $s$ and zero-padding $p$
(writing $\tilde{\vect{x}}$ for the input bordered by $p$ zeros on each
side), the output channel $c_{\mathrm{out}}$ is
\begin{equation}\label{eq:cnn-convlayer}
  y_{c_{\mathrm{out}}}(i,j)
  \;=\; b_{c_{\mathrm{out}}}
  \;+\; \sum_{c=1}^{C_{\mathrm{in}}} \sum_{m=0}^{k_h-1} \sum_{n=0}^{k_w-1}
        W_{c_{\mathrm{out}},c}(m,n)\;
        \tilde{x}_{c}\big(s\,i + m,\; s\,j + n\big),
\end{equation}
where $i,j$ index the output positions. Three properties of
\eqref{eq:cnn-convlayer} explain the effectiveness of the operation. Each
output value depends only on a $k_h \times k_w$ window of the input
(\emph{local connectivity}); the same weights
$W_{c_{\mathrm{out}},c}$ are used at every position $(i,j)$ (\emph{weight
sharing}), which makes the layer \emph{equivariant} to translation — a
shifted input produces a correspondingly shifted output — and reduces the
parameter count dramatically; and stacking such layers composes local
detectors into progressively more global and abstract
ones~\cite{goodfellow2016book}. Algorithm~\ref{alg:cnn-forward} spells out
the layer's forward pass as literal loops over
\eqref{eq:cnn-convlayer} — production implementations replace the loops
with im2col-plus-GEMM or FFT-based routines, but compute exactly the same
map.

\begin{algorithm}[htbp]
  \caption{Forward pass of a convolutional layer
    (stride $s$, padding $p$)}
  \label{alg:cnn-forward}
  \begin{algorithmic}[1]
    \Require input $\vect{x} \in \R^{C_{\mathrm{in}} \times H \times W}$,
      weights $W \in \R^{C_{\mathrm{out}} \times C_{\mathrm{in}} \times
      k_h \times k_w}$, biases $\vect{b}$, activation $\varphi$
    \State $\tilde{\vect{x}} \gets$ zero-pad $\vect{x}$ with a border of
      $p$ zeros on each spatial side
    \State $H_{\mathrm{out}} \gets \lfloor (H + 2p - k_h)/s \rfloor + 1$;
      \quad $W_{\mathrm{out}} \gets \lfloor (W + 2p - k_w)/s \rfloor + 1$
      \Comment{\eqref{eq:cnn-outsize}}
    \For{$c_{\mathrm{out}} = 1, \dots, C_{\mathrm{out}}$}
      \For{$i = 0, \dots, H_{\mathrm{out}}-1$;\;
           $j = 0, \dots, W_{\mathrm{out}}-1$}
        \State $u \gets b_{c_{\mathrm{out}}}$
        \For{$c = 1, \dots, C_{\mathrm{in}}$;\;
             $m = 0, \dots, k_h - 1$;\;
             $n = 0, \dots, k_w - 1$}
          \State $u \gets u + W_{c_{\mathrm{out}},c}(m,n)\,
            \tilde{x}_c(s\,i + m,\; s\,j + n)$
            \Comment{windowed sum \eqref{eq:cnn-convlayer}}
        \EndFor
        \State $y_{c_{\mathrm{out}}}(i,j) \gets \varphi(u)$
      \EndFor
    \EndFor
    \State \Return $\vect{y} \in \R^{C_{\mathrm{out}} \times
      H_{\mathrm{out}} \times W_{\mathrm{out}}}$
  \end{algorithmic}
\end{algorithm}

\subsection{Stride, Padding and Output Geometry}

The \emph{stride} $s$ is the step of the sliding window: $s>1$ downsamples
the output. The \emph{padding} $p$ adds a border of zeros; ``valid''
convolution uses $p=0$, while ``same'' padding, $p = \lfloor k/2 \rfloor$
for odd $k$ and $s=1$, preserves the spatial size. For an input of height
$H$, kernel size $k$, padding $p$ and stride $s$, the output height is
\begin{equation}\label{eq:cnn-outsize}
  H_{\mathrm{out}}
  \;=\; \left\lfloor \frac{H + 2p - k}{s} \right\rfloor + 1,
\end{equation}
and symmetrically for the width. (With \emph{dilation} $d$, which inserts
$d-1$ gaps between kernel taps, $k$ in \eqref{eq:cnn-outsize} is replaced
by the effective size $k_{\mathrm{eff}} = d(k-1)+1$; dilation enlarges
receptive fields without adding parameters.)

Figure~\ref{fig:cnn-grid} traces \eqref{eq:cnn-convlayer} on a concrete
example: a $5 \times 5$ input, a $3 \times 3$ vertical-edge kernel, stride
$s=1$ and padding $p=0$, so that \eqref{eq:cnn-outsize} gives
$H_{\mathrm{out}} = \lfloor (5 + 0 - 3)/1 \rfloor + 1 = 3$.

\begin{figure}[t]
  \centering
  \begin{tikzpicture}
    % -------- input grid 5x5 (6 mm cells), top-left corner at (0,0)
    \fill[cinput!30] (0,0) rectangle (1.8,-1.8);
    \draw[black!60] (0,-3) grid[step=6mm] (3,0);
    \draw[cinput!80!black, very thick] (0,0) rectangle (1.8,-1.8);
    \foreach \v [count=\k from 0] in
        {1,2,0,1,3, 0,1,2,3,1, 1,0,2,2,0, 2,1,0,1,2, 0,2,1,0,1}{
      \pgfmathtruncatemacro{\r}{int(\k/5)}
      \pgfmathtruncatemacro{\c}{mod(\k,5)}
      \node[font=\small] at (\c*0.6+0.3, -\r*0.6-0.3) {$\v$};
    }
    \node[etiquette, font=\small] at (1.5,-3.45) {input $5\times5$};
    % -------- operator
    \node[font=\large] at (3.9,-1.5) {$\ast$};
    % -------- kernel 3x3, vertically centred on the input
    \begin{scope}[shift={(4.8,-0.6)}]
      \fill[cgate!15] (0,0) rectangle (1.8,-1.8);
      \draw[black!60] (0,-1.8) grid[step=6mm] (1.8,0);
      \foreach \v [count=\k from 0] in {1,0,-1, 1,0,-1, 1,0,-1}{
        \pgfmathtruncatemacro{\r}{int(\k/3)}
        \pgfmathtruncatemacro{\c}{mod(\k,3)}
        \node[font=\small] at (\c*0.6+0.3, -\r*0.6-0.3) {$\v$};
      }
      \node[etiquette, font=\small] at (0.9,-2.25) {kernel $3\times3$};
    \end{scope}
    % -------- operator
    \node[font=\large] at (7.2,-1.5) {$=$};
    % -------- output grid 3x3
    \begin{scope}[shift={(7.8,-0.6)}]
      \fill[coutput!30] (0,0) rectangle (0.6,-0.6);
      \draw[black!60] (0,-1.8) grid[step=6mm] (1.8,0);
      \draw[coutput!80!black, very thick] (0,0) rectangle (0.6,-0.6);
      \foreach \v [count=\k from 0] in {-2,-3,0, -1,-4,1, 0,0,0}{
        \pgfmathtruncatemacro{\r}{int(\k/3)}
        \pgfmathtruncatemacro{\c}{mod(\k,3)}
        \node[font=\small] at (\c*0.6+0.3, -\r*0.6-0.3) {$\v$};
      }
      \node[etiquette, font=\small] at (0.9,-2.25) {output $3\times3$};
    \end{scope}
    % -------- window -> output cell
    \draw[flechep, cinput!80!black] (0.9,0.12) to[bend left=18] (8.1,-0.5);
  \end{tikzpicture}
  \caption{Convolution (cross-correlation) of a $5\times5$ input with a
    $3\times3$ vertical-edge kernel, stride $s=1$, padding $p=0$. The
    highlighted window produces the highlighted output cell:
    $1\cdot1 + 2\cdot0 + 0\cdot(-1) + 0\cdot1 + 1\cdot0 + 2\cdot(-1)
     + 1\cdot1 + 0\cdot0 + 2\cdot(-1) = -2$. Sliding the window over all
    nine valid positions fills the $3\times3$ output map, in accordance
    with \eqref{eq:cnn-outsize}.}
  \label{fig:cnn-grid}
\end{figure}

\subsection{Parameter Efficiency}

The number of parameters of the layer \eqref{eq:cnn-convlayer} is
\begin{equation}\label{eq:cnn-params}
  \#\text{params}
  \;=\; C_{\mathrm{out}} \left( C_{\mathrm{in}}\, k_h\, k_w + 1 \right),
\end{equation}
\emph{independent of the spatial resolution} $H \times W$ — the decisive
advantage over a dense layer, whose parameter count grows with the square
of the image size. A $3\times3$ convolution mapping $64$ channels to $64$
channels costs $36{,}928$ parameters whether the image is $32\times32$ or
$1024\times1024$.

% ----------------------------------------------------------------------------
\section{Pooling}

Pooling summarizes each local neighbourhood of a feature map by a single
value, providing a small amount of translation \emph{invariance} (on top of
the equivariance of convolution) and reducing the spatial resolution. Let
$\mathcal{R}_{ij}$ denote the $k \times k$ window of input positions
associated with output position $(i,j)$ (with stride $s$, typically
$k=s=2$). \emph{Max pooling} keeps the strongest activation,
\begin{equation}\label{eq:cnn-maxpool}
  y_c(i,j) \;=\; \max_{(m,n)\,\in\,\mathcal{R}_{ij}} x_c(m,n),
\end{equation}
whereas \emph{average pooling} keeps the mean response,
\begin{equation}\label{eq:cnn-avgpool}
  y_c(i,j) \;=\; \frac{1}{\lvert \mathcal{R}_{ij} \rvert}
                 \sum_{(m,n)\,\in\,\mathcal{R}_{ij}} x_c(m,n).
\end{equation}
The limiting case of \eqref{eq:cnn-avgpool}, \emph{global average pooling},
collapses each channel's entire $H \times W$ map to one scalar,
\begin{equation}\label{eq:cnn-gap}
  y_c \;=\; \frac{1}{HW} \sum_{i=1}^{H} \sum_{j=1}^{W} x_c(i,j),
\end{equation}
and is the modern replacement for the large dense layers that dominated
the parameter budget of early CNNs: it contributes no parameters, acts as
a structural regularizer, and lets the same network accept inputs of any
size.

% ----------------------------------------------------------------------------
\section{The Canonical Convolutional Architecture}

Assembling the two operations yields the canonical architecture of
Figure~\ref{fig:cnn-archi}: a \emph{feature-extraction} stage alternating
convolution (+ nonlinearity) and pooling, in which the spatial resolution
decreases while the channel count — the richness of the learned
vocabulary — increases; then a \emph{classification} stage, in which the
final maps are flattened (or globally pooled) into a vector, passed
through dense layers, and mapped by a softmax to class probabilities. This
is exactly the structure of LeNet-5~\cite{lecun1998}, which stacked two
conv--pool pairs and two dense layers ($\sim$60k parameters) and
established that such a pipeline can be trained end to end by gradient
descent on raw pixels.

\begin{figure}[t]
  \centering
  \begin{tikzpicture}[node distance=0.55cm]
    \node[blocinput,  minimum width=1.05cm, minimum height=2.6cm,
          font=\scriptsize] (in) {input\\image};
    \node[blochidden, minimum width=1.05cm, minimum height=2.6cm,
          font=\scriptsize, right=of in]  (c1) {conv\\ReLU};
    \node[bloc,       minimum width=0.95cm, minimum height=2.0cm,
          font=\scriptsize, right=of c1]  (p1) {pool};
    \node[blochidden, minimum width=1.05cm, minimum height=2.0cm,
          font=\scriptsize, right=of p1]  (c2) {conv\\ReLU};
    \node[bloc,       minimum width=0.95cm, minimum height=1.4cm,
          font=\scriptsize, right=of c2]  (p2) {pool};
    \node[bloc,       minimum width=1.0cm,  minimum height=2.2cm,
          font=\scriptsize, right=of p2]  (fl) {flatten};
    \node[blochidden, minimum width=1.0cm,  minimum height=1.7cm,
          font=\scriptsize, right=of fl]  (fc) {dense\\ReLU};
    \node[blocoutput, minimum width=1.1cm,  minimum height=1.2cm,
          font=\scriptsize, right=of fc]  (sm) {softmax};
    \foreach \a/\b in {in/c1, c1/p1, p1/c2, c2/p2, p2/fl, fl/fc, fc/sm}
      \draw[fleche] (\a) -- (\b);
    % grouping braces
    \draw[decorate, decoration={brace, mirror, amplitude=5pt}]
      (c1.west |- 0,-1.55) -- (p2.east |- 0,-1.55)
      node[midway, below=7pt, etiquette, font=\small]{feature extraction};
    \draw[decorate, decoration={brace, mirror, amplitude=5pt}]
      (fl.west |- 0,-1.55) -- (sm.east |- 0,-1.55)
      node[midway, below=7pt, etiquette, font=\small]{classification};
  \end{tikzpicture}
  \caption{The canonical convolutional architecture in the lineage of
    LeNet-5~\cite{lecun1998}. In the feature-extraction stage the spatial
    resolution shrinks (decreasing block heights) while the channel count
    grows; the classification stage flattens the final maps and applies
    dense layers followed by a softmax.}
  \label{fig:cnn-archi}
\end{figure}

The historical turning point came when this recipe met large datasets and
GPU computing: AlexNet's 2012 ImageNet victory — five convolutional and
three dense layers, ReLU activations and dropout — cut the top-5 error
from 26.2\% to 15.3\% and opened the modern era of deep
learning~\cite{goodfellow2016book}.

% ----------------------------------------------------------------------------
\section{Residual Learning}

Depth is the currency of representational power, yet naively stacking
layers eventually makes even the \emph{training} error worse — the
\emph{degradation problem}. A network of $50$ plain layers underperforms
its $20$-layer counterpart not because of overfitting but because the
optimizer fails to find a solution as good as ``the shallow network plus
identity layers'', although one exists by construction. Residual
learning~\cite{he2016} removes this obstacle by making the identity the
default behaviour of every block: instead of asking a block to learn a
mapping $\mathcal{H}(\vect{x})$ directly, one asks it to learn only the
\emph{residual} $\mathcal{F}(\vect{x}) = \mathcal{H}(\vect{x}) - \vect{x}$
and adds the input back through a shortcut connection,
\begin{equation}\label{eq:cnn-residual}
  \vect{y} \;=\; \mathcal{F}(\vect{x}, \{\mat{W}_i\}) + \vect{x},
\end{equation}
where typically $\mathcal{F}$ consists of two convolution--batch-norm
stages with a ReLU in between (Figure~\ref{fig:cnn-resblock}), and a
learned projection $\mat{W}_s \vect{x}$ replaces the identity when
dimensions change. If the optimal mapping is close to the identity, the
block merely needs to drive $\mathcal{F}$ towards zero — far easier than
reproducing the identity through a stack of nonlinear layers.

\begin{figure}[t]
  \centering
  \begin{tikzpicture}
    \node (xl) at (0,0) {$\vect{x}$};
    \node[blochidden, right=0.95cm of xl, minimum width=2.3cm]
      (f1) {conv $3{\times}3$\\BN, ReLU};
    \node[blochidden, right=0.7cm of f1, minimum width=2.3cm]
      (f2) {conv $3{\times}3$\\BN};
    \node[op, right=0.95cm of f2] (plus) {$+$};
    \node[bloc, right=0.7cm of plus, minimum width=1.1cm] (relu) {ReLU};
    \node[right=0.7cm of relu] (y) {$\vect{y}$};
    \draw[fleche] (xl) -- (f1);
    \draw[fleche] (f1) -- (f2);
    \draw[fleche] (f2) -- (plus);
    \draw[fleche] (plus) -- (relu);
    \draw[fleche] (relu) -- (y);
    % identity shortcut
    \coordinate (tap) at ($(xl.east)+(0.45,0)$);
    \fill (tap) circle (1.3pt);
    \draw[fleche, cmem!80!black] (tap) -- ++(0,1.35) -| (plus.north);
    \node[etiquette, font=\small, text=cmem!80!black]
      at ($(f1.north)!0.5!(f2.north)+(0,1.15)$) {identity shortcut $\vect{x}$};
    \node[etiquette, font=\small]
      at ($(f1.south)!0.5!(f2.south)+(0,-0.35)$)
      {residual branch $\mathcal{F}(\vect{x})$};
  \end{tikzpicture}
  \caption{The residual block of ResNet~\cite{he2016}. The main path
    computes the residual $\mathcal{F}(\vect{x})$ through two
    convolution--batch-norm stages; the shortcut carries $\vect{x}$
    unchanged to the addition node, implementing
    \eqref{eq:cnn-residual}.}
  \label{fig:cnn-resblock}
\end{figure}

\begin{property}[Gradient flow through the identity
  shortcut]\label{prop:cnn-resgrad}
Consider a stack of residual blocks
$\vect{x}_{\ell+1} = \vect{x}_\ell + \mathcal{F}(\vect{x}_\ell)$. Unrolling
from layer $\ell$ to any deeper layer $L$ gives
\begin{equation}\label{eq:cnn-unroll}
  \vect{x}_L \;=\; \vect{x}_\ell
    \;+\; \sum_{i=\ell}^{L-1} \mathcal{F}(\vect{x}_i),
\end{equation}
and the chain rule then yields
\begin{equation}\label{eq:cnn-resgrad}
  \frac{\partial \Loss}{\partial \vect{x}_\ell}
  \;=\;
  \frac{\partial \Loss}{\partial \vect{x}_L}
  \left( \mat{I} \;+\;
         \frac{\partial}{\partial \vect{x}_\ell}
         \sum_{i=\ell}^{L-1} \mathcal{F}(\vect{x}_i) \right).
\end{equation}
The additive identity term $\mat{I}$ in \eqref{eq:cnn-resgrad} provides a
direct gradient path from any layer to any shallower layer: the gradient
cannot vanish even when the Jacobian of the residual branch is small,
because it is never forced through a long product of weight matrices.
\end{property}

This single architectural idea allowed networks of $152$ layers — eight
times deeper than the deepest previous mainstream design yet cheaper in
floating-point operations — to win the 2015 ImageNet competition with a
3.57\% top-5 ensemble error~\cite{he2016}; the same shortcut structure
reappears in virtually every subsequent deep architecture, including the
Transformer.

% ----------------------------------------------------------------------------
\section{The Receptive Field}

The \emph{receptive field} of a unit is the region of the input image that
can influence its value. For a stack of layers where layer $\ell$ has
kernel size $k_\ell$ and stride $s_\ell$, the receptive field $r_\ell$
grows according to the recursion
\begin{equation}\label{eq:cnn-receptive}
  r_\ell \;=\; r_{\ell-1} \;+\; (k_\ell - 1) \prod_{i=1}^{\ell-1} s_i,
  \qquad r_0 = 1,
\end{equation}
where the product accounts for the cumulative downsampling in front of
layer $\ell$. Two consequences guide architecture design. First, small
kernels compose efficiently: two stacked $3\times3$ convolutions reach the
same $5\times5$ receptive field as one $5\times5$ kernel with fewer
parameters ($18C^2$ versus $25C^2$) and one extra nonlinearity — the
observation that underlies VGG's uniform $3\times3$ design. Second,
strides and pooling are multiplicative in \eqref{eq:cnn-receptive}: early
downsampling is the cheapest way to grow the receptive field, which is why
deep units eventually see the whole image and can encode global,
object-level structure.

% ----------------------------------------------------------------------------
\section{A Brief Genealogy of Architectures}

Table~\ref{tab:cnn-genealogy} summarizes the milestones that separate
LeNet-5 from today's networks. Beyond ResNet, three ideas deserve mention.
\emph{Multi-branch design} (GoogLeNet's Inception module) processes each
position at several kernel sizes in parallel and concatenates the results,
using $1\times1$ ``bottleneck'' convolutions to keep the cost low —
22 layers with only $\sim$7M parameters. \emph{Dense connectivity}
(DenseNet) generalizes the shortcut of \eqref{eq:cnn-residual} by
concatenating, rather than adding, the features of \emph{all} preceding
layers, maximizing feature reuse and gradient flow. \emph{Factorized
convolution} (MobileNet) splits a standard convolution into a per-channel
(depthwise) spatial filter followed by a $1\times1$ (pointwise) channel
mixer; on a feature map with $M$ input channels, $N$ output channels and a
$D_K \times D_K$ kernel, the cost ratio relative to the standard layer is
\begin{equation}\label{eq:cnn-separable}
  \frac{\text{separable}}{\text{standard}}
  \;=\; \frac{1}{N} + \frac{1}{D_K^2},
\end{equation}
about an $8$--$9\times$ saving for $3\times3$ kernels — the enabling
arithmetic of mobile and embedded vision.

\begin{table}[t]
  \centering
  \caption{Milestones of convolutional architecture design.}
  \label{tab:cnn-genealogy}
  \begin{tabular}{@{}llll@{}}
    \toprule
    Year & Architecture & Key idea & Scale \\
    \midrule
    1980 & Neocognitron & S/C-cell hierarchy, weight sharing & --- \\
    1998 & LeNet-5~\cite{lecun1998} & end-to-end conv--pool--dense & 60k \\
    2012 & AlexNet & ReLU, dropout, GPU training & 60M \\
    2014 & VGG & uniform $3\times3$ depth & 138M \\
    2014 & GoogLeNet & Inception multi-branch, $1\times1$ bottlenecks & 7M \\
    2015 & ResNet~\cite{he2016} & identity shortcut \eqref{eq:cnn-residual} & 25M \\
    2017 & DenseNet & concatenated dense connectivity & 8M \\
    2017 & MobileNet & depthwise separable conv.\ \eqref{eq:cnn-separable} & 4M \\
    2019 & EfficientNet & compound depth/width/resolution scaling & 5--66M \\
    2022 & ConvNeXt & modernized ResNet, $7\times7$ depthwise & 29M+ \\
    \bottomrule
  \end{tabular}
\end{table}

The through-line of this genealogy is that every leap either improved
\emph{gradient flow} (ReLU, batch normalization, the shortcut of
\eqref{eq:cnn-resgrad}) or improved the \emph{allocation of computation}
(bottlenecks, separable filters, compound scaling). The convolutional
prior itself — locality, weight sharing, hierarchy — has remained intact
from the Neocognitron to ConvNeXt, and it transfers beyond images: 1D
convolutions over sequences and 3D convolutions over videos and volumes
are the direct analogues of \eqref{eq:cnn-convlayer} with one fewer or one
more spatial index~\cite{goodfellow2016book}.