% ============================================================================
%  Artificial Neural Networks — Methods, Equations and Graphical
%  Representations
%  Author  : Simon-Pierre Boucher — contact@spboucher.ai
%  Chapter 7 : Generative Models (chapters/07-generative.tex)
% ============================================================================
\chapter{Generative Models}\label{chap:generative}

The architectures of the preceding chapters were, for the most part,
\emph{discriminative}: they learn a mapping from an input $\vect{x}$ to a
target $\vect{y}$. Generative models pursue a more ambitious goal — to learn
a representation of the data distribution $p_{\mathrm{data}}(\vect{x})$
itself, so that new samples can be drawn from it. The major families differ
in \emph{how} they represent this density: approximately, through a
variational lower bound (variational autoencoders); implicitly, through a
sampling procedure trained adversarially (generative adversarial networks);
or exactly, through the chain-rule factorization (autoregressive models).
This chapter presents each family with its objective function, its
gradients, and its computational structure, and closes with the iterative
denoising paradigm — diffusion models — that dominates image synthesis
today.

\begin{definition}[Generative model]\label{def:gen-model}
A generative model is a parametric family $p_\theta(\vect{x})$ together
with a sampling procedure, trained so that $p_\theta \approx
p_{\mathrm{data}}$, where $p_{\mathrm{data}}$ is the (unknown) distribution
from which the training set $\{\vect{x}_1,\dots,\vect{x}_N\}$ was drawn.
\end{definition}

% ============================================================================
\section{Autoencoders}\label{sec:gen-ae}

The autoencoder is the conceptual ancestor of the modern deep generative
model. It consists of an \emph{encoder} $f_{\mathrm{enc}} \colon \R^{d}
\to \R^{m}$, which maps an input to a low-dimensional \emph{latent code},
and a \emph{decoder} $f_{\mathrm{dec}} \colon \R^{m} \to \R^{d}$, which
attempts to reconstruct the input from that code:
\begin{equation}
  \vect{z} = f_{\mathrm{enc}}(\vect{x}),
  \qquad
  \hat{\vect{x}} = f_{\mathrm{dec}}(\vect{z}),
  \qquad m \ll d .
  \label{eq:gen-ae}
\end{equation}
Both maps are neural networks — typically mirror-image MLPs or
convolutional networks — trained jointly to minimize the
\emph{reconstruction loss}
\begin{equation}
  \Loss_{\mathrm{AE}}
  = \frac{1}{N}\sum_{i=1}^{N}
    \bigl\lVert \vect{x}_i -
    f_{\mathrm{dec}}\bigl(f_{\mathrm{enc}}(\vect{x}_i)\bigr)
    \bigr\rVert_2^2 ,
  \label{eq:gen-ae-loss}
\end{equation}
or a cross-entropy loss when the inputs are modeled as Bernoulli variables
(e.g.\ binarized pixels). Because the bottleneck dimension $m$ in
\eqref{eq:gen-ae} is much smaller than the input dimension $d$, the network
cannot learn the identity map; it is forced to discover a compressed
representation that preserves exactly the information needed to reconstruct
the data. Figure~\ref{fig:gen-ae} shows the characteristic hourglass shape.

\begin{figure}[ht]
  \centering
  \begin{tikzpicture}[scale=0.92, transform shape]
    % input layer: 6 neurons
    \foreach \i in {1,...,6}
      \node[ninput]  (x\i)  at (0,   3.0-\i)  {};
    % encoder hidden: 4 neurons
    \foreach \i in {1,...,4}
      \node[nhidden] (e\i)  at (2.0, 2.0-\i)  {};
    % latent: 2 neurons (memory color)
    \foreach \i in {1,2}
      \node[neuron, fill=cmem!25, draw=cmem!70!black]
                     (z\i)  at (4.0, 1.0-\i)  {};
    % decoder hidden: 4 neurons
    \foreach \i in {1,...,4}
      \node[nhidden] (d\i)  at (6.0, 2.0-\i)  {};
    % output layer: 6 neurons
    \foreach \i in {1,...,6}
      \node[noutput] (y\i)  at (8.0, 3.0-\i)  {};
    % connections
    \foreach \i in {1,...,6} \foreach \j in {1,...,4}
      \draw[black!30, semithick] (x\i) -- (e\j);
    \foreach \i in {1,...,4} \foreach \j in {1,2}
      \draw[black!30, semithick] (e\i) -- (z\j);
    \foreach \i in {1,2} \foreach \j in {1,...,4}
      \draw[black!30, semithick] (z\i) -- (d\j);
    \foreach \i in {1,...,4} \foreach \j in {1,...,6}
      \draw[black!30, semithick] (d\i) -- (y\j);
    % column labels
    \node[etiquette] at (0, 2.6)  {$\vect{x}$};
    \node[etiquette] at (4.0, 0.6) {$\vect{z}$};
    \node[etiquette] at (8.0, 2.6) {$\hat{\vect{x}}$};
    % braces
    \draw[decorate, decoration={brace, mirror, amplitude=5pt}, thick]
      (-0.4,-3.6) -- (3.4,-3.6)
      node[midway, below=7pt, font=\small]
      {encoder $f_{\mathrm{enc}}$};
    \draw[decorate, decoration={brace, mirror, amplitude=5pt}, thick]
      (4.6,-3.6) -- (8.4,-3.6)
      node[midway, below=7pt, font=\small]
      {decoder $f_{\mathrm{dec}}$};
    \node[font=\small] at (4.0, -2.1) {latent code};
  \end{tikzpicture}
  \caption{The autoencoder: an hourglass architecture. The encoder
    compresses the input $\vect{x}$ into a latent code $\vect{z}$ (violet)
    of much smaller dimension; the decoder reconstructs
    $\hat{\vect{x}}$ from $\vect{z}$. Training minimizes the reconstruction
    loss~\eqref{eq:gen-ae-loss}.}
  \label{fig:gen-ae}
\end{figure}

\begin{remark}\label{rem:gen-ae-not-generative}
A plain autoencoder is \emph{not} yet a generative model in the sense of
Definition~\ref{def:gen-model}: nothing constrains the geometry of the
latent space, so decoding an arbitrary $\vect{z}$ generally does not
produce a realistic sample. Imposing a probabilistic structure on the
latent space is precisely the contribution of the variational autoencoder.
\end{remark}

% ============================================================================
\section{Variational Autoencoders}\label{sec:gen-vae}

The variational autoencoder (VAE) of Kingma and
Welling~\cite{kingma2014vae} turns the autoencoder into a proper
latent-variable model. It posits a prior $p(\vect{z}) =
\mathcal{N}(\vect{0}, \mat{I})$ and a decoder that defines a conditional
likelihood $p_\theta(\vect{x} \mid \vect{z})$, so that the model density is
\begin{equation}
  p_\theta(\vect{x})
  = \int p_\theta(\vect{x} \mid \vect{z})\, p(\vect{z})\, d\vect{z} .
  \label{eq:gen-lvm}
\end{equation}
The integral in \eqref{eq:gen-lvm} is intractable, so the VAE introduces an
\emph{approximate posterior} $q_\phi(\vect{z} \mid \vect{x}) =
\mathcal{N}\bigl(\vect{\mu}_\phi(\vect{x}),
\operatorname{diag}(\vect{\sigma}_\phi^2(\vect{x}))\bigr)$ — the
probabilistic encoder — and maximizes a lower bound on the log-likelihood.

\subsection{The evidence lower bound}

For any $q_\phi$, the log-likelihood decomposes exactly as
\begin{equation}
  \log p_\theta(\vect{x})
  = \Loss_{\mathrm{ELBO}}(\theta, \phi; \vect{x})
  + \KL\bigl(q_\phi(\vect{z} \mid \vect{x}) \,\big\Vert\,
             p_\theta(\vect{z} \mid \vect{x})\bigr) ,
  \label{eq:gen-elbo-decomp}
\end{equation}
and since the Kullback--Leibler divergence is non-negative, the first term
— the \emph{evidence lower bound} (ELBO) — bounds $\log p_\theta(\vect{x})$
from below:
\begin{equation}
  \Loss_{\mathrm{ELBO}}(\theta, \phi; \vect{x})
  = \underbrace{\E_{q_\phi(\vect{z} \mid \vect{x})}
      \bigl[\log p_\theta(\vect{x} \mid \vect{z})\bigr]}_{\text{reconstruction}}
  \;-\;
  \underbrace{\KL\bigl(q_\phi(\vect{z} \mid \vect{x})
      \,\big\Vert\, p(\vect{z})\bigr)}_{\text{regularization}} .
  \label{eq:gen-elbo}
\end{equation}
The two terms of \eqref{eq:gen-elbo} recover, and generalize, the two
ingredients of the plain autoencoder: the first is a (probabilistic)
reconstruction objective, while the second pulls the encoder's output
distribution toward the prior, organizing the latent space so that samples
$\vect{z} \sim \mathcal{N}(\vect{0},\mat{I})$ decode to realistic data.
By \eqref{eq:gen-elbo-decomp}, maximizing the ELBO simultaneously raises
the likelihood and tightens the posterior approximation.

For diagonal Gaussians the regularization term of \eqref{eq:gen-elbo} has
a closed form. With $q_\phi = \mathcal{N}(\vect{\mu},
\operatorname{diag}(\vect{\sigma}^2))$ and $p = \mathcal{N}(\vect{0},
\mat{I})$ in $m$ dimensions,
\begin{equation}
  \KL\bigl(q_\phi(\vect{z} \mid \vect{x}) \,\big\Vert\,
           \mathcal{N}(\vect{0},\mat{I})\bigr)
  = -\frac{1}{2}\sum_{j=1}^{m}
    \bigl(1 + \log \sigma_j^2 - \mu_j^2 - \sigma_j^2\bigr) ,
  \label{eq:gen-kl}
\end{equation}
so this term costs nothing to estimate: it is computed analytically from
the encoder outputs.

\subsection{The reparameterization trick}

One obstacle remains: the reconstruction term of \eqref{eq:gen-elbo} is an
expectation over $\vect{z} \sim q_\phi(\vect{z} \mid \vect{x})$, and the
sampling operation blocks the gradient with respect to $\phi$. The
\emph{reparameterization trick}~\cite{kingma2014vae} rewrites the sample as
a deterministic, differentiable function of the parameters plus exogenous
noise:
\begin{equation}
  \vect{z}
  = \vect{\mu}_\phi(\vect{x})
  + \vect{\sigma}_\phi(\vect{x}) \odot \vect{\varepsilon},
  \qquad
  \vect{\varepsilon} \sim \mathcal{N}(\vect{0}, \mat{I}) .
  \label{eq:gen-reparam}
\end{equation}
The randomness now enters only through $\vect{\varepsilon}$, which does not
depend on $\phi$; gradients flow through $\vect{\mu}_\phi$ and
$\vect{\sigma}_\phi$ by the ordinary chain rule, yielding an unbiased,
low-variance pathwise estimator of $\nabla_\phi \Loss_{\mathrm{ELBO}}$.
Figure~\ref{fig:gen-vae} traces the full computation.

\begin{figure}[ht]
  \centering
  \begin{tikzpicture}[scale=0.88, transform shape]
    \node[blocinput, minimum width=1cm]  (x)   at (0,0)      {$\vect{x}$};
    \node[blochidden, minimum width=1.9cm, minimum height=1.9cm]
                                         (enc) at (2.3,0)
      {Encoder\\ $q_\phi(\vect{z}\mid\vect{x})$};
    \node[bloc, minimum width=1.15cm]    (mu)  at (4.9, 1.0) {$\vect{\mu}$};
    \node[bloc, minimum width=1.15cm]    (sg)  at (4.9,-1.0) {$\log\vect{\sigma}^2$};
    \node[bloc, minimum width=1.6cm, densely dashed]
                                         (eps) at (4.9,-2.7)
      {$\vect{\varepsilon}\sim\mathcal{N}(\vect{0},\mat{I})$};
    \node[op] (odot) at (7.0,-1.0) {$\odot$};
    \node[op] (plus) at (8.1, 0)   {$+$};
    \node[mem, minimum width=1cm]        (z)   at (9.4,0)    {$\vect{z}$};
    \node[blochidden, minimum width=1.9cm, minimum height=1.9cm]
                                         (dec) at (11.7,0)
      {Decoder\\ $p_\theta(\vect{x}\mid\vect{z})$};
    \node[blocoutput, minimum width=1cm] (xh)  at (14.0,0)   {$\hat{\vect{x}}$};
    % arrows
    \draw[fleche] (x)   -- (enc);
    \draw[fleche] ([yshift=5.5mm]enc.east) -- ++(0.35,0) |- (mu.west);
    \draw[fleche] ([yshift=-5.5mm]enc.east) -- ++(0.35,0) |- (sg.west);
    \draw[fleche] (sg)  -- node[etiquette, above]{$\exp(\cdot/2)$} (odot);
    \draw[fleche] (eps.east) -| (odot);
    \draw[fleche] (odot) -- (plus);
    \draw[fleche] (mu.east) -| (plus);
    \draw[fleche] (plus) -- (z);
    \draw[fleche] (z)   -- (dec);
    \draw[fleche] (dec) -- (xh);
  \end{tikzpicture}
  \caption{The variational autoencoder with the reparameterization
    trick~\eqref{eq:gen-reparam}. The encoder outputs the parameters
    $(\vect{\mu}, \log\vect{\sigma}^2)$ of the approximate posterior; the
    latent sample is assembled as $\vect{z} = \vect{\mu} + \vect{\sigma}
    \odot \vect{\varepsilon}$, so that gradients flow through
    $\vect{\mu}$ and $\vect{\sigma}$ while the stochasticity is confined
    to the exogenous noise $\vect{\varepsilon}$ (dashed).}
  \label{fig:gen-vae}
\end{figure}

With the reparameterization in place, a VAE training step is ordinary
stochastic gradient ascent on the ELBO, summarized in
Algorithm~\ref{alg:gen-vae}: encode, sample through
\eqref{eq:gen-reparam}, decode, and differentiate the two terms of
\eqref{eq:gen-elbo} jointly with respect to both parameter sets.

\begin{algorithm}[htbp]
  \caption{VAE training step (stochastic gradient ascent on the ELBO)}
  \label{alg:gen-vae}
  \begin{algorithmic}[1]
    \Require minibatch $\{\vect{x}^{(i)}\}_{i=1}^{m}$, encoder parameters
      $\phi$, decoder parameters $\theta$, learning rate $\eta$
    \For{$i = 1, \dots, m$}
      \State $\bigl(\vect{\mu}^{(i)}, \log\vect{\sigma}^{2(i)}\bigr)
        \gets \mathrm{Encoder}_\phi\!\left(\vect{x}^{(i)}\right)$
      \State sample $\vect{\varepsilon}^{(i)} \sim
        \mathcal{N}(\vect{0}, \mat{I})$;\quad
        $\vect{z}^{(i)} \gets \vect{\mu}^{(i)}
        + \vect{\sigma}^{(i)} \odot \vect{\varepsilon}^{(i)}$
        \Comment{reparameterization \eqref{eq:gen-reparam}}
      \State $\hat{\vect{x}}^{(i)} \gets
        \mathrm{Decoder}_\theta\!\left(\vect{z}^{(i)}\right)$
      \State $\Loss^{(i)} \gets
        \log p_\theta\!\left(\vect{x}^{(i)} \mid \vect{z}^{(i)}\right)
        - \KL\!\left( q_\phi(\vect{z} \mid \vect{x}^{(i)})
        \,\Vert\, p(\vect{z}) \right)$
        \Comment{ELBO \eqref{eq:gen-elbo}, KL \eqref{eq:gen-kl}}
    \EndFor
    \State $\Loss \gets \frac{1}{m}\sum_{i=1}^{m} \Loss^{(i)}$
    \State $\theta \gets \theta + \eta\, \nabla_\theta \Loss$;\quad
      $\phi \gets \phi + \eta\, \nabla_\phi \Loss$
      \Comment{ascent; gradients by backpropagation}
  \end{algorithmic}
\end{algorithm}

\begin{remark}\label{rem:gen-vae-blur}
Because the reconstruction term of \eqref{eq:gen-elbo} is typically a
Gaussian (squared-error) likelihood averaged over the posterior, VAE
samples tend to be slightly blurry: the decoder learns to output the
conditional \emph{mean} of all plausible reconstructions. This is the
characteristic failure mode recorded in
Table~\ref{tab:gen-comparison}.
\end{remark}

% ============================================================================
\section{Generative Adversarial Networks}\label{sec:gen-gan}

Generative adversarial networks (GANs), introduced by Goodfellow et
al.~\cite{goodfellow2014gan}, abandon explicit densities altogether. A
\emph{generator} $G$ maps noise $\vect{z} \sim p_{\vect{z}} =
\mathcal{N}(\vect{0},\mat{I})$ to a sample $G(\vect{z})$, and a
\emph{discriminator} $D(\vect{x}) \in [0,1]$ estimates the probability
that its input came from the data rather than from $G$. The two networks
play a two-player \emph{minimax game}:
\begin{equation}
  \min_G \max_D \; V(D, G)
  = \E_{\vect{x} \sim p_{\mathrm{data}}}\bigl[\log D(\vect{x})\bigr]
  + \E_{\vect{z} \sim p_{\vect{z}}}
      \bigl[\log\bigl(1 - D(G(\vect{z}))\bigr)\bigr] .
  \label{eq:gen-minimax}
\end{equation}
The discriminator is trained to tell real from fake; the generator is
trained to fool it. Figure~\ref{fig:gen-gan} shows the adversarial
arrangement: crucially, the generator never sees the data directly — its
only training signal is the gradient that flows \emph{through} the
discriminator.

\begin{figure}[ht]
  \centering
  \begin{tikzpicture}[scale=0.92, transform shape]
    \node[bloc, minimum width=1.9cm] (z) at (0,1.0)
      {$\vect{z}\sim\mathcal{N}(\vect{0},\mat{I})$};
    \node[blochidden, minimum width=2.2cm, minimum height=1.1cm]
      (G) at (3.2,1.0) {Generator\\ $G$};
    \node[bloc, minimum width=2.3cm] (xf) at (6.5,1.0)
      {fake $\tilde{\vect{x}} = G(\vect{z})$};
    \node[blocinput, minimum width=2.3cm] (xr) at (6.5,-1.0)
      {real $\vect{x}\sim p_{\mathrm{data}}$};
    \node[blochidden, minimum width=2.4cm, minimum height=1.1cm]
      (D) at (10.0,0) {Discriminator\\ $D$};
    \node[blocoutput, minimum width=1.7cm] (out) at (13.2,0)
      {real / fake};
    \draw[fleche] (z)  -- (G);
    \draw[fleche] (G)  -- (xf);
    \draw[fleche] (xf.east) -- ++(0.55,0) |- ([yshift=3mm]D.west);
    \draw[fleche] (xr.east) -- ++(0.55,0) |- ([yshift=-3mm]D.west);
    \draw[fleche] (D)  -- (out);
    % adversarial gradient (dashed), routed above the fake-sample block
    \draw[flechep, draw=coutput!80!black]
      (D.north) -- ++(0,1.45) -| (G.north)
      node[pos=0.25, above, etiquette, text=coutput!80!black]
      {adversarial gradient $\nabla_{\theta_G} \Loss$};
  \end{tikzpicture}
  \caption{The generative adversarial network. The generator maps noise
    $\vect{z}$ to a fake sample $\tilde{\vect{x}}$; the discriminator
    receives both real and fake samples and outputs the probability that
    its input is real. The generator's only learning signal is the
    adversarial gradient (dashed, red) backpropagated through the
    discriminator, per the minimax objective~\eqref{eq:gen-minimax}.}
  \label{fig:gen-gan}
\end{figure}

\subsection{The optimal discriminator and the Jensen--Shannon divergence}

The minimax game \eqref{eq:gen-minimax} has a precise
distribution-matching interpretation.

\begin{theorem}[Optimal discriminator]\label{thm:gen-dstar}
Let $p_g$ denote the distribution of $G(\vect{z})$ for fixed $G$. The
discriminator maximizing $V(D,G)$ in \eqref{eq:gen-minimax} is
\begin{equation}
  D^{*}(\vect{x})
  = \frac{p_{\mathrm{data}}(\vect{x})}
         {p_{\mathrm{data}}(\vect{x}) + p_g(\vect{x})} ,
  \label{eq:gen-dstar}
\end{equation}
and substituting $D^{*}$ into $V$ gives the generator's effective
objective
\begin{equation}
  C(G) = \max_D V(D,G)
  = -\log 4
  + 2\,\mathrm{JSD}\bigl(p_{\mathrm{data}} \,\big\Vert\, p_g\bigr) ,
  \label{eq:gen-jsd}
\end{equation}
where $\mathrm{JSD}$ is the Jensen--Shannon divergence. Hence $C(G)$ is
minimized if and only if $p_g = p_{\mathrm{data}}$, where $D^{*} \equiv
\tfrac{1}{2}$ and $C(G) = -\log 4$.
\end{theorem}

\begin{proof}[Proof sketch]
For fixed $G$, $V(D,G) = \int \bigl[p_{\mathrm{data}}(\vect{x}) \log
D(\vect{x}) + p_g(\vect{x}) \log(1 - D(\vect{x}))\bigr] d\vect{x}$;
pointwise maximization of $a \log t + b \log(1-t)$ over $t \in (0,1)$
yields $t^{*} = a/(a+b)$, which is \eqref{eq:gen-dstar}. Substituting back
and completing each term to a KL divergence against the mixture
$(p_{\mathrm{data}} + p_g)/2$ gives \eqref{eq:gen-jsd};
see~\cite{goodfellow2014gan}.
\end{proof}

\subsection{Training in practice: non-saturating and Wasserstein losses}

Early in training, $D$ rejects fakes easily, $D(G(\vect{z})) \approx 0$,
and the generator's term $\log(1 - D(G(\vect{z})))$ in
\eqref{eq:gen-minimax} saturates — its gradient vanishes exactly when the
generator most needs guidance. The standard remedy, proposed already
in~\cite{goodfellow2014gan}, is the \emph{non-saturating} generator loss:
instead of minimizing $\E[\log(1 - D(G(\vect{z})))]$, the generator
maximizes
\begin{equation}
  \Loss_G^{\mathrm{NS}}
  = \E_{\vect{z} \sim p_{\vect{z}}}\bigl[\log D(G(\vect{z}))\bigr] ,
  \label{eq:gen-nonsat}
\end{equation}
which has the same fixed points but provides strong gradients precisely
when the discriminator is confident. The resulting estimation procedure,
Algorithm~\ref{alg:gen-gan}, alternates $k$ ascent steps on the
discriminator's objective with one non-saturating update of the generator.

\begin{algorithm}[htbp]
  \caption{GAN alternating training with the non-saturating generator
    loss}
  \label{alg:gen-gan}
  \begin{algorithmic}[1]
    \Require generator $G$ (parameters $\theta_g$), discriminator $D$
      (parameters $\theta_d$), discriminator steps $k$, batch size $m$,
      learning rates $\eta_d, \eta_g$
    \While{not converged}
      \For{$j = 1, \dots, k$} \Comment{discriminator updates}
        \State sample $\{\vect{x}^{(1)}, \dots, \vect{x}^{(m)}\}$ from the
          data, $\{\vect{z}^{(1)}, \dots, \vect{z}^{(m)}\}$ from
          $p_{\vect{z}}$
        \State $\Loss_D \gets \frac{1}{m}\sum_{i=1}^{m}
          \Bigl[ \log D\!\left(\vect{x}^{(i)}\right)
          + \log\!\left(1 - D\!\left(G(\vect{z}^{(i)})\right)\right)
          \Bigr]$
          \Comment{value of \eqref{eq:gen-minimax}}
        \State $\theta_d \gets \theta_d + \eta_d\,
          \nabla_{\theta_d} \Loss_D$
          \Comment{gradient \emph{ascent}}
      \EndFor
      \State sample $\{\vect{z}^{(1)}, \dots, \vect{z}^{(m)}\}$ from
        $p_{\vect{z}}$
      \State $\Loss_G^{\mathrm{NS}} \gets \frac{1}{m}\sum_{i=1}^{m}
        \log D\!\left(G(\vect{z}^{(i)})\right)$
        \Comment{non-saturating loss \eqref{eq:gen-nonsat}}
      \State $\theta_g \gets \theta_g + \eta_g\,
        \nabla_{\theta_g} \Loss_G^{\mathrm{NS}}$
    \EndWhile
  \end{algorithmic}
\end{algorithm}

A deeper pathology is that the Jensen--Shannon divergence in
\eqref{eq:gen-jsd} is poorly behaved when $p_{\mathrm{data}}$ and $p_g$
have (nearly) disjoint supports — as is typical for high-dimensional data
concentrated on low-dimensional manifolds — contributing to unstable
dynamics and \emph{mode collapse}, in which $G$ maps many latent vectors
onto a few high-scoring outputs. The Wasserstein GAN (WGAN) replaces the
JSD with the Wasserstein-1 distance, which by Kantorovich--Rubinstein
duality is
\begin{equation}
  W\bigl(p_{\mathrm{data}}, p_g\bigr)
  = \sup_{\lVert f \rVert_{L} \le 1}\;
    \E_{\vect{x} \sim p_{\mathrm{data}}}\bigl[f(\vect{x})\bigr]
  - \E_{\vect{x} \sim p_g}\bigl[f(\vect{x})\bigr] ,
  \label{eq:gen-wgan}
\end{equation}
where the supremum ranges over 1-Lipschitz functions. A \emph{critic}
network (a discriminator without the final sigmoid) approximates the
supremum in \eqref{eq:gen-wgan}, with the Lipschitz constraint enforced by
weight clipping or, in later refinements, a gradient penalty. Because $W$
remains finite and provides usable gradients even for disjoint supports,
WGAN training is markedly more stable and less prone to mode collapse.

% ============================================================================
\section{Autoregressive Models and a Comparison of Families}
\label{sec:gen-ar}

A third route to generation requires neither latent variables nor an
adversary: factor the joint density exactly by the chain rule,
\begin{equation}
  p_\theta(\vect{x})
  = \prod_{i=1}^{d} p_\theta\bigl(x_i \mid x_1, \dots, x_{i-1}\bigr) ,
  \label{eq:gen-ar}
\end{equation}
and train a network to model each conditional by maximum likelihood — the
factorization already encountered for sequence models, now applied to
arbitrary data (pixels in raster order, audio samples, discrete tokens).
Training is stable and parallelizable, and the likelihood
\eqref{eq:gen-ar} is exact; the price is sampling, which is inherently
sequential — one dimension at a time. Table~\ref{tab:gen-comparison}
summarizes the trade-offs among the three families
(see also~\cite{goodfellow2016book}).

\begin{table}[ht]
  \centering
  \small
  \begin{tabular}{@{}l l l l l@{}}
    \toprule
    Family & Training objective & Sampling & Likelihood & Typical failure \\
    \midrule
    VAE & ELBO \eqref{eq:gen-elbo}, stable
        & one pass, fast & lower bound & blurry samples \\
    GAN & minimax \eqref{eq:gen-minimax}, unstable
        & one pass, fast & implicit (none) & mode collapse \\
    Autoregressive & exact MLE \eqref{eq:gen-ar}, stable
        & sequential, slow & exact & slow sampling \\
    \bottomrule
  \end{tabular}
  \caption{Comparison of the three classical generative families. Each
    optimizes a different surrogate of the same goal, $p_\theta \approx
    p_{\mathrm{data}}$, and each pays for its strengths with a
    characteristic weakness.}
  \label{tab:gen-comparison}
\end{table}

% ============================================================================
\section{Toward Diffusion Models}\label{sec:gen-diffusion}

The current state of the art in image, audio and video synthesis belongs
to a fourth family that turns generation into \emph{iterative denoising}.
A fixed forward process gradually destroys the data with Gaussian noise
over $T$ steps, according to a variance schedule $\beta_1, \dots,
\beta_T$:
\begin{equation}
  q(\vect{x}_t \mid \vect{x}_{t-1})
  = \mathcal{N}\bigl(\vect{x}_t;\;
      \sqrt{1 - \beta_t}\, \vect{x}_{t-1},\; \beta_t \mat{I}\bigr) .
  \label{eq:gen-diff-forward}
\end{equation}
Iterating \eqref{eq:gen-diff-forward} and writing $\alpha_t = 1 -
\beta_t$, $\bar{\alpha}_t = \prod_{s=1}^{t} \alpha_s$, the noisy state at
any timestep is available in closed form directly from the clean sample:
\begin{equation}
  q(\vect{x}_t \mid \vect{x}_0)
  = \mathcal{N}\bigl(\vect{x}_t;\;
      \sqrt{\bar{\alpha}_t}\, \vect{x}_0,\;
      (1 - \bar{\alpha}_t)\, \mat{I}\bigr)
  \;\Longleftrightarrow\;
  \vect{x}_t
  = \sqrt{\bar{\alpha}_t}\, \vect{x}_0
  + \sqrt{1 - \bar{\alpha}_t}\, \vect{\varepsilon},
  \quad \vect{\varepsilon} \sim \mathcal{N}(\vect{0}, \mat{I}) .
  \label{eq:gen-diff-closed}
\end{equation}
As $t \to T$, $\bar{\alpha}_t \to 0$ and the data dissolves into pure
noise. Generation runs the process in reverse: a learned Markov chain
$p_\theta(\vect{x}_{t-1} \mid \vect{x}_t)$ starts from
$\mathcal{N}(\vect{0},\mat{I})$ and removes the noise step by step, and a
variational argument in the spirit of \eqref{eq:gen-elbo} reduces its
training to a strikingly simple regression — predict the noise
$\vect{\varepsilon}$ injected in \eqref{eq:gen-diff-closed}:
\begin{equation}
  \Loss_{\mathrm{simple}}
  = \E_{t,\, \vect{x}_0,\, \vect{\varepsilon}}
    \Bigl[\bigl\lVert \vect{\varepsilon}
      - \vect{\varepsilon}_\theta(\vect{x}_t, t)
      \bigr\rVert_2^2\Bigr] .
  \label{eq:gen-diff-loss}
\end{equation}
Diffusion models thus combine the stable, likelihood-based training of the
VAE lineage with sample quality surpassing GANs, at the cost of an
iterative sampling procedure — a trade-off that a rapidly evolving
literature (accelerated samplers, latent-space diffusion, flow matching,
consistency models) continues to push toward few-step and even one-step
generation.