spb/artificial-neural-networks-book Public

1% ============================================================================2%  Artificial Neural Networks — Methods, Equations and3%  Graphical Representations4%  Author : Simon-Pierre Boucher — contact@spboucher.ai5%  Book format (parts, preface, chapters, bibliography)6%  File   : main.tex — preamble, cover, preface, abstract, TOC, bibliography7% ============================================================================8\documentclass[11pt,a4paper]{book}910% ----------------------------------------------------------------------------11% Encoding, language, typography12% ----------------------------------------------------------------------------13\usepackage[utf8]{inputenc}14\usepackage[T1]{fontenc}15\usepackage{lmodern}16\usepackage[english]{babel}17\usepackage{microtype}1819% ----------------------------------------------------------------------------20% Mathematics21% ----------------------------------------------------------------------------22\usepackage{amsmath,amssymb,amsthm}23\usepackage{mathtools}24\usepackage{bm}2526% ----------------------------------------------------------------------------27% Graphics: TikZ + pgfplots (no external images)28% ----------------------------------------------------------------------------29\usepackage{tikz}30\usetikzlibrary{positioning,arrows.meta,calc,shapes.geometric,fit,31                backgrounds,decorations.pathreplacing}32\usepackage{pgfplots}33\pgfplotsset{compat=1.17}3435% ----------------------------------------------------------------------------36% Algorithms37% ----------------------------------------------------------------------------38\usepackage{algorithm}39\usepackage{algpseudocode}4041% ----------------------------------------------------------------------------42% Page layout43% ----------------------------------------------------------------------------44\usepackage[a4paper,top=2.6cm,bottom=2.8cm,inner=2.8cm,outer=2.4cm,45            headheight=14.5pt]{geometry}46\usepackage{fancyhdr}47\usepackage{booktabs}48\usepackage{array}4950\pagestyle{fancy}51\fancyhf{}52\fancyhead[LE]{\small\itshape\nouppercase{\leftmark}}53\fancyhead[RO]{\small\itshape\nouppercase{\rightmark}}54\fancyfoot[C]{\small\thepage}55\renewcommand{\headrulewidth}{0.4pt}56\fancypagestyle{plain}{\fancyhf{}\fancyfoot[C]{\small\thepage}%57  \renewcommand{\headrulewidth}{0pt}}5859% ----------------------------------------------------------------------------60% Hyperlinks61% ----------------------------------------------------------------------------62\usepackage[colorlinks=true,63            linkcolor=blue!50!black,64            citecolor=green!40!black,65            urlcolor=violet!70!black,66            pdftitle={Artificial Neural Networks — Methods, Equations and67                      Graphical Representations},68            pdfauthor={Simon-Pierre Boucher}]{hyperref}6970% ----------------------------------------------------------------------------71% Named palette (used by ALL figures)72% ----------------------------------------------------------------------------73\definecolor{cinput}{RGB}{31,119,180}    % blue   — inputs74\definecolor{chidden}{RGB}{255,127,14}   % orange — hidden layers75\definecolor{coutput}{RGB}{214,39,40}    % red    — outputs76\definecolor{cgate}{RGB}{44,160,44}      % green  — gates77\definecolor{cmem}{RGB}{148,103,189}     % violet — memory / state7879% ----------------------------------------------------------------------------80% Global TikZ styles (reused across all chapters)81% ----------------------------------------------------------------------------82\tikzset{83  neuron/.style   = {circle, draw=black!70, minimum size=8mm,84                     inner sep=0pt, font=\small},85  ninput/.style   = {neuron, fill=cinput!25,  draw=cinput!80!black},86  nhidden/.style  = {neuron, fill=chidden!30, draw=chidden!80!black},87  noutput/.style  = {neuron, fill=coutput!25, draw=coutput!80!black},88  bloc/.style     = {rectangle, rounded corners=2pt, draw=black!70,89                     fill=black!5, minimum width=2.1cm, minimum height=8mm,90                     align=center, font=\small},91  blocinput/.style  = {bloc, fill=cinput!15,  draw=cinput!70!black},92  blochidden/.style = {bloc, fill=chidden!20, draw=chidden!70!black},93  blocoutput/.style = {bloc, fill=coutput!15, draw=coutput!70!black},94  gate/.style     = {bloc, fill=cgate!20, draw=cgate!70!black,95                     minimum width=9mm, minimum height=7mm},96  mem/.style      = {bloc, fill=cmem!15, draw=cmem!70!black},97  op/.style       = {circle, draw=black!70, fill=white, inner sep=1pt,98                     minimum size=5.5mm, font=\small},99  fleche/.style   = {-{Stealth[length=2.5mm]}, thick},100  flechep/.style  = {fleche, dashed},101  etiquette/.style= {font=\scriptsize, text=black!60},102}103104% ----------------------------------------------------------------------------105% Theorem environments (numbered per chapter)106% ----------------------------------------------------------------------------107\theoremstyle{plain}108\newtheorem{theorem}{Theorem}[chapter]109\newtheorem{property}[theorem]{Property}110\theoremstyle{definition}111\newtheorem{definition}[theorem]{Definition}112\theoremstyle{remark}113\newtheorem{remark}[theorem]{Remark}114115% ----------------------------------------------------------------------------116% Notation macros (absolute consistency across chapters)117% ----------------------------------------------------------------------------118\newcommand{\vect}[1]{\bm{#1}}          % vector (bold)119\newcommand{\mat}[1]{\bm{#1}}           % matrix (bold)120\newcommand{\Loss}{\mathcal{L}}         % loss function121\newcommand{\E}{\mathbb{E}}             % expectation122\newcommand{\R}{\mathbb{R}}             % reals123\newcommand{\KL}{D_{\mathrm{KL}}}       % Kullback–Leibler divergence124\DeclareMathOperator{\softmax}{softmax}125\DeclareMathOperator*{\argmin}{arg\,min}126\DeclareMathOperator*{\argmax}{arg\,max}127\newcommand{\transp}{^{\top}}           % transpose128129\numberwithin{equation}{chapter}130131% ============================================================================132\begin{document}133134% ----------------------------------------------------------------------------135% COVER / TITLE PAGE136% ----------------------------------------------------------------------------137\begin{titlepage}138  \centering139  \vspace*{1.2cm}140  {\scshape\Large The Deep Learning Series\par}141  \vspace{0.4cm}142  \rule{0.72\textwidth}{0.8pt}\par143  \vspace{0.9cm}144  {\Huge\bfseries Artificial Neural\\[0.25em] Networks\par}145  \vspace{0.55cm}146  {\Large Methods, Equations and Graphical Representations\par}147  \vspace{0.9cm}148  \rule{0.72\textwidth}{0.8pt}\par149  \vspace{1.3cm}150151  % Decorative TikZ mini-network152  \begin{tikzpicture}[scale=0.9, transform shape]153    \foreach \i in {1,2,3}154      \node[ninput]  (i\i) at (0, 1.5-\i*1.0) {};155    \foreach \j in {1,...,4}156      \node[nhidden] (h1\j) at (2.4, 2.0-\j*1.0) {};157    \foreach \j in {1,...,4}158      \node[nhidden] (h2\j) at (4.8, 2.0-\j*1.0) {};159    \foreach \k in {1,2}160      \node[noutput] (o\k) at (7.2, 1.0-\k*1.0) {};161    \foreach \i in {1,2,3} \foreach \j in {1,...,4}162      \draw[black!30, semithick] (i\i) -- (h1\j);163    \foreach \i in {1,...,4} \foreach \j in {1,...,4}164      \draw[black!30, semithick] (h1\i) -- (h2\j);165    \foreach \i in {1,...,4} \foreach \k in {1,2}166      \draw[black!30, semithick] (h2\i) -- (o\k);167  \end{tikzpicture}168  \par169  \vspace{1.5cm}170171  {\Large\itshape Simon-Pierre Boucher\par}172  \vspace{0.35cm}173  {\large\href{mailto:contact@spboucher.ai}{\texttt{contact@spboucher.ai}}\par}174  \vfill175  {\large \today\par}176  \vspace*{0.8cm}177\end{titlepage}178179\frontmatter180181% ----------------------------------------------------------------------------182% PREFACE183% ----------------------------------------------------------------------------184\chapter*{Preface}185\addcontentsline{toc}{chapter}{Preface}186187Neural networks have moved, in a single generation, from a contested188scientific hypothesis to the engine of modern artificial intelligence. Yet189the field is often taught as a collection of recipes, where the underlying190mathematics is scattered across papers and the architectures are described191by ambiguous sketches. This book was written against that fragmentation.192Its organizing principle is simple and strict: \emph{every method is193presented with rigorous equations and a faithful graphical representation}.194Every figure in this book is drawn natively in TikZ or pgfplots — no195imported images — so that each diagram is exactly as precise as the196equations it illustrates.197198The book is self-contained and can serve as a standalone course text. It199assumes only linear algebra, multivariate calculus and elementary200probability. Notation is kept absolutely consistent from the first chapter201to the last: vectors are bold lowercase ($\vect{x}$), matrices bold202uppercase ($\mat{W}$), losses are written $\Loss$, and the Hadamard product203is $\odot$ throughout.204205\vspace{0.6cm}206\begin{flushright}207\itshape Simon-Pierre Boucher\\208\href{mailto:contact@spboucher.ai}{contact@spboucher.ai}209\end{flushright}210211% ----------------------------------------------------------------------------212% ABSTRACT213% ----------------------------------------------------------------------------214\chapter*{Abstract}215\addcontentsline{toc}{chapter}{Abstract}216217This book offers a rigorous and self-contained treatment of the major218methods of artificial neural networks, from Rosenblatt's perceptron to219contemporary generative models. Each method is presented under a twofold220requirement: on one hand, a complete mathematical formulation —221forward-propagation equations, gradients, objective functions — in a222notation unified across chapters; on the other hand, a native graphical223representation (TikZ/pgfplots) that makes the underlying computational224structure visible.225226Chapter~\ref{chap:foundations} lays the foundations: the artificial neuron,227the perceptron, activation functions, the multilayer perceptron and the228universal approximation theorem. Chapter~\ref{chap:training} covers229training: backpropagation of gradients, adaptive optimizers (SGD, Momentum,230Adam), learning-rate scheduling and initialization.231Chapter~\ref{chap:regularization} addresses regularization: $L_1$/$L_2$232penalties, dropout, batch and layer normalization, early stopping.233Chapters~\ref{chap:cnn} and~\ref{chap:rnn} detail convolutional networks234(convolution, pooling, residual blocks, receptive fields) and recurrent235networks (RNN, LSTM, GRU, bidirectional and encoder--decoder236architectures). Chapter~\ref{chap:attention} is devoted to the attention237mechanism and the Transformer architecture, and238Chapter~\ref{chap:modern} to its modern descendants: Vision Transformers,239mixtures of experts, efficient attention and selective state-space models.240Chapter~\ref{chap:gnn} develops graph neural networks (message passing,241GCN, GraphSAGE, GAT, GIN) and Chapter~\ref{chap:energy} the energy-based242family: Hopfield networks, Boltzmann machines, restricted Boltzmann243machines and deep belief networks. Chapter~\ref{chap:generative} presents244the core generative models — autoencoders, variational autoencoders and245generative adversarial networks — while Chapter~\ref{chap:advgen} covers246normalizing flows, diffusion models and deep autoregressive models.247Finally, Chapter~\ref{chap:bio} treats biologically inspired computation248(spiking networks, self-organizing maps, reservoir computing) and249Chapter~\ref{chap:emerging} the emerging architectures: capsule networks,250neural ordinary differential equations, memory-augmented networks and251Kolmogorov--Arnold networks.252253\vspace{0.8cm}254\noindent\textbf{Keywords}: neural networks, deep learning,255backpropagation, regularization, convolutional networks, recurrent256networks, LSTM, attention, Transformer, variational autoencoder, GAN,257diffusion.258259% ----------------------------------------------------------------------------260% TABLES261% ----------------------------------------------------------------------------262\tableofcontents263264\mainmatter265266% ----------------------------------------------------------------------------267% CHAPTERS268% ----------------------------------------------------------------------------269\part{Foundations and Learning}270\include{chapters/01-foundations}271\include{chapters/02-training}272\include{chapters/03-regularization}273274\part{Core Architectures}275\include{chapters/04-cnn}276\include{chapters/05-rnn}277\include{chapters/06-attention}278\include{chapters/07-modern-transformers}279280\part{Graphs, Energy and Memory}281\include{chapters/08-gnn}282\include{chapters/09-energy}283284\part{Generative Models}285\include{chapters/10-generative}286\include{chapters/11-advanced-generative}287288\part{Specialized and Emerging Architectures}289\include{chapters/12-bio-inspired}290\include{chapters/13-emerging}291292\backmatter293294% ----------------------------------------------------------------------------295% BIBLIOGRAPHY296% ----------------------------------------------------------------------------297\begin{thebibliography}{99}298\addcontentsline{toc}{chapter}{Bibliography}299300\bibitem{rosenblatt1958}301F.~Rosenblatt.302\newblock The perceptron: A probabilistic model for information storage and303  organization in the brain.304\newblock \emph{Psychological Review}, 65(6):386--408, 1958.305306\bibitem{rumelhart1986}307D.~E. Rumelhart, G.~E. Hinton, and R.~J. Williams.308\newblock Learning representations by back-propagating errors.309\newblock \emph{Nature}, 323:533--536, 1986.310311\bibitem{lecun1998}312Y.~LeCun, L.~Bottou, Y.~Bengio, and P.~Haffner.313\newblock Gradient-based learning applied to document recognition.314\newblock \emph{Proceedings of the IEEE}, 86(11):2278--2324, 1998.315316\bibitem{hochreiter1997}317S.~Hochreiter and J.~Schmidhuber.318\newblock Long short-term memory.319\newblock \emph{Neural Computation}, 9(8):1735--1780, 1997.320321\bibitem{cho2014}322K.~Cho, B.~van Merriënboer, C.~Gulcehre, D.~Bahdanau, F.~Bougares,323  H.~Schwenk, and Y.~Bengio.324\newblock Learning phrase representations using {RNN} encoder--decoder for325  statistical machine translation.326\newblock In \emph{Proceedings of EMNLP}, pages 1724--1734, 2014.327328\bibitem{srivastava2014}329N.~Srivastava, G.~Hinton, A.~Krizhevsky, I.~Sutskever, and330  R.~Salakhutdinov.331\newblock Dropout: A simple way to prevent neural networks from overfitting.332\newblock \emph{Journal of Machine Learning Research}, 15:1929--1958, 2014.333334\bibitem{ioffe2015}335S.~Ioffe and C.~Szegedy.336\newblock Batch normalization: Accelerating deep network training by337  reducing internal covariate shift.338\newblock In \emph{Proceedings of ICML}, pages 448--456, 2015.339340\bibitem{kingma2015adam}341D.~P. Kingma and J.~Ba.342\newblock Adam: A method for stochastic optimization.343\newblock In \emph{Proceedings of ICLR}, 2015.344345\bibitem{he2016}346K.~He, X.~Zhang, S.~Ren, and J.~Sun.347\newblock Deep residual learning for image recognition.348\newblock In \emph{Proceedings of CVPR}, pages 770--778, 2016.349350\bibitem{vaswani2017}351A.~Vaswani, N.~Shazeer, N.~Parmar, J.~Uszkoreit, L.~Jones, A.~N. Gomez,352  Ł.~Kaiser, and I.~Polosukhin.353\newblock Attention is all you need.354\newblock In \emph{Advances in Neural Information Processing Systems~30},355  pages 5998--6008, 2017.356357\bibitem{kingma2014vae}358D.~P. Kingma and M.~Welling.359\newblock Auto-encoding variational {B}ayes.360\newblock In \emph{Proceedings of ICLR}, 2014.361362\bibitem{goodfellow2014gan}363I.~Goodfellow, J.~Pouget-Abadie, M.~Mirza, B.~Xu, D.~Warde-Farley,364  S.~Ozair, A.~Courville, and Y.~Bengio.365\newblock Generative adversarial nets.366\newblock In \emph{Advances in Neural Information Processing Systems~27},367  pages 2672--2680, 2014.368369\bibitem{goodfellow2016book}370I.~Goodfellow, Y.~Bengio, and A.~Courville.371\newblock \emph{Deep Learning}.372\newblock MIT Press, 2016.373374\bibitem{hopfield1982}375J.~J. Hopfield.376\newblock Neural networks and physical systems with emergent collective377  computational abilities.378\newblock \emph{Proceedings of the National Academy of Sciences},379  79(8):2554--2558, 1982.380381\bibitem{ackley1985}382D.~H. Ackley, G.~E. Hinton, and T.~J. Sejnowski.383\newblock A learning algorithm for {B}oltzmann machines.384\newblock \emph{Cognitive Science}, 9(1):147--169, 1985.385386\bibitem{kohonen1990}387T.~Kohonen.388\newblock The self-organizing map.389\newblock \emph{Proceedings of the IEEE}, 78(9):1464--1480, 1990.390391\bibitem{maass1997}392W.~Maass.393\newblock Networks of spiking neurons: The third generation of neural394  network models.395\newblock \emph{Neural Networks}, 10(9):1659--1671, 1997.396397\bibitem{jaeger2001}398H.~Jaeger.399\newblock The ``echo state'' approach to analysing and training recurrent400  neural networks.401\newblock GMD Report 148, German National Research Center for Information402  Technology, 2001.403404\bibitem{hinton2002}405G.~E. Hinton.406\newblock Training products of experts by minimizing contrastive407  divergence.408\newblock \emph{Neural Computation}, 14(8):1771--1800, 2002.409410\bibitem{hinton2006}411G.~E. Hinton, S.~Osindero, and Y.-W. Teh.412\newblock A fast learning algorithm for deep belief nets.413\newblock \emph{Neural Computation}, 18(7):1527--1554, 2006.414415\bibitem{graves2014}416A.~Graves, G.~Wayne, and I.~Danihelka.417\newblock Neural {T}uring machines.418\newblock arXiv:1410.5401, 2014.419420\bibitem{rezende2015}421D.~J. Rezende and S.~Mohamed.422\newblock Variational inference with normalizing flows.423\newblock In \emph{Proceedings of ICML}, pages 1530--1538, 2015.424425\bibitem{oord2016wavenet}426A.~van~den Oord, S.~Dieleman, H.~Zen, K.~Simonyan, O.~Vinyals, A.~Graves,427  N.~Kalchbrenner, A.~Senior, and K.~Kavukcuoglu.428\newblock {WaveNet}: A generative model for raw audio.429\newblock arXiv:1609.03499, 2016.430431\bibitem{dinh2017}432L.~Dinh, J.~Sohl-Dickstein, and S.~Bengio.433\newblock Density estimation using {Real NVP}.434\newblock In \emph{Proceedings of ICLR}, 2017.435436\bibitem{kipf2017}437T.~N. Kipf and M.~Welling.438\newblock Semi-supervised classification with graph convolutional networks.439\newblock In \emph{Proceedings of ICLR}, 2017.440441\bibitem{hamilton2017}442W.~L. Hamilton, R.~Ying, and J.~Leskovec.443\newblock Inductive representation learning on large graphs.444\newblock In \emph{Advances in Neural Information Processing Systems~30},445  pages 1024--1034, 2017.446447\bibitem{shazeer2017}448N.~Shazeer, A.~Mirhoseini, K.~Maziarz, A.~Davis, Q.~Le, G.~Hinton, and449  J.~Dean.450\newblock Outrageously large neural networks: The sparsely-gated451  mixture-of-experts layer.452\newblock In \emph{Proceedings of ICLR}, 2017.453454\bibitem{sabour2017}455S.~Sabour, N.~Frosst, and G.~E. Hinton.456\newblock Dynamic routing between capsules.457\newblock In \emph{Advances in Neural Information Processing Systems~30},458  pages 3856--3866, 2017.459460\bibitem{velickovic2018}461P.~Veličković, G.~Cucurull, A.~Casanova, A.~Romero, P.~Liò, and462  Y.~Bengio.463\newblock Graph attention networks.464\newblock In \emph{Proceedings of ICLR}, 2018.465466\bibitem{chen2018}467R.~T.~Q. Chen, Y.~Rubanova, J.~Bettencourt, and D.~Duvenaud.468\newblock Neural ordinary differential equations.469\newblock In \emph{Advances in Neural Information Processing Systems~31},470  pages 6571--6583, 2018.471472\bibitem{xu2019}473K.~Xu, W.~Hu, J.~Leskovec, and S.~Jegelka.474\newblock How powerful are graph neural networks?475\newblock In \emph{Proceedings of ICLR}, 2019.476477\bibitem{ho2020}478J.~Ho, A.~Jain, and P.~Abbeel.479\newblock Denoising diffusion probabilistic models.480\newblock In \emph{Advances in Neural Information Processing Systems~33},481  pages 6840--6851, 2020.482483\bibitem{ramsauer2021}484H.~Ramsauer, B.~Schäfl, J.~Lehner, P.~Seidl, M.~Widrich, T.~Adler,485  L.~Gruber, M.~Holzleitner, M.~Pavlović, G.~K. Sandve, V.~Greiff,486  D.~Kreil, M.~Kopp, G.~Klambauer, J.~Brandstetter, and S.~Hochreiter.487\newblock Hopfield networks is all you need.488\newblock In \emph{Proceedings of ICLR}, 2021.489490\bibitem{dosovitskiy2021}491A.~Dosovitskiy, L.~Beyer, A.~Kolesnikov, D.~Weissenborn, X.~Zhai,492  T.~Unterthiner, M.~Dehghani, M.~Minderer, G.~Heigold, S.~Gelly,493  J.~Uszkoreit, and N.~Houlsby.494\newblock An image is worth $16\times16$ words: Transformers for image495  recognition at scale.496\newblock In \emph{Proceedings of ICLR}, 2021.497498\bibitem{gu2023}499A.~Gu and T.~Dao.500\newblock Mamba: Linear-time sequence modeling with selective state spaces.501\newblock arXiv:2312.00752, 2023.502503\bibitem{liu2024kan}504Z.~Liu, Y.~Wang, S.~Vaidya, F.~Ruehle, J.~Halverson, M.~Soljačić,505  T.~Y. Hou, and M.~Tegmark.506\newblock {KAN}: Kolmogorov--Arnold networks.507\newblock arXiv:2404.19756, 2024.508509\end{thebibliography}510511\end{document}512