% ============================================================================
%  Artificial Neural Networks — Methods, Equations and
%  Graphical Representations
%  Author : Simon-Pierre Boucher — contact@spboucher.ai
%  Book format (parts, preface, chapters, bibliography)
%  File   : main.tex — preamble, cover, preface, abstract, TOC, bibliography
% ============================================================================
\documentclass[11pt,a4paper]{book}

% ----------------------------------------------------------------------------
% Encoding, language, typography
% ----------------------------------------------------------------------------
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{lmodern}
\usepackage[english]{babel}
\usepackage{microtype}

% ----------------------------------------------------------------------------
% Mathematics
% ----------------------------------------------------------------------------
\usepackage{amsmath,amssymb,amsthm}
\usepackage{mathtools}
\usepackage{bm}

% ----------------------------------------------------------------------------
% Graphics: TikZ + pgfplots (no external images)
% ----------------------------------------------------------------------------
\usepackage{tikz}
\usetikzlibrary{positioning,arrows.meta,calc,shapes.geometric,fit,
                backgrounds,decorations.pathreplacing}
\usepackage{pgfplots}
\pgfplotsset{compat=1.17}

% ----------------------------------------------------------------------------
% Algorithms
% ----------------------------------------------------------------------------
\usepackage{algorithm}
\usepackage{algpseudocode}

% ----------------------------------------------------------------------------
% Page layout
% ----------------------------------------------------------------------------
\usepackage[a4paper,top=2.6cm,bottom=2.8cm,inner=2.8cm,outer=2.4cm,
            headheight=14.5pt]{geometry}
\usepackage{fancyhdr}
\usepackage{booktabs}
\usepackage{array}

\pagestyle{fancy}
\fancyhf{}
\fancyhead[LE]{\small\itshape\nouppercase{\leftmark}}
\fancyhead[RO]{\small\itshape\nouppercase{\rightmark}}
\fancyfoot[C]{\small\thepage}
\renewcommand{\headrulewidth}{0.4pt}
\fancypagestyle{plain}{\fancyhf{}\fancyfoot[C]{\small\thepage}%
  \renewcommand{\headrulewidth}{0pt}}

% ----------------------------------------------------------------------------
% Hyperlinks
% ----------------------------------------------------------------------------
\usepackage[colorlinks=true,
            linkcolor=blue!50!black,
            citecolor=green!40!black,
            urlcolor=violet!70!black,
            pdftitle={Artificial Neural Networks — Methods, Equations and
                      Graphical Representations},
            pdfauthor={Simon-Pierre Boucher}]{hyperref}

% ----------------------------------------------------------------------------
% Named palette (used by ALL figures)
% ----------------------------------------------------------------------------
\definecolor{cinput}{RGB}{31,119,180}    % blue   — inputs
\definecolor{chidden}{RGB}{255,127,14}   % orange — hidden layers
\definecolor{coutput}{RGB}{214,39,40}    % red    — outputs
\definecolor{cgate}{RGB}{44,160,44}      % green  — gates
\definecolor{cmem}{RGB}{148,103,189}     % violet — memory / state

% ----------------------------------------------------------------------------
% Global TikZ styles (reused across all chapters)
% ----------------------------------------------------------------------------
\tikzset{
  neuron/.style   = {circle, draw=black!70, minimum size=8mm,
                     inner sep=0pt, font=\small},
  ninput/.style   = {neuron, fill=cinput!25,  draw=cinput!80!black},
  nhidden/.style  = {neuron, fill=chidden!30, draw=chidden!80!black},
  noutput/.style  = {neuron, fill=coutput!25, draw=coutput!80!black},
  bloc/.style     = {rectangle, rounded corners=2pt, draw=black!70,
                     fill=black!5, minimum width=2.1cm, minimum height=8mm,
                     align=center, font=\small},
  blocinput/.style  = {bloc, fill=cinput!15,  draw=cinput!70!black},
  blochidden/.style = {bloc, fill=chidden!20, draw=chidden!70!black},
  blocoutput/.style = {bloc, fill=coutput!15, draw=coutput!70!black},
  gate/.style     = {bloc, fill=cgate!20, draw=cgate!70!black,
                     minimum width=9mm, minimum height=7mm},
  mem/.style      = {bloc, fill=cmem!15, draw=cmem!70!black},
  op/.style       = {circle, draw=black!70, fill=white, inner sep=1pt,
                     minimum size=5.5mm, font=\small},
  fleche/.style   = {-{Stealth[length=2.5mm]}, thick},
  flechep/.style  = {fleche, dashed},
  etiquette/.style= {font=\scriptsize, text=black!60},
}

% ----------------------------------------------------------------------------
% Theorem environments (numbered per chapter)
% ----------------------------------------------------------------------------
\theoremstyle{plain}
\newtheorem{theorem}{Theorem}[chapter]
\newtheorem{property}[theorem]{Property}
\theoremstyle{definition}
\newtheorem{definition}[theorem]{Definition}
\theoremstyle{remark}
\newtheorem{remark}[theorem]{Remark}

% ----------------------------------------------------------------------------
% Notation macros (absolute consistency across chapters)
% ----------------------------------------------------------------------------
\newcommand{\vect}[1]{\bm{#1}}          % vector (bold)
\newcommand{\mat}[1]{\bm{#1}}           % matrix (bold)
\newcommand{\Loss}{\mathcal{L}}         % loss function
\newcommand{\E}{\mathbb{E}}             % expectation
\newcommand{\R}{\mathbb{R}}             % reals
\newcommand{\KL}{D_{\mathrm{KL}}}       % Kullback–Leibler divergence
\DeclareMathOperator{\softmax}{softmax}
\DeclareMathOperator*{\argmin}{arg\,min}
\DeclareMathOperator*{\argmax}{arg\,max}
\newcommand{\transp}{^{\top}}           % transpose

\numberwithin{equation}{chapter}

% ============================================================================
\begin{document}

% ----------------------------------------------------------------------------
% COVER / TITLE PAGE
% ----------------------------------------------------------------------------
\begin{titlepage}
  \centering
  \vspace*{1.2cm}
  {\scshape\Large The Deep Learning Series\par}
  \vspace{0.4cm}
  \rule{0.72\textwidth}{0.8pt}\par
  \vspace{0.9cm}
  {\Huge\bfseries Artificial Neural\\[0.25em] Networks\par}
  \vspace{0.55cm}
  {\Large Methods, Equations and Graphical Representations\par}
  \vspace{0.9cm}
  \rule{0.72\textwidth}{0.8pt}\par
  \vspace{1.3cm}

  % Decorative TikZ mini-network
  \begin{tikzpicture}[scale=0.9, transform shape]
    \foreach \i in {1,2,3}
      \node[ninput]  (i\i) at (0, 1.5-\i*1.0) {};
    \foreach \j in {1,...,4}
      \node[nhidden] (h1\j) at (2.4, 2.0-\j*1.0) {};
    \foreach \j in {1,...,4}
      \node[nhidden] (h2\j) at (4.8, 2.0-\j*1.0) {};
    \foreach \k in {1,2}
      \node[noutput] (o\k) at (7.2, 1.0-\k*1.0) {};
    \foreach \i in {1,2,3} \foreach \j in {1,...,4}
      \draw[black!30, semithick] (i\i) -- (h1\j);
    \foreach \i in {1,...,4} \foreach \j in {1,...,4}
      \draw[black!30, semithick] (h1\i) -- (h2\j);
    \foreach \i in {1,...,4} \foreach \k in {1,2}
      \draw[black!30, semithick] (h2\i) -- (o\k);
  \end{tikzpicture}
  \par
  \vspace{1.5cm}

  {\Large\itshape Simon-Pierre Boucher\par}
  \vspace{0.35cm}
  {\large\href{mailto:contact@spboucher.ai}{\texttt{contact@spboucher.ai}}\par}
  \vfill
  {\large \today\par}
  \vspace*{0.8cm}
\end{titlepage}

\frontmatter

% ----------------------------------------------------------------------------
% PREFACE
% ----------------------------------------------------------------------------
\chapter*{Preface}
\addcontentsline{toc}{chapter}{Preface}

Neural networks have moved, in a single generation, from a contested
scientific hypothesis to the engine of modern artificial intelligence. Yet
the field is often taught as a collection of recipes, where the underlying
mathematics is scattered across papers and the architectures are described
by ambiguous sketches. This book was written against that fragmentation.
Its organizing principle is simple and strict: \emph{every method is
presented with rigorous equations and a faithful graphical representation}.
Every figure in this book is drawn natively in TikZ or pgfplots — no
imported images — so that each diagram is exactly as precise as the
equations it illustrates.

The book is self-contained and can serve as a standalone course text. It
assumes only linear algebra, multivariate calculus and elementary
probability. Notation is kept absolutely consistent from the first chapter
to the last: vectors are bold lowercase ($\vect{x}$), matrices bold
uppercase ($\mat{W}$), losses are written $\Loss$, and the Hadamard product
is $\odot$ throughout.

\vspace{0.6cm}
\begin{flushright}
\itshape Simon-Pierre Boucher\\
\href{mailto:contact@spboucher.ai}{contact@spboucher.ai}
\end{flushright}

% ----------------------------------------------------------------------------
% ABSTRACT
% ----------------------------------------------------------------------------
\chapter*{Abstract}
\addcontentsline{toc}{chapter}{Abstract}

This book offers a rigorous and self-contained treatment of the major
methods of artificial neural networks, from Rosenblatt's perceptron to
contemporary generative models. Each method is presented under a twofold
requirement: on one hand, a complete mathematical formulation —
forward-propagation equations, gradients, objective functions — in a
notation unified across chapters; on the other hand, a native graphical
representation (TikZ/pgfplots) that makes the underlying computational
structure visible.

Chapter~\ref{chap:foundations} lays the foundations: the artificial neuron,
the perceptron, activation functions, the multilayer perceptron and the
universal approximation theorem. Chapter~\ref{chap:training} covers
training: backpropagation of gradients, adaptive optimizers (SGD, Momentum,
Adam), learning-rate scheduling and initialization.
Chapter~\ref{chap:regularization} addresses regularization: $L_1$/$L_2$
penalties, dropout, batch and layer normalization, early stopping.
Chapters~\ref{chap:cnn} and~\ref{chap:rnn} detail convolutional networks
(convolution, pooling, residual blocks, receptive fields) and recurrent
networks (RNN, LSTM, GRU, bidirectional and encoder--decoder
architectures). Chapter~\ref{chap:attention} is devoted to the attention
mechanism and the Transformer architecture, and
Chapter~\ref{chap:modern} to its modern descendants: Vision Transformers,
mixtures of experts, efficient attention and selective state-space models.
Chapter~\ref{chap:gnn} develops graph neural networks (message passing,
GCN, GraphSAGE, GAT, GIN) and Chapter~\ref{chap:energy} the energy-based
family: Hopfield networks, Boltzmann machines, restricted Boltzmann
machines and deep belief networks. Chapter~\ref{chap:generative} presents
the core generative models — autoencoders, variational autoencoders and
generative adversarial networks — while Chapter~\ref{chap:advgen} covers
normalizing flows, diffusion models and deep autoregressive models.
Finally, Chapter~\ref{chap:bio} treats biologically inspired computation
(spiking networks, self-organizing maps, reservoir computing) and
Chapter~\ref{chap:emerging} the emerging architectures: capsule networks,
neural ordinary differential equations, memory-augmented networks and
Kolmogorov--Arnold networks.

\vspace{0.8cm}
\noindent\textbf{Keywords}: neural networks, deep learning,
backpropagation, regularization, convolutional networks, recurrent
networks, LSTM, attention, Transformer, variational autoencoder, GAN,
diffusion.

% ----------------------------------------------------------------------------
% TABLES
% ----------------------------------------------------------------------------
\tableofcontents

\mainmatter

% ----------------------------------------------------------------------------
% CHAPTERS
% ----------------------------------------------------------------------------
\part{Foundations and Learning}
\include{chapters/01-foundations}
\include{chapters/02-training}
\include{chapters/03-regularization}

\part{Core Architectures}
\include{chapters/04-cnn}
\include{chapters/05-rnn}
\include{chapters/06-attention}
\include{chapters/07-modern-transformers}

\part{Graphs, Energy and Memory}
\include{chapters/08-gnn}
\include{chapters/09-energy}

\part{Generative Models}
\include{chapters/10-generative}
\include{chapters/11-advanced-generative}

\part{Specialized and Emerging Architectures}
\include{chapters/12-bio-inspired}
\include{chapters/13-emerging}

\backmatter

% ----------------------------------------------------------------------------
% BIBLIOGRAPHY
% ----------------------------------------------------------------------------
\begin{thebibliography}{99}
\addcontentsline{toc}{chapter}{Bibliography}

\bibitem{rosenblatt1958}
F.~Rosenblatt.
\newblock The perceptron: A probabilistic model for information storage and
  organization in the brain.
\newblock \emph{Psychological Review}, 65(6):386--408, 1958.

\bibitem{rumelhart1986}
D.~E. Rumelhart, G.~E. Hinton, and R.~J. Williams.
\newblock Learning representations by back-propagating errors.
\newblock \emph{Nature}, 323:533--536, 1986.

\bibitem{lecun1998}
Y.~LeCun, L.~Bottou, Y.~Bengio, and P.~Haffner.
\newblock Gradient-based learning applied to document recognition.
\newblock \emph{Proceedings of the IEEE}, 86(11):2278--2324, 1998.

\bibitem{hochreiter1997}
S.~Hochreiter and J.~Schmidhuber.
\newblock Long short-term memory.
\newblock \emph{Neural Computation}, 9(8):1735--1780, 1997.

\bibitem{cho2014}
K.~Cho, B.~van Merriënboer, C.~Gulcehre, D.~Bahdanau, F.~Bougares,
  H.~Schwenk, and Y.~Bengio.
\newblock Learning phrase representations using {RNN} encoder--decoder for
  statistical machine translation.
\newblock In \emph{Proceedings of EMNLP}, pages 1724--1734, 2014.

\bibitem{srivastava2014}
N.~Srivastava, G.~Hinton, A.~Krizhevsky, I.~Sutskever, and
  R.~Salakhutdinov.
\newblock Dropout: A simple way to prevent neural networks from overfitting.
\newblock \emph{Journal of Machine Learning Research}, 15:1929--1958, 2014.

\bibitem{ioffe2015}
S.~Ioffe and C.~Szegedy.
\newblock Batch normalization: Accelerating deep network training by
  reducing internal covariate shift.
\newblock In \emph{Proceedings of ICML}, pages 448--456, 2015.

\bibitem{kingma2015adam}
D.~P. Kingma and J.~Ba.
\newblock Adam: A method for stochastic optimization.
\newblock In \emph{Proceedings of ICLR}, 2015.

\bibitem{he2016}
K.~He, X.~Zhang, S.~Ren, and J.~Sun.
\newblock Deep residual learning for image recognition.
\newblock In \emph{Proceedings of CVPR}, pages 770--778, 2016.

\bibitem{vaswani2017}
A.~Vaswani, N.~Shazeer, N.~Parmar, J.~Uszkoreit, L.~Jones, A.~N. Gomez,
  Ł.~Kaiser, and I.~Polosukhin.
\newblock Attention is all you need.
\newblock In \emph{Advances in Neural Information Processing Systems~30},
  pages 5998--6008, 2017.

\bibitem{kingma2014vae}
D.~P. Kingma and M.~Welling.
\newblock Auto-encoding variational {B}ayes.
\newblock In \emph{Proceedings of ICLR}, 2014.

\bibitem{goodfellow2014gan}
I.~Goodfellow, J.~Pouget-Abadie, M.~Mirza, B.~Xu, D.~Warde-Farley,
  S.~Ozair, A.~Courville, and Y.~Bengio.
\newblock Generative adversarial nets.
\newblock In \emph{Advances in Neural Information Processing Systems~27},
  pages 2672--2680, 2014.

\bibitem{goodfellow2016book}
I.~Goodfellow, Y.~Bengio, and A.~Courville.
\newblock \emph{Deep Learning}.
\newblock MIT Press, 2016.

\bibitem{hopfield1982}
J.~J. Hopfield.
\newblock Neural networks and physical systems with emergent collective
  computational abilities.
\newblock \emph{Proceedings of the National Academy of Sciences},
  79(8):2554--2558, 1982.

\bibitem{ackley1985}
D.~H. Ackley, G.~E. Hinton, and T.~J. Sejnowski.
\newblock A learning algorithm for {B}oltzmann machines.
\newblock \emph{Cognitive Science}, 9(1):147--169, 1985.

\bibitem{kohonen1990}
T.~Kohonen.
\newblock The self-organizing map.
\newblock \emph{Proceedings of the IEEE}, 78(9):1464--1480, 1990.

\bibitem{maass1997}
W.~Maass.
\newblock Networks of spiking neurons: The third generation of neural
  network models.
\newblock \emph{Neural Networks}, 10(9):1659--1671, 1997.

\bibitem{jaeger2001}
H.~Jaeger.
\newblock The ``echo state'' approach to analysing and training recurrent
  neural networks.
\newblock GMD Report 148, German National Research Center for Information
  Technology, 2001.

\bibitem{hinton2002}
G.~E. Hinton.
\newblock Training products of experts by minimizing contrastive
  divergence.
\newblock \emph{Neural Computation}, 14(8):1771--1800, 2002.

\bibitem{hinton2006}
G.~E. Hinton, S.~Osindero, and Y.-W. Teh.
\newblock A fast learning algorithm for deep belief nets.
\newblock \emph{Neural Computation}, 18(7):1527--1554, 2006.

\bibitem{graves2014}
A.~Graves, G.~Wayne, and I.~Danihelka.
\newblock Neural {T}uring machines.
\newblock arXiv:1410.5401, 2014.

\bibitem{rezende2015}
D.~J. Rezende and S.~Mohamed.
\newblock Variational inference with normalizing flows.
\newblock In \emph{Proceedings of ICML}, pages 1530--1538, 2015.

\bibitem{oord2016wavenet}
A.~van~den Oord, S.~Dieleman, H.~Zen, K.~Simonyan, O.~Vinyals, A.~Graves,
  N.~Kalchbrenner, A.~Senior, and K.~Kavukcuoglu.
\newblock {WaveNet}: A generative model for raw audio.
\newblock arXiv:1609.03499, 2016.

\bibitem{dinh2017}
L.~Dinh, J.~Sohl-Dickstein, and S.~Bengio.
\newblock Density estimation using {Real NVP}.
\newblock In \emph{Proceedings of ICLR}, 2017.

\bibitem{kipf2017}
T.~N. Kipf and M.~Welling.
\newblock Semi-supervised classification with graph convolutional networks.
\newblock In \emph{Proceedings of ICLR}, 2017.

\bibitem{hamilton2017}
W.~L. Hamilton, R.~Ying, and J.~Leskovec.
\newblock Inductive representation learning on large graphs.
\newblock In \emph{Advances in Neural Information Processing Systems~30},
  pages 1024--1034, 2017.

\bibitem{shazeer2017}
N.~Shazeer, A.~Mirhoseini, K.~Maziarz, A.~Davis, Q.~Le, G.~Hinton, and
  J.~Dean.
\newblock Outrageously large neural networks: The sparsely-gated
  mixture-of-experts layer.
\newblock In \emph{Proceedings of ICLR}, 2017.

\bibitem{sabour2017}
S.~Sabour, N.~Frosst, and G.~E. Hinton.
\newblock Dynamic routing between capsules.
\newblock In \emph{Advances in Neural Information Processing Systems~30},
  pages 3856--3866, 2017.

\bibitem{velickovic2018}
P.~Veličković, G.~Cucurull, A.~Casanova, A.~Romero, P.~Liò, and
  Y.~Bengio.
\newblock Graph attention networks.
\newblock In \emph{Proceedings of ICLR}, 2018.

\bibitem{chen2018}
R.~T.~Q. Chen, Y.~Rubanova, J.~Bettencourt, and D.~Duvenaud.
\newblock Neural ordinary differential equations.
\newblock In \emph{Advances in Neural Information Processing Systems~31},
  pages 6571--6583, 2018.

\bibitem{xu2019}
K.~Xu, W.~Hu, J.~Leskovec, and S.~Jegelka.
\newblock How powerful are graph neural networks?
\newblock In \emph{Proceedings of ICLR}, 2019.

\bibitem{ho2020}
J.~Ho, A.~Jain, and P.~Abbeel.
\newblock Denoising diffusion probabilistic models.
\newblock In \emph{Advances in Neural Information Processing Systems~33},
  pages 6840--6851, 2020.

\bibitem{ramsauer2021}
H.~Ramsauer, B.~Schäfl, J.~Lehner, P.~Seidl, M.~Widrich, T.~Adler,
  L.~Gruber, M.~Holzleitner, M.~Pavlović, G.~K. Sandve, V.~Greiff,
  D.~Kreil, M.~Kopp, G.~Klambauer, J.~Brandstetter, and S.~Hochreiter.
\newblock Hopfield networks is all you need.
\newblock In \emph{Proceedings of ICLR}, 2021.

\bibitem{dosovitskiy2021}
A.~Dosovitskiy, L.~Beyer, A.~Kolesnikov, D.~Weissenborn, X.~Zhai,
  T.~Unterthiner, M.~Dehghani, M.~Minderer, G.~Heigold, S.~Gelly,
  J.~Uszkoreit, and N.~Houlsby.
\newblock An image is worth $16\times16$ words: Transformers for image
  recognition at scale.
\newblock In \emph{Proceedings of ICLR}, 2021.

\bibitem{gu2023}
A.~Gu and T.~Dao.
\newblock Mamba: Linear-time sequence modeling with selective state spaces.
\newblock arXiv:2312.00752, 2023.

\bibitem{liu2024kan}
Z.~Liu, Y.~Wang, S.~Vaidya, F.~Ruehle, J.~Halverson, M.~Soljačić,
  T.~Y. Hou, and M.~Tegmark.
\newblock {KAN}: Kolmogorov--Arnold networks.
\newblock arXiv:2404.19756, 2024.

\end{thebibliography}

\end{document}