\documentclass[pdf]
{beamer}
\mode<presentation>{}
%% preamble
\title{CS 4995 Lecture 2: \\ Multilayer Perceptrons \& Backpropagation}
\author{Richard Zemel}
\date{}
\usetheme{CambridgeUS}

\usepackage[round]{natbib}
\usepackage{bbm}
\usepackage{tikz}

\input{../../misc/macros.tex}



\newcommand{\xx}{{\bf x}}
\newcommand{\XX}{{\bf X}}
\newcommand{\ww}{{\bf w}}
\newcommand{\WW}{{\bf W}}
\newcommand{\zz}{{\bf z}}
\newcommand{\vv}{{\bf v}}
\newcommand{\hh}{{\bf h}}
\newcommand{\phivec}{{\boldsymbol \phi}}



\begin{document}

%% title frame
\begin{frame}
  \titlepage
\end{frame}

\begin{frame}{Limits of Linear Classification}
  \begin{itemize}
  \item Single neurons (linear classifiers) are very limited in expressive power.
  \item {\bf XOR} is a classic example of a function that's not linearly separable.
    \begin{center}
      \includegraphics[width=0.3 \textwidth]{figures/xor_data.png}
    \end{center}
  \item There's an elegant proof using convexity.
  \end{itemize}
\end{frame}


\begin{frame}{Limits of Linear Classification}

	{\bf Convex Sets}
	
	\begin{center}
	\includegraphics[width=0.15 \textwidth]{figures/convex_set.png}
	\end{center}
	
	\begin{itemize}
		\item A set $\mathcal{S}$ is \alert{convex} if any line segment connecting points in $\mathcal{S}$ lies entirely within $\mathcal{S}$. Mathematically,
		\[ \mathbf{x}_1, \mathbf{x}_2 \in \mathcal{S} \quad \Longrightarrow \quad \lambda \mathbf{x}_1 + (1-\lambda) \mathbf{x}_2 \in \mathcal{S} \quad \textrm{for } 0 \leq \lambda \leq 1. \]
		\item A simple inductive argument shows that for $\mathbf{x}_1, \ldots, \mathbf{x}_N \in \mathcal{S}$, \alert{weighted averages}, or \alert{convex combinations}, lie within the set:
		\[ \lambda_1 \mathbf{x}_1 + \cdots + \lambda_N \mathbf{x}_N \in \mathcal{S} \quad \textrm{for } \lambda_i > 0, \ \lambda_1 + \cdots \lambda_N = 1. \]
	\end{itemize}
\end{frame}

\begin{frame}{Limits of Linear Classification}
	
	{\bf Showing that XOR is not linearly separable}
	
	\begin{small}
	\begin{itemize}
		\item Half-spaces are obviously convex.
		\item Suppose there were some feasible hypothesis. If the positive examples are in the positive half-space, then the green line segment must be as well.
		\item Similarly, the red line segment must line within the negative half-space.
		\begin{center}
		\includegraphics[width=0.2 \textwidth]{figures/xor_contradiction.png}
		\end{center}
		\item But the intersection can't lie in both half-spaces. Contradiction!
	\end{itemize}
	\end{small}
\end{frame}


\begin{frame}{Limits of Linear Classification}

	{\bf A more troubling example}
	
	\begin{center}
	\includegraphics[width=0.4 \textwidth]{figures/pattern_A.pdf}
	\includegraphics[width=0.4 \textwidth]{figures/pattern_B.pdf}
	\end{center}
	
	\begin{small}
	\begin{itemize}
        \item These images represent 16-dimensional vectors. White = 0, black = 1.
		\item Want to distinguish patterns A and B in all possible translations (with wrap-around)
		\item Translation invariance is commonly desired in vision!
		\pause
		\item Suppose there's a feasible solution. The average of all translations of A is the vector $(0.25, 0.25, \ldots, 0.25)$. Therefore, this point must be classified as A.
		\item Similarly, the average of all translations of B is also $(0.25, 0.25, \ldots, 0.25)$. Therefore, it must be classified as B. Contradiction!
	\end{itemize}
	\end{small}
	
	\flushright {\tiny Credit: Geoffrey Hinton}
\end{frame}

\begin{frame}{Limits of Linear Classification}
	\begin{itemize}
		\item Sometimes we can overcome this limitation using feature maps, just like for linear regression. E.g., for {\bf XOR}:
		\[ \featureMap(\inputVec) = \begin{pmatrix} \inputJ{1} \\ \inputJ{2} \\ \inputJ{1} \inputJ{2} \end{pmatrix} \]
		\begin{center}
		\begin{tabular}{cc|ccc|c}
		$\inputJ{1}$ & $\inputJ{2}$ & $\featureJ{1}(\inputVec)$ & $\featureJ{2}(\inputVec)$ & $\featureJ{3}(\inputVec)$ & $\target$ \\
		\hline
		0 & 0 & 0 & 0 & 0 & 0 \\
		0 & 1 & 0 & 1 & 0 & 1 \\
		1 & 0 & 1 & 0 & 0 & 1 \\
		1 & 1 & 1 & 1 & 1 & 0
		\end{tabular}
		\end{center}
		
		\vspace{0.5em}
		\item This is linearly separable. (Try it!)
		\item Not a general solution: it can be hard to pick good basis functions. Instead, we'll use neural nets to learn nonlinear hypotheses directly. 
	\end{itemize}
\end{frame}



\begin{frame}{Multilayer Perceptrons}
  \begin{columns}
    \begin{column}{0.4 \linewidth}
      \begin{itemize}
      \item We can connect lots of units together into a \alert{directed acyclic graph}. 
      \item This gives a \alert{feed-forward neural network}. That's in contrast to \alert{recurrent neural networks}, which can have cycles. (We'll talk about those later.)
      \item Typically, units are grouped together into \alert{layers}.
      \end{itemize}
    \end{column}
    \begin{column}{0.6 \linewidth}
      \includegraphics[width=\linewidth]{figures/mlp.png}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{Multilayer Perceptrons}
  \begin{footnotesize}
  \begin{itemize}
  \item Each layer connects $N$ input units to $M$ output units. 
    \item In the simplest case, all input units are connected to all output units. We call this a \alert{fully connected layer}. We'll consider other layer types later.
  \item Note: the inputs and outputs for a layer are distinct from the inputs and outputs to the network.
  \end{itemize}
  \vspace{-1.5em}
  \begin{columns}
    \begin{column}{0.6 \linewidth}
      \begin{itemize}
        \item Recall from softmax regression: this means we need an $M \times N$ weight matrix.
      \item The output units are a function of the input units:
        \[ \predictions = \function(\inputVec) = \activationFunction \left( \weightMat \inputVec + \biases \right) \]
      \item A multilayer network consisting of fully connected layers is called a \alert{multilayer perceptron}. Despite the name, it has nothing to do with perceptrons!
      \end{itemize}
    \end{column}
    \begin{column}{0.4 \linewidth}
        \begin{center}
          \includegraphics[width=\linewidth]{figures/fc_layer.png}
        \end{center}
    \end{column}
  \end{columns}
  \end{footnotesize}
\end{frame}

\begin{frame}{Multilayer Perceptrons}
  
  {\bf Some activation functions:}

  \vspace{2em}

  \begin{small}
  \begin{columns}
    \begin{column}{0.3 \linewidth}
      \includegraphics[width=\linewidth]{figures/act_lin.png}
      \begin{center}
        {\bf Linear}
      \end{center}
      \[ y = z \]
    \end{column}

    \begin{column}{0.3 \linewidth}
      \includegraphics[width=\linewidth]{figures/act_relu.png}
      \begin{center}
        {\bf Rectified Linear Unit} \\
        {\bf (ReLU)}
      \end{center}
      \[ y = \max(0, z) \]
    \end{column}

    \begin{column}{0.3 \linewidth}
      \includegraphics[width=\linewidth]{figures/act_soft_relu.png}
      \begin{center}
        {\bf Soft ReLU}
      \end{center}
      \[ y = \log 1 + e^z \]
    \end{column}
  \end{columns}
  \end{small}

\end{frame}

\begin{frame}{Multilayer Perceptrons}

  {\bf Some activation functions:}

  \vspace{2em}
  \begin{small}
  \begin{columns}
    \begin{column}{0.3 \linewidth}
      \includegraphics[width=\linewidth]{figures/act_threshold.png}
      \begin{center}
        {\bf Hard Threshold}
      \end{center}
      \[ y = \left\{ \begin{array}{ll} 1 & \textrm{if } z > 0 \\ 0 & \textrm{if } z \leq 0 \end{array} \right. \]
    \end{column}

    \begin{column}{0.3 \linewidth}
      \includegraphics[width=\linewidth]{figures/act_logistic.png}
      \begin{center}
        {\bf Logistic}
      \end{center}
      \[ y = \frac{1}{1+e^{-z}} \]
    \end{column}

    \begin{column}{0.3 \linewidth}
      \includegraphics[width=\linewidth]{figures/act_tanh.png}
      \begin{center}
        {\bf Hyperbolic Tangent} \\
        {\bf (tanh)}
      \end{center}
      \[ y = \frac{e^{z} - e^{-z}}{e^{z} + e^{-z}} \]
    \end{column}
  \end{columns}
  \end{small}
\end{frame}



\begin{frame}{Multilayer Perceptrons}

  {\bf Designing a network to compute XOR:}

  \vspace{1em}
  Assume hard threshold activation function

  \begin{center}
    \includegraphics[width=0.35 \linewidth]{figures/xor.png}
  \end{center}
\end{frame}

\begin{frame}{Multilayer Perceptrons}
  \begin{center}
    \includegraphics[width=0.5 \linewidth]{figures/xor_completed.png}
  \end{center}
\end{frame}

\begin{frame}{Multilayer Perceptrons}
  \begin{columns}
    \begin{column}{0.75 \linewidth}
      \begin{itemize}
      \item Each layer computes a function, so the network computes a composition of functions:
        \begin{align*}
          \hiddensL{1} &= \functionL{1}(\inputVec) \\
          \hiddensL{2} &= \functionL{2}(\hiddensL{1}) \\
                       & \hspace{0.5em} \vdots \\
          \predictions &= \functionL{\numLayers}(\hiddensL{\numLayers-1})
        \end{align*}
      \item Or more simply:
        \[ \predictions = \functionL{\numLayers} \circ \cdots \circ \functionL{1}(\inputVec). \]
      \item Neural nets provide modularity: we can implement each layer's computations as a black box.
      \end{itemize}
    \end{column}
    \begin{column}{0.25 \linewidth}
      \includegraphics[width=\linewidth]{figures/layer_functions.png}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{Feature Learning}
  \begin{itemize}
  \item Neural nets can be viewed as a way of learning features:
    \begin{center}
      \includegraphics[width=0.45 \textwidth]{figures/network_as_features.png}
    \end{center}
    \pause

  \vspace{-1em}
  \item The goal:
    \begin{center}
      \includegraphics[width=0.6 \textwidth]{figures/features_cartoon.png}
    \end{center}
  \end{itemize}
\end{frame}


\begin{frame}{Feature Learning}
  Input representation of a digit : 784 dimensional vector.
\begin{center}
  \includegraphics[width=0.8\textwidth]{figures/digit_array.png}
\end{center}
   
\end{frame}

\begin{frame}{Feature Learning}
	Each first-layer hidden unit computes $\sigma(\ww_i^T \xx)$
	
	\vspace{0.5em}
	Here is one of the weight vectors (also called a \alert{feature}). 
	
	\vspace{0.5em}
	It's reshaped into an image, with gray = 0, white = +, black = -.
	
	\vspace{0.5em}
	To compute $\ww_i^T \xx$, multiply the corresponding pixels, and sum the result.
	
	\vspace{2em}
	\begin{center}
	\includegraphics[width=0.15 \textwidth]{figures/one_filter.png}
	\end{center}
	
\end{frame}


\begin{frame}{Feature Learning}
  There are 256 first-level features total. Here are some of them.
  \vspace{2em}
	\begin{center}
	\includegraphics[width=0.4\textwidth]{figures/mnist_filters.png}
	\end{center}
\end{frame}


\begin{frame}{Expressive Power}
  \begin{itemize}
  \item We've seen that there are some functions that linear classifiers can't represent. Are deep networks any better?
  \item Any sequence of \emph{linear} layers can be equivalently represented with a single linear layer. 
    \[ \predictions = \underbrace{\weightMatL{3} \weightMatL{2} \weightMatL{1}}_{\triangleq \weightMat^\prime} \inputVec \]
    \begin{itemize}
    \item Deep linear networks are no more expressive than linear regression!
    \item Linear layers do have their uses --- stay tuned!
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Expressive Power}
  \begin{itemize}
    \item Multilayer feed-forward neural nets with \emph{nonlinear} activation functions are \alert{universal approximators}: they can approximate any function arbitrarily well.
    \item This has been shown for various activation functions (thresholds, logistic, ReLU, etc.)
      \begin{itemize}
      \item Even though ReLU is ``almost'' linear, it's nonlinear enough!
      \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Expressive Power}

  {\bf Universality for binary inputs and targets:}
  \begin{itemize}
    \item Hard threshold hidden units, linear output
    \item Strategy: $2^\nfeat$ hidden units, each of which responds to one particular input configuration
      \begin{center}
        \includegraphics[width=0.8 \linewidth]{figures/universal.png}
      \end{center}
    \item Only requires one hidden layer, though it needs to be extremely wide!
  \end{itemize}
\end{frame}

\begin{frame}{Expressive Power}
  \begin{itemize}
    \item What about the logistic activation function?
    \item You can approximate a hard threshold by scaling up the weights and biases:
      \begin{columns}
        \begin{column}{0.4 \textwidth}
          \begin{center}
            \includegraphics[width=0.8 \textwidth]{figures/scaling_1.png}
          \end{center}
          \[ y = \sigma(x) \]
        \end{column}
        \begin{column}{0.4 \textwidth}
          \begin{center}
            \includegraphics[width=0.8 \textwidth]{figures/scaling_5.png}
          \end{center}
          \[ y = \sigma(5x) \]
        \end{column}
      \end{columns}

      \vspace{0.5em}
    \item This is good: logistic units are differentiable, so we can tune them with gradient descent. (Stay tuned!)
  \end{itemize}
\end{frame}

\begin{frame}{Expressive Power}
  \begin{itemize}
  \item Limits of universality
    \pause
    \begin{itemize}
      \item You may need to represent an exponentially large network.
      \item If you can learn any function, you'll just overfit.
      \item Really, we desire a \emph{compact} representation!
    \end{itemize}
    \pause
  \item We've derived units which compute the functions AND, OR, and NOT. Therefore, any Boolean circuit can be translated into a feed-forward neural net. 
    \begin{itemize}
      \item This suggests you might be able to learn \emph{compact} representations of some complicated functions
      %\item The view of neural nets as ``differentiable computers'' is starting to take hold. More about this when we talk about recurrent neural nets.
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Overview}
  \begin{itemize}
  \item We've seen that multilayer neural networks are powerful. But how can we actually learn them?
  \item Backpropagation is the central algorithm in this course.
    \begin{itemize}
      \item It's is an algorithm for computing gradients.
      \item Really it's an instance of reverse mode automatic differentiation, which is much more broadly applicable than just neural nets.
        \begin{itemize}
          \item This is ``just'' a clever and efficient use of the Chain Rule for derivatives.
          \item We'll see how to implement an automatic differentiation system next week.
        \end{itemize}
    \end{itemize}
  \end{itemize}
\end{frame}

\begin{frame}{Recap: Gradient Descent}

  \begin{small}
  \begin{itemize}
    \item {\bf Recall:} gradient descent moves opposite the gradient (the direction of steepest descent)
  \end{itemize}

  \vspace{-2em}
  \begin{center}
    \includegraphics[width=0.3 \textwidth]{figures/gradient_descent.png}
  \end{center}

  \vspace{-1em}
  \begin{itemize}
  \item Weight space for a multilayer neural net: one coordinate for each weight or bias of the network, in \emph{all} the layers
  \item Conceptually, not any different from what we've seen so far --- just higher dimensional and harder to visualize!
  \item We want to compute the cost gradient $\deriv \cost / \deriv \weights$, which is the vector of partial derivatives. 
    \begin{itemize}
    \item This is the average of $\deriv \loss / \deriv \weights$ over all the training examples, so in this lecture we focus on computing $\deriv \loss / \deriv \weights$.
    \end{itemize}
  \end{itemize}
  \end{small}
\end{frame}

\begin{frame}{Univariate Chain Rule}
  \begin{itemize}
    \item We've already been using the univariate Chain Rule.
    \item Recall: if $f(x)$ and $x(t)$ are univariate functions, then
      \[ \frac{\deriv}{\deriv t} f(x(t)) = \frac{\deriv f}{\deriv x} \frac{\deriv x}{\deriv t}. \]
  \end{itemize}
\end{frame}

\begin{frame}{Univariate Chain Rule}
  
  {\bf Recall: Univariate logistic least squares model}

  \begin{align*}
    \intermediate &= \weightUni \inputUni + \bias \\
    \prediction &= \sigmoid(\intermediate) \\
    \loss &= \frac{1}{2} (\prediction - \target)^2 \\
  \end{align*}

  Let's compute the loss derivatives.
\end{frame}

\begin{frame}{Univariate Chain Rule}
  
  {\bf How you would have done it in calculus class}
  \begin{scriptsize}
  \begin{minipage}{0.45 \linewidth}
    \begin{align*}
      \loss &= \frac{1}{2} ( \sigmoid(\weightUni \inputUni + \bias) - \target)^2 \\
      \frac{\partial \loss}{\partial \weightUni} &= \frac{\partial}{\partial \weightUni} \left[ \frac{1}{2} ( \sigmoid(\weightUni \inputUni + \bias) - \target)^2 \right] \\
            &= \frac{1}{2} \frac{\partial}{\partial \weightUni} ( \sigmoid(\weightUni \inputUni + \bias) - \target)^2 \\
            &= (\sigma(\weightUni \inputUni + \bias) - \target) \frac{\partial}{\partial \weightUni} (\sigma(\weightUni \inputUni + \bias) - \target) \\
            &= (\sigma(\weightUni \inputUni + \bias) - \target) \sigma^\prime (\weightUni \inputUni + \bias) \frac{\partial}{\partial \weightUni} (\weightUni \inputUni + \bias) \\
            &= (\sigma(\weightUni \inputUni + \bias) - \target) \sigma^\prime (\weightUni \inputUni + \bias) \inputUni
    \end{align*}
  \end{minipage}
  \begin{minipage}{0.45 \linewidth}
    \begin{align*}
    \frac{\partial \loss}{\partial \bias} 
    &= \frac{\partial}{\partial \bias} \left[ \frac{1}{2} ( \sigmoid(\weightUni \inputUni + \bias) - \target)^2  \right] \\
    &= \frac{1}{2} \frac{\partial}{\partial \bias} ( \sigmoid(\weightUni \inputUni + \bias) - \target)^2 \\
    &= ( \sigmoid(\weightUni \inputUni + \bias) - \target) \frac{\partial}{\partial \bias} ( \sigmoid(\weightUni \inputUni + \bias) - \target)  \\
    &= (\sigma(\weightUni \inputUni + \bias) - \target) \sigma^\prime (\weightUni \inputUni + \bias) \frac{\partial}{\partial \bias} (\weightUni \inputUni + \bias) \\
    &= (\sigma(\weightUni \inputUni + \bias) - \target) \sigma^\prime (\weightUni \inputUni + \bias) 
  \end{align*}
  \end{minipage}
  \end{scriptsize}

  \vspace{1em}
  What are the disadvantages of this approach?
\end{frame}


\begin{frame}{Univariate Chain Rule}
  
  {\bf A more structured way to do it}
  
  \vspace{2em}
  \begin{footnotesize}
  \begin{minipage}{0.45 \linewidth}
    {\bf Computing the loss:}
    \begin{align*}
      \intermediate &= \weightUni \inputUni + \bias \\
      \prediction &= \sigmoid(\intermediate) \\
      \loss &= \frac{1}{2} (\prediction - \target)^2 \\
    \end{align*}
  \end{minipage}
  \begin{minipage}{0.45 \linewidth}
    {\bf Computing the derivatives:}
    \begin{align*}
      \frac{\deriv \loss}{\deriv \prediction} &= \prediction - \target \\
      \frac{\deriv \loss}{\deriv \intermediate} &= \frac{\deriv \loss}{\deriv \prediction} \, \logistic^\prime(\intermediate) \\
      \frac{\partial \loss}{\partial \weightUni} &= \frac{\deriv \loss}{\deriv \intermediate} \, \inputUni \\
      \frac{\partial \loss}{\partial \bias} &= \frac{\deriv \loss}{\deriv \intermediate}
    \end{align*}
  \end{minipage}
  \end{footnotesize}

  \vspace{2em}
  Remember, the goal isn't to obtain closed-form solutions, but to be able to write a program that efficiently computes the derivatives.

\end{frame}

\begin{frame}{Univariate Chain Rule}
  \begin{itemize}
  \item We can diagram out the computations using a \alert{computation graph}.
  \item The nodes represent all the inputs and computed quantities, and the edges represent which nodes are computed directly as a function of which other nodes.
  \end{itemize}
  \begin{center}
    \includegraphics[width=0.35 \textwidth]{figures/unreg_computation_graph.png}
  \end{center}
\end{frame}


\begin{frame}{Univariate Chain Rule}

  {\bf A slightly more convenient notation:}

  \begin{small}
  \begin{itemize}
  \item Use $\costDeriv{\prediction}$ to denote the derivative $\deriv \loss / \deriv \prediction$, sometimes called the \alert{error signal}.
  \item This emphasizes that the error signals are just values our program is computing (rather than a mathematical operation).
  \item This is not a standard notation, but I couldn't find another one that I liked.
  \end{itemize}
  \end{small}

  \vspace{1em}
  \begin{small}
  \begin{minipage}{0.45 \linewidth}
    {\bf Computing the loss:}
    \begin{align*}
      \intermediate &= \weightUni \inputUni + \bias \\
      \prediction &= \sigmoid(\intermediate) \\
      \loss &= \frac{1}{2} (\prediction - \target)^2 \\
    \end{align*}
  \end{minipage}
  \begin{minipage}{0.45 \linewidth}
    {\bf Computing the derivatives:}
    \begin{align*}
      \costDeriv{\prediction} &= \prediction - \target \\
      \costDeriv{\intermediate} &= \costDeriv{\prediction} \, \logistic^\prime(\intermediate) \\
      \costDeriv{\weightUni} &= \costDeriv{\intermediate} \, \inputUni \\
      \costDeriv{\bias} &= \costDeriv{\intermediate}
    \end{align*}
  \end{minipage}
  \end{small}

\end{frame}

\begin{frame}{Multivariate Chain Rule}
  
  {\bf Problem:} what if the computation graph has \alert{fan-out} $> 1$? \\
 
  This requires the \alert{multivariate Chain Rule}!

  \vspace{1em}
  \begin{minipage}{0.45 \textwidth}

    
    \begin{center}
      {\bf $L_2$-Regularized regression}
      
      \vspace{0.5em}
      \includegraphics[width=0.9 \linewidth]{figures/reg_computation_graph.png}
    \end{center}

    \vspace{-2em}
    \begin{footnotesize}
    \begin{align*}
      \intermediate &= \weightUni \inputUni + \bias \\
      \prediction &= \sigmoid(\intermediate) \\
      \loss &= \frac{1}{2} (\prediction - \target)^2 \\
      \regularizer &= \frac{1}{2} w^2 \\
      \loss_{\rm reg} &= \loss + \weightCost \regularizer
    \end{align*}
    \end{footnotesize}
  \end{minipage}
  \begin{minipage}{0.45 \textwidth}
    \begin{center}
      {\bf Multiclass logistic regression}

      \vspace{0.5em}
      \includegraphics[width=0.6 \linewidth]{figures/multiway_graph.png}
    \end{center}

    \vspace{-2em}
    \begin{footnotesize}
    \begin{align*}
      \intermediateK{\classIdxTwo} &= \sum_{\featIdx} \weightKJ{\classIdxTwo}{\featIdx} \inputJ{\featIdx} + \biasK{\classIdxTwo} \\
      \predictionK{\classIdx} &= \frac{e^{\intermediateK{\classIdx}}}{\sum_{\classIdxTwo} e^{\intermediateK{\classIdxTwo}}} \\
      \loss &= -\sum_{\classIdx} \targetK{\classIdx} \log \predictionK{\classIdx}
    \end{align*}
    \end{footnotesize}

  \end{minipage}
\end{frame}



\begin{frame}{Multivariate Chain Rule}
  \begin{itemize}
    \item Suppose we have a function $f(x, y)$ and functions $x(t)$ and $y(t)$. (All the variables here are scalar-valued.) Then
      \begin{columns}
        \begin{column}{0.7 \linewidth}
          \[ \frac{\deriv}{\deriv t} f(x(t), y(t)) = \frac{\partial f}{\partial x} \frac{\deriv x}{\deriv t} + \frac{\partial f}{\partial y} \frac{\deriv y}{\deriv t} \]
        \end{column}
        \begin{column}{0.2 \linewidth}
          \includegraphics[width=\linewidth]{figures/toy_multivariate_graph.png}
        \end{column}
      \end{columns}
    \item Example:
      \begin{align*}
        f(x, y) &= y + e^{xy} \\
        x(t) &= \cos t \\
        y(t) &= t^2
      \end{align*}
    \item Plug in to Chain Rule:
      \begin{align*}
        \frac{\deriv f}{\deriv t} &= \frac{\partial f}{\partial x} \frac{\deriv x}{\deriv t} + \frac{\partial f}{\partial y} \frac{\deriv y}{\deriv t} \\
        &= \left( y e^{xy} \right) \cdot \left( -\sin t \right) + \left( 1 + xe^{xy} \right) \cdot 2t
      \end{align*}
  \end{itemize}
\end{frame}

\begin{frame}{Multivariable Chain Rule}
  
  \begin{itemize}
  \item In the context of backpropagation:
    \vspace{1em}
    \begin{center}
      \includegraphics[width=0.35 \linewidth]{figures/label_equation.png}
      \hspace{0.1 \linewidth}
      \includegraphics[width=0.4 \linewidth]{figures/multivariate_context.png}
    \end{center}
    \vspace{1em}
  \item In our notation:
    \[ \costDeriv{t} = \costDeriv{x} \, \frac{\deriv x}{\deriv t} + \costDeriv{y} \, \frac{\deriv y}{\deriv t} \]
  \end{itemize}
  
\end{frame}


\begin{frame}{Backpropagation}
  
  {\bf Full backpropagation algorithm:}
  \begin{itemize}
    \begin{small}
  \item[] Let $v_1, \ldots, v_N$ be a \alert{topological ordering} of the computation graph (i.e.~parents come before children.)
  \item[] $v_N$ denotes the variable we're trying to compute derivatives of (e.g.~loss). 
    \end{small}
  \end{itemize}

  \begin{center}
    \includegraphics[width=0.8 \linewidth]{figures/passes.png}
  \end{center}

\end{frame}


\begin{frame}{Backpropagation}
  
  {\bf Example:} univariate logistic least squares regression

  \vspace{1em}
  \begin{columns}
    \begin{column}{0.4 \linewidth}
      \includegraphics[width=\linewidth]{figures/reg_computation_graph.png}
      
      \vspace{1em}
      {\bf Forward pass:}
      \begin{footnotesize}
        \begin{align*}
          \intermediate &= \weightUni \inputUni + \bias \\
          \prediction &= \sigmoid(\intermediate) \\
          \loss &= \frac{1}{2} (\prediction - \target)^2 \\
          \regularizer &= \frac{1}{2} w^2 \\
          \loss_{\rm reg} &= \loss + \weightCost \regularizer
        \end{align*}
      \end{footnotesize}
    \end{column}
    
    \begin{column}{0.25 \linewidth}
      {\bf Backward pass:}
      
      \begin{footnotesize}
        \begin{align*}
          \costDeriv{\loss_{\rm reg}} &= 1 \\
          \costDeriv{\regularizer} &= \costDeriv{\loss_{\rm reg}}\, \frac{\deriv \loss_{\rm reg}}{\deriv \regularizer} \\
                                      &= \costDeriv{\loss_{\rm reg}} \, \weightCost  \\
          \costDeriv{\loss} &= \costDeriv{\loss_{\rm reg}}\, \frac{\deriv \loss_{\rm reg}}{\deriv \loss} \\
                                      &= \costDeriv{\loss_{\rm reg}} \\
          \costDeriv{\prediction} &= \costDeriv{\loss}\, \frac{\deriv \loss}{\deriv \prediction} \\
                                      &= \costDeriv{\loss}\, (\prediction - \target) \\
        \end{align*}
      \end{footnotesize}
    \end{column}

    \begin{column}{0.25 \linewidth}
      \begin{footnotesize}
        \begin{align*}
          \costDeriv{\intermediate} &= \costDeriv{\prediction} \, \frac{\deriv \prediction}{\deriv \intermediate} \\
                                    &= \costDeriv{\prediction} \, \logistic^\prime(\intermediate) \\
          \textcolor[rgb]{0.9, 0, 0.9}{\costDeriv{\weightUni}} & \textcolor[rgb]{0.9, 0, 0.9}{= \costDeriv{\intermediate} \, \frac{\partial \intermediate}{\partial \weightUni} + \costDeriv{\regularizer} \frac{\deriv \regularizer}{\deriv \weightUni}}  \\
                                   & \textcolor[rgb]{0.9, 0, 0.9}{= \costDeriv{\intermediate} \, \inputUni + \costDeriv{\regularizer} \, \weightUni } \\
          \costDeriv{\bias} &= \costDeriv{\intermediate} \, \frac{\partial \intermediate}{\partial \bias} \\
                                    &= \costDeriv{\intermediate}
        \end{align*}
      \end{footnotesize}
    \end{column}
  \end{columns}
\end{frame}

\begin{frame}{Backpropagation}
  
  {\bf Multilayer Perceptron} (multiple outputs):

  \begin{columns}
    \begin{column}{0.45 \linewidth}

      \includegraphics[width=0.8 \linewidth]{figures/mlp_graph.png}

      \vspace{-0.5em}
      {\bf Forward pass:}

      \vspace{-1em}
      \begin{footnotesize}
      \begin{align*}
        %\intermediates &= \weightMatL{1} \inputVec + \biasesL{1} \\
        %\hiddens &= \logistic(\intermediates) \\
        %\predictions &= \weightMatL{2} \hiddens + \biasesL{2} \\
        %\loss &= \| \targets - \predictions \|^2 \\
        \intermediateK{\hidIdx} &= \sum_{\featIdx} \weightLIJ{1}{\hidIdx}{\featIdx} \inputJ{\featIdx} + \biasLI{1}{\hidIdx} \\
        \hiddenI{\hidIdx} &= \logistic(\intermediateK{\hidIdx}) \\
        \predictionK{\outputIdx} &= \sum_{\hidIdx} \weightLIJ{2}{\outputIdx}{\hidIdx} \hiddenI{\hidIdx} + \biasLI{2}{\outputIdx} \\
        \loss &= \frac{1}{2} \sum_\outputIdx (\predictionK{\outputIdx} - \targetK{\outputIdx})^2
      \end{align*}
      \end{footnotesize}
    \end{column}

    \begin{column}{0.45 \linewidth}
      {\bf Backward pass:}

      \vspace{-1em}
      \begin{footnotesize}
      \begin{align*}
        \costDeriv{\loss} &= 1 \\
        \costDeriv{\predictionK{\outputIdx}} &= \costDeriv{\loss} \, (\predictionK{\outputIdx} - \targetK{\outputIdx}) \\
        \costDeriv{\weightLIJ{2}{\outputIdx}{\hidIdx}} &= \costDeriv{\predictionK{\outputIdx}} \, \hiddenI{\hidIdx} \\
        \costDeriv{\biasLI{2}{\outputIdx}} &= \costDeriv{\predictionK{\outputIdx}} \\
        \costDeriv{\hiddenI{\hidIdx}} &= \sum_{\outputIdx} \costDeriv{\predictionK{\outputIdx}} \weightLIJ{2}{\outputIdx}{\hidIdx} \\
        \costDeriv{\intermediateK{\hidIdx}} &= \costDeriv{\hiddenI{\hidIdx}} \, \logistic^\prime(\intermediateK{\hidIdx}) \\
        \costDeriv{\weightLIJ{1}{\hidIdx}{\featIdx}} &= \costDeriv{\intermediateK{\hidIdx}} \, \inputJ{\featIdx} \\
        \costDeriv{\biasLI{1}{\hidIdx}} &= \costDeriv{\intermediateK{\hidIdx}} \\
      \end{align*}
      \end{footnotesize}
    \end{column}
  \end{columns}
  
\end{frame}

\begin{frame}{Vector Form}
  \begin{itemize}
  \item Computation graphs showing individual units are cumbersome.
  \item As you might have guessed, we typically draw graphs over the vectorized variables.
    \begin{center}
      \includegraphics[width=0.4 \linewidth]{figures/mlp_graph_vectorized.png}
    \end{center}
  \item We pass messages back analogous to the ones for scalar-valued nodes.
  \end{itemize}
\end{frame}

\begin{frame}{Vector Form}
  \begin{itemize}
  \item Consider this computation graph:

    \vspace{0.2em}
    \begin{center}
      \includegraphics[width=0.4 \linewidth]{figures/vectorize_the_graph.png}
    \end{center}
  \item Backprop rules:
    \[ \costDeriv{z_j} = \sum_k \costDeriv{y_k} \, \frac{\partial y_k}{\partial z_j} \hspace{4em} 
      \costDeriv{\mathbf{z}} = \frac{\partial \mathbf{y}}{\partial \mathbf{z}}^\transpose \costDeriv{\mathbf{y}}, \]
  where $\partial \mathbf{y} / \partial \mathbf{z}$ is the \alert{Jacobian matrix}:
  \[ \frac{\partial \mathbf{y}}{\partial \mathbf{z}} = 
      \begin{pmatrix}
        \frac{\partial y_1}{\partial z_1} & \cdots & \frac{\partial y_1}{\partial z_n} \\ 
        \vdots & \ddots & \vdots \\
        \frac{\partial y_m}{\partial z_1} & \cdots & \frac{\partial y_m}{\partial z_n}
      \end{pmatrix} \]
  \end{itemize}
\end{frame}

\begin{frame}{Vector Form}
  Examples
  \begin{itemize}
  \item Matrix-vector product
    \begin{footnotesize}
    \[ \mathbf{z} = \mathbf{W} \mathbf{x} \qquad \frac{\partial \mathbf{z}}{\partial \mathbf{x}} = \mathbf{W} \qquad \costDeriv{\mathbf{x}} = \mathbf{W}^\transpose \costDeriv{\mathbf{z}} \]
    \end{footnotesize}
  \item Elementwise operations
    \begin{footnotesize}
    \[ \mathbf{y} = \exp(\mathbf{z}) \qquad \frac{\partial \mathbf{y}}{\partial \mathbf{z}} = \begin{pmatrix} \exp(z_1) && 0 \\ & \ddots & \\ 0 && \exp(z_D) \end{pmatrix} \qquad \costDeriv{\mathbf{z}} = \exp(\mathbf{z}) \circ \costDeriv{\mathbf{y}} \]
    \end{footnotesize}
  \item Note: we never explicitly construct the Jacobian. It's usually simpler and more efficient to compute the VJP directly.
  \end{itemize}
\end{frame}


\begin{frame}{Vector Form}
  
  {\bf Full backpropagation algorithm (vector form):}
  \begin{itemize}
    \begin{small}
  \item[] Let $\mathbf{v}_1, \ldots, \mathbf{v}_N$ be a \alert{topological ordering} of the computation graph (i.e.~parents come before children.)
  \item[] $\mathbf{v}_N$ denotes the variable we're trying to compute derivatives of (e.g.~loss). It's a scalar, which we can treat as a 1-D vector.
    \end{small}
  \end{itemize}

  \begin{center}
    \includegraphics[width=0.8 \linewidth]{figures/passes_vec.png}
  \end{center}

\end{frame}



\begin{frame}{Vector Form}
  
  {\bf MLP example in vectorized form:}
  
  \vspace{1em}
  \begin{columns}
    \begin{column}{0.45 \linewidth}
      \includegraphics[width=0.9 \linewidth]{figures/mlp_graph_vectorized.png}

      {\bf Forward pass:}
      \begin{align*}
        \intermediates &= \weightMatL{1} \inputVec + \biasesL{1} \\
        \hiddens &= \logistic(\intermediates) \\
        \predictions &= \weightMatL{2} \hiddens + \biasesL{2} \\
        \loss &= \frac{1}{2} \| \targets - \predictions \|^2
      \end{align*}
    \end{column}
    \begin{column}{0.45 \linewidth}
      {\bf Backward pass:}

      \vspace{-1.5em}
      \begin{align*}
        \costDeriv{\loss} &= 1 \\
        \costDeriv{\predictions} &= \costDeriv{\loss} \, (\predictions - \targets) \\
        \costDeriv{\weightMatL{2}} &= \costDeriv{\predictions} \hiddens^\transpose \\
        \costDeriv{\biasesL{2}} &= \costDeriv{\predictions} \\
        \costDeriv{\hiddens} &= \weightMat^{(2) \transpose} \costDeriv{\predictions} \\
        \costDeriv{\intermediates} &= \costDeriv{\hiddens} \circ \logistic^\prime(\intermediates) \\
        \costDeriv{\weightMatL{1}} &= \costDeriv{\intermediates} \inputVec^\transpose \\
        \costDeriv{\biasesL{1}} &= \costDeriv{\intermediates}
      \end{align*}
    \end{column}
  \end{columns}

\end{frame}

% \begin{frame}{Backpropagation}
  
%   {\bf Backprop as message passing:}
%   \begin{center}
%     \includegraphics[width=0.55 \linewidth]{figures/message_passing.png}
%   \end{center}

%   \begin{itemize}
%     \item Each node receives a bunch of messages from its children, which it aggregates to get its error signal. It then passes messages to its parents.
%     \item This provides modularity, since each node only has to know how to compute derivatives with respect to its arguments, and doesn't have to know anything about the rest of the graph.
%   \end{itemize}

% \end{frame}


\begin{frame}{Computational Cost}
  \begin{itemize}
    \item Computational cost of forward pass: one \alert{add-multiply operation} per weight
      \[ \intermediateK{\hidIdx} = \sum_{\featIdx} \weightLIJ{1}{\hidIdx}{\featIdx} \inputJ{\featIdx} + \biasLI{1}{\hidIdx} \]
    \item Computational cost of backward pass: two add-multiply operations per weight
      \begin{align*}
        \costDeriv{\weightLIJ{2}{\outputIdx}{\hidIdx}} &= \costDeriv{\predictionK{\outputIdx}} \, \hiddenI{\hidIdx} \\
        \costDeriv{\hiddenI{\hidIdx}} &= \sum_{\outputIdx} \costDeriv{\predictionK{\outputIdx}} \weightLIJ{2}{\outputIdx}{\hidIdx}
      \end{align*}
    \item Rule of thumb: the backward pass is about as expensive as two forward passes.
    \item For a multilayer perceptron, this means the cost is linear in the number of layers, quadratic in the number of units per layer.
  \end{itemize}
\end{frame}


\begin{frame}{Closing Thoughts}
  \begin{small}
  \begin{itemize}
  \item Backprop is used to train the overwhelming majority of neural nets today. %Nobody has come up with an alternative that can compete with it.
    \begin{itemize}
      \item Even optimization algorithms much fancier than gradient descent (e.g.~second-order methods) use backprop to compute the gradients.
    \end{itemize}
    \item Despite its practical success, backprop is believed to be neurally implausible.
      \begin{itemize}
        \item No evidence for biological signals analogous to error derivatives.
        \item All the biologically plausible alternatives we know about learn much more slowly (on computers).
        \item So how on earth does the brain learn?
      \end{itemize}
  \end{itemize}
\end{small}
\end{frame}

%\begin{frame}{Levels of Abstraction}
%
%  \begin{center}
%    The psychological profiling [of a programmer] is mostly the ability to shift levels of abstraction, from low level to high level. To see something in the small and to see something in the large.
%  \end{center}
%  \flushright -- Don Knuth
%\end{frame}
%
%\begin{frame}{Levels of Abstraction}
%
%  When you design neural networks and machine learning algorithms, you'll need to think at multiple levels of abstraction.
%
%  \vspace{1.5em}
%  \begin{center}
%    \includegraphics[width=0.5 \textwidth]{figures/abstraction.png}
%  \end{center}
%\end{frame}

\begin{frame}{Closing Thoughts}
  \begin{center}
    The psychological profiling [of a programmer] is mostly the ability to shift levels of abstraction, from low level to high level. To see something in the small and to see something in the large.
  \end{center}
  \flushright -- Don Knuth
  \begin{itemize}
    \item By now, we've seen three different ways of looking at gradients:
      \begin{itemize}
      \item {\bf Geometric:} visualization of gradient in weight space
      \item {\bf Algebraic:} mechanics of computing the derivatives
      \item {\bf Implementational:} efficient implementation on the computer
      \end{itemize}
    \item When thinking about neural nets, it's important to be able to shift between these different perspectives!
  \end{itemize}
\end{frame}



\end{document}

