\documentclass[a4paper,10pt]{scrartcl}

\usepackage[utf8x]{inputenc}


\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{amssymb}
\usepackage{amsthm}
\usepackage{graphicx}
\usepackage[cm]{fullpage}

\usepackage{graphicx}
\usepackage[usenames]{color}
\usepackage{subcaption}

\usepackage{tikz}
\usetikzlibrary{arrows}
\usetikzlibrary{circuits.ee.IEC}
\usetikzlibrary{backgrounds}
\usetikzlibrary{decorations.pathmorphing}
\usetikzlibrary{decorations.markings}
\usetikzlibrary{graphs}
\usetikzlibrary{shapes.geometric}
\usetikzlibrary{calc}
\usetikzlibrary{patterns}

\usepackage{lettrine}



\usepackage{upgreek}
\usepackage{xparse}

\newcommand\Tau{\mathcal{T}}
\usepackage{csquotes} % enquotes
%tabulky
\usepackage{tabularx}
\setlength{\extrarowheight}{3pt}

\title{VIR - Bonus Exercise - Batch Norm}
\author{Jiří Jirák}
\date{\today}


\begin{document}
\maketitle

\section{Task}
Consider mini-batch $\mathbf{x} = [1\;3]^\text{T}$, affine function $y_i({\hat{x}_i}) = \gamma \, \hat{x}_i + \beta  $ where $\gamma$ and $\beta$ are the learnable parameters starting as $\gamma = 4 $, $\beta = -2$. Consider Loss function defined as $\mathcal{L} = \sum_{i=1}^n y_i^2$.
\begin{enumerate}
\item 
%--------------------------
Draw the computational graph of given mini-batch, perform a forward pass. \\
Hint: consider $\mu_B = \frac{1}{n} \sum_{k=1}^n x_k$, $\sigma_B = \sqrt{ \frac{1}{n} \sum_{k=1}^n (x_k-\mu_B)^2}$, $\hat{x}_i = \frac{{x}_i - \mathbf{\mu_B}}{\sigma_B}$.
%--------------------------
\item
%-------------------------- 
Calculate $\frac{\partial \mathcal{L}(\mathbf{\hat{x}})}{\partial \gamma}$,  $\frac{\partial \mathcal{L}(\mathbf{\hat{x}})}{\partial \beta}$ and update learnable parameters using learning rate $\alpha = 0.3$.
%--------------------------
\end{enumerate}

\section{Solution}
\begin{enumerate}
\item 
% --------------------------
The computational graph is presented on Figure \ref{fig:sol1}, values from the forward pass are also presented there.
% --------------------------
\begin{figure}[h!]
	\centering
	\includegraphics[width=0.95\textwidth]{./figs/bonus_vir.pdf}
	\caption{Computational graph - solution}
	\label{fig:sol1}
\end{figure}
% --------------------------

\item 
% --------------------------
Calculate
\begin{align*}
\frac{\partial \mathcal{L}(\mathbf{y})}{\partial \gamma} &= \frac{\partial \mathcal{L}(\mathbf{y})}{\partial y_i } \; \frac{\partial y_i}{\partial \gamma} = 2\,y_i \, \hat{x_i} \; \\
\frac{\partial \mathcal{L}(\mathbf{y})}{\partial \beta} &= \frac{\partial \mathcal{L}(\mathbf{y})}{\partial y_i } \; \frac{\partial y_i}{\partial \beta} = 2\,y_i. \; \\
\end{align*}
Now update parameters for i $\in \{1,2\}$, following formula
\begin{align*}
\gamma_{\text{new}} &= \gamma - \alpha \, \frac{\partial \mathcal{L}(\mathbf{y})}{\partial \gamma} \\
\beta_{\text{new}} &= \beta - \alpha \, \frac{\partial \mathcal{L}(\mathbf{y})}{\partial \beta} .\\
\end{align*}
The requested update is
\begin{align*}
i=1: \quad \gamma &= 4-0.3\cdot 2 \cdot (-6) \cdot (-1) = 0.4 , \quad \beta = -2-0.3\cdot 2\cdot (-6)= 1.6\\
i=2: \quad \gamma &= 0.4-0.3\cdot 2 \cdot 2 \cdot 1 = -0.8 , \quad \beta = 1.6-0.3\cdot 2\cdot 2= 0.4.\\
\end{align*}
Therefore after update, learnable parameters are $\gamma = -0.8$, $\beta = 0.4$.
% --------------------------
\end{enumerate}

\end{document}