\chapter{Assignment- Differential Equations}
\section{MCQ}
\begin{enumerate}
	\item $\left. \right. $
	\begin{answer}
		\begin{align*}
			\int \frac{y}{\sqrt{1+y^{2}}} d y+\int \frac{x}{\sqrt{1+x^{2}}} d x=c \Rightarrow \sqrt{1+y^{2}}+\sqrt{1+x^{2}}=c
		\end{align*}
		So the correct answer is \textbf{Option (c)}
	\end{answer}
	\item $\left. \right. $
	\begin{answer}
		\begin{align*}
		x d y&=-\cot y d x \Rightarrow \tan y d y=-\frac{d x}{x}\\
		\Rightarrow \int \tan y d y&=-\int \frac{d x}{x}+\log A \Rightarrow \log \sec y=-\log x+\log A \Rightarrow \log \sec y=\log \frac{A}{x}\\
		\Rightarrow x&=A \cos y \Rightarrow \sqrt{2}=A \cos \frac{\pi}{4} \Rightarrow A=2 \Rightarrow x=2 \cos y
		\end{align*}
		So the correct answer is \textbf{Option (a)}
	\end{answer}
	\item $\left. \right. $
	\begin{answer}
		\begin{align*}
		\frac{d y}{d x}&=-\frac{x}{y+1} \Rightarrow x d x+y d y+d y=0 \Rightarrow \frac{x^{2}}{2}+\frac{y^{2}}{2}+y=C_{1} \Rightarrow x^{2}+y^{2}+2 y=2 C_{1}\\
		\Rightarrow(x-0)^{2}+(y+1)^{2}&=2 C_{1}+1=C
		\intertext{which is family of circles with different radii.}
		\end{align*}
		So the correct answer is \textbf{Option (a)}
	\end{answer}
	\item $\left. \right. $
\begin{answer}
	\begin{align*}
	x y \frac{d y}{d x}&=3 y^{2}+x^{2} \Rightarrow \frac{d y}{d x}=\frac{3 y^{2}+x^{2}}{x y}\text{ which is a homogeneous equation.}\\
	\text{Put }y&=v x \Rightarrow \frac{d y}{d x}=v+x \frac{d v}{d x} \Rightarrow v+x \frac{d v}{d x}=\frac{3 v^{2} x^{2}+x^{2}}{v x^{2}}=\frac{3 v^{2}+1}{v}\\
	\Rightarrow x \frac{d v}{d x}&=\frac{3 v^{2}+1}{v}-v=\frac{2 v^{2}+1}{v} \Rightarrow \frac{v}{2 v^{2}+1} d v=\frac{d x}{x}\\
	\Rightarrow \frac{1}{4} \ln \left(2 v^{2}+1\right)&=\ln x+\ln c^{\prime} \Rightarrow \ln \left(2 v^{2}+1\right)=\ln x^{4}+\ln c \Rightarrow\left(2 v^{2}+1\right)=c x^{4}\\
	\Rightarrow\left(2 \frac{y^{2}}{x^{2}}+1\right)&=c x^{4}\\
	y=2\text{ when }x&=1 \Rightarrow\left(2 \frac{4}{1}+1\right)=c .1 \Rightarrow c=9 \Rightarrow 2 y^{2}+x^{2}=9 x^{6}
	\end{align*}
		So the correct answer is \textbf{Option (a)}
\end{answer}
	\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\frac{d y}{d x}=\frac{x+2 y-1}{x+2 y+1}\text{. Let }x+2 y&=z \Rightarrow 1+2 \frac{d y}{d x}=\frac{d z}{d x} \Rightarrow \frac{d z}{d x}=\frac{3 z-1}{z+1}\\
	\Rightarrow \int\left(\frac{1}{3}+\frac{1}{4}-\frac{1}{3 z-1}\right) d z&=\int d x+c \Rightarrow \frac{z}{3}+\frac{4}{9} \log (3 z-1)=x+c\\
	\Rightarrow 3 x-3 y+a&=2 \log (3 x+6 y-1)
	\end{align*}
		So the correct answer is \textbf{Option (b)}
\end{answer}
	\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\text{I.F. }=e^{\int 3 d x}&=e^{3 x} \Rightarrow y \times e^{3 x}=\int e^{2 x} \times e^{3 x} d x+c \Rightarrow y \times e^{3 x}=\frac{e^{5 x}}{5}+c \Rightarrow y=c_{1} e^{2 x}+c_{2} e^{-3 x}
	\end{align*}
		So the correct answer is \textbf{Option (c)}
\end{answer}
\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\frac{d y}{d x}+\frac{1}{x} y&=x^{3} \Rightarrow \mathrm{I} . F .=e^{\int \frac{1}{x} d x}=e^{\ln x}=x\\
	\Rightarrow y \times x&=\int x^{3} \times x d x+c \Rightarrow y \times x=\frac{x^{5}}{5}+c \Rightarrow y=\frac{x^{4}}{5}+\frac{c}{x}\\
\text{	Since }y&=1\text{ at }x=1 \Rightarrow 1=\frac{1}{5}+\frac{c}{1} \Rightarrow c=\frac{4}{5} \Rightarrow y=\frac{x^{4}}{5}+\frac{4}{5 x}
	\end{align*}
		So the correct answer is \textbf{Option (d)}
\end{answer}
	\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\frac{d x}{d t}+\frac{K_{2}}{K_{1}} x&=\frac{K_{3}}{K_{1}} \Rightarrow \frac{d x}{d t}+A x=B \Rightarrow I . F .=e^{\int A d t}=e^{A t} \Rightarrow x \times e^{A t}=\int\left(B \times e^{A t}\right) d t+c\\
	\Rightarrow x \times e^{A t}&=B \times \frac{e^{A t}}{A}+c \Rightarrow x=c_{1}+c e^{-A t}\\
	\text{Since }x&=0\text{ at }t=0 \Rightarrow c_{1}+c=0 \Rightarrow c_{1}=-c \Rightarrow x=c\left(1-e^{-A t}\right)
	\end{align*}
		So the correct answer is \textbf{Option (a)}
\end{answer}
	\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\text{Divide by }y t \Rightarrow \frac{1}{y} \frac{d y}{d t}+\frac{1}{t} \log y&=e^{t}\text{, put }\log y=z \Rightarrow \frac{1}{y} \frac{d y}{d t}=\frac{d z}{d t}\\
	\Rightarrow \frac{d z}{d t}+\frac{1}{t} \cdot z&=e^{t} \Rightarrow I \cdot F=e^{\int_{t}^{1} d t}=e^{\log t}=t\\
	\Rightarrow z t&=\int t e^{t} d t+c \Rightarrow t \log y=t e^{t}-e^{t}+c
	\end{align*}
		So the correct answer is \textbf{Option (d)}
\end{answer}
	\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\left(1+y^{2}\right) d x&=\left(\tan ^{-1} y-x\right) d y \Rightarrow \frac{d x}{d y}=\frac{\tan ^{-1} y-x}{1+y^{2}}\\
	\Rightarrow \frac{d x}{d y}+\frac{x}{1+y^{2}}&=\frac{\tan ^{-1} y}{1+y^{2}}.\text{ This is a linear differential equation.}\\
	I . F .&=e^{\int \frac{1}{1+y^{2}} d y}=e^{\tan ^{-1} y}\\
	\text{	Its solution is }x . e^{\tan ^{-1} y}&=\int e^{\tan ^{-1} y} \frac{\tan ^{-1} y}{1+y^{2}} d y+c\\
	\text{Put }\tan ^{-1} y&=t\text{ on R.H.S. so that }\frac{1}{1+y^{2}} d y=d t\\
	x . e^{\tan ^{-1} y}&=\int e^{t} t d t+C=t . e^{t}-e^{t}+C=e^{\tan ^{-1} y}\left(\tan ^{-1} y-1\right)+C\\
	\Rightarrow x&=\left(\tan ^{-1} y-1\right)+c e^{-\tan ^{-1} y}
	\end{align*}
		So the correct answer is \textbf{Option (d)}
\end{answer}
	\item $\left. \right. $
\begin{answer}
	\begin{align*}
	y \log y \frac{d x}{d y}+x-\log y&=0 \Rightarrow \frac{d x}{d y}+\frac{x}{y \log y}=\frac{1}{y}\\
	I . F .&=e^{\int \frac{1}{y \log y} d y}=e^{\log (\log y)}=\log y\\
	\text{Its solution is }x \cdot \log y&=\int \frac{1}{y}(\log y) d y+c \Rightarrow x \cdot \log y=\frac{1}{2}(\log y)^{2}+c
	\end{align*}
		So the correct answer is \textbf{Option (a)}
\end{answer}
\item $\left. \right. $
\begin{answer}
	\begin{align*}
	M&=\left(e^{y}+2\right) \sin x, N=-e^{y} \cos x \Rightarrow \frac{\partial M}{\partial y}=e^{y} \sin x, \frac{\partial N}{\partial x}=e^{y} \sin x\\
	\Rightarrow \frac{\partial M}{\partial y}&=\frac{\partial N}{\partial x} \Rightarrow \int\left(e^{y}+2\right) \sin x d x+0=c^{\prime} \Rightarrow\left(e^{y}+2\right) \cos x=-c^{\prime}=c
	\end{align*}
\end{answer}
\item $\left. \right. $
\begin{answer}
	\begin{align*}
	M&=\left(y^{4}+2 y\right), N=\left(x y^{3}+2 y^{4}-4 x\right)\\
	\Rightarrow \frac{\partial M}{\partial y}&=4 y^{3}+2, \frac{\partial N}{\partial x}=y^{3}-4 \Rightarrow \frac{\frac{\partial N}{\partial x}-\frac{\partial M}{\partial y}}{M}=-\frac{3}{y}=f(y)\\
\text{	then I.F. }&=e^{\int \frac{3}{y} d y}=e^{-3 \log y}=\frac{1}{y^{3}}\\
\text{Multiplying by }&\frac{1}{y^{3}}\text{ we get }\frac{1}{y^{3}}\left(y^{4}+2 y\right) d x+\frac{1}{y^{3}}\left(x y^{3}+2 y^{4}-4 x\right) d y=0\text{ which is an exact}
	\end{align*}
	So the correct answer is \textbf{Option (c)}
\end{answer}
\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\left(D^{2}-3 D+2\right)&=0 \Rightarrow D=1,2 \Rightarrow C . F .=c_{1} e^{t}+c_{2} e^{2 t}\\
	P . I .&=\frac{1}{\left(D^{2}-3 D+2\right)} e^{3 t}=\frac{1}{\left(3^{2}-3 \times 3+2\right)} e^{3 t}=\frac{1}{2} e^{3 t}
	\end{align*}
		So the correct answer is \textbf{Option (a)}
\end{answer}
\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\left(D^{2}+2 D+101\right)&=0 \Rightarrow D=\frac{-2 \pm \sqrt{4-4 \times 101}}{2}=\frac{-2 \pm 20 i}{2}\\&=-1 \pm 10 iC.F. =e^{-x}(A \cos 10 x+B \sin 10 x)\\
	P.I. &=\frac{1}{\left(D^{2}+2 D+101\right)} 10.4 e^{x}=\frac{1}{(1+2 \times 1+101)} 10.4 e^{x}=\frac{10.4}{104} e^{x}=0.1 e^{x}\\
	\Rightarrow y&=e^{-x}(A \cos 10 x+B \sin 10 x)+0.1 e^{x}\\
	\because y(0)&=1.1 \Rightarrow 0=1(A+0)+0.1=1.1 \Rightarrow A=1 .\\
	\Rightarrow y&=e^{-x}(\cos 10 x+B \sin 10 x)+0.1 e^{x}\\
	\Rightarrow \frac{d y}{d x}&=e^{-x}(-10 \sin 10 x+10 B \cos 10 x)-e^{-x}(\cos 10 x+B \sin 10 x)+0.1 e^{x}\\
	\left.\because \frac{d y}{d x}\right|_{x=0}&=-0.9 \Rightarrow-0.9=1(-0+10 B)-1(1+0)+0.1 \Rightarrow 10 B=1-0.1-0.9 \Rightarrow B=0 .\\
	\Rightarrow y&=e^{-x} \cos 10 x+0.1 e^{x}
	\end{align*}
		So the correct answer is \textbf{Option (a)}
\end{answer}
\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\left(D^{2}+4\right)=0 \Rightarrow D=+2 i,-2 i \Rightarrow C . F .=A \cos 2 t+B \sin 2 t
	\end{align*}
	So the correct answer is \textbf{Option (c)}
\end{answer}
\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\left(D^{2}-1\right)&=0 \Rightarrow D=+1,-1 \Rightarrow C . F .=c_{1} e^{t}+c_{2} e^{-t}\\
	P.I. &=\frac{1}{\left(D^{2}-1\right)} 2\left(\frac{e^{t}+e^{-t}}{2}\right)=\frac{1}{\left(D^{2}-1\right)}\left(e^{t}+e^{-t}\right)=t \frac{1}{2 D}\left(e^{t}+e^{-t}\right)=\frac{t}{2}\left(e^{t}-e^{-t}\right)\\
	\Rightarrow y(t)&=c_{1} e^{t}+c_{2} e^{-t}+\frac{t}{2}\left(e^{t}-e^{-t}\right)\\
	y(0)&=0 \Rightarrow c_{1}+c_{2}=0 \\
	\frac{d y}{d t}&=c_{1} e^{t}-c_{2} e^{-t}+\frac{1}{2}\left(e^{t}-e^{-t}\right)+\frac{t}{2}\left(e^{t}+e^{-t}\right),\left.\frac{d y}{d t}\right|_{t=0}=0 \Rightarrow c_{1}-c_{2}=0\\
\text{	Thus }c_{1}&=c_{2}=0 \Rightarrow y(t)=\frac{t}{2}\left(e^{t}-e^{-t}\right)=t \sinh (t)
	\end{align*}
	So the correct answer is \textbf{Option (d)}
\end{answer}
\item $\left. \right. $
\begin{answer}
	\begin{align*}
	\left(D^{2}-1\right)&=0 \Rightarrow D=+1,-1 \Rightarrow C . F .=C_{1} e^{t}+C_{2} e^{-t}\\
	P.I. &=\frac{1}{\left(D^{2}-1\right)} 2\left(\frac{e^{t}-e^{-t}}{2}\right)=\frac{1}{\left(D^{2}-1\right)}\left(e^{t}-e^{-t}\right)=t \frac{1}{2 D}\left(e^{t}-e^{-t}\right)=\frac{t}{2}\left(e^{t}+e^{-t}\right)\\
	\Rightarrow y(t)&=c_{1} e^{t}+c_{2} e^{-t}+\frac{t}{2}\left(e^{t}+e^{-t}\right)\\
	y(0)&=0 \Rightarrow c_{1}+c_{2}=0\\
	\frac{d y}{d t}&=c_{1} e^{t}-c_{2} e^{-t}+\frac{1}{2}\left(e^{t}+e^{-t}\right)+\left.\frac{t}{2}\left(e^{t}-e^{-t}\right) \Rightarrow \frac{d y}{d t}\right|_{t=0}=0 \Rightarrow c_{1}-c_{2}=0\\
\text{	Thus }c_{1}&=c_{2}=0 \Rightarrow y(t)=\frac{t}{2}\left(e^{t}+e^{-t}\right)=t \cosh (t)
	\end{align*}
		So the correct answer is \textbf{Option (c)}
\end{answer}
\end{enumerate}
\section{NAT}
\begin{enumerate}
	\item $\left. \right. $
	\begin{answer}
		\begin{align*}
		\frac{d x}{d t}&=x^{2} \Rightarrow \int \frac{d x}{x^{2}}=\int d t \Rightarrow \frac{x^{-2+1}}{-2+1}=t+C \Rightarrow \frac{-1}{x}=t+C\\
		\Rightarrow x(0)&=1 \Rightarrow \frac{-1}{1}=0+C \Rightarrow C=-1 \Rightarrow \frac{-1}{x}=t-1 \Rightarrow x=\frac{1}{1-t}\text{ as }t \rightarrow 1, x\text{ blows up.}\\
		\end{align*}
			So the correct answer is 1
	\end{answer}
		\item $\left. \right. $
	\begin{answer}
		\begin{align*}
		M&=\left(x y^{2}+\lambda x^{2} y\right), N=(x+y) x^{2} d y \Rightarrow \frac{\partial M}{\partial y}=2 x y+\lambda x^{2}, \frac{\partial N}{\partial x}=3 x^{2}+2 x y\\
		\Rightarrow \frac{\partial M}{\partial y}&=\frac{\partial N}{\partial x} \Rightarrow \lambda=3
		\end{align*}
			So the correct answer is 3
	\end{answer}
		\item $\left. \right. $
	\begin{answer}
		\begin{align*}
			P.I. &=\frac{1}{D^{2}+D+1} \cdot \cos 2 x=\frac{1}{-2^{2}+D+1} \cdot \cos 2 x=\frac{1}{D-3} \cdot \cos 2 x\\
			\Rightarrow P . I .&=\frac{D+3}{D^{2}-9} \cdot \cos 2 x=\frac{D+3}{-2^{2}-9} \cdot \cos 2 x=\frac{1}{13}(2 \sin 2 x-3 \cos 2 x)
		\end{align*}
		So the correct answer is 13
	\end{answer}
		\item $\left. \right. $
\begin{answer}
	\begin{align*}
	P \cdot I .&=\frac{1}{D^{2}+2 D+2} \sin x=\frac{1}{-1+2 D+2} \sin x=\frac{2 D-1}{4 D^{2}-1} \sin x=-\frac{1}{5}(2 D-1) \sin x\\
	\Rightarrow P \cdot I .&=-\frac{1}{5}(2 \cos x-\sin x)=\frac{1}{5}(\sin x-2 \cos x)
	\end{align*}
		So the correct answer is 2
\end{answer}
		\item $\left. \right. $
		\begin{answer}
			\begin{align*}
			P \cdot I \cdot&=\frac{1}{D^{2}-5 D+6} e^{t} \cos 2 t=e^{t} \frac{1}{(D+1)^{2}-5(D+1)+6} \cos 2 t\\
			P \cdot I \cdot&=e^{t} \frac{1}{D^{2}-3 D+2} \cos 2 t=e^{t} \frac{1}{-4-3 D+2} \cos 2 t\\
			P \cdot I \cdot&=-e^{t} \frac{1}{3 D+2} \cos 2 t=-e^{t} \frac{3 D-2}{9 D^{2}-4} \cos 2 t\\
			P \cdot I \cdot&=-e^{t} \frac{3 D-2}{9 \times-4-4} \cos 2 t=\frac{e^{t}}{40}(3 D-2) \cos 2 t=-\frac{e^{t}}{20}(3 \sin 2 t+\cos 2 t)
			\end{align*}
				So the correct answer is 3
		\end{answer}
		\item $\left. \right. $
\begin{answer}
	\begin{align*}
	P . I .&=\frac{1}{D^{2}-4 D+4} \cdot x^{3} e^{2 x}=e^{2 x} \frac{1}{(D+2)^{2}-4(D+2)+4} \cdot x^{3}\\
	\Rightarrow P . I .&=e^{2 x} \frac{1}{D^{2}} \cdot x^{3}=e^{2 x} \frac{1}{D}\left(\frac{x^{4}}{4}\right)=e^{2 x} \frac{x^{5}}{20}
	\end{align*}
		So the correct answer is 5
\end{answer}
		\item $\left. \right. $
	\begin{answer}
		\begin{align*}
		 P.I. &=\left[D^{2}+5 D+4\right]^{-1}(3-2 x)=\frac{1}{4}\left[1+\frac{5}{4} D+\frac{5}{4} D^{2}\right]^{-1}(3-2 x)\\
		\Rightarrow P.I. &=\frac{1}{4}\left[1-\frac{5}{4} D-\frac{5}{4} D^{2}\right](3-2 x)=\frac{1}{4}\left[3-2 x-\frac{5}{4} \times-2\right]=\frac{1}{8}[11-4 x]
		\end{align*}
		So the correct answer is 11
	\end{answer}
		\item $\left. \right. $
	\begin{answer}
		\begin{align*}
		\left(D^{2}+1\right)&=0 \Rightarrow D=\pm i \Rightarrow y=c_{1} e^{i x}+c_{2} e^{-i x}=A \cos x+B \sin x\\
		\because y(0)&=1 \Rightarrow 1=A \times 1+B \times 0 \Rightarrow A=1 \Rightarrow \dot{y}=-A \sin x+B \cos x \\
		\Rightarrow \dot{y}(0)&=1 \Rightarrow 1=-A \times 0+B \times 1 \Rightarrow B=1 \Rightarrow y=\cos x+\sin x\\
		\text{For maxima, }y^{\prime}&=-\sin x+\cos x=0 \Rightarrow \sin x=\cos x \Rightarrow x=45^{\circ}\\
		y^{\prime \prime}&=-\cos x-\sin x, \quad y^{\prime \prime}<0 \text { for } x=45^{\circ} \\
		\Rightarrow y(\max )&=\cos 45^{\circ}+\sin 45^{\circ}=\frac{1}{\sqrt{2}}+\frac{1}{\sqrt{2}}=\frac{2}{\sqrt{2}}=\sqrt{2}
		\end{align*}
			So the correct answer is 1.41
	\end{answer}
		\item $\left. \right. $
		\begin{answer}
			\begin{align*}
				\left(D^{2}+2 \alpha D+1\right)=0 \Rightarrow m_{1}, m_{1}=\frac{-2 \alpha \pm \sqrt{4 \alpha^{2}-4}}{2} \quad \because m_{1}=m_{1} \Rightarrow \alpha=1
			\end{align*}
			So the correct answer is 1
		\end{answer}
\item $\left. \right. $
	\begin{answer}
		\begin{align*}
		\left(D^{2}+3 D+2\right)&=0 \Rightarrow D=-1,-2 \Rightarrow x(t)=C_{1} e^{-t}+C_{2} e^{-2 t}\\
		 \Rightarrow x(1)&=\frac{10}{e}=C_{1} e^{-1}+C_{2} e^{-2} \Rightarrow C_{1}+C_{2} e^{-1}=10\text{ and }C_{1}+C_{2}=20\\
		\Rightarrow C_{1}&=\frac{10 e-20}{e-1} ; C_{2}=\frac{10 e}{e-1} \\
		x(2)&=\left(\frac{10 e-20}{e-1}\right) e^{-2}+\left(\frac{10 e}{e-1}\right) e^{-4}=0.8566
		\end{align*}
			So the correct answer is 0.8566
	\end{answer}
\end{enumerate}

CSIR- Mathematical Physics/chapter/Assignments/Assignment Differential Equations solutions.tex


% !TEX root = Thesis.tex
\section{Relational Model}
	The relational data model, in its most basic form, is built upon sets and tuples.  Each of these sets consist of a set of finite values.  Tuples are constructed from these sets to form relations.

	\begin{defn}[Named Tuple]
	\label{def:named-tuple}
		A named tuple \(\tuple\) is an instance of a relation \(\relation\), consisting of values corresponding to the attributes of \(\relation\).  For example,
	\end{defn}

	\begin{ex}
		Given a tuple \(\tuple = \{\text{code}: \text{``CDPS 101''}, \text{title}: \text{``Human-Mutant Relations''}, \text{subject}: \text{``CDPS''}\}\), we denote the attributes of \(\tuple\) as \(\attributes{\tuple} = \{\text{code}, \text{title}, \text{subject}\}\).  The values are \(\tuple\lbrack\text{code}\rbrack = \text{``CDPS 101''}\), \(\tuple\lbrack\text{title}\rbrack = \text{``Human-Mutant Relations''}\), and \(\tuple\lbrack\text{subject}\rbrack = \text{``CDPS''}\).
	\end{ex}

	\begin{defn}[Relation]
	\label{def:relation}
		A relation \(\relation\) is a set of named tuples, \(\relation = \{\tuple_1, \tuple_2, \dotsc, \tuple_n\}\), such that all the named tuples share the same attributes.
		\begin{equation}
			\forall \tuple, \tuple' \in \relation, \attributes{\tuple} = \attributes{\tuple'}
		\end{equation}
	\end{defn}

	\begin{ex}
	\label{ex:relation}
		An example Course relation, \(\relation\), would be
		\[
			\relation = \left\{
				\begin{array}{llllll}
					\{\text{code} &: \text{``CDPS 101''}, &\text{title} &: \text{``Human-Mutant Relations''}, &\text{subject} &: \text{``CDPS''}\}, \\
					\{\text{code} &: \text{``CDPS 201''}, &\text{title} &: \text{``Humans and You''}, &\text{subject} &: \text{``CDPS''}\}, \\
					\{\text{code} &: \text{``MATH 360''}, &\text{title} &: \text{``Complex Analysis''}, &\text{subject} &: \text{``MATH''}\}
				\end{array}
			\right\}
		\]
	\end{ex}

	Relations are typically referred to, and represented, as tables.  \cref{ex:relation} is shown in its tabular form in \cref{tbl:course-relation}.

	\begin{table}
		\centering

		\begin{tabular}{lll}
			\toprule
			code & title & subject \\
			\midrule
			CDPS 101 & Human-Mutant Relations & CDPS \\
			CDPS 201 & Humans and You & CDPS \\
			MATH 360 & Complex Analysis & MATH \\
			\bottomrule
		\end{tabular}

		\caption{Tabular representation of the Course relation}
		\label{tbl:course-relation}
	\end{table}

	\begin{defn}[Keys]
	\label{def:keys}
		Keys are constraints imposed on relations.	A key constraint \(\key\) on a relation \(\relation\) is a subset of \(\attributes{\relation}\) which must uniquely identify a tuple.	Formally, we say \(\relation\) satisfies the key constraint \(\key\), denoted as \(\relation \models \key\), subject to
		\[
			\forall \tuple, \tuple' \in \relation, \tuple \not= \tuple' \implies \tuple\lbrack\key\rbrack \not= \tuple'\lbrack\key\rbrack
		\]

		For example, in \cref{tbl:course-relation}, the relation satisfies the key constraint \(\{\text{code}\}\) or \(\{\text{title}\}\), but not \(\{\text{subject}\}\).
	\end{defn}

	\begin{defn}[Foreign Keys]
	\label{def:foreign-keys}
		\glsreset{fk} % Want to spell it out in the definition
		A \gls{fk} constraint applies to two relations, \(\relation_1, \relation_2\).  It asserts that values of certain attributes of \(\relation_1\) must appear as values of some corresponding attributes of \(\relation_2\).  A \gls{fk} constraint is written as
		\[
			\theta = \relation_1(\attribute_{1,1}, \attribute_{1,2}, \dotsc, \attribute_{1,k}) \rightarrow \relation_2(\attribute_{2,1}, \attribute_{2,2}, \dotsc, \attribute_{2,k})
		\]

		where \(\attribute_{1,i} \subseteq \attributes{\relation_1}\) and \(\attribute_{2,i} \subseteq \attributes{\relation_2}\).  We say \((\relation_1, \relation_2)\) satisfies \(\theta\), denoted as \((\relation_1, \relation_2) \models \theta\), if for every tuple \(\tuple \in \relation_1\), there exists a tuple \(\tuple' \in \relation_2\) such that \(\tuple\lbrack\attribute_{1,1}, \attribute_{1,2}, \dotsc, \attribute_{1,k}\rbrack = \tuple'\lbrack\attribute_{2,1}, \attribute_{2,2}, \dotsc, \attribute_{2,k}\rbrack\).

		We say \(\relation_1\) is the source, while \(\relation_2\) is the target.

		\begin{ex}
			Suppose we have a relation \(\rel{Course}{code, title, subject}\).	 We impose a \gls{fk} constraint of
			\begin{equation}
				\theta = \rel{Course}{subject} \rightarrow \rel{Subject}{id}
			\end{equation}

			which asserts \((\text{Course}, \text{Subject}) \models \theta\).  Therefore, if
			\[
				\tuple = \{\text{code}: \text{``CDPS 101''}, \text{title}: \text{``Human-Mutant Relations''}, \text{subject}: \text{``CDPS''}\}
			\]

			then \(\exists! \tuple' \in \text{Subject}\) such that \(\tuple'\lbrack\text{id}\rbrack = \text{``CDPS''}\).
		\end{ex}
	\end{defn}

	\begin{defn}[Relational Database]
	\label{def:relational-database}
		A relational database, \(\db\), is a named collection of relations (\cref{def:relation}), keys (\cref{def:keys}), and foreign key constraints (\cref{def:foreign-keys}).

		We use \(\name{\db}\) to denote the name of \(\db\), \(\relations{\db}\) the list of relations in \(\db\), \(\keys{\db}\) the list of key constraints of \(\db\), and \(\fks{\db}\) the list of foreign key constraints of \(\db\).
	\end{defn}

	\subsection{Schema Group}
		\begin{defn}[Schema Graph]
		\label{def:schema-graph}
			If we view relations as vertices, and \gls{fk} constraints as edges, a database \(\db\) can be viewed as a schema graph, \(\sgraph{}\), formally defined as
			\begin{align}
				\text{vertices} &: \text{V}(\sgraph) = \relations{\db} \\
				\text{edges} &: \text{E}(\sgraph) = \fks{\db}
			\end{align}
		\end{defn}

		\begin{ex}
			Given the schema in \cref{tbl:schema} and the \gls{fk} constraints in \cref{eqn:fk-constraints}, we produce the schema graph in \cref{fig:schema-graph}.

			\begin{table}
				\centering

				\begin{tabular}{ll}
					\toprule
					Relation & Attributes \\
					\midrule
					Course & \underline{code}, title, subject \\
					Section & \underline{id}, actual, campus, capacity, credits, levels, registration\_start, \\
					 & registration\_end, semester, sec\_code, sec\_number, year, course \\
					Schedule & \underline{id}, date\_start, date\_end, day, schedtype, hour\_start, hour\_end, \\
					 & min\_start, min\_end, classtype, location, section\_id \\
					Instructor & \underline{id}, name \\
					Teaches & \underline{id}, schedule\_id, instructor\_id, position \\
					\bottomrule
				\end{tabular}

				\caption{Subset of mycampus dataset schema}
				\label{tbl:schema}
			\end{table}

			\begin{figure}
				\begin{align*}
					\rel{Schedule}{section\_id} &\rightarrow \rel{Section}{id} \\
					\rel{Section}{course} &\rightarrow \rel{Course}{code} \\
					\rel{Teaches}{schedule\_id} &\rightarrow \rel{Schedule}{id} \\
					\rel{Teaches}{instructor\_id} &\rightarrow \rel{Instructor}{id} \\
				\end{align*}

				\caption{\gls{fk} constraints on schema in \cref{tbl:schema}}
				\label{eqn:fk-constraints}
			\end{figure}

			\begin{figure}
				\centering

				\begin{dot2tex}[neato]
					digraph G {
						node [shape=plaintext]; Course; Section; Schedule; Instructor; Teaches;

						Schedule -> Section;
						Section -> Course;
						Teaches -> Schedule;
						Teaches -> Instructor;
					}
				\end{dot2tex}

				\caption{Graph representation of relations (\cref{tbl:schema}) and \gls{fk} (\cref{eqn:fk-constraints})}
				\label{fig:schema-graph}
			\end{figure}
		\end{ex}

		The relational data model is particularly powerful for analytic queries.

	\subsection{Entity Group}
		\begin{defn}[Entity Group]
		\label{def:entity-group}
			An entity group is a forest, \(\egraph\), of tuples interconnected by join conditions defined by the \gls{fk} constraints in the schema graph \(\sgraph\).

			Given two vertices \((\tuple, \tuple') \in \text{V}(\egraph)\), \(\exists (\relation_1, \relation_2) \in \relations{\db}\) such that \(\tuple \in \relation_1\), \(\tuple'\in \relation_2\), and \((\relation_1, \relation_2)\in \sgraph\).  That is, \(\tuple\) and \(\tuple'\) belong to two relations that are directly connected by the schema graph.
		\end{defn}

		\begin{ex}
			Let \(\relation_1(\attribute_{11}, \attribute_{12}, \dotsc, \attribute_{1k}) \to \relation_2(\attribute_{21}, \attribute_{22}, \dotsc, \attribute_{2k})\) be the \gls{fk} that connects \((\relation_1, \relation_2)\).  We assert that \(\tuple\lbrack\attribute_{11}, \attribute_{12}, \dotsc, \attribute_{1k}\rbrack = \tuple'\lbrack\attribute_{21}, \attribute_{22}, \dotsc, \attribute_{2k}\rbrack\).
		\end{ex}

		Entity groups define complex, structured objects that include more information than individual tuples in the relations.

		\begin{ex}
			The information in \cref{tbl:hmr-properties-rel} all relates to the Course titled Human-Mutant Relations, however no single tuple in the database has all of this information as a result of database normalization.

			\begin{table}
				\centering

				\begin{tabular}{ll}
					\toprule
					Attribute & Value \\
					\midrule
					code & CDPS 101 \\
					title & Human-Mutant Relations \\
					subject & Community Development \& Policy Studies \\
					\bottomrule
				\end{tabular}

				\caption{Properties of the Course titled Human-Mutant Relations.}
				\label{tbl:hmr-properties-rel}
			\end{table}

			We require an entity group to join together all pieces of information related to this course.  An example of this is given in \cref{fig:hmr-entity-group}, where an example entity group comprised of a Course and Subject is shown.

			\begin{figure}
				\centering

				\begin{dot2tex}[dot]
digraph G {
	node [shape=plaintext]; "Human-Mutant Relations"; "Community Development & Policy Studies";

	"Human-Mutant Relations" -> "Community Development & Policy Studies";
}
				\end{dot2tex}

				\caption{Human-Mutant Relations entity group}
				\label{fig:hmr-entity-group}
			\end{figure}
		\end{ex}

	\subsection{Benefits and Drawbacks of the Relational Model}
		To better understand the motivation behind this work, it is important to examine both the strong and weak points of the relational model.

		\subsubsection{Benefits}
			The enforcement of constraints is essential to the relational model.  Types of constraints include uniqueness and \glspl{fk}.  The first constraint maintains uniqueness.

			The Course relation (\cref{tbl:course-relation}) has the attribute \texttt{code} as its primary key.	In order for other relations to reference a specific named tuple, the \texttt{code} attribute must be unique.

			\begin{ex}[Unique Constraint]
			\label{ex:unique-constraint}
				Attempt to insert another course with a \texttt{code} of ``CDPS 101.''

				\begin{singlespaced}
					\begin{pygments}{sql}
INSERT INTO course
VALUES      ('CDPS 101',
             'Mutant-Human Relations',
             'CDPS');
					\end{pygments}
				\end{singlespaced}

				The \gls{rdbms} enforces the primary key constraint on the \texttt{code} attribute, rejecting the insertion.

				\begin{verbatim}
Error: column code is not unique
				\end{verbatim}
			\end{ex}

			With the uniqueness of named tuples guaranteed (as demonstrated in \cref{ex:unique-constraint}), we must ensure that any named tuples that are referenced actually exist.  If they do not, the database must not permit the operation to continue.  Doing so leads to dangling references.

			\begin{ex}[Referential Integrity]
				Attempt to insert the tuple (``CHEM 101'', ``Introductory Chemistry'', ``'CHEM'') in the Course relation.

				\begin{singlespaced}
					\begin{pygments}{sql}
INSERT INTO course
VALUES      ('CHEM 101',
             'Introductory Chemistry',
             'CHEM');
					\end{pygments}
				\end{singlespaced}

				Again we see the \gls{rdbms} protecting the integrity of the data.

				\begin{verbatim}
Error: foreign key constraint failed
				\end{verbatim}
			\end{ex}

			The relational model, in addition to enforcing consistency, is capable of providing higher-level views of the data through aggregation.

			\begin{ex}[Aggregation]
				Find the number of sections offered for the subject named ``Community Development \& Policy Studies.''

				\begin{singlespaced}
					\begin{pygments}{sql}
SELECT Count(*)
FROM   section
       JOIN course
         ON section.course = course.code
       JOIN subject
         ON subject.id = course.subject
WHERE  subject.name = 'Community Development & Policy Studies';
					\end{pygments}
				\end{singlespaced}
			\end{ex}

			Information stored within a properly designed database is normalized.  That is, no information is repeated.

			\begin{ex}[Normalization]
				For example, suppose Emma Frost became headmistress and the subject named ``Community Development \& Policy Studies'' was renamed to ``Community Destruction \& Policy Studies.''  If this information were not normalized, each course in this subject needs to be updated.  Since this information is normalized, the following query will suffice.

				\begin{singlespaced}
					\begin{pygments}{sql}
UPDATE subject
SET    name = 'Community Destruction & Policy Studies'
WHERE  id = 'CDPS';
					\end{pygments}
				\end{singlespaced}
			\end{ex}

			The above examples are some of the most important reasons for choosing the relational model over others.	Unfortunately, the relational model is not without its downsides.

		\subsubsection{Drawbacks}
			While the relational model excels at ensuring data consistency, aggregation, and reporting; it is not suitable for every task.	A user must be familiar with the schema to issue queries.  This requires specific domain knowledge of the data.

			A casual user is unlikely to determine the correct join path, know the exact name of the tables and attributes, etc.  This is in contrast to the document model where the data is semi-structured or unstructured, requiring minimal domain knowledge.

			The relational model is also rigid in structure.  If a relation is modified, every query referencing said relation may require a rewrite.  Even a simple attribute being renamed (e.g.~\(\rho_{\text{name/alias}}(\text{Person})\)) is capable of modifying the join paths.  This rigidity places additional cognitive burden on users.

			In addition to having a rigid structure, most relational database management systems lack flexible string matching options.	 Assuming basic SQL-92 compliance, a \gls{rdbms} only supports the \texttt{LIKE} predicate \cite{sql-11}.

			\begin{ex}[\texttt{LIKE} Predicate]
				Find all courses with a title that contains ``man.''

				\begin{singlespaced}
					\begin{pygments}{sql}
SELECT *
FROM   course
WHERE  title LIKE '%man%';
					\end{pygments}
				\end{singlespaced}
			\end{ex}

			A couple of limitations to the \texttt{LIKE} predicate exist.  First, it only supports basic substring matching.  If a user accidentally searches for all courses with a title containing ``men,'' nothing would be found.

			Second, unless the predicate is applied to the end of the string and the column is indexed, performance will be poor.  The \gls{rdbms} must scan the entire relation to answer the query, resulting in performance of \(\mathcal{O}(n)\), where \(n\) is the number of named tuples in the relation.


% Copyright 2018-2021 Melvin Eloy Irizarry-Gelpí
\setcounter{chapter}{4}
\chapter{Capacitors}
%
...
%
\section{Preliminary}
%
Resistance is a property of the components in an electric circuit. You can use units of ohm to measure electric resistance. One ohm of electric resistance is equivalent to
\begin{equation}
	1 \text{ ohm} = 1 \text{ V/A}
\end{equation}
Another property is capacitance. You can measure capacitance in units of farads (F). One farad of electric capacitance is equivalent to
\begin{equation}
	1 \text{ F} = 1 \text{ C/V}
\end{equation}
In some ways, capacitance measures the amount of electric charge per unit voltage.

A component in a circuit with an electric resistance is called a \textbf{resistor}. Similarly, a component in a circuit with a capacitance is called a \textbf{capacitor}. A circuit with both capacitors and resistors is known as an RC circuit. The simplest RC circuit has one resistor and one capacitor connected in series to a battery (a DC source).

A particular quantity $\tau_{C}$ is given by multiplying resistance by capacitance:
\begin{equation}
	\tau_{C} = R C
	\label{eq.05C.tauC}
\end{equation}
This quantity is known as the \textbf{capacitive time constant}. If resistance is in ohms and capacitance is in farads, then $\tau_{C}$ is in seconds. The capacitive time constant is important because it describes how quickly voltage and current change in circuits with resistors and capacitors. During a \textbf{charging event}, the \textbf{electric current} drops in an exponential way:
\begin{equation}
	I(t) = I_{0} \exp{\left(- \frac{t}{\tau_{C}} \right)}
	\label{eq.05C.It}
\end{equation}
During a \textbf{discharging event}, the \textbf{voltage} drops in an exponential way:
\begin{equation}
	V(t) = V_{0} \exp{\left(- \frac{t}{\tau_{C}} \right)}
	\label{eq.05C.Vt}
\end{equation}
%
\section{Experiment}
%
There are two kinds of experiments: the charging event and the discharging event. In each experiment you measure the current flowing between the resistor and the capacitor, and the voltage across the capacitor.

For the \textbf{charging event}, the data collection is started with the switch off (i.e. incomplete circuit) and shortly after turned on. You should analyze the \textbf{current} data for such events, as it follows the behavior in (\ref{eq.05C.It}).

For the \textbf{discharging event}, the data collection is started with the switch on (i.e. complete circuit) and shortly after turned off. You should analyze the \textbf{voltage} data for such events, as it follows the behavior in (\ref{eq.05C.Vt}).

There are 9 runs of data:
\begin{enumerate}
	\item Charging capacitor with $R = 10$ ohm and $C = 0.025$ F.
	\item Discharging capacitor with $R = 10$ ohm and $C = 0.025$ F.
	\item Charging capacitor with $R = 51$ ohm and $C = 0.025$ F.
	\item Discharging capacitor with $R = 51$ ohm and $C = 0.025$ F.
	\item Charging capacitor with $R = 68$ ohm and $C = 0.025$ F.
	\item Discharging capacitor with $R = 68$ ohm and $C = 0.025$ F.
	\item Discharging capacitor with $R = 22 \times 10^{3}$ ohm and $C = 10^{-5}$ F.
	\item Discharging capacitor with $R = 47 \times 10^{3}$ ohm and $C = 10^{-5}$ F.
	\item Discharging capacitor with $R = 100 \times 10^{3}$ ohm and $C = 10^{-5}$ F.
\end{enumerate}
Each time the resistance or the capacitance changes, the time constant $\tau_{C}$ will also change.
%
\section{Analysis}
%
The analysis is pretty simple: you need to make a graph of the desired quantity versus time and then include an \textbf{exponential} fit. The exponential fit is of the form:
\begin{equation}
	y = A e^{-Bx}
	\label{eq.05C.exp.fit}
\end{equation}
Comparing (\ref{eq.05C.exp.fit}) with (\ref{eq.05C.It}) and also (\ref{eq.05C.Vt}) leads to identifying the experimental estimate of $\tau_{C}$ with $1 / B$.

The trouble is that it takes some effort to prepare the data in such a way that the exponential fit is appropriate and accurate. You need to do two things:
\begin{enumerate}
	\item Truncate the time region before the switch is turned on/off.
	\item Truncate the long time tail after the exponential drop.
\end{enumerate}
For the first truncation, just find the time value where the voltage (in discharging events) or the current (in charging events) suddenly jumps. Remove all the values before this time and perhaps one or two events right after the exponential drop begins.

For the second truncation, one good rule of thump is to use the expected value of $\tau_{C}$. You should keep a time region that is about $5\tau_{C}$ long in time, starting from the beginning of the exponential drop.
%
\section{My Data}
%
In the spreadsheet that I shared with you, there is a sheet called ``Results''. This sheet has a table with eight columns and nine rows. Each of the rows corresponds to one of the nine runs mentioned above. Here are the columns:
\begin{itemize}
	\item Column 1: run label.
	\item Column 2: value of resistance (in ohm) for the resistor used.
	\item Column 3: value of the capacitance (in F) for the capacitor used.
	\item Column 4: Expected value of $\tau_{C}$ (in s). Use (\ref{eq.05C.tauC}) with columns 2 and 3 to calculate these values.
	\item Column 5: Expected value of $1 / \tau_{C}$ (in 1/s). Just take the reciprocal of the values in column 4.
	\item Column 6: Fit value of $1 / \tau_{C}$ (in 1/s). This correspond to the $B$ value in the exponential fit. See (\ref{eq.05C.exp.fit}).
	\item Column 7: Fit value of $\tau_{C}$ (in s). This is just the reciprocal of the values in column 6.
	\item Column 8: The percent difference between the theoretical values in column 4 and the experimental values in column 7.
\end{itemize}
For the percent difference, use
\begin{equation}
	\text{Percent Difference } = 100 \times (\text{experiment } - \text{ theory}) / \text{theory}
\end{equation}
In my case I found mixed agreement, with the largest disagreements not being larger than 20\%.
%
\section{Your Data}
%
Your data is structured in the same way as my data, but the time delay for turning on/off the switch is different.
%
\section{Your Lab Report}
%
In your lab report you should include:
\begin{enumerate}
	\item One voltage versus time graph. Choose one from runs 2, 4, or 6. Include the exponential fit line, and also the equation.
	\item One current versus time graph. Choose one from runs 1, 3, or 5. Include the exponential fit line, and also the equation.
	\item Another voltage versus time graph. Choose one from runs 7, 8, or 9. Include the exponential fit line, and also the equation.
	\item A table in your printed lab report similar to my table in the ``Results'' sheet from the shared spreadsheet.
\end{enumerate}
You should also include answers to the following questions:
\begin{enumerate}
	\item Are the fit values for $\tau_{C}$ for runs 1, 3, and 5 consistent with the corresponding values for runs 2, 4, and 6?
	\item Note that some runs involve current measurements and the other runs involve voltage measurements. In general, which runs give fit values for $\tau_{C}$ that are closer to the expected values: charging events (current) or discharging events (voltage)?
	\item Why are the current measurements in runs 7, 8 and 9 so small?
\end{enumerate}


\subsection{Implementation of Ruder style transfer for video}
\label{sec:Ruder style transfer}
As a part of our project we have implemented some of the components described in the paper by Ruder et al. \cite{Ruder:1}. Here we try to reduce the noise that appears when styling each image independently by introducing a temporal constraint in order to penalize the model for straying too far away from the previous image.\newline\newline
The temporal constraint we have implemented is an additional loss-component: \newline

\begin{equation}
\mathcal{L}_{temporal}(x, \omega, c) = \frac{1}{D}\sum_{k=1} c_k \cdot (x_k - \omega_k)^2
\end{equation}
Since we are using the style and content-loss used in Gatys style transfer, our total loss-function with a shortterm temporal constraint is this:\newline
\begin{equation}
\mathcal{L}_{shortterm}(p^{(i)}, a, x) = \alpha \mathcal{L}_{content} + \beta \mathcal{L}_{style} + \gamma \mathcal{L}_{temporal}(x^{(i)}, \omega_{i-1}^i(x^{i-1}, c^{i-1, i}))
\end{equation}
We use different weights for different resolutions, generally increasing the style-loss and decreasing the temporal constraint with increasing resolution.\newline
As well as using this shortterm temporal loss function, we also initialize every frame after the first stylized frame with the previously stylized frame \textit{i-1} warped to the current image \texit{i}. This is done by using the flow between the content image $i-1$ and $i$.
To calculate this flow we need a flow algorithm. In our implementation we have used cv2's DeepFlow function. The calculated flow between frames is also used to create our c-matrix using the inequalities described in section \ref{sec:usage_of_optical_flow}. These inequalities result in c-matrix masks like these:\newline
\begin{figure}[!ht]
\begin{center}
\includegraphics[scale=0.5]{report/Method/images/shrek_flow.png}
\caption{A frame from the shrek video and the flow image calculated, indicating what parts of the image are moving and should be recalculated}
\label{fig:architecture}
\end{center}
\end{figure}
\newline
To calculate these values for each frame, we have primarily used the numpy library. We have also implemented the long-term temporal constraint described in section \ref{sec:long_term_temporal_loss}. However, a lot of additional computation is needed for this. With our current access to hardware and the negligible results we got from testing our implementation, we decided to not use this in our results.\newline\newline
With the implemented temporal constraint, we are able to have some consistency between frames, resulting in less noise: \newline
\begin{figure}[!ht]
\begin{center}
\includegraphics[scale=0.11]{report/Method/images/ruder.png}
\caption{Two consecutive frames produced by Ruders method using La Muse (picasso) as style refernce, source video is from Parasite}
\label{fig:architecture}
\end{center}
\end{figure}Here we can see the same artifacts staying in the same place in both adjacent frames, while the person to the left has moved. The results from this implementation can be seen in \ref{seq:ruder_result}


%\subsection{Variational Inference in Sparse GPs Revisited}
%
%In this section we review inference in sparse GPs (as presented in Titsias).
%The posterior in an augmented model is $p(\f, \u | \y) = p(\f | \u, \y) p(\u | \y)$.
%The key property of such augmented GPs  is the notion of \textit{sufficient statistics}: given the inducing points $\u$, the latent values $\f$ are independent with any other set of latent values (e.g. the test set).
%In the optimal setting when $\u$ is the sufficient statistics of $\f$, it should hold that $p(\f | \u, \y) = p(\f | \u)$ as $\y$ is only the noisy version of $\f$.
%This leads to choosing a variational approximation of the posterior which factorizes as $q(\f, \u | \y) = p(\f | \u) q(\u | \y)$.
%Since the conditional $p(\f | \u)$ is known, variational inference becomes learning an optimal posterior $q(\u | \y)$ only. \\
%
We approximate the posterior over the latent variables $\vec{g}, \vec{h}, \vec{u}, \vec{v}$ given observations $\vec{y}$ using variational inference \citep{jordan-variational-99}.
In section \ref{sec:variationalLowerBound} we derive a lower bound of the marginal likelihood which has the key property of factorizing over the data points and outputs. Section \ref{sec:svi}  takes advantage of this factorization to
derive stochastic variational inference, allowing the model to scale to very large datasets.
Section \ref{sec:complexity} compares the complexity of the model with previous multi-output methods.

\subsection{VARIATIONAL LOWER BOUND \label{sec:variationalLowerBound}}
\newcommand{\ug}{\u_g}
\newcommand{\uh}{\u^h}
\newcommand{\mgj}{\m_j}
\newcommand{\mhi}{\m^h_i}
\newcommand{\Sgj}{\S_j}
\newcommand{\Shi}{\S^h_i}
In variational inference, we find the ``closest'' approximate distribution to the true posterior in terms of the $\KL$ divergence.
We first observe that the true posterior distribution can be written as:
\begin{align}
p(\g, \h, \u, \v | \y) = p(\g | \u, \y) p(\h | \v, \y) p(\u, \v | \y).
\end{align}
Here we recall the modeling assumption that each set of  inducing variables is the sufficient statistics of the corresponding latent process.
This motivates replacing the true posteriors over $\g$ and $\h$ with their conditional distributions given the inducing variables, leading to a distribution of the form:
\begin{align}
q(\g, \h, \u, \v | \y)
= p(\g|\u) p(\h|\v) q(\u,\v),
\end{align}
with
\begin{align}
q(\u,\v) &= \prod_{j=1}^Q \underbrace{\Normal(\u_j; \mgj, \Sgj)}_{q(\u_j)} \prod_{i=1}^P  \underbrace{\Normal(\v_i; \mhi, \Shi)}_{q(\v_i)}.
\end{align}
This technique has been used by \citet{titsias2009variational} and \citet{hensmangaussian}
to derive variational inference algorithms for the single output case.
Since the conditionals $p(\g | \u)$ and $p(\h | \v)$ are known (Equations \eqref{eq:gu} and \eqref{eq:hv}),
we only need to
learn $q(\u, \v)$ so as to minimize the divergence between the approximate posterior and
the true posteriors.
The quality of approximation depends entirely on the posterior over the inducing variables, thus underlining their pivotal role in the model as previously discussed.

\noindent
To find the best $q(\u, \v)$, we  optimize the evidence lower bound (ELBO) of the log marginal:
\begin{align}
%\nonumber
%\log p(\y) \ge& \int q(\u, \v) \log \frac{p(\y | \u, \v) p(\u, \v)}{q(\u, \v)} \der \u \der \v \\
%\nonumber
%=& \int q(\u, \v) \log p(\y | \u, \v)  \der \u \der \v
%+ \int q(\u, \v) \log \frac{p(\u, \v)}{q(\u, \v)} \der \u \der \v \\
\nonumber
&\log p(\y) \ge \int q(\u, \v) \log p(\y | \u, \v)  \der \u \der \v \\
&- \sum_{j=1}^Q \KL[q(\u_j) || p(\u_j)] - \sum_{i=1}^P \KL[q(\v_i) || p(\v_i)],
\label{eq:elbo}
\end{align}
which is derived using Jensen's inequality and the fact that both of $q(\u, \v)$ and $p(\u, \v)$ fully factorize.
Since $q(\u_j), q(\v_i), p(\u_j), p(\v_i)$ are all multivariate Gaussian distributions, the KL divergence terms are analytically tractable.
To compute the expected likelihood term in the ELBO we first see that
\begin{align}
\nonumber
\log \text{ } &p(\y | \u, \v)
% &= \log \Eb{p(\y | \g, \h)}_{p(\g,\h | \u, \v)} \\
% \nonumber
\ge \Eb{\log p(\y | \g, \h)}_{p(\g,\h | \u, \v)}  \\
&= \sum_{i=1}^P \sum_{n=1}^N \Eb{\log p(y_{in} | \g_n, h_{in}) }_{p(\g | \u) p(\h_i | \v_i)}
\end{align}
where $\g_n = \{g_{jn} = (\g_j)_n\}_{j=1}^Q$.
The inequality is due to Jensen's inequality and the equality is due to %the fact that the likelihood fully factorizes.
the factorization of the likelihood.
\newcommand{\Ahi}{\A^h_i}
\newcommand{\Zi}{\Z_i}

The ELBO can be computed by first solving for the individual expectations
%$l_{in} \define \Eb{\log p(y_{in} | \g_n, h_{in}) }_{p(\g_n | \u) p(\h_in | \v_i)}$
$ \Eb{\log p(y_{in} | \g_n, h_{in}) }$ over ${p(\g | \u) p(\h_i | \v_i)}$ and
then substituting these into Equation \eqref{eq:elbo} (see the supplementary material for details).
Hence the resulting lower bound is given by:
\begin{align}
\nonumber
%&\log p(\y)
 & \calL =
\sum_{i,n}
\bigg( \log  \Normal(y_{in}; \tilde{\mu}_{in}, \beta_i^{-1})
          - \frac{1}{2} \beta_i \sum_{j=1}^Q w_{ij}^2 \tilde{k}_{jnn} \\ \nonumber
         &- \frac{1}{2} \beta_i \tilde{k}^h_{inn}
         - \frac{1}{2} \beta_i \sum_{j=1}^Q \trace w_{ij}^2 \S_j \mat{\Lambda}_{jn} - \beta_i \frac{1}{2} \trace \S^h_i \mat{\Lambda}_{in}
\bigg) \\
\nonumber
& - \sum_{j=1}^Q \bigg( \frac{1}{2} \log |\K_{jzz} \S_j^{-1}| + \frac{1}{2} \trace \K_{jzz}^{-1}  \left( \m_j \m_j^T + \S_j \right)\bigg) \\
\nonumber
& - \sum_{i=1}^P  \bigg( \frac{1}{2} \log |\K_{izz}(\Shi)^{-1}| \\
& \quad \qquad + \frac{1}{2} \trace \K_{izz}^{-1} \left( \mhi (\mhi)^T   +   \Shi \right) \bigg)  \text{,}
  \label{eq:L}
\end{align}
%
where $\K_{jzz} = k(\Zj, \Zj)$, $\K_{izz} = k(\Zhi, \Zhi)$, and:
\begin{align}
\tilde{\mu}_{in}
%&= \sum_{j=1}^Q w_{ij} k(\x_n, \Zj)k(\Zj,\Zj)^{-1}\m_j + k(\x_n, \Zhi)k(\Zhi,\Zhi)^{-1}\mhi \\
&= \sum_{j=1}^Q w_{ij} \A_j(n,:) \m_j + \Ahi(n,:) \mhi \text{,}\\
\mat{\Lambda}_{jn}
%&= k(\Zj,\Zj)^{-1} k(\Zj, \x_n) k(\x_n, \Zj) k(\Zj,\Zj)^{-1} \\
&= \A_j(n,:)^T \A_j(n,:)  \text{,}\\
\mat{\Lambda}_{in}
%&= k(\Zhi,\Zhi)^{-1} k(\Zhi, \x_n) k(\x_n, \Zhi) k(\Zhi,\Zhi)^{-1}
&= \Ahi(n,:)^T \Ahi(n,:)  \text{,}
\end{align}
with $\tilde{k}_{jnn} = (\tilde{\K}_j)_{nn}$;
$\tilde{k}^h_{inn} = (\tilde{\K}^h_i)_{nn}$;
$\mu_{jn} = (\Mu_j)_n$;
 $\mu^h_{in} = (\Mu^h_i)_n$;
and we have defined the auxiliary matrices $\A_j = k(\X,\Zj) \K_{jzz}^{-1}$ and $\Ahi = k(\X_i,\Zhi)\K_{izz}^{-1}$ and
used $\A_j(n,:)$ to denote the $n$-th row vector of $\A_j$.
Notice that this ELBO  generalizes the bound for standard GP regression derived in \cite{hensmangaussian}, which can be recovered by setting $P = Q = 1$, $w_{ij} = 1$ and $h_i(\x) = 0$.

The novelty of the  variational lower bound in Equation  \eqref{eq:L} is that it decomposes across both inputs and outputs.
This enables the use of stochastic optimization methods, which
allow the model to handle  very large datasets for which existing GP-based multi-output models are simply
impractical.
%
\subsection{STOCHASTIC VARIATIONAL INFERENCE \label{sec:svi}}
\newcommand{\oi}{\vec{o}_i}
% some notation

So far in the description of the model and inference  we have
 implicitly assumed that every output has full observations at all inputs $\X$.
To discern where learning occurs for each output, we make the missing data scenario more explicit.
Specifically, each output $i$ can have observations at a different set of inputs $\X_i$.
We shall use $\oi$ to denote the indices of $\X_i$ (in the set $\X$) and use the indexing operator $\B(\oi)$ to select the rows corresponding to $\oi$ from any arbitrary matrix $\B$.
We also overload $\y_i$ as the observed targets of output $i$.
%
\subsubsection{Learning the Parameters of the Variational Distribution}
%------------------------------------------
% derivatives of q(u_j)
\newcommand{\Lgj}{\calL^g_j}
\newcommand{\ynoj}{\y_i^{\backslash j}}
\newcommand{\Kjzz}{\mat{K}_{jzz}}
 We can obtain the derivatives of the ELBO in Equation  \eqref{eq:L}
  wrt the variational parameters for optimization.
The derivatives of $\calL$ wrt the parameters of $q(\u_j)$ are given by:
\begin{align}
\deriv{\calL}{\m_j}
=& \sum_{i=1}^P \beta_i w_{ij} \A_j(\oi)^T \ynoj \\
\nonumber
&- \bigg[\Kjzz^{-1} + \sum_{i=1}^P \beta_i w_{ij}^2 \A_j(\oi)^T \A_j(\oi) \bigg] \m_j \text{,} \\
\deriv{\calL}{\S_j}
=& \frac{1}{2} \S_j^{-1} - \frac{1}{2} \bigg[ \Kjzz^{-1} + \sum_{i=1}^P \beta_i w_{ij}^2 \A_j(\oi)^T \A_j(\oi) \bigg],
\end{align}
where $\y_i^{\backslash j} = \y_i - \Ahi(\oi) \mhi - \sum_{j' \neq j} w_{ij'} \A_{j'}(\oi) \m_{j'}$.
%\noindent As a function of the parameters of $q(\u_j)$, the lowerbound $\calL$ is:
%\begin{align}
%\nonumber
%\calL^g_j \define&
% \sum_{i=1}^P \log \Normal(\y_i; \sum_{j=1}^Q w_{ij} \A_j(\oi,:) \m_j + \Ahi \mhi, \beta_i^{-1} \I)  \\
% \nonumber
% &- \frac{1}{2} \sum_{i=1}^P \bigg(\beta_i \trace w_{ij}^2 \tilde{\K}_j(\oi,\oi)
% + \beta_i \trace w_{ij}^2 \S_j \A_j(\oi,:)^T \A_j(\oi,:) \bigg)
% \\
% &- \frac{1}{2} \log |k(\Zj,\Zj) \S_j^{-1}| -\frac{1}{2} \trace k(\Zj,\Zj)^{-1} (\m_j \m_j^T + \S_j) ,
%\end{align}
%where $\A_j = k(\X,\Zj)k(\Zj,\Zj)^{-1}$, which gives $\A_j(\oi,:) = k(\X_i,\Zj) k(\Zj,\Zj)^{-1}$, and
%$\Ahi = k(\X_i,\Zhi)k(\Zhi,\Zhi)^{-1}$. \\

%-------------------------------------------
%  derivatives of q(v_i)
\newcommand{\Lhi}{\calL^h_i}
\newcommand{\Kizz}{\mat{K}_{izz}}
\noindent The derivatives of $\calL$ wrt the parameters of $q(\v_i)$ are given by:
\newcommand{\ynoh}{\y_i^{\backslash h}}
\begin{align}
\nonumber
\deriv{\calL}{\mhi}
= & \beta_i \Ahi(\oi)^T \ynoh  \\
&- \bigg[\Kizz^{-1} +  \beta_i \Ahi(\oi)^T \Ahi(\oi) \bigg] \m_i \text{,} \\
\deriv{\calL}{\Shi}
=& \frac{1}{2} \S_i^{-1} - \frac{1}{2} \bigg[ \Kizz^{-1} + \beta_i \Ahi(\oi)^T \Ahi(\oi) \bigg] ,
\end{align}
where $\ynoh = \y_i - \sum_{j=1}^Q w_{ij} \A_j(\oi,:) \m_j$.
%\noindent As a function of the parameters of $q(\v_i)$, the lower bound $\calL$ is:
%\begin{align}
%\nonumber
%\Lhi \define&
% \log \Normal(\y_i; \sum_{j=1}^Q w_{ij} \A_j(\oi,:) \m_j + \Ahi \mhi, \beta_i^{-1} \I)
% - \frac{1}{2} \beta_i \trace \tilde{\K}^h_i(\oi,\oi)
% - \frac{1}{2} \beta_i \trace \Shi (\Ahi)^T \Ahi
% \\
%  &- \frac{1}{2} \log |k(\Zhi,\Zhi) (\Shi)^{-1}| -\frac{1}{2} \trace k(\Zhi,\Zhi)^{-1} (\mhi (\mhi)^T + \Shi) ,
%\end{align}

% comment on computation
\noindent It can be seen that the derivatives of the parameters of $q(\v_i)$ only involve the observations of the output $i$.
The derivatives of the parameters of $q(\u_j)$ involve the observations of all outputs but is a sum of contributions from individual outputs.
Computation of the derivatives can therefore be easily distributed or parallelized.
%Therefore, computation of the derivatives can be distributed or parallelized easily.
%This property further enhances the scalability of the model, allowing it to handle big data problems with a very large number of inputs and outputs.

Since the optimal distributions $q(\u_j)$ and $q(\v_i)$ are in the exponential family, it is more convenient to use stochastic variational inference \citep{hensman2012fast,hensmangaussian} to perform update of their canonical parameters.
This works by taking a step of length $l$ in the direction of the natural gradient approximated by mini-batches of the data.
For instance, consider $q(\u_j)$ whose canonical parameters are $\Phi_1 = \S_j^{-1} \m_j$ and $\Phi_2 = -\frac{1}{2}\S_j^{-1}$.
Their stochastic update equations at time $t + 1$ are given by:
\begin{align}
\nonumber
& \Phi_{1(t+1)} = \S_{j(t)}^{-1} \m_{j(t)}  \\
 & \quad \qquad +  l \bigg(\sum_{i=1}^P \beta_i w_{ij} \A_j(\oi)^T \ynoj - \S_{j(t)}^{-1} \m_{j(t)} \bigg) \\
& \Phi_{2(t+1)} = -\frac{1}{2} \S_{j(t)}^{-1} + l \bigg(\frac{1}{2} \S_{j(t)}^{-1} - \frac{1}{2} \mat{\Lambda} \bigg),
\end{align}
\normalsize
where $\mat{\Lambda} = \Kjzz^{-1} + \sum_{i=1}^P \beta_i w_{ij}^2 \A_j(\oi)^T \A_j(\oi)$.
%
\subsubsection{Inducing Inputs and Hyper-parameters \label{sec:hyperparameters}}
To learn the hyperparameters, which in this model include the mixing weights, the covariance hyperparameters of the latent processes, and the noise precision of each output, we follow standard practice in GP inference.
For this model this involves taking derivatives of the ELBO and applying standard stochastic gradient descent in alternative steps with the variational parameters, much like a variational EM algorithm.
The derivatives are given in the supplementary material.

Learning of the inducing inputs, which was not considered in the single output case in \citet{hensmangaussian}, is also possible in our stochastic optimization approach.
In the supplementary material, we show that
%with the help of vectorized operations,
the additional cost of computing the derivatives of the lower bound wrt the inducing inputs is not significantly
higher than the cost of updating the variational parameters.
%can be computed with a cost \emph{independent of the input dimensionality}.
This makes optimizing the inducing locations a practical option, which can be critical
in high-dimensional problems. %where fixing the inducing inputs may hinder the performance of the model.
Indeed, our experiments on a large scale multi-output problem show that automatic learning of the inducing inputs can lead to significant performance gain with little overhead in computation.
%
\subsection{COMPLEXITY ANALYSIS \label{sec:complexity}}
In this section we analyze the complexity of the model and compare it to existing multi-output approaches.
For consistency, we first unify common notations used for all models.
We use $P$ as the number of outputs; $N$ as the number of inputs; $Q$ as the number of shared latent processes; and $M_t$ as the \emph{total} number of inducing inputs.
It is worth noting that $M_t = (P + Q) \times M$ in our COGP model, assuming that each sparse process has equal number of inducing points.
Also, COGP has $P$ additional individual processes, one for each output.

The complexity of COGP can be read off by inspecting the ELBO  in Equation \eqref{eq:L}, with the key observation that it contains
a sum over the outputs as well as over the inputs.
This means a mini-batch containing a small subset of the inputs and outputs can be used for stochastic optimization.
Technically, the cost is $\calO(M^3)$ or $\calO(N_bM^2)$, where $N_b$ is the size of the mini-batches, depending on which is larger between $M$ and $N_b$.
In practice, we may use $N_b > M$ as a large batch size (e.g.~$N_b = 1000$) helps reduce stochasticity of the optimization.
However, here we use $\calO(M^3)$  for easier comparison with other models whose time and storage demands are
given in Table \ref{tab:complexity}.
We see that COGP has
a computational  complexity that is independent of the size of the inputs and outputs,
which makes it the only method capable of handling large scale problems.
%
\setlength{\tabcolsep}{4pt}
\begin{table}[t]
\caption{Comparison of the time and storage complexity of approximate inference of  multi-output GP models.
A\&W, 2009 refers to \citet{alvarez-lawrence-nips-08}. COGP is the only method with complexity \emph{independent} of the
 number of inputs $N$ and outputs $P$, thus it can scale to very large datasets.}
\label{tab:complexity}
\begin{center}
\small
\begin{tabular}{lcc}
\toprule
\textbf{METHOD} & \textbf{TIME} & \textbf{STORAGE} \\ \hline
COGP, this paper  & $\calO(M^3)$ & $\calO(M^2)$ \\
SLFM, \citep{teh-et-al-aistats-05} & $\calO(QNM_t^{2})$ & $\calO(QNM_t)$ \\
MTGP, \citep{bonilla-et-al-nips-08} & $\calO(PNM_t^2)$ & $\calO(PNM_t)$\\
CGP-FITC (A\&W, 2009)& $\calO(PNM_t^2)$ & $\calO(PNM_t)$ \\
GPRN, \citep{wilson-et-al-icml-12} & $\calO(PQN^3)$ & $\calO(PQN^2)$ \\
\bottomrule
\end{tabular}
\end{center}
\end{table}

\subsection{PREDICTION}
The predictive distribution of the $i$-th output for a test input $\x_*$ is given by:
\begin{align}
p(\fstar | \y, \x_*) = \Normal(\fstar; \sum_{j=1}^Q w_{ij} \mu_{j*} + \mu^h_{i*}, w_{ij}^2 s_{j*} + s^h_{i*}),
\end{align}
where $\mu_{j*}$ and $s_{j*}$ are the mean and variance of the prediction for $g_{j*} = g_j(\x_*)$, i.e. $p(g_{j*} | \y, \x_*) = \Normal(g_{j*}; \mu_{j*}, s_{j*})$.
Likewise, $\mu^h_{i*}$ and $s^h_{i*}$ are the mean and variance of the prediction for $h_{i*} = h_i(\x_*)$, $p(h_{i*} | \y, \x_*) = \Normal(h_{i*}; \mu^h_{i*}, s^h_{i*})$.
These predictive means and variances are given by:
\begin{align}
\mu_{j*} &= \vec{k}_{j*z} \K_{jzz}^{-1} \m_j  \text{,}\\
s_{j*} &= k_{j**} - \vec{k}_{j*z} \left(\K_{jzz}^{-1} - \K_{jzz}^{-1} \S_j \K_{jzz}^{-1} \right) \vec{k}_{j*z}^T  \text{,}\\
\mu^h_{i*} &= \vec{k}_{i*z} \K_{izz}^{-1} \mhi \text{,} \\
s^h_{i*} &= k_{i**} - \vec{k}_{i*z} \left(\K_{izz}^{-1} - \K_{izz}^{-1} \S_i \K_{izz}^{-1} \right) \vec{k}_{i*z}^T \text{,}
\end{align}
where $k_{j**} = k_j(\x_*, \x_*)$, $k_{i**} = k^h_i(\x_*, \x_*)$,  $\vec{k}_{j*z}$ is the covariance between $\x_*$ and $\Zj$, and $\vec{k}_{i*z}$ is the covariance between $\x_*$ and $\Zhi$.

%\noindent Also as a consequence of sufficient statistics, the approximate prediction for test targets $\vfstar$ at test inputs $\X_*$ is
%\begin{align}
%\nonumber
%p(\vfstar | \y, \X_*) &= \int p(\vfstar | \f, \u, \X_*) q(\f, \u | \y) \der \f \der \u \\
%\nonumber
%&= \int p(\vfstar | \u) q(\u| \y) p(\f | \u) \der \f \der \u \\ \nonumber
%&= \int p(\vfstar | \u) q(\u| \y) \der \u \\
%\label{eq:sorprediction}
%&= \Normal(\vfstar; \bs{\mu_*},\vec{s_*})
%\end{align}
%where,
%\begin{align}
%\nonumber
%\bs{\mu_*} &= \K_{*z} \K_{zz}^{-1}\m \\
%\nonumber
%\S_* &= \K_{**} - \K_{*z} \left(\K_{zz}^{-1} - \K_{zz}^{-1} \S \K_{zz}^{-1} \right) \K_{*z}^T.
%\end{align}
% Here $\K_{*z}$ is the covariance matrix between test and inducing inputs, $\K_{zz}$ is the covariance matrix of the inducing inputs.

%\begin{align}
%p(\fstar | \y, \x_*) = \int \Normal(\fstar; \sum_{j=1}^Q w_j g_{j*} + h_{i*}, 0) p(\g_* | \y, \x_*) p(h_{i*} | \y, \x_*) \der \g_* \der h_{i*},
%\end{align}


\chapter{Multi-layer Kernel Machines}
\label{chap_mkm}
%This chapter introduces the concept of MKMs proposed in \cite{saul} et al., followed by an empirical study on some popular datasets cited extensively in deep learning literatures. This chapter is organized as follows; section \ref{chap2_mkm} gives a brief introduction to MKMs and the multi-layer composition of few kernel functions, section \ref{chap2_exp} contains the results of empirical study using MKMs, section \ref{chap2_tsne} talks about the visualization technique used to plot the distribution of digits(from \cite{mnist} dataset) in a low-dimensional space, section \ref{chap2_mkm_mix} contains the results of empirical study on MKMs with mixed kernels and section \ref{chap2_conc} concludes the chapter.

%\label{chap2_mkm}
Multi-layer Kernel Machines (MKMs\nomenclature{MKM}{Multi-layer Kernel Machines}) were first introduced by \cite{saul} et al. In their framework, arc-cosine kernel  which itself is having a layered architecture was used. The architecture of MKMs consists of $L$ layers and in each layer the data is subjected to unsupervised dimensionality reduction methods, viz. kernel PCA(\cite{kpca} et al.),  followed by a supervised feature selection.

The MKM machines use multilayer kernel function, whose description is given in the next section. The rest of the chapter is organized as follows: section \ref{chap2_exp} contains the results of empirical study using MKMs, section \ref{chap2_tsne} describes the visualization technique used to plot the distribution of digits (from \cite{mnist} dataset) in a low-dimensional space, section \ref{chap2_mkm_mix} contains the results of empirical study on MKMs with mixed kernels and  concluding remarks are given in section \ref{chap2_conc}.

\section{Multi-layer Kernel}
Corresponding to each Reproducing kernel Hilbert spaces (RKHS) $\mathcal{F}$ there exists a unique reproducing kernel
$k: \mathcal{X} \times \mathcal{X} \rightarrow \mathcal{F}$,  where $ k(x,y) = \phi(x) \cdot \phi(y), x, y \in \mathcal{X} \; \text{where} \; \mathcal{X}    \text{ is a normed space, } \phi(x), \phi(y) \in \mathcal{F}$ and $\cdot$ is the inner product defined on $\mathcal{F}$.

Iteratively applying the mapping $\phi(\cdot)$ on the inputs $x \textrm{ and } y$ and then taking their inner product, we can obtain an L-layer kernel function as
\[ k^{(L)}(x,y) = \underbrace{\phi(\phi(\ldots \phi(x)))}_{\textrm{L times}} \textrm{ } \cdot \textrm{ } \underbrace{\phi(\phi(\ldots \phi(y)))}_{\textrm{L times}} \]

Thus a $L$ layer kernel machine consists of $L$ RKHS's.

\subsection{Multi-layer Composition of Polynomial and Gaussian Kernels}
\label{chap2_mlother}
A polynomial kernel of degree d is defined as follows
\[k(x, y) = (x \cdot y)^d\]
Consider a two layer composition of polynomial kernel which can be computed as
\begin{equation*}
\begin{aligned}
\phi(\phi(x)) \cdot \phi(\phi(y)) &= \bigg(\phi(x) \cdot \phi(y)\bigg)^d \\
& = {(x \cdot y)^d}^d =  {(x \cdot y)^d}^2
\end{aligned}
\end{equation*}
Thus the higher order compositions of polynomial kernels are simply polynomials of higher degree than the one from which it is constructed.

The two layer composition for gaussian kernels $k(x, y) = e^{-\lambda\norm{x-y}^2} $ is given by
\begin{equation*}
\begin{aligned}
\phi(\phi(x)) \cdot \phi(\phi(y)) &= e^{-\lambda\norm{\phi(x)-\phi(y)}^2} \\
& = e^{-2\lambda(1-k(x, y))}
\end{aligned}
\end{equation*}

\subsection{Multi-layer Composition of Arc-cosine Kernels}
\label{chap2_mlarc}
Let x, y be two inputs in $\mathbb{R}^d$. Define $\theta$ as the angle between them.
\[ \theta = cos^{-1}\left ( \frac{x\cdot y}{\left \| x \right \| \left \| y \right \|} \right ) \]
Then the kernel function computed by the arc-cosine kernel is
\begin{equation}
k_n(x,y) = \frac{1}{\pi}\left \| x \right \|^n \left \| y \right \|^n J_n(\theta)
\label{arc_cosine_kernel}
\end{equation}
where n is called the \textit{degree of the kernel} and
\[ J_n(\theta) = (-1)^n(sin\theta)^{2n+1} \left ( \frac{1}{sin\theta} \frac{\partial}{\partial \theta} \right )^n \left ( \frac{\pi-\theta}{sin\theta} \right ) \]

$J_n(\theta)$ for n=0, 1, 2 is computed as shown below.
\[ J_0(\theta) = \pi-\theta \]
\[ J_1(\theta) = sin\theta + (\pi-\theta)cos\theta \]
\[ J_2(\theta) = 3sin\theta cos\theta + (\pi-\theta)(1+2cos^2\theta) \]
for n=0, it takes the simple form
\[k_0(x,y) =  1- \frac{1}{\pi}cos^{-1}\left ( \frac{x\cdot y}{\left \| x \right \| \left \| y \right \|} \right )  \]
hence the name arc-cosine kernel is given.The kernel function computed by arc-cosine kernel is related to the computation of multi-layer threshold networks as shown below.

Consider a single layer neural network with weights $W_{ij}$ that connects the $j^{th}$ input unit to the $i^{th}$ output unit(see figure \ref{fig_singnet}). The network maps input $x$ to output $f(x)$ by applying a non-linear map
\[ f(x) = g(W \cdot x) \]
where the non-linearity is described by the network`s activation function
\[g_n(z) = \Theta(z)z^n \]
with
\[ \Theta(z) = \frac{1}{2}(1+sign(z)) \]
this activation function is called one-sided polynomial activation function, whose graph for different n values is also shown in figure \ref{fig_singnet}.

\begin{figure}[H]
  \centering
  \captionsetup{justification=centering,margin=0.1cm}
  \includegraphics[width=1.0\linewidth, height=7cm]{figures/singnet}
  \caption{Single layer network and thresholded activation functions(\cite{saul} et al.)}
  \label{fig_singnet}
\end{figure}

Let $f(x)$ and $f(y)$ be the outputs corresponding to inputs $x$ and $y$. Then the inner product of $f(x)$ and $f(y)$ is
\[ f(x)\cdot f(y) = \sum_{i=1}^m \Theta(w_i\cdot x) \Theta(w_i\cdot y)(w_i\cdot x)^n (w_i\cdot y)^n\]
Here $w_i$ is the $i^{th}$ row of weight matrix $W$ and $m$ is the no of output units. Assume $W_{ij}$ are Gaussian distributed with zero mean and unit variance and the network has an infinite number of output units. Then
\[ lim_{m\rightarrow \infty} \frac{2}{m}f(x)\cdot f(y) = k_n(x,y) \]
where
\begin{equation}
k_n(x,y) = 2\int dw \frac{e^{-\frac{\norm{w}^2}{2}}}{(2\pi )^{d/2}} \Theta (w\cdot x) \Theta (w\cdot y) (w\cdot x)^n (w\cdot y)^n
\label{arc_cosine_integral}
\end{equation}
The kernel function obtained in equation \ref{arc_cosine_integral} can be converted into an alternate form as in equation \ref{arc_cosine_kernel} with the derivation shown in \cite{saul} et al.

The multi-layer composition of arc-cosine kernel can be recursively computed as
\[ k^{(L+1)}_n(x,y) = \frac{1}{\Pi} \Bigg[ k^{(L)}(x,x) \textrm{ } k^{(L)}(y,y)\Bigg]^{\frac{n}{2}} J_n(\theta_n^{(L)}) \]
where $\theta_n^{(L)}$ is the angle between images of $x \textrm{ and } y$ in the feature space after L layer composition
\[ \theta_n^{(L)} = \textrm{cos}^{-1}\Bigg( k^{(L)}(x,y) \Bigg[ k^{(L)}(x,x) \textrm{ } k^{(L)}(y,y)\Bigg]^{\frac{-1}{2}} \Bigg) \]
In the above formulation, we have assumed that the arc-cosine kernels have the same degree $n$ at every layers of recursion. We can also use kernels of different degree at different layers. The intuition behind the multi-layer composition in case of arc-cosine kernels is, if the base kernel $k(x,y) = \phi(x) \cdot \phi(y)$ can mimic the computation of a single-layer network, then the iterated mapping in $k^{(L)}(x,y)$ can mimic the computation of multi-layer network(\cite{saul} et al.).

\subsection{MKM Architecture}
 The architecture of MKMs for solving classification tasks  is similar to that of neural network based deep learning machines, with unsupervised feature extraction(using Kernel PCA, \cite{kpca} et al.) followed by supervised feature selection in each layer. Figure \ref{fig_mkm} shows the architecture of an MKM consisting of $L$ layers of non-linear transformations.

\begin{figure}[h]
  \centering
  \captionsetup{justification=centering,margin=0.1cm}
  \includegraphics[scale=0.5]{figures/mkm}
  \caption{An MKM with L layers of transformations. Each layer consists of unsupervised feature extraction(using kernel PCA) followed by supervised feature selection.}
  \label{fig_mkm}
\end{figure}

The supervised feature selection allows us to retain only those set of features that are most informative for the pattern recognition task. The number of features to be passed to the next layer determines the width of that layer. The optimal layer width is computed by ranking the features based on their importance and selecting the required set of feature with the help of a lightweight classifier (detailed procedure is given in the experiments section). This procedure determines the architecture of the network in a greedy, layer-by-layer fashion. In their implementation \cite{saul} et al. used exhaustive search in the range 10 to 300 to determine the optimal set of features. The output from the final layer can be passed to any classifier. Though any kernel can be used for the kernel PCA based feature extraction, \cite{saul} et al. emphasized on using arc-cosine kernels due to their similarity with deep learning architecture of neural networks and the inclusion of multiple layers is significant  only in case of arc-cosine kernels.

\section{Experiments on Arc-cosine Kernel MKMs}
\label{chap2_exp}
Empirical study was conducted on four datasets, in which three were created from \cite{mnist} dataset of handwritten digits by adding noise in the background and one was a binary classification problem on shape images. The detailed experimental set up and a short description about datasets used is given below.
\subsection{Experimental Set-Up}
In the training phase, for each layer we set apart 10000 datapoints for training the model and 2000 datapoints for cross validating kernel parameters. For the kernel PCA we chose 3000 datapoints  from the training set of 10000 images randomly. In each layer after extracting features with KPCA, we train a lightweight classifier like kNN and test the performance on the held out 2000 images. The best kernel parameter was chosen based on this performance values. With that parameter value, we did feature extraction on the entire dataset and then passed it to feature selection module. The feature selection was done by using univariate feature selection method available in \cite{scikit} library. This method produces a ranking of features with a univariate statistical test. Based on this rank we can choose the required number of important features. We chose top 5 percent features based on this rank, since empirically it was giving a consistant performance.

In the final classification stage SVMs with arc-cosine kernels are used. The metric used for comparing the performance is, percentage loss in test dataset. Percentage loss is estimated as
\[ \textrm{loss in percentage} = \Bigg(1 - \frac{\# \textrm{correct classifications}}{\# \textrm{datapoints}} \Bigg) \times 100 \]

\subsection{Mnist-back-rand Dataset}
The \textit{mnist-back-rand} dataset was created by filling the image background with random pixel values. Each pixel value of the background was generated uniformly between 0 and 255. Image size is 28$\times$28 and the dataset contains 12000 training and 50000 testing images.
\subsection{Mnist-back-image Dataset}
The \textit{mnist-back-image} dataset was generated by filling the image background with random image patches. The patches were extracted randomly from a set of 20 images downloaded from the internet. The dataset contains contains 12000 training and 50000 testing images, each of size 28$\times$28.
\subsection{Mnist-rot-back-image Dataset}
The \textit{mnist-rot-back-image} is a rotated variant of \textit{mnist-back-image} where the rotation angle is generated uniformly between $0$ and $2\pi$. Image size and number of samples are also the same as \textit{mnist-back-image} dataset.
\subsection{Rectangles-image Dataset}
For \textit{rectangles-image} dataset, the classification task is to identify whether a rectangle contained in an image has larger width or length. The dataset is constructed by uniformly sampling the height and width of the rectangles. Then random image patches are added in the background, where image patches are extracted from one of the 20 images used by \textit{mnist-back-image}. Each image is of size 28$\times$28 and each dataset contains 12000 training images and 50000 testing images.

Figure \ref{samples} contains some sample images from first three dataset and Figure \ref{chap2_rect} contains the samples from \textit{rectangles-image} dataset.
\begin{figure}[h]
  \centering
  \captionsetup{justification=centering,margin=0.1cm}
  \includegraphics[scale=0.55]{figures/data_sample1}
  \caption{sample images from \textit{mnist-back-rand}(first row), \textit{mnist-back-image}(second row) and \textit{mnist-rot-back-image}(third row) datasets.}
  \label{samples}
\end{figure}

\begin{figure}[h]
  \centering
  \captionsetup{justification=centering,margin=0.1cm}
  \includegraphics[scale=0.55]{figures/rect_image}
  \caption{sample images from \textit{rectangles-image} dataset.}
  \label{chap2_rect}
\end{figure}

\subsection{Results and Analysis}
\renewcommand{\arraystretch}{2.1}
\begin{table*}
\centering
\begin{tabular}{|c|c|c|c|c|c|c|c|}
  \hline
  \multirow{2}{*}{\textbf{Dataset}} & \multicolumn{7}{ |c| }{\textbf{Loss in Percentage}} \\
  \cline{2-8}
  & $\textrm{SVM}_{\textrm{RBF}}$ & $\textrm{SVM}_{\textrm{Poly}}$ & NNet & DBN-3 & SAA-3 & DBN-1 & MKMs\\
  \hline
  \textit{back-rand} & 14.58 & 16.62 & 20.04 & \textbf{6.73} & 11.28 & 9.80 & 10.55\\
  \hline
  \textit{back-image} & 22.61 & 24.01 & 27.41 & 16.31 & 23.00 & \textbf{16.15} & 21.39\\
  \hline
  \textit{rot-back-image} & 55.18 & 56.41 & 62.16 & \textbf{47.39} & 51.93 & 52.21 & 51.61\\
  \hline
  \textit{rect-image} & 24.04 & 24.05 & 33.20 & 23.69 & 24.05 & \textbf{22.50} & 23.01\\
  \hline
\end{tabular}
\caption{Experimental Results of MKMs with Arc-cosine Kernels}
\label{tab_results_mkm}
\end{table*}
\renewcommand{\arraystretch}{1}

Table \ref{tab_results_mkm} also contains best results obtained from other models(\cite{dbn} et al.) like SVM with RBF kernel($\textrm{SVM}_{\textrm{RBF}}$), SVM with polynomial kenel($\textrm{SVM}_{\textrm{Poly}}$), single hidden layer feed-forward neural network(NNet\nomenclature{NNet}{Neural Network with one hidden layer}), Deep Belif Networks(DBN)\nomenclature{DBN}{Deep Belif Networks} with 1 hidden layer(DBN-1), DBN with 3 hidden layer(DBN-3) and 3 hidden layer Stacked Autoassociator Network(SAA-3). The first three models comes under shallow architectures and the remaining are deep architectures. From the table, it can be observed that MKMs outperforms all the remaining models except Deep Belif Networks(DBN). Compared to DBN the architecture, parameter tuning and optimization  are fairly simple in MKMs.

The change in the performance of the classifier when each adding layer to the model was also analyzed. Figures \ref{mkm_rand} and \ref{mkm_image} shows the variations in the classifier performance when model complexity was increased by adding more layers(on \textit{mnist-back-rand} and \textit{mnist-back-image} datasets respectively). The results indicates that in some cases better representation can be obtained even with a less complex model (eg: in case of \textit{mnist-back-image} dataset, where the performance degrades when model complexity increases).

\begin{figure}
  \centering
  \captionsetup{justification=centering,margin=0.1cm}
  \includegraphics[scale=0.65]{figures/mkm_rand}
  \caption{Change in classifier performance on \textit{mnist-back-rand} dataset when increasing the number of layers.}
  \label{mkm_rand}
\end{figure}

\begin{figure}
  \centering
  \captionsetup{justification=centering,margin=0.1cm}
  \includegraphics[scale=0.65]{figures/mkm_image}
  \caption{Change in classifier performance on \textit{mnist-back-image} dataset when increasing the number of layers.}
  \label{mkm_image}
\end{figure}

\section{Visualizing Features using tSNE}
\label{chap2_tsne}
In order to visualize the features learned by the MKM, t-distributed Stochastic Neighbor Embedding(tSNE, proposed by \cite{tsne} et al.) was used. tSNE\nomenclature{tSNE}{t-distributed Stochastic Neighbur Embedding} is a tool for visualizing high-dimensional data. It converts the similarities between datapoints to joint probabilities and tries to minimize the Kullback-Leibler divergence(KL divergence) between the joint probabilities of the low-dimensional embedding and the high dimensional data(\cite{tsne} et al.). tSNE is a variant of stochastic neighbor embedding \nomenclature{SNE}, that is much easier to optimize and produce significantly better visualizations by reducing the tendency of the  points to get accumulated  at the center of the map.

Suppose $X = \{x_1, x_2, \ldots, x_n\}$ are the high-dimensional datapoints, and $Y = \{y_1, y_2, \ldots, y_n\}$ the corresponding low-dimensional map points. In SNE, we first convert high-dimensional representation of datapoints into pairwise similarities that can be interpreted as conditional probabilities. Intuitively, the similarity of a datapoint $x_i$ to $x_j$ is interpreted as the probability of $x_i$ picking up $x_j$ as its neighbor with a Gaussian centered at $x_i$(denoted as $p_{j/i}$). Mathematically this is formulated as
\[ p_{j/i} = \frac{exp(-\norm{x_i-x_j}^2/2 \sigma_i^2}{\sum_{k\neq i} exp(-\norm{x_i - x_k}^2/2 \sigma_i^2)}  \]

where $\sigma_i$ is the variance of the Gaussian centered at $x_i$. For every $x_i$ we calculate corresponding $\sigma_i$ which will vary according to the density of datapoints around $x_i$(if the region around $x_i$ is dense $\sigma_i$ will be low and vice versa).

Similarly for the low-dimensional representations $y_i$ and $y_j$ corresponding to $x_i$ and $x_j$ respectively, we compute the conditional probability(denoted as $q_{j/i}$) by setting the variance of the Gaussian to $\frac{1}{\sqrt{2}}$. Thus
\[ q_{j/i} = \frac{exp(-\norm{y_i-y_j}^2}{\sum_{k\neq i} exp(-\norm{y_i - y_k}^2)}  \]

Since we are only interested in modelling pairwise similarity, we set $p_{i/i} = q_{i/i} = 0$. The crux of SNE as stated by \cite{tsne} et al. is ``if the map points $y_i$ and $y_j$ correctly model the similarity between high-dimensional datapoints $x_i$ and $x_j$ then the conditional probabilities $p_{j/i}$ and $q_{j/i}$ will be equal''. In this case, the cost function can be formulated to minimize the difference between conditional probabilities $p_{j/i}$ and $q_{j/i}$ using KL divergence. The cost function $C$ is defined as
\begin{equation}
C = \sum_{i}\textrm{KL}(\textrm{P}_i \big|\big| \textrm{Q}_i) = \sum_{i}\sum_{j} p_{j/i} log \frac{p_{j/i}}{q_{j/i}}
\label{cost_tsne}
\end{equation}
where $\textrm{P}_i$ is the conditional probability distribution over all other datapoints given datapoint $x_i$ and $\textrm{Q}_i$ is the conditional probability distribution over all other map points given the map point $y_i$.

The minimization of the cost function in \ref{cost_tsne} is performed by using gradient descent method. The gradient is computed as
\[ \frac{\partial C}{\partial y_i} = 2\sum_j(p_{j/i}-q_{j/i} + p_{i/j}-q_{i/j})(y_i - y_j) \]
Though SNE can produce good quality visualizations, it has the following limitations.
\begin{itemize}
\item Since the KL divergence is not symmetric, different types of error in the pairwise distances in low-dimensional map are not weighted equally(there is a large cost for using widely separated map points to represent nearby datapoints, but the cost is small for using nearby map points to represent widely separated datapoints).
\item crowding of map points in the center of the map.
\end{itemize}

tSNE is a variant of SNE which improves the visualization quality by alleviating the above mentioned problems.
\begin{itemize}
\item It uses a symmetric version of the SNE cost function by employing a joint probability distribution instead of the conditional probability distribution used by SNE. This also results in simpler gradients.
\item It uses student-t distribution to compute the similarity between map points. tSNE employs a heavy-tailed distribution for the map points to alleviate both the crowding problem and optimization problems of SNE.
\end{itemize}

In the symmetric version of the  tSNE the cost function is computed as shown below.
\begin{equation*}
C = \textrm{KL}(\textrm{P} \big|\big| \textrm{Q}) = \sum_{i}\sum_{j} p_{ij} log \frac{p_{ij}}{q_{ij}}
\end{equation*}
where $P$ and $Q$ are the joint probability distribution functions in the high-dimensional and low-dimensional space respectively. Here also we set $p_{ii}$ and $q_{ii}$ to zero. The symmetry in the cost function is achieved due to the fact that $p_{ij} = p_{ji}$ and $q_{ij} = q_{ji}$ $\forall i,j$. The joint probability in high-dimensional space($p_{ij}$) and low-dimensional space($q_{ij}$) are computed as
\[ p_{ij} = \frac{exp(-\norm{x_i-x_j}^2/2 \sigma^2}{\sum_{k\neq l} exp(-\norm{x_k - x_l}^2/2 \sigma^2)}  \]
\[ q_{ij} = \frac{exp(-\norm{y_i-y_j}^2}{\sum_{k\neq l} exp(-\norm{y_k - y_l}^2)}  \]
The gradient of the symmetric SNE has a fairly simple form
\[ \frac{\partial C}{\partial y_i} = 4\sum_j(p_{ij}-q_{ij})(y_i - y_j) \]
Symmetric SNE still faces problems when a datapoint $x_i$ is an outlier; when $x_i$ is an outlier $\norm{x_i-x_j}^2$ is large for all $x_j$ with $x_i$, hence $p_{ij}$ are extremely low, so the location of its low-dimensional map point $y_i$ has very little effect on the cost function. This problem is addressed in tSNE by defining $p_{ij} = \frac{p_{j/i} + p_{i/j}}{2n}$, which ensures that $\sum_j p_{ij} \ge \frac{1}{2n}$ $\forall \textrm{ } x_i$, hence each datapoint $x_i$ makes a significant contribution to the cost function. The crowding problem is addressed in tSNE by using a student t-distribution with one degree of freedom as the heavy-tailed distribution in the low-dimensional map. Thus the joint probability $q_{ij}$ is computed as
\[ q_{ij} = \frac{(1+\norm{y_i-y_j}^2)^-1}{\sum_{k \neq l} (1+\norm{y_k-y_l}^2)^-1} \]
Then, finally the gradient of the cost function is obtained as
\[ \frac{\partial C}{\partial y_i} = 4\sum_j(p_{ij}-q_{ij})(y_i - y_j)(1+\norm{y_i-y_j}^2)^-1) \]

tSNE was employed in our experiments to understand the changes in the data distribution before and after the feature learning process. Figure \ref{tsne_rand_raw} shows the tSNE embedding of the raw data from \textit{mnist-back-rand} dataset and figure \ref{tsne_rand_mkm} shows the same embedding applied on the features produced by MKM with arc-cosine kernels. The tSNE embedding of features learned by MKMs are crowded near the center, hence it was difficult to interpret the separability.

\begin{sidewaysfigure}
  \centering
  \captionsetup{justification=centering,margin=0.1cm}
  \includegraphics[scale=0.45]{figures/tsne_rand_rawdata}
  \caption{tSNE embedding of raw data from \textit{mnist-back-rand} dataset.}
  \label{tsne_rand_raw}
\end{sidewaysfigure}

\begin{sidewaysfigure}
  \centering
  \captionsetup{justification=centering,margin=0.1cm}
  \includegraphics[scale=0.45]{figures/tsne_rand_mkmbest}
  \caption{tSNE embedding of features obtained by MKM from \textit{mnist-back-rand} dataset.}
  \label{tsne_rand_mkm}
\end{sidewaysfigure}

\section{MKMs with Mixed Kernels}
\label{chap2_mkm_mix}
We tried different kernel functions in different layers of MKMs (Gaussian, arc-cosine, polynomial etc.) while performing KPCA. It had been seen that mixing a layer of Gaussian or polynomial kernel in between successive layers of arc-cosine kernel improves the result. This further supported the belief that, with more similarity information (by using more kernels) in hand, we can build better representations. Table \ref{tab_results_mix} shows the result obtained with the mixed kernel models. Because of the improved performance, we used  Multi-layer Multiple Kernel Machines(ML-MKL) model for our analysis, whose  discussion is given in the next chapter.

\renewcommand{\arraystretch}{2.1}
\begin{table}
\centering
\begin{tabular}{|c|c|c|c|c|c|c|c|}
  \hline
  \multirow{2}{*}{\textbf{Dataset}} & \multicolumn{7}{ |c| }{\textbf{Loss in Percentage}} \\
  \cline{2-8}
  & $\textrm{SVM}_{\textrm{RBF}}$ & $\textrm{SVM}_{\textrm{Poly}}$ & NNet & DBN-3 & SAA-3 & DBN-1 & $\textrm{MKMs}_\textrm{(mix)}$\\
  \hline
  \textit{back-rand} & 14.58 & 16.62 & 20.04 & \textbf{6.73} & 11.28 & 9.80 & 9.54\\
  \hline
  \textit{back-image} & 22.61 & 24.01 & 27.41 & 16.31 & 23.00 & \textbf{16.15} & 20.94\\
  \hline
  \textit{rot-back-image} & 55.18 & 56.41 & 62.16 & \textbf{47.39} & 51.93 & 52.21 & 54.03\\
  \hline
  \textit{rect-image} & 24.04 & 24.05 & 33.20 & 23.69 & 24.05 & \textbf{22.50} & 25.04\\
  \hline
\end{tabular}
\caption{Experimental Results of MKMs with Mixed Kernels}
\label{tab_results_mix}
\end{table}
\renewcommand{\arraystretch}{1}

As in MKMs with arc-cosine kernel, we chose top 5 percent features based on univariate test score for the mixed kernel MKMs. The classifier used in the output layer was an SVM\nomenclature{SVM}{Support Vector Machines} with arc-cosine kernel. The results indicates that MKMs with mixed kernels improved the classifier performance for \textit{mnist-back-rand} and \textit{mnist-back-image} datasets, whereas the classification accuracy declined in the case of \textit{mnist-rot-back-image} and \textit{rectangles-image} dataset. The tSNE embedding of the features produced by MKMs with mixed kernels is shown in figure \ref{tsne_rand_mkmmix} (for \textit{mnist-back-rand} dataset). The best result in this dataset was obtained from a model consisting of three layers with a Gaussian kernel in the middle layer. The visualization indicates that, MKMs with mixed kernels has good separability between different classes.
\begin{sidewaysfigure}
  \centering
  \captionsetup{justification=centering,margin=0.1cm}
  \includegraphics[scale=0.45]{figures/tsne_rand_mkmmix}
  \caption{tSNE embedding of features obtained by MKM with mixed kernels from \textit{mnist-back-rand} dataset.}
  \label{tsne_rand_mkmmix}
\end{sidewaysfigure}

\section{Conclusion}
\label{chap2_conc}
In this chapter we analysed the MKMs framework which has the characteristics of deep learning algorithms by employing  kernel methods. The empirical study indicated that MKMs are performing comparable with that of  popular deep learning algorithms like DBN, SAA etc. We also analysed the performnace of MKMs with mixed kernels. The mixing produces better results when other kernels are sandwiched between two arc-cosine kernels.

The visualization of features learned by MKMs are difficult to interpret due to crowding of datapoints near the center, but for MKMs with mixed kernels the visualization indicates that the separation is good enough between different classes, even though the samples from same class is distributed in different locations.


\documentclass[a4paper,11pt]{article}

\usepackage{physics}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{amsmath}
\usepackage{amsthm, mathtools}
%\usepackage{hyperref}
\usepackage{color}
\usepackage{jheppub}
\usepackage[T1]{fontenc} % if needed

\newcommand{\be}{\begin{equation}}
\newcommand{\ee}{\end{equation}}
\newcommand{\bes}{\begin{equation*}}
\newcommand{\ees}{\end{equation*}}
\newcommand{\bea}{\begin{flalign*}}
\newcommand{\eea}{\end{flalign*}}

%\linespread{1.0}
%\setlength{\parindent}{0em}
%\setlength{\parskip}{0.8em}

\title{\textbf{Methods for Data}}
\author{Aditya Vijaykumar}
\affiliation{International Centre for Theoretical Sciences, Bengaluru, India.}
\emailAdd{aditya.vijaykumar@icts.res.in}
\abstract{}

\begin{document}
\maketitle
\section{A Few Basics}

Physicists believe in a \textit{cause and effect} philosophy of the world. If we are given a fair coin, we can assign equal probabilities to \textit{head} and \textit{tail} and predict the outcomes given the number of times the coin is tossed. We could deal with unfair coins equally well, and given the properties of the unfair coin  (\textit{ie.} probabilities of landing heads/tails/neither) are specified, we can make deductions about the outcomes of our \textit{experiment}.

But, generally, we have information about the observed effects, from which we are supposed to ascertain what models/scenarios these effects would have arisen from. Consider tossing a single coin ten times. The nature of the coin is not specified, but we find that all ten times the coin lands heads up. Are we to believe that the coin is \textit{biased}? Not quite. The most one can do is make some inference based on the data and our prior knowledge (\textit{ie}. we say that the coin is \textit{most likely} biased), adding a caveat that we reserve the right to alter our result if new information pops up in the future (\textit{ie.} we have the data from more tosses of the same coin).

This makes the whole problem of \textit{data analysis} open-ended. Through this exploration, we hope to understand the mathematical rules governing the analyses.

\subsection{Operations on Probabilities}
All the definitions of probability made below are with respect to some background information denoted by $ I $.
\begin{itemize}
	\item \textbf{Sum Rule} - Probability of an X being true and that of it being false should add up to 1.
	\begin{equation*}
	P(X | I) + P (\overline{X} | I) = 1
	\end{equation*}
	\item \textbf{Product Rule} - Probability of both $ X $ and $ Y $ being true should be the product of probability of $ Y $ being true and the probability of $ X $ being true given $ Y $ is true.
	\begin{equation*}
	P(X,Y| I) = P(X|Y,I) \cross P(Y|I) = P(Y|X,I) \cross P(X|I)
	\end{equation*}
	The symmetry property of the AND operation on $ X $ and $ Y $ means that we can interchange $ X $ and $ Y $ on the LHS.
	\item \textbf{Bayes' Theorem} - The probability of $ X $ conditioned on $ Y $ is proportional to the probability of $ Y $ conditioned on $ X $.
	\begin{equation*}
	P(X|Y,I)  =\dfrac{ P(Y|X,I) \cross P(X|I)}{P(Y|I)}
	\end{equation*}
	The proof follows from the second and third expressions in the product rule. The magic of Bayes' theorem becomes evident when we replace $ X $ with \textit{hypothesis} and $ Y $ with \textit{data}.
	\begin{equation*}
	\underbrace{P(hypothesis|data,I)}_{\text{posterior}} \propto \underbrace{ P(data|hypothesis,I)}_{\text{likelihood}} \cross \underbrace{ P(data|hypothesis,I)}_{\text{prior}}
	\end{equation*}
	The statement of the Bayes' theorem now becomes a powerful tool for inversion! Some definitions follow,
	\begin{itemize}
		\item \textbf{Prior} - This encodes our degree of ignorance about the hypothesis before we have even touched the data.
		\item \textbf{Likelihood Function} - This encodes how well data agrees with a particular model (\textit{Details Later}).
		\item \textbf{Posterior} - This gives us the degree of knowledge of the truth of the model in light of the data.
		\item  \textbf{Evidence} - $ P(data| I) $ which we omitted while writing the expression of the Bayes' theorem. In most cases, it will remain a proportionality constant.
	\end{itemize}

	\item \textbf{Marginalization} - The probability of $ X $ is the same as the probability of $ X $ given $ Y_k $, summed over all $ k $. Here, $ \{Y_k\} $ is the set of all possibilities in $ Y $.
	\begin{equation*}
	P(X| I) = \sum_k P(X|Y_k, I)
	\end{equation*}
	A continuous, integral form of the equation can be written as follows,
	\begin{equation*}
	P(X|I) = \int_{-\infty}^\infty P(X,Y|I) \dd{Y}
	\end{equation*}
\end{itemize}
\end{document}


In game theory, the default assumption is that all players are perfectly rational and that this is common knowledge between all players. From this assumption, often without additional arguments, game theorists assume that \emph{backward induction} can be used to determine outcomes and strategies of perfect-information games. Backward induction is an algorithm that computes the outcome of a game assuming that all players want to maximize their payoffs from all states of the game.

In this section we shall investigate what rationality is. We shall also examine why common knowledge of rationality implies correctness of backward induction.

\paragraph*{Backward induction}
The idea of backward induction is straightforward: When a game tree has been constructed and payoffs have been assigned to the leaves, use a bottom-up approach to determine the \emph{inductive outcome}. This approach can also determine the strategies of the players.

Starting in the last stages (the nodes before the leaves) of the game tree, determine the action which yields maximal payoff and save the payoffs in the current node, so that the maximal payoffs propagate upwards in the tree until the root is encountered.

\subsection{Definitions and Assumptions}
In order to talk about rationality in perfect-information games, some important definitions and assumptions must be presented. Note that the Liar's Dice example is not used in this section, as it is not a game of perfect information.

\paragraph*{Strategy} A player participating in a game chooses a strategy, as defined previously in \secref{seq:perfect-information}. This involves that a player, when choosing a strategy and deciding what action to take at some node $v$, act as though $v$ has been reached.

\paragraph*{Rationality} Rationality can now be defined as \textit{choosing an action which yields maximum payoff at any node $v$}. In other word, a player will never choose a strategy that results in less payoff than another strategy, independent of which node he/she resides in.
It is important to understand that, even if $v$ is not reached, a player must still act rationally in $v$ for backward induction to work. This is also linked to the definition of strategy, as the player acts as though $v$ has been reached.

\paragraph*{Common knowledge of Rationality} The simple definition of rationality is enough, to show that backward induction works. Therefore the notion of common knowledge in epistemic logic is used.
Essentially common knowledge of rationality boils down to all players being rational and all players knowing this fact. This piece of information is also known to all players, which results in an infinite chain of knowledge, which defines common knowledge.

\subsection{Results}
As stated before, it's important to assume that common knowledge of rationality holds, in order for backward induction to work.
Intuitively, this requirement makes perfect sense, which is illustrated in \figref{prg:lec5}:
Assume that the black nodes in the game tree is where the player Ann has to select an action. She could either end the game by taking action 1, and receive 3 utils as payoff, or continue the game by choosing action 2. By backward induction it is obvious that she should end the game by choosing action 1, because she knows that Bob is rational and will choose payoffs $A:2$, $E:4$. By choosing action 2, Ann would not optimize her payoff, because she would only receive 2 utils. Let's say Ann doesn't know that Bob will act rationally. If Ann considers it possible that Bob is not rational, Bob could possibly choose the payoffs $A:7$, $E:1$, which then would maximize Anns payoff. Therefore the common knowledge of rationality is needed.

\begin{figure}[htbp]
\centering
	\begin{tikzpicture}[every node/.style={draw,circle}, align=center, node distance=1.9cm]
	%Nodes
	\node[draw,fill=black]   	(q1)   {};
	\node[draw, below right=1cm of q1,style={draw=none}]      (q2)   {$_{E:2}^{A:3}$};
	\node[draw, below left of=q1]     (q3)   {};
	\node[draw,fill=none, below right of=q3,style={draw=none}]    (q4)   {$_{E:4}^{A:2}$};
	\node[draw,fill=none, below left of=q3,style={draw=none}]      (q5)   {$_{E:1}^{A:7}$};
	íges
	\draw [line] (q1) -- (q2) node [label={[shift={(-0.6,-0.5)}]2},very near start, right, draw=none, fill=none] (TextNode) {1};
	\draw [line] (q1) -- (q3) node [very near start, left, draw=none, fill=none] (TextNode) {2};
	\draw [line] (q3) -- (q4) node [very near start, right, draw=none, fill=none] (TextNode) {1};
	\draw [line] (q3) -- (q5) node [very near start, left, draw=none, fill=none] (TextNode) {2};
	\end{tikzpicture}
	\caption{Game tree for a perfect-information game between Ann and Bob.}
	\label{prg:lec5}
\end{figure}

Here, it can be seen that common knowledge of rationality implies backward induction. To proof that this is actually the case, one must look into epistemic logic to try and model the notion of common knowledge of rationality.

First step is to create a possible worlds model $M$, where $\Omega$ is the set of all states of the world.
Let $s$  be a function defined by
$$
s(\omega) = \times_i S_i
$$
where $S_i$ is the set of all strategies of player $i$. The function $s$ is therefore a function that maps a state $\omega$ in the world model, to a tuple of the players strategies at $\omega$.
Let the set of states in $M$, where the strategies of $s(\omega)$ are rational for all players in the game, be denoted $R$. Let the set of states where $R$ is common knowledge be denoted $CR$. At some states in the $M$, the strategy represented in the state leads to the inductive outcome. Let $I$ denote these states.
In \cite{aumann1995a} a detailed proof is given to show that
$$
CR \subset I,
$$
which means that common knowledge of rationality implies correctness of backward induction.

\subsection{Discussion}
It is worth to investigate a play of a game where some nodes are not reached. Previously we assumed that a player would act rationally at every point in the game, but let's assume that we only demand rationality at the nodes that are actually reached in the game tree.
If a node is never reached, then one could possibly say that any decision made in that node is actually rational, but it's obvious that this prevents backward induction from working correctly. Therefore it is important to assume rationality at all nodes.

Common knowledge of rationality is a quite rare scenario in practice, given that the players are human. Often human players are not rational at all or at least it is not common knowledge. In these cases it would be irrational to choose a strategy based on the result of the backwards induction, because a player do not know the reasons for his/hers opponents choices.

From a human perspective, perfect-information games with common knowledge of rationality are not fun to play, as the course of the game is completely known before the first move is made. It would be a complete waste of time to play a game and reach an outcome that was determined beforehand.