%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%                                                                 %
%  CERNLIB manual in LaTeX form   	                          %
%                                                                 %
%  Michel Goossens (for translation into LaTeX)                   %
%  Version 2.00                                                   %
%  Last Mod.  6 Oct 1992  1030   MG                               %
%                                                                 %
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\documentstyle[epsfig,11pt,fleqn,amssym,cernlib,cerndoc]{cernman}
\newcommand{\Title}{CERN Program Library}%       Title for document
\newmathalphabet*{\mathtt}{cmtt}{m}{n}
\newmathalphabet*{\mathbf}{cmr}{b}{n}
%\romanfont{times}
%\PScommands% Initialize PS boxes
%\makeindex
\begin{document}
%  ==================== Front material ============================
%\include{cernlibf}
%\cleardoublepage
%  ==================== Body of text ==============================
\pagenumbering{arabic}
\setcounter{page}{1}

%\chapter{General Information}

%%%%%%   Catalog of Program packages and entries%%%%

\def\Rtnr{Catalog}%Dummy routine name to appear at bottom of page
\include{crnlbcat}
\cleardoublepage

\let\LARGE\large
\let\Large\large

% Here come the different files to be included

%%     A part     %%

\Sectitle{Arithmetic Routines}

\include{a105}

%%     B part     %%

\Sectitle{Elementary Functions}

\include{b100}
\include{b101}
\include{b102}

%%     C part     %%

\Sectitle{Equations and Special Functions}

\include{c200}
\include{c201}
\include{c202}
\include{c205}
\include{c207}
\include{c208}
\include{c209}
\include{c210}
\include{c300}
\include{c301}
\include{c302}
\include{c303}
\include{c304}
\include{c306}
\include{c307}
\include{c308}
\include{c309}
\include{c310}
\include{c312}
\include{c313}
\include{c315}
\include{c316}
\include{c317}
\include{c318}
\include{c320}
\include{c321}
\include{c322}
\include{c323}
\include{c324}
\include{c327}
\include{c328}
\include{c330}
\include{c331}
%\include{c332}
\include{c333}
\include{c334}
\include{c335}
\include{c336}
\include{c337}
\include{c338}
\include{c339}
\include{c340}
%\include{c341}
\include{c342}
\include{c343}
\include{c344}
\include{c345}
\include{c346}
\include{c347}
\include{c348}
\include{c349}
%\include{c351}

%%     D part     %%

\Sectitle{Integration, Minimization, Non-linear Fitting}

\include{d101}
\include{d103}
\include{d104}
\include{d105}
\include{d107}
\include{d108}
\include{d110}
\include{d111}
\include{d113}
\include{d114}
\include{d115}
\include{d151}
\include{d201}
\include{d202}
\include{d203}
\include{d209}
\include{d300}
\include{d302}
\include{d401}
\include{d506}
\include{d507}
\include{d508}
\include{d509}
\include{d510}
\include{d601}
\include{d700}
\include{d701}
\include{d702}
\include{d703}
\include{d704}
%\include{d999}

%%     E part     %%

\Sectitle{Interpolation, Approximations, Linear Fitting}

\include{e100}
\include{e102}
\include{e103}
\include{e104}
\include{e105}
\include{e106}
\include{e207}
\include{e208}
\include{e211}
\include{e220}
\include{e221}
\include{e230}
\include{e250}
\include{e255}
\include{e401}
\include{e406}
\include{e407}
\include{e410}
%\include{e999}

%%     F part     %%

\Sectitle{Matrices, Vectors and Linear Equations}

\include{f001}
\include{f002}
\include{f003}
\include{f004}
\include{f010}
\include{f011}
\include{f012}
\include{f105}
\include{f106}
\include{f112}
\include{f116}
\include{f117}
\include{f118}
\include{f121}
\include{f122}
\include{f123}
\include{f150}
\include{f202}
\include{f220}
\include{f221}
\include{f222}
\include{f223}
\include{f224}
\include{f225}
\include{f230}
\include{f406}
\include{f500}
\include{f600}

%%     G part     %%

\Sectitle{Statistical Analysis and Probability}

\include{g100}
\include{g101}
\include{g102}
\include{g103}
\include{g104}
\include{g105}
\include{g106}
\include{g110}
\include{g111}
\include{g900}
\include{g901}
%\include{g999}

%%     H part     %%

\Sectitle{Operation Research Techniques and Management Science}

\include{h100}
\include{h300}

%%     I part     %%

\Sectitle{Input/Output}

\include{i101}
\include{i202}
\include{i302}
\include{i303}
%\include{i999}

%%     J part     %%

\Sectitle{Output and Graphical Data Presentation}

\include{j200}
\include{j401}
\include{j403}
\include{j509}
\include{j511}
\include{j530}
%\include{j551}
%\include{j999}

%%     K part     %%

%%     L part     %%

\Sectitle{Executive Routines}

\include{l210}
\include{l400}

%%     M part     %%

\Sectitle{Data Handling}

\include{m101}
\include{m103}
\include{m104}
\include{m107}
\include{m108}
\include{m109}
\include{m214}
\include{m215}
\include{m216}
\include{m218}
\include{m220}
\include{m224}
\include{m231}
\include{m232}
\include{m233}
\include{m250}
\include{m251}
\include{m400}
\include{m409}
\include{m410}
\include{m416}
\include{m421}
\include{m422}
\include{m423}
\include{m426}
\include{m427}
\include{m428}
\include{m429}
\include{m431}
\include{m432}
\include{m433}
\include{m434}
\include{m436}
\include{m437}
\include{m438}
\include{m439}
\include{m440}
\include{m441}
\include{m442}
\include{m501}
\include{m502}
\include{m503}
\include{m506}
\include{m507}
\include{m508}

%%     N part     %%

\Sectitle{Debugging, Error Handlng}

\include{n001}
\include{n002}
\include{n100}
\include{n103}
\include{n105}
\include{n203}

%%     Q part     %%

\Sectitle{Service or Housekeeping Programming Aids}

\include{q100}
\include{q120}
\include{q121}
\include{q122}
\include{q123}
\include{q124}
\include{q180}
\include{q210}
\include{q901}
\include{q902}
\include{q904}
%\include{q999}

%%     R part     %%

\Sectitle{Logical and Symbolic}

\include{r205}

%%     T part     %%

\Sectitle{Magnet and Beam Design, Electronics}

\include{t604}

%%     U part     %%

\Sectitle{Quantum Mechanics, Particle Physics}

\include{u100}
\include{u101}
\include{u102}
\include{u110}
\include{u501}
%\include{u999}

%%     V part     %%
\Sectitle{Random Numbers and General Purpose Utilities}

\include{v100}
\include{v101}
\include{v102}
\include{v103}
\include{v104}
\include{v105}
\include{v106}
\include{v107}
\include{v108}
\include{v109}
\include{v110}
\include{v111}
\include{v112}
\include{v113}
\include{v114}
\include{v130}
\include{v150}
\include{v151}
\include{v202}
\include{v300}
\include{v301}
\include{v302}
\include{v304}
\include{v306}
\include{v401}
\include{v700}
%\include{v999}

%%     W part     %%

\Sectitle{High Energy Physics Simulation, Kinematics, Phase Space}

\include{w150}
\include{w151}
\include{w505}
\include{w515}
%\include{w999}

%%     X part     %%

%\Sectitle{Particle Detection, Measurement, Reconstruction}

%\include{x999}

%%     Y part     %%

\Sectitle{Statistical Data Analysis and Presentation}

\include{y201}
\include{y250}
\include{y251}
%\include{y999}

%%     Z part     %%

\Sectitle{Miscellaneous System-Dependent Facilities}

\include{z001}
\include{z007}
\include{z008}
\include{z009}
\include{z020}
\include{z029}
\include{z034}
\include{z035}
\include{z036}
\include{z037}
\include{z041}
\include{z042}
\include{z044}
\include{z100}
\include{z203}
\include{z204}
\include{z262}
\include{z264}
\include{z265}
\include{z267}
\include{z300}
\include{z301}
\include{z303}
\include{z304}
\include{z305}
\include{z306}
\include{z307}
\include{z308}
\include{z309}
\include{z310}
\include{z311}
\include{z312}

%  ==================== Backmaterial ===========================
\bibliographystyle{myunsrt} % style for bibliography
\bibliography{/user/goossens/cnasall/cnasbibl}   % Master BibTeX file for CNAS docs

\input{\jobname.ind} % index

\end{document}


\documentclass[a4paper]{article}
\title{Implementation of Gouraud shading in GLSL}
\author{Hiroka IHARA (The Univ. of Tokyo)}
\date{2016/10/05}
\usepackage{bm}
\begin{document}
\maketitle
\section{Basics for per-vertex shading}
\[
I_{diffuse} = k_{diffuse} \cdot I_{light} \cdot max \left( \bm{N} \cdot \bm{L}, 0 \right)
\]
where $\bm{N}$ is the normalized vertex normal in viewspace, and $\bm{L}$ is the normalized viewspace vector pointing to the light.
\[
I_{ambient} = k_{ambient} \cdot I_{light}
\]
\[
I_{specular} = k_{specular} \cdot I_{light} \cdot max \left( \bm{R} \cdot \bm{V}, 0 \right)^{k_{reflectivity}}
\]
where $\bm{R}$ is calculated as $2\bm{N}\left(\bm{N}\cdot\bm{L}\right)-\bm{L}$ or simply $reflect\left(\bm{N},-\bm{L}\right)$,
and $\bm{V}$ is the normalized viewspace vector pointing to the view, which is equivalent to the negated projection vector.
\end{document}


\chapter{A LIBRARY FOR LOCAL DIFFERENTIAL PRIVACY}

\section{Introduction in Local DP}

As we mentioned in previous chapters, there are two major forms of Differential Privacy. Having analyzed and tested the first one, \emph{Global D.P.}, it is now time to examine \emph{Local D.P.}, by explaining some possible protocols, as well as building our own.

In Local D.P., there is a significant difference compared to Global DP: there is \emph{no trusted curator} between the data and the users, as they just want to send their data, while already being anonymized. Thus, an algorithm must perturb the data before sending it to the untrusted curator, who will then transmit it to the analysts.

In order to achieve that goal, the user must randomize the value before making it public (i.e. sending it to the untrusted curator). Then, the curator which collects the data (we will reference to him as aggregator moving forward), collects the data and tries to retrieve their original values, with a goal of producing the most accurate results possible.

Thus, each LDP algorithm has the following steps:

\begin{itemize}
    \item Each user encodes, and then perturbs the private value that he wants to make public
    \item Each user sends out the result of the perturbation process, with that being only the final value, as they keep the intermediate results for themselves
    \item The untrusted data curator collects each user's value, and implements some kind of aggregation in order to retrieve the stats that he wants from the data given to him.
\end{itemize}

In comparison with Global D.P., the Local model has advantages, as well as disadvantages.
Its main advantages are:
\begin{itemize}
    \item The user is not forced to trust the data curator, as only the perturbed value is reported
    \item Simpler implementation of the algorithms, due to the district steps taken by both sides.
\end{itemize}

while the main disadvantages are the following:

\begin{itemize}
    \item The noise added should be larger than the Global model, in order to satisfy the definition, thus the number of people in the dataset should be significant for accurate results to be produced.
    \item Because this is not always possible, many real-world applications use extremely high values of epsilon compared to what we got used to during our testing in the Global models.
\end{itemize}

During this Thesis, concern was raised for the main disadvantage of L.D.P., and thus\emph{ we will present a new protocol aiming to reduce the need for many users, while still covering the definition.} However, the definition for L.D.P. is quite different than the Global model one's.

\section{Definition of Local DP}

Having a general idea in how Local D.P. functions, it is now time to give a strict definition that we are going to depend our work on moving forward.

We can say that an algorithm $A$ satisfies ε-Local Differential Privacy, if and only if for any input $v_1$, $v_2$, we have

$$ \forall y \in Range(A):\ Pr[A(v_1) = y] \leq e^{\epsilon} * Pr[A(v_2) = y] $$

where $Range(A)$ denotes the set of all possible outputs of the algorithm $A$.

As mentioned in Chapter 2, this definition can have many interpretations by different algorithms or protocols, but each one must produce a probabilistic space whose elements must satisfy the above equation.

\section{Simple Application of LDP}

The most simple of L.D.P. protocols is already mentioned in this Thesis, and is no other than the \emph{Randomized Response} protocol. This algorithm implements the three steps mentioned in the introduction, as the user chooses a value (Yes or No), perturbs it (by the flipping of the coins), reports the perturbed value, with the sole job of the aggregator being to collect, normalize and report the values provided. It meets the definition of L.D.P., as the fraction of a pair of probabilities in the space of possible outputs (Yes, No) has always the ceiling of a real number.

Our goal is to now find this ceiling, and thus denote the level of privacy that randomized response offers. In order to do this, we are going to select the possibility of the user having chose the answer "Yes". A simple case analysis shows that $Pr[Yes | Truth] = \frac{3}{4}$, and of course $Pr[Yes | False] = \frac{1}{4}$. Thus, by the definition of L.D.P., we have

\begin{align*}
    \frac{Pr[Yes | Truth]}{Pr[Yes | False]} = \frac{\frac{3}{4}}{\frac{1}{4}} = 3 = e^\epsilon \Longleftrightarrow \epsilon = ln(3)
\end{align*}

Thus, R.R. offers  $ln(3)$-differential privacy to its users. This is quite a good setting, but the restriction is that the user can only report 2 values, something not suitable for modern problems and surveys.

In R.R., we care about the total true answers of the users, and not the individual responses. Thus the metric we are going to use is the \emph{absolute difference of the sum of the 2 vectors: the one with the truthful answers, and the one with the reported answers}. We are going to divide this result with the number of the users, in order to get the scale of the error depending on the size of the vector that was reported. The metric is expressed from the following function:

\begin{align*}
    \text{Error} = \frac{|\sum \text{true\_values} - \sum \text{reported\_values}| }{\text{number\ of \ users}}
\end{align*}

As always, during the creation of probabilistic distributions, one run is not enough, because of the extreme amount of noise that can occur. Thus, for each number of users we are going to run the R.R. protocol 100 times, and the final accuracy error will be produced by the mean value of those runs.

Having implemented R.R. in Python, we can now display the accuracy error of R.R. as the number of users rises. The results of the testings are shown bellow in \textbf{Figure 4.1}.

\begin{figure}[!htb]\centering
    \includegraphics[width=0.8\textwidth]{images/rr_results.png}
    \caption{Accuracy Error in R.R for increasing values of epsilon}
\end{figure}

We observe that the plot behaves as expected: the protocol produces a logarithmic curve for the accuracy error, while for a large number of users (over 3000), the error stabilizes bellow $0.1$.


\documentclass[12pt,a4paper]{article}

\usepackage{amsmath}
\usepackage[font=footnotesize]{caption}
\usepackage[section]{algorithm}
\captionsetup[algorithm]{font=footnotesize}
\usepackage[numbered]{algo}

% Following packages only required for this sample, not for using algo.sty in the first place!
\usepackage{color}
\usepackage{listings}
\lstset{language=TeX}

\definecolor{dkgreen}{rgb}{0,0.6,0}
\definecolor{dkgray}{rgb}{0.25,0.25,0.25}

\lstset{%
  	backgroundcolor=\color{white},
  	basicstyle=\footnotesize,
  	breakatwhitespace=false,
  	breaklines=true,
  	captionpos=b,
  	commentstyle=\color{dkgreen},
  	deletekeywords={...},
  	escapeinside={\%*}{*)},
  	extendedchars=true,
  	frame=single,
  	keepspaces=true,
  	keywordstyle=\color{blue},
  	language=Octave,
  	morekeywords={*,...},
  	numbers=left,
  	numbersep=5pt,
  	numberstyle=\tiny\color{dkgray},
  	rulecolor=\color{dkgray},
  	showspaces=false,
  	showstringspaces=false,
  	showtabs=false,
  	stepnumber=1,
  	tabsize=2,
  	title=\lstname
}

\begin{document}

	\section{Quick Shift}

	To use the \lstinline!alog.sty! package for a book, update the following lines of \lstinline!algo.sty!:

\begin{lstlisting}
% Set counter to include chapter:
% \renewcommand{\thealgorithm}{\thechapter .\arabic{algo}}
\renewcommand{\thealgorithm}{\thesection .\arabic{algo}}
\end{lstlisting}

	\begin{algorithm}[h]
		\begin{algo}{QS}{\label{algo:related-work-qs}\qinput{color image $I$}\qoutput{superpixel segmentation $S$}}
			\qfor $n = 1$ \qto $N$\\
				initialize $t(x_n) = \boldsymbol 0$\qrof\\
			\qfor $n = 1$ \qto $N$\\
				\qcom{$N_R(x_n)$ is the set of all pixels in the local neighborhood of size $R \times R$ around pixel $x_n$:}\\
				calculate $p(x_n) = \sum_{x_m \in N_R(x_n)} \exp\left(\frac{-d(x_n,x_m)^2}{(2/3) R}\right)$ \qrof\\
			\qfor $n = 1$ \qto $N$\\
				set $t(x_n) = \arg\max_{x_m \in N_R(x_n): p(x_m) > p(x_n)} \{p(x_m)\}$\qrof\\
				\qcom{$t$ maps each pixel to its neighbor $x_m$ with highest $p(x_m)$ if $p(x_m) > p(x_n)$;}\\
			\qcom{$t$ can be interpreted as forest, where all pixels $x_n$ with $t(x_n) = \boldsymbol 0$ are roots.}\\
			derive superpixel segmentation $S$ from $t$\\
			\qreturn $S$
		\end{algo}
		\caption{The superpixel algorithm \textbf{QS} proposed in \cite{QuickShift}.}
		\label{fig:related-work-qs-algorithm}
	\end{algorithm}

	\begin{thebibliography}{1}
		\bibitem{QuickShift}
		A. Vedaldi,
		S. Soatto,
		\emph{Quick shift and kernel methods for mode seeking},
		ECCV,
		2008.
	\end{thebibliography}

\end{document}


%%%
%%% Statistics
%%%
\part{Statistics}
\begin{frame}
\thispagestyle{empty}
\textbf{\huge{Statistics}}
\end{frame}

\begin{frame}{Statistics Contents}
 \tableofcontents
\end{frame}

\section{Statistic}
\begin{frame}[fragile]{Statistic I}
Arithmetic mean and some additional statistics \index{Mean!mean} \index{Describe!summarize} \index{Describe!detail} \index{Mean} \index{Statistic}
\begin{lstlisting}
  mean height
  summarize height
  summarize height, detail
\end{lstlisting}

  \begin{tikzpicture}[transform shape, rotate=10, overlay]
\node at (8,-1.5) [mybox] (box) {%
    \begin{minipage}[t!]{0.35\textwidth}
    \tiny\textcolor{black}{\texttt{mean returns the standard error: $s/\sqrt{n}$. summarize returns the standard deviation: $\sqrt{\sum(x-\bar{x})^2)/n}$.}}
    \end{minipage}
    };
\end{tikzpicture}
\end{frame}

\begin{frame}[fragile]{Statistic II}
Minimum, maximum, arithmetic mean, median, number of observations and quartils. \index{Minimum} \index{Minimum!min} \index{Maximum} \index{Maximum!max} \index{Mean!mean} \index{Mean} \index{Mean!arithmetic mean} \index{Mean!Median} \index{Quantile} \index{tabstat} \index{Range} \index{Range!range} \index{Quantile!q}
\begin{lstlisting}
  tabstat height, statistic(min max mean median p50)
  tabstat height, statistic(min max range mean count q) by(county)
\end{lstlisting}

  \begin{tikzpicture}[transform shape, rotate=10, overlay]
\node at (8,-1.5) [mybox] (box) {%
    \begin{minipage}[t!]{0.35\textwidth}
    \tiny\textcolor{black}{\texttt{range = max - min}}
    \end{minipage}
    };
\end{tikzpicture}
\end{frame}

\begin{frame}[fragile]{Statistics III}
Standard deviation, standard error, variance und interquartil range. \index{Standard deviation} \index{Standard error} \index{Variance} \index{Interquartil range} \index{tabstat} \index{Standard deviation!sd} \index{Variance!var} \index{Standard error!sem} \index{Interquartil range!iqr} \index{Quantile!q} \index{Skewness} \index{Kurtosis} \index{Skewness!skewness} \index{Kurtosis!kurtosis}
\begin{lstlisting}
 tabstat height, statistic(sd sem var q iqr)
\end{lstlisting}
Skewness and kurtosis
\begin{lstlisting}
  tabstat height, statistics(skewness kurtosis)
\end{lstlisting}

  \begin{tikzpicture}[transform shape, rotate=10, overlay]
\node at (8,-1.5) [mybox] (box) {%
    \begin{minipage}[t!]{0.35\textwidth}
    \tiny\textcolor{black}{\texttt{iqr = q3 - q1}}
    \end{minipage}
    };
\end{tikzpicture}
\end{frame}

\section{Tables}
\begin{frame}[fragile]{Tables} \index{Table!tab} \index{Table!tab, summarize()} \index{Table}
We already had \texttt{summarize} and \texttt{tab}, let us combine them
\begin{lstlisting}
  tab county, summarize(height)
\end{lstlisting}
\end{frame}

\subsection{Cross tabs}
\begin{frame}[fragile]{Cross tabs} \index{Table!tab} \index{Table!tab1} \index{Table!tab2} \index{Table!cross tab}
Create a cross tab with
\begin{lstlisting}
  tab school county
\end{lstlisting}
And some more tables
\begin{lstlisting}
  ** tabs of each variable
  tab1 sex county school
  ** cross tabs of ab bc ac
  tab2 sex county school
\end{lstlisting}
\end{frame}

\subsection{Chi, V and Phi}
\begin{frame}[fragile]{Cross tabs II} \index{Table!cross tab} \index{Table!$\chi^2$} \index{Table!chi-square} \index{Table!chi} \index{Table!tab} \index{Table!Cramers V} \index{Table!tab exp} \index{Table!tab col} \index{Table!tab row}
Let us recapitulate Statistics I, we will do some tests \footnote{or see e.g. \textcite{Krebs10} oder \textcite{Agresti09}.}
\begin{lstlisting}
** chi^2
tab sex county, chi
** cramers v
tab sex county, V

** manual
tab sex county, exp col row
** phi and for 2x2 tabs V
di (1013*1002 - 925*1131) / (2144*1927*1938*2133)^(1/2)
** chi^2
di 4071 *(-.00753735)^2
** usual way to compute V
di (.23128021 / 4071 * (2 - 1) )^(1/2) // V
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{Numlabel}
You can add numeric values to your variable labels \index{Label!numlabel}
\begin{lstlisting}
  ** numeric labels on for all variables
  numlabel _all, add
  ** and off
  numlabel _all, remove
\end{lstlisting}
\end{frame}

\section{Inference}
\begin{frame}[fragile]{T-Test} \index{T-Test!ttest} \index{T-Test}
And finally let us have a look at some test-statistics
\begin{lstlisting}
  ** compare
  tab sex, sum(bmi)
  ttest bmi, by(sex)
\end{lstlisting}
\end{frame}

\begin{frame}[fragile]{And much more to come \dots} \index{Regression!regress} \index{Regression!logit} \index{Regression!mlog}
Regression
\begin{lstlisting}
  regress
  logit
  mlog
\end{lstlisting}
\end{frame}

\begin{frame}
\thispagestyle{empty}
 To be continued.
\end{frame}


\documentclass{article}
\usepackage[utf8]{inputenc}
\usepackage{amsmath, amssymb, amsthm}
\usepackage{geometry}
\usepackage{graphicx}

\geometry{top=1in, bottom=1in, left=1in, right=1in}

\title{Analysis of Music in Machine Learning}
\author{Eben Kadile}
\date{July 2020}

\begin{document}

\maketitle

\section{Objective}

We want to better understand the temporal and statistical structure of music, and how machine learning systems detect this structure and exploit it for music prediction and synthesis

\section{Regression}

Because fitting a linear regression model is a convex optimization problem, we would like to start by looking at the result of fitting the next set of notes in piano roll music based on the last set of notes.

\begin{figure}
    \centering
    \includegraphics{figures/regression_weights.png}
    \caption{Here we plot the weights of the trained 1-step logistic regression model. Red indicates positivity and blue indicates negativity. As can be inferred, only notes 27 through 75 appear in the dataset. If a note is played in one time step, it is likely to be played in the next, and the probability of a note far away from the current note being played is low.}
\end{figure}

\begin{figure}
    \centering
    \includegraphics{figures/regression_bias.png}
    \caption{Here we plot the bias of the trained 1-step logistic regression model. The color scale is different from that of the weights.}
\end{figure}

Using the information encoded in the weights and bias, we should be able to recover the fact that all the songs were in C major, but I don't know how to do that yet.

\section{Latent LDS}

One can initialize a latent LDS so that it forgets everything at every time step and behaves like the regression model. Hopefully, this initialization is good for finding a superior solution to the problem using the LDS. We can also use the weights of the trained latent LDS to initialize a tanh RNN. First, we investigate the various ways the hidden weights of the latent LDS can be initialized.

\begin{figure}
    \centering
    \includegraphics{figures/jsb_lds_losses.png}
    \caption{Loss of different latent LDS models compared to regression.}
\end{figure}

\begin{figure}
    \centering
    \includegraphics{figures/jsb_lds_accs.png}
    \caption{Accuracy of different latent LDS models compared to regression.}
\end{figure}

As we can see from these comparisons, scaled down critical (power-law) initialization performs the best while overfitting the least (having weights high in magnitude seems to encourage overfitting, especially for tanh RNNs).

We will plot the weights and hidden eigenvalues of this model.

\begin{figure}
    \centering
    \includegraphics{figures/scrit_lds_in_weights.png}
    \caption{Input weights of scaled down critical initialization.}
\end{figure}

\begin{figure}
    \centering
    \includegraphics{figures/scrit_lds_hid_weights.png}
    \caption{Hidden weights of scaled down critical initialization.}
\end{figure}

\begin{figure}
    \centering
    \includegraphics{figures/scrit_lds_out_weights.png}
    \caption{Output weights of scaled down critical initialization.}
\end{figure}

\begin{figure}
    \centering
    \includegraphics{figures/scrit_lds_eigs.png}
    \caption{Eigenvalues of hidden weight matrix after training of scaled down power law LDS.}
\end{figure}

\section{RNNs}

A tanh RNN can be initialized using the weights of the trained LDS. This increases performance noticeably, but in the case of some initializations it drastically increases overfitting.

\end{document}