\documentclass{memoir}
\usepackage{notestemplate}

%\logo{~/School-Work/Auxiliary-Files/resources/png/logo.png}
%\institute{Rice University}
%\faculty{Faculty of Whatever Sciences}
%\department{Department of Mathematics}
%\title{Class Notes}
%\subtitle{Based on MATH xxx}
%\author{\textit{Author}\\Gabriel \textsc{Gress}}
%\supervisor{Linus \textsc{Torvalds}}
%\context{Well, I was bored...}
%\date{\today}

\begin{document}

% \maketitle

% Notes taken on ??

\section{Character Theory}
\label{sec:character_theory}

Character theory will serve as a very convenient bookkeeping tool for representations of \(G\) when \(G\) is finite. We still keep \(K = \C\).

\begin{defn}
	Let \((V,\rho)\) be a \(\C\)-representation of \(G\) of finite degree \(n\). Choose any basis of \(V\) and express \(\rho_g\) as a matrix in \(GL_n(\C)\), for all \(g \in G\). The \textbf{character of \((V,\rho)\)}, denoted \(X_V\) is the function
	\begin{align*}
		X_V:G\to \C\\
		g\mapsto \textrm{Tr}(\rho g)
	\end{align*}
	We say that \(X_V\) is \textbf{irreducible} if \((V,\rho)\) is irreducible.
\end{defn}
It turns out that characters detect irreducibility. Let \(X_V,\psi_W\) be given. We define a scalar by
\begin{align*}
	\langle X_V,\psi_W \rangle := \frac{1}{\left| G \right| }\sum_{g \in G} \overline{X_V(g)}\psi_W(g).
\end{align*}
\begin{prop}
	Let \(V\) be a representation of \(G\). Then
	\begin{align*}
		V \text{ is irreducible }\iff\langle X_V,X_V \rangle =1.
	\end{align*}
\end{prop}

Of course, we need to be sure that our construction of characters is well-defined. It turns out that they capture the properties of our representation well and are unique.

\begin{prop}
	\begin{enumerate}
		\item The definition of \(X_V\) is independent of choice of basis of \(V\)
		\item If \(V \cong W\), then \(X_V = X_W\)
		\item If \(g,h \in G\) are conjugate, then \(X_V(g) = X_V(h)\)
	\end{enumerate}
\end{prop}

\begin{defn}
	The \textbf{character table} of \(G\) is defined as
	\begin{align*}
		\begin{bmatrix} X_{V_1}(g_1) & X_{V_1}(g_2) & \ldots & X_{V_1}(g_k)\\
		\vdots & & & \vdots \\
	X_{V_\ell}(g_1) & X_{V_\ell}(g_2) & \ldots & X_{V_\ell}(g_k)\end{bmatrix}
	\end{align*}
\end{defn}
The number of irreducible characters of \(G\) is the same as the number of conjugacy classes of elements of \(G\). Furthermore, the character table is a square matrix with entries in \(\C\) when the rows are indexed by irreducible representations of \(G\) and the columns are indexed by conjugacy classes representations of elements of \(G\).\\

In this case, \((X_{V_i}(g_j))\) is an invertible matrix.

% Examples

\begin{prop}
	Let \((V,\rho )\) be a representation of \(G\), and take \(g \in G\). Then
	\begin{enumerate}
		\item \(X_V(e) = \textrm{dim}(V)\)
		\item \(X_V(g)\) is a sum of roots of unity
		\item \(X_{V\oplus W}(g) = X_{V}(g) + X_{W}(g)\)
		\item \(X_V(g^{-1}) = \overline{X_V(g)}\)
		\item \(\overline{X_V}\) is a character of \(G\)
	\end{enumerate}
\end{prop}

Let \(X_1,\ldots,X_r\) be irreducible characters of a finite group \(G\). Define
\begin{align*}
	\langle X_i, X_j \rangle := \frac{1}{\left| G \right| } \sum_{g \in G} \overline{X_i(g)}X_j(g)
\end{align*}

\begin{thm}
	\begin{enumerate}
		\item \(\langle X_i, X_j \rangle = \delta _{ij}\)
		\item
			\begin{align*}
				\sum_{i=1}^{r} \overline{X_i(x)}X_i(y) = \begin{cases}
					\left| C_G(x) \right| & x,y \text{ conjugate in }G\\
					0 & \text{otherwise}
				\end{cases}
			\end{align*}
	\end{enumerate}
\end{thm}
			Here, \(C_G(x)\) is the centralizer of \(x \in G\), that is, \(C_G(x) = \textrm{Stab}_x(G) = \left\{g \in G \mid g x g^{-1} = x \right\} \).

% Example

\begin{thm}
	If \(V,W\) are irreducible representations of \(G\), then
	\begin{align*}
		\langle X_V,X_V \rangle = 1\\
		\langle X_V,X_W \rangle = 0 \text{ when }V\not\cong W
	\end{align*}
\end{thm}
This shows us that characters completely determine representations, and forthermore characters completely determine irreducibility.
\end{document}

Abstract Algebra - Introductory/Algebra II/Notes/source/Lecture13 - CharacterThry.tex


\hypertarget{lppl-module}{%
\section{LPPL Module}\label{lppl-module}}

\begin{Shaded}
\begin{Highlighting}[]
\ImportTok{import}\NormalTok{ LPPL}

\BuiltInTok{help}\NormalTok{(LPPL.Model)}
\end{Highlighting}
\end{Shaded}

\begin{verbatim}
Help on class Model in module LPPL:

class Model(builtins.object)
 |  Methods defined here:
 |
 |  __init__(self, source, code, start, end, tt_ratio, eo_ratio)
 |      Argument
 |      =======
 |              source: Callable, `tushare.get_hist_data` or                            `pandas_datareader.data.get_data_yahoo`
 |              code: str
 |              start: str
 |              end: str
 |              tt_ratio: float, ratio of train and test data
 |              et_ratio: float, ratio of extended and original time
 |
 |  __repr__(self)
 |      Return `repr(self)`.
 |
 |  plot(self, *args, **kwargs)
 |      Plot self.
 |
 |  plot_origin(self, file_name='')
 |      Plot original figure.
 |
 |  plot_predict(self, file_name='')
 |      Plot the prediction.
 |
 |  predict(self, t)
 |      Prediction.
 |
 |      Argument
 |      =======
 |              t: float or numpy.array
 |
 |  ----------------------------------------------------------------------
 |  Data descriptors defined here:
 |
 |  __dict__
 |      dictionary for instance variables (if defined)
 |
 |  __weakref__
 |      list of weak references to the object (if defined)

\end{verbatim}

\hypertarget{different-data-sources}{%
\subsection{Different Data Sources}\label{different-data-sources}}

\hypertarget{pandas_datareaderdataget_data_yahoo}{%
\subsubsection{\texorpdfstring{\texttt{pandas\_datareader.data.get\_data\_yahoo}}{pandas\_datareader.data.get\_data\_yahoo}}\label{pandas_datareaderdataget_data_yahoo}}

\begin{Shaded}
\begin{Highlighting}[]
\ImportTok{from}\NormalTok{ pandas\_datareader.data }\ImportTok{import}\NormalTok{ get\_data\_yahoo}

\NormalTok{code }\OperatorTok{=} \StringTok{\textquotesingle{}000001.SS\textquotesingle{}}
\NormalTok{start, end }\OperatorTok{=} \StringTok{\textquotesingle{}2017{-}4{-}1\textquotesingle{}}\NormalTok{, }\StringTok{\textquotesingle{}2018{-}4{-}1\textquotesingle{}}
\NormalTok{tt\_ratio }\OperatorTok{=} \FloatTok{0.5}
\NormalTok{eo\_ratio }\OperatorTok{=} \FloatTok{1.2}

\NormalTok{m1 }\OperatorTok{=}\NormalTok{ LPPL.Model(get\_data\_yahoo, code, start, end, tt\_ratio, eo\_ratio)}
\NormalTok{m1.plot()}
\end{Highlighting}
\end{Shaded}

\includegraphics{1216cf99ce60f340d488c6eaaf44cee6e75c6be3.png}

\includegraphics{843c1bf01e4c541f9f66f7a8acef1e8af88bc3a6.png}

\hypertarget{tushareget_hist_data}{%
\subsubsection{\texorpdfstring{\texttt{tushare.get\_hist\_data}}{tushare.get\_hist\_data}}\label{tushareget_hist_data}}

\begin{Shaded}
\begin{Highlighting}[]
\ImportTok{from}\NormalTok{ tushare }\ImportTok{import}\NormalTok{ get\_hist\_data}

\NormalTok{code }\OperatorTok{=} \StringTok{\textquotesingle{}600848\textquotesingle{}}
\NormalTok{start, end }\OperatorTok{=} \StringTok{\textquotesingle{}2017{-}4{-}1\textquotesingle{}}\NormalTok{, }\StringTok{\textquotesingle{}2018{-}4{-}1\textquotesingle{}}
\NormalTok{tt\_ratio }\OperatorTok{=} \FloatTok{0.5}
\NormalTok{eo\_ratio }\OperatorTok{=} \FloatTok{1.2}

\NormalTok{m2 }\OperatorTok{=}\NormalTok{ LPPL.Model(get\_hist\_data, code, start, end, tt\_ratio, eo\_ratio)}
\NormalTok{m2.plot()}
\end{Highlighting}
\end{Shaded}

\includegraphics{1796a3b29c2587ce45432662697b9097fb77a20b.png}

\includegraphics{1df43c6d1f7a8d2666f954d47bda0b45048c7cb1.png}

\hypertarget{others}{%
\subsubsection{Others}\label{others}}

Require callable function with \texttt{code}, \texttt{start}, and
\texttt{end} arguments.


\section{PolyPasswordHasher: A New Technique for Password Verification}
\label{SEC:design}

The goal of PolyPasswordHasher is to make cracking individual passwords
infeasible.  It provides a way of preventing an attacker from validating a
password hash.  At its core, PolyPasswordHasher aims to protect password hashes
by combining a share (derived using a threshold cryptosystem) with a salted
password hash and then storing this combined value in the password database.
Neither the share nor the password hash is stored on disk and, as we discuss in
more detail below, an attacker cannot recover either piece with only the
password database.  Cracking a password that is stored by PolyPasswordHasher
requires that the attacker know a threshold of passwords; this effectively
makes the passwords in a database interdependent.  Our goal is to ensure that
so long as an attacker does not know a threshold of passwords, no password in a
database can be cracked.

To make passwords interrelate, \PPH functions differently from a salted hash.  A
typical salted hash database stores a Username, Salt, and a Salted Hash.  A \PPH
database, rather than storing a salted hash, stores the secure hash XORed with
the share (\sxh).  The resulting \PPH database also holds an extra field called the
share number.  This field indicates which share was XORed with which salted
hash.

\begin{figure}
    \includegraphics[width=1\linewidth]{./images/Verify_account.pdf}
    \caption{Account verification. }
    \label{FIGURE:account-verification}
\end{figure}

When prompted to validate a password, the server XORs the salted hash with the
stored data and determines whether the result is a valid share of the threshold
%cryptosystem.  For example, in Figure~\ref{FIGURE:basic-pph-algorithm}, Bob
cryptosystem.  For example, in Figure~\ref{FIGURE:account-verification}, Alice
has provided her password (`fsh\$t!kz') and the username `Alice'.
The server and computes the salted hash of her password (`4298f44d...') and
reconstructs Alice's share (`3e773b6f...') using her share number
(2).  The server XORs the share and Alice's salted password hash together
and compares them to the value stored in the \sxh field in the password
database (`7cefcf22...').  If they match, then the password
provided was correct.

Account creation involves creating a share and XORing it with
the salted hash of the password before storing it on disk
For example, in Figure~\ref{FIGURE:basic-pph-algorithm}, Bob registers an
account with his password (`Tr4mP0l1ne') and username (`Bob').  The
server computes a salt (`0x5a17') and the salted hash of Bob's
password (`4153f0aa...').  The server knows the secret and knows Bob's
share number, 4. It computes Bob's share (`66ef2279...') and this value is XORed
with Bob's password’s salted hash; that value is compared to the value
stored in the database (`27bcd2d3...'). If these match, then the password
provided was correct.
%cryptosystem.  For example, in Figure~\ref{FIGURE:basic-pph-algorithm}, Bob

\begin{figure}
    \includegraphics[width=1\linewidth]{./images/basic-pph-algorithm.pdf}
    \caption{The basic PolyPasswordHasher algorithm showing how an entry is created}
    \label{FIGURE:basic-pph-algorithm}
\end{figure}

In PolyPasswordHasher, each share protects a salted hash and each salted hash
protects a share, unless a threshold of shares is known (as is shown in
Figure~\ref{FIGURE:pph-interdependency-2}).  Suppose that an attacker has
obtained the password database and knows some set of account passwords (x,y,z).
For each known password, (illustrated in Figure~\ref{FIGURE:pph-interdependency-new})
the hacker can compute the salted hash and XOR this with the database entry to
obtain the corresponding share.  If the attacker does not have a threshold of shares,
the attacker cannot generate a share for another account (e.g., `share a').  As a
result, the attacker cannot access share a's password's salted hash and cannot crack
a's password.  Or, suppose that a server has a threshold of correct passwords.
The server now has enough information to validate a threshold of shares and can use those to
recover the secret. The server could then reconstruct any share and thus,
recover the salted hash for any account's password -- an important step in
validating passwords.  Because shares protect passwords, an attacker who does
not have an adequate number of correct passwords (and thus shares) cannot
feasibly crack passwords individually.

\begin{figure}
    \includegraphics[width=1\linewidth]{./images/pph-interdependency-2.pdf}
    \caption{An attacker: (1) Cannot obtain salted hashes (needed to crack
    passwords) without a threshold  of shares AND (2) Cannot obtain a share without
    knowing the salted hash (password) for a \thresholdaccount.}
    \label{FIGURE:pph-interdependency-2}
\end{figure}

\begin{figure*}
    \includegraphics[width=1\linewidth]{./images/pph-interdependency-new.pdf}
    \caption{If an attacker knows some number of shares he can get the
    corresponding salted hashes; if he knows the salted hashes, he can get the
    corresponding shares; and, if he has a threshold of these, he can get the
    secret.}
    \label{FIGURE:pph-interdependency-new}
\end{figure*}

The basic scheme, as outlined so far, covers PolyPasswordHasher’s core
functions.  In the remaining subsections we address how \PPH checks passwords
without having a threshold of correct passwords (such as after a restart) and
how it ensures that all
accounts, even those that could be created by an attacker, are protected.  To
more fully describe the full \PPH algorithm, we begin, in
Section~\ref{SUBSEC:normal-operation}, with how PolyPasswordHasher functions in
situations where the server has already validated a threshold of correct
passwords and how, after reaching this threshold, the server proceeds to login
normal users, in a phase we call normal operation.  Following this,
Section~\ref{SUBSEC:bootstrapping} discusses how \PPH performs differently when
a threshold of correct passwords have not yet been provided and needs to
acquire that threshold (e.g. after a reboot).
Section~\ref{SUBSEC:bootstrap-transitioning} discusses how the system
transitions between bootstrapping and normal operation and in
Section~\ref{SUBSEC:handling-rare-events}, we describe how unlikely situations,
such as the loss of a large number of account passwords, are handled.

\subsection{Normal operation of PolyPasswordHasher}
\label{SUBSEC:normal-operation}

Given that a server with a threshold of shares can effectively recover any
salted hash, if every account protected a share, every password would play an
important role in protecting the security of the database.  However, not all
accounts should necessarily be trusted to protect shares.  For example, a forum
may allow any user to register an account and any party, including an attacker,
to register any number of user accounts.  If these accounts each protected a
share, an attacker could easily crack the password database.

PolyPasswordHasher enables one group of passwords, which we call protector
accounts, to protect the remaining passwords in the password database; these we
call \thresholdlessaccounts.  Bob's account (who we started following in the
previous example) is an example of a \thresholdaccount
(Figure~\ref{FIGURE:protected-protector}).  An administrator will define a
threshold value of \thresholdaccounts so that if an attacker does not know a
threshold of protector passwords, Bob's salted hash cannot be recovered.  Bob's
password hash protects his share and in turn, Bob's share helps to protect the
other shares (such as shares 1,2), which in turn protect their corresponding
password hashes (for root and Alice). These shares serves to protect the full
password database, including \thresholdlessaccounts (Trudy and Luke).

\begin{figure}
    \includegraphics[width=1\linewidth]{./images/pph-store.pdf}
    \caption{A PolyPasswordHasher store with protector and \thresholdlessaccounts. \Thresholdlessaccounts are displayed with a share number of SH. }
    \label{FIGURE:protected-protector}
\end{figure}

Since a \thresholdlessaccount  does not protect a share, it is not XORed with one.
Nonetheless, these accounts are still protected by shares, or more precisely,
the secret.  \PPH uses the secret as an encryption key and with it, encrypts the
password's salted hash.  For these accounts, the resulting value (but not the
secret or salted hash) is stored in the password database.  For example, in
Figure~\ref{FIGURE:protected-protector}, Trudy's password hash (a9e32543...) is
encrypted using the secret (9d380eb6...); the result (8198a5fd...) is stored in the
database.  There are two important points to note.  First, if an attacker knows
another \thresholdlessaccount's password (e.g., Luke's), this does not
substantially help with cracking Trudy's password.  The attacker would need to
break the symmetric encryption algorithm.  Also, knowing Luke's
password would not substantially help the attacker crack the secret or other
shares in the password.  Once again, the attacker would need to break the
symmetric encryption algorithm.  Thus, \thresholdlessaccounts cannot be
effectively cracked unless an attacker knows a threshold of \thresholdaccount
passwords.

\subsubsection{PolyPasswordHasher's algorithm}
\label{SUBSUBSEC:pph-algorithm}

Algorithm~\ref{ALG:acc-creation} details the processes of creating a user
accounts.  The relevant operations for normal operation (lines 2-13)
are discussed in this section.  Bootstrapping operations (lines 14-23) are
deferred to the next section.

\begin{algorithm}
\footnotesize
\begin{algorithmic}[1]\Function{createAccount}{username, salt, saltedPasswordHash, isProtectorAccount}
    \vspace{.1cm}
    \State \codecomment{\footnotesize// check whether we are under normal operation}

    \If{normalOperation} \codecomment{\footnotesize// Section \ref{SEC:design} and
    \ref{SUBSEC:normal-operation}}

        \If{isProtectorAccount}

            \State \codecomment{\footnotesize// Obtain a share from the share cryptosystem}
            \State shareNumber, share = SecretShares.getShare()

            \State \codecomment{\footnotesize// Combine the share with the hash}
            \State passwordEntry = share $\oplus$ saltedHash
            \State shareID = shareNumber

        \vspace{.11cm}
        \Else~\codecomment{\footnotesize// Shielded Account
        (Section~\ref{SUBSEC:normal-operation})}
            \State Key = SecretShares.getSecret()
            \State passwordEntry = AES.encrypt(saltedPasswordHash, Key)
            \State shareID = \THRESHOLDLESS
        \EndIf

    \vspace{.1cm}
    \Else~\codecomment{\footnotesize// Bootstrapping (Section~\ref{SUBSEC:bootstrapping})}

        \If{isProtectorAccount}
            \State raise AccountCreationError
        \Else
            \State shareID = BOOTSTRAP
        \EndIf
        \State passwordEntry = saltedPasswordHash
    \EndIf

    \State \codecomment{\footnotesize// Isolated Validation (Section~\ref{SUBSEC:bootstrapping})}
    \State isolatedCheckBits = saltedPasswordHash.getSuffix(IC\_BITS)
    \State passwordEntry += isolatedCheckBits

    \State store (username, salt, shareID, passwordEntry)
    \EndFunction

\end{algorithmic}
\caption{\small Account creation pseudocode.
\label{ALG:acc-creation}}
\end{algorithm}

{\bf Creating Accounts}.  Adding an account works as is shown in
Algorithm~\ref{ALG:acc-creation}.  As the algorithm describes, the username,
salt, the password’s salted hash, and a boolean are used to indicate whether or
not the account should be a \thresholdaccount. If the account is a
\thresholdaccount, an unused share is found (line 6) and the salted hash is
XORed with it (line 8) before the share number and resulting value are stored
(line 23).

Since a \thresholdlessaccount does not protect a share, those accounts are
instead encrypted with the secret (lines 11-12).  The share field is set to a
special value that indicates a \thresholdlessaccount (line 13).  As before,
this information is then stored in the database (line 23).

It is important to prevent users, particularly \thresholdaccounts, from using bad
passwords.  Similar to many deployed systems, PolyPasswordHasher employs simple
techniques to weed out extremely bad passwords.  When creating an account,
users input a password they want to use, but \PPH checks this password to ensure
that it is not too weak (e.g., ``letmein'' or ``password'').  This is done by
checking the requested password against a list of commonly used passwords (the
64K most popular). The proposed password will be rejected if it is on
that list.  PolyPasswordHasher also enforces constraints on the password length
and composition (number of lowercase and uppercase letters, numbers, and
symbols).  The purpose is not to ensure that passwords are immensely strong,
but to prevent the use of \thresholdaccount  passwords that are trivial to
guess.  These constraints are already common in many systems, such as those
that aim to prevent passwords from being cracked by brute force over the
network.

{\bf Process for Verifying Passwords}. Assuming that the server holds the
right number of valid passwords (and thus shares), it will be able to recover
the remaining shares, as shown in Algorithm~\ref{ALG:acc-verification}.
To verify a \thresholdaccount  (lines 4-5), the server first computes the salted
hash of the user’s password and XORs it with the passwordEntry. If the
passwordEntry XORed with the salted hash is the share, then the password is
correct.

For a \thresholdlessaccount, password verification differs because the salted hash
is encrypted instead of being XORed with a share.  Lines 8-10 of
Algorithm~\ref{ALG:acc-verification} describe how to compare the provided password
hash.  The password hash is encrypted with the secret and, if the encrypted
value matches the \sxh field, the correct password was provided.

\begin{algorithm}
\footnotesize
\begin{algorithmic}[1]\Function{verifyAccount}{username, saltedPasswordHash, shareID, passwordEntry}

    \If{normalOperation}
        \If{isProtector(shareID)}\codecomment{\footnotesize // Section~\ref{SEC:design}}
            \State share = passwordHash $\oplus$ passwordEntry
            \State return SecretShares.computeShare(shareID) == share
        \Else \codecomment{\footnotesize// \thresholdlessaccount ,
    Section~\ref{SUBSEC:normal-operation}}

            \State \codecomment{\footnotesize// Encrypt the obtained hash and compare}
            \State Key = SecretShares.getSecret()
            \State encryptedHash = AES.encrypt(passwordHash, Key)
            \State return passwordEntry == encryptedHash

        \EndIf
        \vspace{.1cm}

    \Else \codecomment{\footnotesize// Bootstrapping, Section~\ref{SUBSEC:bootstrapping}}

        \State \codecomment{\footnotesize// verify account created during bootstrap}
        \If{shareID == BOOTSTRAP}
            \State return passwordHash == passwordEntry
        \EndIf

        \If{not passwordHash.endsWith(isolatedCheckBits)}
            \State return False
        \EndIf
        \If{isProtector(shareID)}
            \State share = passwordHash $\oplus$ passwordEntry
            \State SecretShares.cacheShare(share)
        \EndIf
        \State return True

    \EndIf

    \EndFunction
\end{algorithmic}
\caption{\small Account verification pseudocode.\label{ALG:acc-verification}}
\end{algorithm}

{\bf Changing a user's password / password recovery} Changing a password
is a similar process to that used in a salted hash system, for both \thresholdaccounts
and \thresholdlessaccounts. Similar to existing systems, the procedure for
changing a password may require validating the existing password before
allowing a replacement to be generated. Changing the password involves creating
a new entry with the same username and share number. (This is nearly identical
to account creation.)  A new salt is generated and hashed along with the new
password.  Since the hash has changed, the fourth field will also change. Once
the new password entry is computed, it replaces the original stored data.  The
user may then log in normally with their new account.

\subsection{Bootstrapping After a Reboot}
\label{SUBSEC:bootstrapping}

When a system restarts, \PPH cannot validate or create accounts as it normally
would, because a threshold of valid accounts has not been reached. Because \PPH
stores shares in memory, not on disk, these are lost during reboot.  This means
that a server does not know the secret and cannot compute arbitrary shares.  As
a result, when bootstrapping, neither \thresholdaccount nor \thresholdlessaccounts
may be verified or created using the methods described above; account creation and verification
procedures are different when the system is bootstrapping.

%\begin{figure}
%    \includegraphics[width=1\linewidth]{./images/bootstrap}
%    \caption{Bootstrapping is similar to verifying an account except that we
%    use the obtained shares to restore the secret and interpolate the rest of the
%    shares.}
%    \label{FIGURE:bootstrap}
%\end{figure}

\subsubsection{Bootstrap account creation}
\label{SUBSUBSEC:bootstrap-account}

New accounts may also be created during bootstrapping. To do this, the new
account is added to the database along with the salted hash.  While the system
is bootstrapping, the new account is available to use.  In the interim, these
passwords will be created (line 19 of Algorithm~\ref{ALG:acc-creation}) and
validated (lines 18-19 of Algorithm~\ref{ALG:acc-verification}) in the same
manner as passwords stored in a system that uses salted hashes.

Although it would be easy to support \thresholdaccount creation during
bootstrap, PolyPasswordHasher does not do so (line 16 of
Algorithm~\ref{ALG:acc-creation}).  The reason is that an attacker who can
read the password database would also be able to read the salted password hash.
If the attacker can later read the password database during normal operation,
the attacker could use that salted password hash to recover the share.

\subsubsection{Isolated Validation}
\label{SUBSUBSEC:isolated-validation}

When started, a \PPH system does not have enough protector passwords (and thus,
shares) to recover the secret.  At this point it is not possible to validate
accounts following the same process as \PPH's normal operation
(\ref{SUBSEC:normal-operation}).  Here we describe how \partialverification  can
be used to check logins even without the secret.  \Partialverification is a
process that leaks a configurable number of bits of the salted hash by using a
slow hash algorithm.  It implements a mechanism wherein it is possible, but
extremely unlikely, that an attacker could access an account using an incorrect
password.  In the next section we discuss the \partialverification mechanism,
but defer discussion of its algorithm until
Section~\ref{SUBSEC:security-properties}.

As it bootstraps, \PPH collects shares from protector logins. The number of
logins required to recover the secret will have been configured by the system
administrator, with the threshold normally set to a low number (e.g., three).
Once the threshold has been reached, \PPH will finish bootstrapping
(Section~\ref{SUBSEC:bootstrapping}). Meanwhile, before this threshold is
reached, \partialverification makes use of an isolated-check bits field to authenticate
user passwords.

In this scheme, illustrated in the upper right corner of
Figure~\ref{FIGURE:isolated-verification}, the password database contains
isolated-check bits.  These bits are used to verify logins before a threshold
is reached and the same process is used for both protector and shielded
accounts.  The user's password is hashed using the \partialverification hash
function.  This function returns a small number of bits of the hash (such as 24
bits of a SHA256 hash) and typically involves many iterations of a secure hash
function.  If the isolated-check bits field of the password database match the
\partialverification  hash function’s output (line 12 of
Algorithm~\ref{ALG:acc-verification}), the user is allowed to log in.

\begin{figure}
    \includegraphics[width=1\linewidth]{./images/pph-with-partial-bytes.pdf}
    \caption{This figure shows validation using \partialverification. The
    isolated-check bits are stored on disk. This allows verification of accounts
    before a threshold of correct passwords is provided.  }
    \label{FIGURE:isolated-verification}
\end{figure}

\begin{figure}
    \includegraphics[width=1\linewidth]{./images/bootstrap.pdf}
    \caption{Wilson's account was created during bootstrap, so a regular
    salted hash is stored instead.}
        \label{FIGURE:bootstrap-verification}
\end{figure}

For example, Figure~\ref{FIGURE:bootstrap-verification} illustrates
verification while the system is bootstrapping.  Bootstrap accounts, such
as Wilson's account are validated in an identical way to a salted hash
system.  Wilson's password is salted and hashed and this is compared
with the value stored in the database.  If it matches, Wilson is logged in.

\Thresholdaccounts and \thresholdlessaccounts, like those of Trudy or Alice may
also log in while the system is bootstrapping
(Figure~\ref{FIGURE:bootstrap-verification}) using \partialverification.  The
\partialverification hash function is computed over the provided password and
the \partialbytes field is checked.  If these values match, the user is allowed
to log in. Bootstrap accounts, such as Wilson's account are validated in an
identical way to a salted hash system.  Wilson's password is salted and hashed
and this is compared with the value stored in the database.  If it matches,
Wilson is logged in.

\Partialverification represents a tradeoff for administrators.  Using
\partialverification makes the system available immediately after a reboot.
However, when using a small number of \partialbytes, there is the potential
for an incorrect authentication during the bootstrapping phase.  Alternatively
a large number of \partialbytes impacts the confidentiality of the password
database in the event of a theft, by making the passwords easier to crack.
Thus in some scenarios different settings are appropriate, possibly
even for \thresholdaccounts and \thresholdlessaccounts in the same database.
We explore this tradeoff more in Section~\ref{SUBSEC:security-properties}

\subsection{Transitioning From Bootstraping to Normal Operation}
\label{SUBSEC:bootstrap-transitioning}

As the system bootstraps, the server batches shares from \thresholdaccounts
(line 14 of Algorithm~\ref{ALG:acc-verification}).   After the system has a
threshold of shares, it is possible to recover the secret. The server performs
Lagrange interpolation, which allows the server to not only recover the secret
but also, to generate arbitrary shares (both are needed to check \thresholdlessaccounts).

Recall that accounts created during the bootstrap phase contain BOOTSTRAP for
their share number and have a salted hash stored in the database.  When the
system recovers the secret and transitions to normal operation, these accounts
will have their password hashes encrypted with the secret and their share
number field set to \THRESHOLDLESS.  This transforms these accounts to the same
state as \thresholdlessaccounts that are created in normal operation.

Also, all account logins that were processed during bootstrapping are now
checked to be certain that the correct passwords were provided.  This step is
needed because it is possible that an attacker could find a password that is
invalid and yet, matches the \partialbytes.  (We examine the feasibility and
computational cost of this in Section~\ref{SEC:evaluation})  The salted
password hash for \thresholdlessaccount logins are cached in memory and the
passwords are verified once a threshold is reached.

For a \thresholdaccount , if a provided password hash matches the \partialbytes,
but the password is not correct, the recovered share is not invalid.  The
incorrect share will be detected when a threshold of shares are obtained
because the integrity check on the secret will fail.  If this integrity check
fails, the administrator is notified.  The system can still enter normal
operation once a threshold of valid shares are obtained.

\subsection{Handling Rare Events}
\label{SUBSEC:handling-rare-events}

At an administrator's behest, accounts may be switched between shielded
and protector without user intervention. To do this, the server must be in
normal operation.  The server can then recover the salted secure hash. This
salted secure hash can then be re-encoded (using a share for an account that
becomes a \thresholdaccount; or encrypted, for an account that becomes
shielded) and the new entry can be stored. This transitions the account
from a protector to shielded (or vice versa).

{\bf Recovering data if all \thresholdaccounts are lost}.  If not enough
known threshold users can be verified, the salted hashes are lost forever and
accounts cannot be validated using the technique we described in
Section~\ref{SUBSUBSEC:isolated-validation}.  However, this does not mean that
the system is unusable because \partialverification allows users to log in.
Furthermore, mechanisms like root password recovery that are done through the console
will still work, and will allow any data on the system to be accessed.

{\bf Trusting a \thresholdaccount with multiple shares}.  A single
account may optionally provide access to multiple shares.
To do this, a single user can have multiple rows in the table.  Each
row will have a different \sxh entry and thus protect a different share.
Each entry must also have a different salt to ensure that a different
hash value is XORed with each share.  When the user provides their password,
the value can be used to recover the share in each row by XORing the salted
password hash with each share.

{\bf Detecting an \partialbytes match of an incorrect password}.  If the
system is in normal operation, it will detect that the password does not match.
However, PolyPasswordHasher will always check the \partialbytes for an entered
password (for clarity, not depicted in Algorithm~\ref{ALG:acc-verification}).
If the password is incorrect, but the \partialbytes match, this indicates that
an attacker has almost certainly stolen the password database but has not (yet)
cracked the password.  This generates an alert to the administrator to notify
her of the likely breach.


\documentclass{article}
\usepackage{amsmath}
\usepackage{amsfonts}
\usepackage{xcolor}

\author{Salvador Castagnino \\ scastagnino@itba.edu.ar}
\date{}
\title{Chapter 4 - Elementary Hilbert Space Theory}

\begin{document}

\maketitle

\section*{Exercise Solutions}

\begin{exercise}\textbf{Exercise 4.4.}
    We start by assuming that $\{u_n\}_{n \in \mathbb{N}}$ is a countable maximal orthonormal system in $H$, we want to see that $H$ is separable. Observe that it sufices to find a countable set $A$ such that $P \subset \overline{A}$, with $P$ the set of all finite linear combinations of elements in $\{u_n\}_{n \in \mathbb{N}}$. Define the set $A$ to be,
    \[
        A = \Bigg\{ \sum_{n=1}^{N} \left( q_n + i p_n \right)  u_n : q_n,p_n \in \mathbb{Q},\ N \ge 1\Bigg\}
    \]
    which is clearly countable. Take $x \in P$, we have $x = \sum_{n=1}^{N} c_n u_n $ for some $c_n$ complex numbers and some $N \ge 1$ . Observe that for every $1 \le n \le N$ there exist $\{q_{nk}\}_{k \in \mathbb{N}}$ and $\{p_{nk}\}_{k \in \mathbb{N}}$ sequences of rational numbers such that $ \left( q_{nk} + i p_{nk} \right) \rightarrow c_n$ as $k$ goes to infinity. Then let $\epsilon > 0$ we can ask for a $k$ large enough such that $|\left( q_{nk} + i p_{nk} \right) - c_n| < \frac{\epsilon}{N}$ for all $1 \le n \le N$, we in turn have
    \[
        \|\sum_{n=1}^{N} \left( q_{nk} + i p_{nk} \right)u_n - \sum_{n=1}^{N} c_n u_n \| \le \sum_{n=1}^{N} |\left( q_{nk} + i p_{nk} \right) - c_n| < \sum_{n=1}^{N} \frac{\epsilon}{N} < \epsilon
    \]
    Given that every element in $P$ can be aproximated by elements in $A$ we have that, $P \subset \overline{A}$, which concludes the proof.

\bigbreak

Now suppose that $\{x_n\}_{n \in \mathbb{N}}$ is countable and dense in $H$, we are going to build a countable maximal orthonormal system. Let $A$ be the set such that
\[
   \begin{cases}
      x_1 \in A \\
      x_{n+1} \in A \text{ iff } x_{n+1} \notin [ x_1, ..., x_n ]
   \end{cases}
\]
where $[x_1, ..., x_n]$ denotes the span of $\{x_1, ..., x_n\}$, let's see that $A$ is linearly independent. Suppose that for some $v_1, ..., v_{m+1}$ in $A$ and $\alpha_1, ..., \alpha_m$ nonzero complex numbers we have that
\[
    \sum_{j=1}^{m} \alpha_j v_j = v_{m+1}
\]
Every $v_j$ can be expressed as $x_{n_j}$, so let $v_k$ be such that $n_k$ is the greates index between all $n_j$, we then have
\[
    v_k = \frac{v_{m+1}}{\alpha_k} - \sum_{1 \le n \neq k \le m} \frac{\alpha_n v_n}{\alpha_k}
\]
which clashes with the construction of A, thus A is linearly independent. By \textbf{Ex4.3} we can build from A a countable orthnormal set $\{u_n\}$ such that $[v_1, ..., v_n] = [u_1, ..., u_n]$ for all $n \ge 1$. Given that every $x_n$ can be expressed as a linear combination of elements in $A$, one can see that every $x_n$ will be in $[u_1, ..., u_m]$ for some $m$. Then we can see that every $x_n$ is an element of $\overline{P}$, with P the set of finite linear combinations of elements in $\{u_n\}$, which in turn imples that $P$ is dense in $H$ and concludes the proof.
\end{exercise}

\bigbreak

\begin{exercise}\textbf{Exercise 4.6.}
    Let $U = \{u_n\}_{n \in \mathbb{N}}$ be an orthonormal set in $H$, let's prove the assertion \textbf{(a)}. Boundedness of $U$ is more than clear and closedness and non-compacntess can be easily derived from that fact that $\|u_n - u_m\| = \sqrt{2}$ for all $1 \le  n < m$.

\bigbreak

To prove assertion \textbf{(b)} (we go directly with the general case), we start by supposing that $\sum_{n=1}^{\infty} \delta_n^2 < \infty$ and proving that $S$ is compact. In ordert to do this, let $\{x_n\}_{n \in \mathbb{N}}$ be a sequence in $S$, we will construct a convergent subsequence.
\begin{itemize}
    \item Let $y_{n1} = \hat{x}_n \left( 1 \right)$, notation as in the book. Given that the closed disk $\overline{B} \left( 0,\delta_1 \right)$ is compact and $y_{n1} \in \overline{B} \left( 0, \delta_1 \right) $, we can find $y_{n_k1}$ a convergent subsequence of $y_{n1}$ which converges to a $y_1$ in that disk. We replace our original sequence $x_n$ for the new subsequence $x_{n_k}$, define $z_1 = x_{n_1}$  and proceed be repeating what we just did but for the second terms (we also rename the new sequence $x_{n_k}$ as $x_n$ for the sake of simplicity).
    \item Notation as above we can find a convergent subsequence $y_{n_k2}$ of the sequence of second terms which converges in $\overline{B} \left( 0,\delta_2 \right) $ to a $y_2$ in that disk. We again define $z_2 = x_{n_1}$ and replace $x_n$ by it's subsequence $x_{n_k}$. Observe that the sequence of first terms still converges as it's a subsequence of a convergent sequence.
\end{itemize}
Repeating this process an arbirary number of times we build a sequence $z_n$ in S and a sequence of numbers $y_n$ in $\overline{B} \left( 0,\delta_n \right) $ for each $n$. Given that $0 \le |y_n| \le \delta_n$ for all $n \ge 1$ we have that $\sum_{n=1}^{\infty} |y_n|^2 < \infty $ and thus there exists a $y \in S$ such that $\hat{y} \left( n \right) = y_n$ for all $n \ge 1$. By the definition of the sequences $z_n$ and $y_n$, given $\epsilon > 0$ we can find $N, M \in \mathbb{N}$ large enough such that,
\begin{equation}
    \sum_{n=N}^{\infty} |\hat{z}_m \left( n \right)  - y_n|^2 \le \sum_{n=N}^{\infty} 4\delta_n^2   < \frac{\epsilon}{2}
\end{equation}
\begin{equation}
    |\hat{z}_m \left( n \right)  - y_n|^2 < \frac{\epsilon}{2^{n+1}} \quad \forall\ 1 \le n < N
\end{equation}
for all $m \ge M$. With this in mind we can see that $\sum_{n=1}^{\infty} |\hat{z}_m \left( n \right) - y_n|^2 =  \|z_m-y\|^2 < \epsilon$ for all $m \ge M$ which shows that $z_n$ converges to $y$ in $S$ and concludes the proof.

\bigbreak

Now suppose that $S$ is compact and that $\sum_{n=1}^{\infty} \delta_n^2 = \infty $, we will get to a contradiction. Define $x_k = \sum_{n=1}^{\infty} c_{kn}u_n $ with
\[
    c_{kn} =
    \begin{cases}
        \delta_n & \text{if } n \le k \\
        0 & \text{else}
    \end{cases}
\]
Clearly $x_k \in S$ for all $k \ge 1$  and $\|x_k\|^2 \rightarrow \sum_{n=1}^{\infty} \delta_n^2 = \infty$ but given that $\|x_k\| \le \|x_k-x\| + \|x\|$ for every $x \in S$ no subsequence of $x_k$ can converge in $S$ which implies that $S$ is not compact, contradicting our assumption and concluding the proof.

\bigbreak

Finally to prove assertion \textbf{(c)} we observe that $\frac{r}{2}u_n \in B \left( 0,r \right)$ for all $r > 0$ and all $n \ge 1$. With this in mind, and the fact that $\|u_n\|$ has no covergent subsequence, the sequences $\|\frac{r}{2}u_n\| $ have no convergent subsequence in $\overline{B} \left( 0,r \right) $ which in turn shows that $0$ has no neighbourhood with compact closure in $H$ which concludes the proof.
\end{exercise}

\bigbreak

\begin{exercise}\textbf{Exercise 11.}
We claim that the set $ E = \{ f_n \}_{n \in \mathbb{N}}$  with $f_n \left( x \right) = \sin \left( nx \right) I_{[0, \pi+\frac{\pi}{n}]}$ (where $I_A$ stands for the indicator function of $A$) is closed in $L^2 \left( T \right) $ and has no element of smallest norm. It can be easily verified that $E$ is a subset of $L^2 \left( T \right) $, we proceed to prove the other assertions.

\bigbreak

We start by proving that E is closed, to do this we will prove that the sequence $\{f_n\}_{n \in \mathbb{N}}$ has no convergent subsequence in $L^2 \left( T \right) $ and thus $E$ equals it's closure. Before starting the proof, observe that most properties of $L^2 \left( \mu \right) $ spaces hold for $L^2 \left( T \right) $ as their norms are just a scalar multiplication appart, we won't give the proofs for them here. Suppose that there exists a subsequence $\{f_{n_j}\}_{j \in \mathbb{N}}$ convergent in $L^2 \left( T \right) $, by \textcolor{blue}{Th3.12} this subsequence has a subsequence $\{f_{n_k}\}_{ k \in \mathbb{N}$ which converges pointwise a.e in $[0, 2\pi]$. Given that $f_n \left( x \right) = \sin \left( nx \right) $ for all $x \in [0,\pi]$, this conclusion contradicts the result obtained in Ex4.10 saying that the set in which $\sin \left( n_k x \right) $ converges has null measure and proves our assertion.

\bigbreak

Now, to prove that $E$ has no element of smallest norm let's prove that $\inf_{n \ge 1}\|f_n\|_2 = \frac{1}{2}$ and that this infimum is not attained in $E$. To do this we start by observing that
\begin{equation}
    \bigg\{ \frac{1}{2\pi} \int_0^\pi \sin^2 \left( nx \right) \: dx \bigg\}^\frac{1}{2} = \frac{1}{2}
\end{equation}
for all $n \in \mathbb{N}$ and that $f_n \left( x \right) = \sin \left( nx \right) I_{[0,\pi]} + \sin \left( nx \right) I_{(\pi, \pi + \frac{\pi}{n}]}$. Now using (3) we can write
\[
    \|f_n\|_2 = \bigg\{ \frac{1}{2\pi} \int_0^\pi \sin^2 \left( nx \right) \: dx + \frac{1}{2\pi} \int_\pi^{\pi + \frac{\pi}{n}} \sin^2 \left( nx \right) \: dx \bigg\}^\frac{1}{2} > \frac{1}{2}
\]
which shows that $\frac{1}{2}$ is smaller than the norm of every element in $E$. With this in mind and the fact that $\sin \left( x \right) $ is a bounded function using Minkowski's inequality we get,
\[
    0 < \|f_n\|_2 - \frac{1}{2}\le  \|\sin \left( nx \right) I_{(\pi, \pi + \frac{\pi}{n}]}\|_2 \rightarrow 0
\]
This final assertion shows that $E$ has no element of smallest norm which concludes the proof.

\end{exercise}

\section*{Useful Properties}

\end{document}


\documentclass[12pt]{article}

\usepackage[utf8]{inputenc}
\usepackage{latexsym,amsfonts,amssymb,amsthm,amsmath}
\usepackage[makeroom]{cancel}
\usepackage {tikz}
\usetikzlibrary {positioning}

\setlength{\parindent}{0in}
\setlength{\oddsidemargin}{0in}
\setlength{\textwidth}{6.5in}
\setlength{\textheight}{8.8in}
\setlength{\topmargin}{0in}
\setlength{\headheight}{18pt}

\title{Complexity Theory - Assignment 1}
\author{Kishlaya Jaiswal, Satya Prakash Nayak}

\begin{document}

\maketitle

\vspace{0.5in}

\subsection*{Exercise 1}
\begin{proof}
We shall prove the contrapositive that $P = NP \implies EXP = NEXP$. So assume $P=NP$ then it suffices to show $NEXP \subseteq EXP$.

Let $L \in NEXP$, then there exist a non-deterministic turing machine $M$ with running time $T(x) = O(2^{|x|^c})$ that decides $L$.
Define $L_p = \{x10^{T(x)} \mid x \in L\}$.\\

We claim that $L_p \in NP$. For any $y \in L_p$, we have $|y| = |x| + T(x) + 1$. Define NTM $M'$ that does the following: on input $y$ it throws out the last $10^{T(x)}$ parts and run $M$ on $x$ (remaining part) and does whatever $M$ does. Since $M$ runs in $T(x)$ time, $M'$ is a NP machine. Therefore, $L_p \in NP \implies L_p \in P$, so there exists a deterministic poly-time machine $D$ which decides $L_p$. Now we can construct a deterministic machine $D'$ which on input $x$, will pad it with $10^{T(x)}$ and check whether $x10^{T(x)} \in L_p$ using the machine $D$ on input $x$. Hence, $L \in EXP$.
\end{proof}

\subsection*{Exercise 2}
\begin{proof}
We shall show $SAT \leq_p HALT$.

Let $N$ be the poly-time $NTM$ that decides $SAT$. Consider the modified $NTM$ $M$ which on input $x$, accepts if $N$ accepts $x$ and otherwise if $N$ rejects then $M$ goes into a loop, and hence never halts. Let $\alpha$ be the encoding for this machine $M$.

Given a SAT-instance $\phi$ encoded as $x$, consider the string $(\alpha, x)$ as an instance of $HALT$ problem. (Note that: $M_\alpha = M$)

Suppose $\phi \in SAT$, then there exists a satisfying assignment and so $N$ accepts $x$ $\implies$ $M = M_\alpha$ halts and accepts $x$ and therefore $(\alpha, x) \in HALT$. Conversely, $\phi \not \in SAT$, then $N$ rejects $x$ $\implies$ $M = M_\alpha$ doesn't halt on $x$ and therefore $(\alpha, x) \not \in HALT$.

Thus, $HALT$ is $NP$-hard.
\newline

Furthermore, $HALT \not \in NP$ because otherwise, if there exists a $NTM$ which decides $HALT$, then given a machine $M_\alpha$ and an input $x$, we could decide if $M_\alpha$ halts on input $x$, hence solving the HALTING problem. So $HALT$ is not $NP$-complete.
\end{proof}

\subsection*{Exercise 3}
\begin{proof}
$L_1 \in NP \implies \exists$ a polynomial $p_1$ and a poly-time predicate $B_1$ such that $x \in L_1 \iff \exists w, |w| \leq p_1(|x|) \wedge B_1(w,x) = 1$

$L_2 \in NP \implies \exists$ a polynomial $p_2$ and a poly-time predicate $B_2$ such that $x \in L_2 \iff \exists w, |w| \leq p_2(|x|) \wedge B_2(w,x) = 1$
\newline

To show $L_1 \cup L_2 \in NP$, consider the polynomial $p = p_1 + p_2$ and the poly-time predicate $B = B_1 \vee B_2$, then: $x \in L_1 \cup L_2 \iff (x \in L_1) \vee (x \in L_2) \iff (\exists w, |w| \leq p_1(|x|) \wedge B_1(w,x) = 1) \vee (\exists w, |w| \leq p_2(|x|) \wedge B_2(w,x) = 1) \iff \exists w, |w| \leq p(|x|) \wedge B(w,x) = 1$
\newline

To show $L_1 \cap L_2 \in NP$, consider the polynomial $p = p_1 + p_2$ and the poly-time predicate $B(w_1 \# w_2) = B_1(w_1) \wedge B_2(w_2)$, then: $x \in L_1 \cap L_2 \iff (x \in L_1) \wedge (x \in L_2) \iff (\exists w_1, |w_1| \leq p_1(|x|) \wedge B_1(w_1,x) = 1) \wedge (\exists w_2, |w_2| \leq p_2(|x|) \wedge B_2(w_2,x) = 1) \iff \exists w, w = w_1 \# w_2, |w| \leq p(|x|) \wedge B(w,x) = 1$
\end{proof}

\subsection*{Exercise 4}
\begin{proof}.
It is clear that $TAUT \in coNP$ because given any NO instance $\phi(x)$ of $TAUT$, a short certificate for verification is an assignment of variables $x$ such that $\phi(x) = 0$. Clearly such an assignment can be expressed in $O(n\lg n)$ size (where $n$ = number of variables).

To show that $TAUT$ is $coNP-complete$, we note that, for any two $NP$ languages $A$ and $B$, $A \leq_p B \iff \overline{A} \leq_p \overline{B}$.

Thus, if $L$ is $NP-complete$, then $\overline{L}$ is $coNP-complete$ because firstly, $\overline{L} \in coNP$ by definition and for any $L' \in coNP$, $\overline{L'} \leq_p L \implies L' \leq_p \overline{L}$. So, we conclude that $\overline{3SAT}$ is $coNP-complete$.

To show that $TAUT$ is $coNP-complete$, we show that $\overline{3SAT} \leq_p TAUT$. Given any formula $\varphi$, consider the formula $\overline{\varphi}$. This formula can be constructed in $O(|\varphi|)$ time. Now, $\varphi$ doesn't have a satisfying assignment iff $\overline{\varphi}$ is a tautology.
\newline

Suppose $NP=coNP$. Since $3SAT$ is $NP-complete$, for every $NP$ language $L$, $L \leq_p 3SAT$. But $TAUT \in coNP = NP \implies TAUT \leq_p 3SAT$. Similarly, since $TAUT$ is $coNP-complete$, for every $coNP$ language $L$, $L \leq_p TAUT$. But $3SAT \in NP = coNP \implies 3SAT \leq_p TAUT$.

Conversely, let $L \in NP$, then $L \leq_p 3SAT \leq_p TAUT$. So given a NO instance of the problem $L$, we reduce it to a NO instance of $TAUT$ problem and hence we have a short certificate for the NO instances of $L$ as well implying that $L \in coNP$. Similarly, let $L \in coNP$, then $L \leq_p TAUT \leq_p 3SAT$. So given a YES instance of the problem $L$, we reduce it to a YES instance of $3SAT$ problem and hence we have a short certificate for the YES instances of $L$ as well implying that $L \in NP$. Thus, $NP=coNP$.
\end{proof}

\subsection*{Exercise 5}
\begin{proof}
Suppose unary $NP \subseteq P$.

Let $L \in NEXP$, then there exist a non-deterministic turing machine $M$ with running time $T(x) = 2^{|x|^c}$ that decides $L$.

Define $L_u = \{Unary(x) \mid x \in L\}$.\\

Observe that if $x \in L$, then $x$ is a binary string of length $|x|$ and so the unary representation of $x$ will have size atmost $2^{|x|}$.

We claim that $L_u \in NP$. For that we construct a machine $M'$ which on input $1^y$, converts $y$ into it's binary $x$ and then simulate $M$ on $x$. $M'$ accepts $1^y$ iff $M$ accepts $x$. Since $|1^y| = y < 2^{|x|}$, $M$ requires $2^{|x|} + 2^{|x|^c} = O(2^{|x|^c}) = O(y^c)$ time and hence $L(M') = L_u \in NP$.

Therefore, $L_u \in P$, so there exists a deterministic poly-time machine $D$ which decides $L_u$. Now we can construct a deterministic machine $D'$ which on input $x$, will construct $Unary(x)$ and check if it is in $L_u$ using the machine $D$. Hence, $L \in EXP$.
\end{proof}

\subsection*{Exercise 6}
\begin{proof}
Since $P=NP$, we claim that $P = coNP$: $L \in coNP \implies \overline{L} \in NP \implies \overline{L} \in P \implies L \in coP = P$.

Thus $TAUT \in P$ and so there exists a poly-time algorithm $A$ such that $A(\varphi(y)) = 1$ iff $\varphi(y)$ is a tautology.

We shall show that $\Sigma_2 SAT \in NP$. Consider a YES instance of $\Sigma_2 SAT$, that is a formula $\psi = \exists x \forall y (\phi(x,y) = 1)$ which admits a $x_0$ such that $\phi(x_0,y)$ is a tautology.

Then a short certificate for $\psi \in \Sigma_2 SAT$ is $x_0$; because clearly $|x_0| \leq |\psi|$ and in poly-time we can check if $\phi(x_0,y)$ is a tautology by calling $A(\phi(x_0, y))$.

In other words, $\psi \in \Sigma_2 SAT \iff \exists x_0$ such that $|x_0| \leq |\psi|$ and $A(\phi(x_0,y))=1$.

Hence $\Sigma_2 SAT \in P$.
\end{proof}

\vspace{2in} %Leave more space for comments!

\end{document}


\section{Module 85: Register Allocation}
\begin{flushright}
\textit{(scribed by Jai Javeria)}
\end{flushright}
The Intermediate Representation uses an unlimited number of temporaries. While this simplified code generation and optimization, a machine only has  finite resources and thus we need to map potentially large number of temporaries with the machine registers.

\subsection{Many to One Mapping}
\begin{itemize}
    \item Since there are more temporaries than registers, we would have to assign multiple temporaries to a single register without changing program behavior.
    \item If this is not possible then we have to store('spill') some of the temporaries in the memory.
    \item We can have a static mapping, where a temporary is mapped to the same register for the entire duration of the execution of code. We can also dynamically allocate registers, assigning a temporary to a register at one program point, "spilling" it to memory at another program point and assigning it back to a different registers at a different program point.
    \item A static mapping is simpler to implement that a dynamic one. For now we would focus on it and later think about relaxing this assumption (that one temporary is assigned to only 1 register).
\end{itemize}
\subsection{Register Allocation Example}
Lets take the following code:\\
% \begin{algorithm}[H]
% \SetAlgoLined
\begin{center}
    // Some code\\
    a:=c+d\\
    e:=a+b\\
    f:=e-1\\
    // Some more code\\
\end{center}
The only temporaries are a-f.
Also assume a,e are dead after the  last line and we have 4 registers available for allocation.\\
A possible allocation can be.
\begin{center}
    a, e, f $\rightarrow$ r1\\
    c $\rightarrow$ r2\\
    d $\rightarrow$ r3\\
    b $\rightarrow$ r4\\
\end{center}
The new transformed code would be
\begin{center}
    r1:=r2+r3\\
    r1:=r1+r4\\
    r1:=r1-1\\
\end{center}
How do we know that this is correct?
\begin{itemize}
    \item In the first line a is computed correctly in r1. In the second line e is computed correctly and stored in r1.
    \item But since r1 is overwritten, value of a is no longer available, which is ok since a was dead after the second line in the original code as well.
    \item In the third line, value of f is correctly computed and stored in r1. Value of e is no longer available which is not wrong since it is given e is dead after this line.
\end{itemize}
\textbf{Insight}: We could map a,e,f to the same register r1 because at no program point they are live simultaneously.\\
\textbf{Idea}: Temporaries t1 and t2 can share the same register \textbf{iff} at all program points, atmost one of t1 and t2 is live.
\subsection{Another Example}
Consider the following example:\\
\includegraphics[scale=0.2]{images/85_eg.png}\\
The above image represents the control flow graph of the program under consideration.\\
Executing a liveness analysis on the program we get:\\
\includegraphics[scale=0.2]{images/85_eg2.png}\\
Everything in the purple pen represent the variables that are live at that particular program point.\\
Now we need to find the pairs of variables that are not live simultaneously at any program point. After going through the program points above, we see that the following pairs of variables are not live simultaneously.
\begin{itemize}
    \item (a,b)
    \item (b,d)
    \item (a,d)
    \item (a,e)
\end{itemize}
These pairs of variables can share a register. Moreover, from the pairs (a,b), (b,d), (a,d) we can infer that a,b,d can be allocated to the same register. \\
Next, we discuss an algorithm for register allocation from the ideas that we got from the two examples.
\subsection{Towards a Register Allocation Algorithm}
\begin{itemize}
    \item Construct an undirected graph.
    \item Node for every temporary
    \item There is an edge between t1 and t2 if they are simultaneously live at some program point.
    % \item Color the graph with k colors, where k is the number of registers. If this is possible we
\end{itemize}

The undirected graph made is called as Register Interference Graph (RIG). If 2 temporaries do not have an edge between them then we can allocated them to the same register.\\
The RIG for the second example discussed above is\\
\begin{center}
    \includegraphics[scale=0.2]{images/85_RIGeg2.png}\\
\end{center}
As we can see that the RIG extracts a global picture from the program. Now we do not need to be concerned about any program point.\\
For any register allocation to be valid, any 2 temporaries mapped to the same register by the allocation must not have an edge between them.\\
% Motivated from this we can see Graph Coloring can help us in getting an allocation.\\
Lets just deviate and discuss a theoretical computer science problem.\\
\textbf{Graph Coloring:} An assignment of colours to each node of the graph so that nodes connected by an edge do not have the same color.\\
\textbf{k-colourable Graph:} A graph is k-colourable if it can be coloured with k colours.\\
On giving it a thought, we can see that the problem of graph colouring can help in validating if a register allocation is possible. Denote every register with a unique colour and try to colour the RIG with k colours, where k is the number of registers. If it is possible we get a valid register allocation for each temporary, else we need to spill some temporaries to memory.\\
% Unfortunately graph colouring is a NP hard problem. In the next module we discuss a heuristic based


\documentclass[a4paper,11pt]{article}
\usepackage{geometry}
 \geometry{
 a4paper,
 total={170mm,257mm},
 left=20mm,
 top=20mm,
 }

 \usepackage{amsmath}
 \usepackage{siunitx}
 \usepackage{multirow}
\usepackage{colortbl}
 \usepackage{hhline}

 \usepackage{lipsum}  %%% Lorem ipsum

\setlength{\headheight}{30.0pt}
\setlength{\footskip}{20pt}

\usepackage{hyperref}
\hypersetup{
    colorlinks=True,
    linkcolor={blue!20!black},
    filecolor=magenta,
    urlcolor=cyan,
}

 \usepackage[export]{adjustbox}
\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{fancyhdr}
\usepackage{multicol}

\pagestyle{fancy}
\fancyhf{}
\rhead{\textit{Pul074BEX004}}
\lhead{\textit{Amrit Prasad Phuyal}}
\rfoot{\thepage}

\usepackage{mathpazo} % Palatino font
\usepackage{graphicx}
\usepackage{float}

\input{./CoverPage.tex} %%% cover page
\input{./Matlab.tex} %%% Matlab code

\newcommand\ddfrac[2]{\frac{\displaystyle #1}{\displaystyle #2}}

%%%%%%%%%%%%%%%%%%%%%for matlab observation #1 fig name #2 Caption
\newcommand{\mobs}[2]{
    \begin{figure}[H]
        \centering
        \includegraphics[width=1.07\linewidth]{./FIG/#1.eps}
        \caption{#2}
    \end{figure}

}

\begin{document}

%%%%  COver page
\CP{Digital Signal Processing}{Lab \#2}{Familiarization with basic CT/DT functions}
{Anila  Kansakar}
%%%%%%%%%%%%%%%%%%%%

\pagenumbering{gobble}
\renewcommand{\contentsname}{Table of Contents}
\tableofcontents

\pagebreak
%\listoffigures
% \pagebreak
% \vspace{5em}
\lstlistoflistings
\vspace{10em}
% \pagebreak
\listoffigures
\pagebreak
\pagenumbering{arabic}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Title} {\large Familiarization with basic CT/DT functions }
%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Objective}
Familiarization with basic CT/DT functions

%%%%%%%%%%%%%%%%%%%%%

%Theory
\section{Theory}

\begin{verbatim}
    who         >> List variables in workspace
    whos        >> List variables in workspace, with sizes and types
    input()     >> Read input from user
    disp()      >> Display value of variable
    subplot()   >> Plot multiple graphs in one figure
    figure()    >> Create new figure window
    clear all   >> Clear all variables from workspace, freeing up system memory
    close all   >> Close all figures
    home        >> Send cursor to home position
    hold on     >> Retain plot data for multiple plots in one figure
    grid on     >> Turn on grid lines
    grid off    >> Turn off grid lines
    grid        >> Turn on and off grid lines and set grid spacing
    demo        >> Access product examples in Help browser
    ver         >> Display version of Matlab and operating system information
    lookfor     >> Search for keyword in all help files
    length()    >> Length of largest dimension of array
    pause       >> Stop MATLAB execution temporarily
    plot()      >> 2-D line  plotting
    stem()      >> Plot discrete data as stems
    real()      >> Return real part of complex number
    imag()      >> Return imaginary part of complex number
    zeros()     >> Create array of all zeros
    ones()      >> Create array of  all ones
    exp()       >> Return exponential of complex number
    for         >> Loop to repeat specified numbers of times
    end         >> End  For loop
    if-else     >> If-else statement execute if statement is true
\end{verbatim}

\section {Lab Problems}
%%%%%%%%%%%%Problem 1

\subsection{Problem 1}
\subsection*{Plot the basic signal using Matlab}

%%%%%%%%%%%%% Problem 1aaaaaaaaaaaaaa
\subsubsection{Impulse response}
Codes:
\MAT{./CODES/p1a.m}{Matlab code for plotting Impulse function }
\mobs{impulse}{Plot for Impulse function }

%%%%%%%%%%%%% Problem 1bbbbbbbbbbbb
\subsubsection{Unit step response}
\MAT{./CODES/p1b.m}{Matlab code for plotting Unit Step function }
\mobs{unit}{Plot for Unit Step function }

%%%%%%%%%%%%% Problem 1ccccccccccccccc
\subsubsection{Ramp response}
\MAT{./CODES/p1c.m}{Matlab code for plotting Ramp function }
\mobs{ramp}{Plot for Ramp function}

%%%%%%%%%%%% Problem 1dddddddddddddd
\subsubsection{ Rectangular pulse response}
\MAT{./CODES/p1d.m}{Matlab code for plotting  Rectangular pulse function }
\mobs{rect}{Plot for Rectangular pulse function}

%%%%%%%%%%%%Problem 2
\subsection{Problem 2}
\subsection*{Plot the following continuous-time signals}

%%%%%%%%%%%%% Problem 2aaaa
\subsubsection*{$x(t)=Ce^{at}$ where $C$ and $a$ are real numbers and choose $C$ and $a$ both positive and negative.}
\MAT{./CODES/p2a.m}{Matlab code for C and a  Both real }
\mobs{ca_real}{Plot for for C and a  Both real}

%%%%%%%%%%%%% Problem 2bbbbbb
\subsubsection*{Plot the same signal taking $a$ as pure imaginary number}
\MAT{./CODES/p2b.m}{Matlab code for C real a Imaginary }
\mobs{a_imag}{Plot for C real a Imaginary}

%%%%%%%%%%%%% Problem 2ccccc
\subsubsection*{Consider complex exponential signal as specified in b) where $C$ is expressed in polar form i.e., $C=|C|e^{j\theta}$ and $a$ in rectangular form i.e., $a=r+j\omega_o$. Then function $x(t)$, on simplification, becomes $$ x(t)= |C|e^{rt}[\cos(\omega_o t+\theta)+j\sin(\omega_o t+\theta)]$$
Now, plot the signal for different values of r and comment on the results.\\
i. r=0 \quad \quad ii. r$<$ 0 \quad \quad iii. r$>$0}
\MAT{./CODES/p2c.m}{Matlab code for different value of r}

\subsubsection{For r=0}
\mobs{r0}{Plot for r=0}

\subsubsection{For r$<$0}
\mobs{rm1}{Plot for r$<$0}

\subsubsection{For r$>$0}
\mobs{r1}{Plot for r$>$0}

%%%%%%%%%%%%Problem 3
\subsection{Problem 3}
\subsection*{Plot the DT exponential function $x[n]=a^n$, $a=|a|e^{j\theta}$. Choose the suitable value of $|a|$ and $\theta$.}

\MAT{./CODES/p3.m}{Matlab code for calculation and plot DT exponential function}

\mobs{dt}{Plot for DT exponential function}

% %%%%%%%%%%%%Problem 4

\subsection{Problem 4}
\subsection*{Synthesize the signal from the FS coefficients as $C_0=1$, $C_1=C_{-1}=\ddfrac{1}{4}$, $C_2=C_{-2}=\ddfrac{1}{2}$, $C_3=C_{-3}=\ddfrac{1}{3}$.}
\MAT{./CODES/p4.m}{Matlab code for synthesizing and plotting signal}
\mobs{syn}{Plot for synthesized signal}

% %%%%%%%%%%%%Problem 5

\subsection{Problem 5}
\subsection*{Plot fundamental sinusoidal signal, its higher harmonics up to 5\textsuperscript{th} harmonics and add all of them to see the result. Comment on the result.}
\MAT{./CODES/p5.m}{Matlab code for calculation and plot of Sinusoidal harmonics and thier sum}
\mobs{harmonics}{Plot for Sinusoidal harmonics and thier sum}

%%Discussion and Conclusion
\section{Discussion and Conclusion}
In this Lab we familiarize ourself with Matlab Programming with basic of Continous time and Discrete time signals. We also learn about their basic operations in Matlab including  plotting and analyzing the signals. We learn to use online help for different commands and use them to calculate different DT and CT functions and plot them.

\end{document}

DSP LABs/LAB 2/DSP LAB 2 Amrit Prasad Phuyal.tex


\iffalse
It's unfair to say that mathematicians aren't real doctors, we perform surgeries all the time. In this class we'll introduce the notion of a topological manifold via simplicial (delta) complexes. Spend a day or two doing examples and go over several notions like orientation, cobordism and of course surgery.

Keywords: simplicial complex, manifold, orientation, cobordism, surgery

Type: Lecture
Homework: Recommended
Prereqs: None
\fi

\input{../preamble}
\rhead{\scshape Mathcamp 2017 : All things Manifoldy}
\begin{document}
\title{Mapping class groups}
\author{Apurva Nakade}
\thispagestyle{fancy}
\maketitle

\section{Automorphisms}

\begin{definition}
	An \textbf{automorphism} or a \textbf{self-homeomorphism} of a manifold $X$ is a map $f:X \rightarrow X$ which is a homeomorphism. The set of automorphisms forms a group under composition, denoted ${\mathrm{Homeo}(X)}$.
\end{definition}

This group is usually too big to get a good handle on, so instead we study automorphisms up to deformations i.e. we consider two automorphisms to be the same if one automorphism can be continuously deformed into another.

\begin{definition}
	The group $\mathrm{Homeo}(X)/{deformations}$ is called the \textbf{mapping class group}, denoted $\mathrm{MCG}(X)$.
\end{definition}

\begin{example}\label{thm:MCG}
	Every automorphism of $\R^1$ is a strictly increasing or a strictly decreasing function $f:\R^1 \rightarrow \R^1$. It is possible to deform a strictly increasing function $f_1$ to another strictly increasing function $f_2$ via the path of maps $t.f_1 + (1-t).f_2$ for $t\in[0,1]$, similarly for decreasing functions. And composition of two decreasing functions is an increasing function. Together these imply that $\mathrm{MCG}(\R^1) \cong \Z/2$.
\end{example}

The main object of interest for us is the mapping class group of the torus $\mathrm{MCG}(T)$ i.e. automorphisms of the torus \emph{up to deformations}. Let us fix two non-parallel circles on the torus and call these the \textbf{principal circles}. While there are various choices for these all of which work we'll pick the simples ones and call them the \textbf{red} and the \textbf{blue} circles. See \textbf{Fig.1}.
%
% The first important observation to make is that \emph{up to deformations} an automorphism of a torus is completely determined by where the two principal circles are mapped.
%
%
% \begin{ques}
% 	Are there automorphisms of the torus which map the red lines as in \textbf{Fig.2}?
% \end{ques}

\section{Dehn Twists}
One way to construct non-trivial automorphisms of the torus is via Dehn twists.
\begin{definition}
	A \textbf{Dehn twist}, denoted $D$, is a special automorphism of the cylinder which twists the cylinder as in \textbf{Fig.2}.
\end{definition}

% The direction of the rotation is very important. If the cylinder is placed upright then we think of $D$ as fixing the bottom circle and twisting the cylinder by rotating the top circle counterclockwise. To avoid confusion we'll always indicate which way is \textbf{`up'} in the cylinder.

Dehn twist has the nice property that the two boundary circles are unchanged. We can use Dehn twists to create  non-trivial automorphisms of the torus by cutting out a cylinder, performing a Dehn twist, and glueing it back. This is an example of surgery on the torus! Dehn twists look even more interesting on gluing diagrams. See \textbf{Fig.3}.

We can perform Dehn twists on other cylinders sitting inside a torus which allows us to create more automorphisms of the torus. A theorem of Dehn-Lickorish  says that for genus $g$ surfaces the mapping class group is generated by a small set of Dehn twists. Dehn twists on punctured discs give rise to Braid groups establishing further connections between topology and group theory.

\section{Exercises}
\begin{exercise}
	Use the following exercises to show that the mapping class group of $S^1 = \{ (x,y): x^2 + y^2 = 1\}$ is $\Z/2$.
	\begin{enumerate}
		\item Show that every automorphism can be continuously deformed to one that fixes the point $(1,0)$.
		\item Find two automorphisms of $S^1$ which fix $(1,0)$ which cannot be deformed into each other via automorphisms.
		\item Show that $\mathrm{MCG}(S^1) \cong \Z/2$.
	\end{enumerate}
\end{exercise}

\begin{exercise}
	Find $\mathrm{MCG}(X)$ when $X$ is one of the following spaces
	\begin{enumerate}
		\item Two parallel lines in $\R^2$
		\item Two intersecting lines in $\R^2$
		\item Union of two intersecting circles
		\item The 2 dimensional unit disk $D^2 = \{ (x,y) \in \R^2 : x^2 + y^2 \le 1\}$ \\ This one is non-trivial. Read the wikipedia page on Alexander's trick.
	\end{enumerate}
\end{exercise}

\begin{exercise}
	Based on the above exercise what is the relationship between $\mathrm{MCG}(X \sqcup X)$ and $\mathrm{MCG}(X)$, where $X \sqcup X$ denotes the disjoint union of two copies of $X$.
\end{exercise}

\begin{exercise}
	Describe the homeomorphisms which are inverses of $D_R$ and $D_B$ in the mapping class group of the torus.
\end{exercise}

\begin{exercise}
	Perform a Dehn twist on the cylinder around the equator on $S^2$. What is the corresponding element in $\mathrm{MCG}(S^2)$?
\end{exercise}

\begin{exercise}
	\begin{enumerate}
		\item Verify that $D_B D_R D_B$ and $D_R D_B D_R$ are equal in $\mathrm{MCG}(T)$ by checking what they do to the principal circles.
		\item Consider the $2 \times 2$ matrices $M_B = \begin{bmatrix} 1 & 1 \\ 0 & 1 \end{bmatrix}$ and $M_R = \begin{bmatrix} 1 & 0 \\ -1 & 1 \end{bmatrix}$. Verify that $M_B M_R M_B = M_R M_B M_R$.
		\item Assuming that $\mathrm{MCG}(T)$ is generated by $D_R$ and $D_B$ show that this defines a homomorphism from $\mathrm{MCG}(T)$ to the group $SL_2(\Z)$ of $2 \times 2$ matrices with integer coefficients and determinant 1.
		\item Describe the action of the matrices $M_R$ and $M_B$ on the plane and relate it to the Dehn twists $D_R$ and $D_B$.
	\end{enumerate}
\end{exercise}

\begin{exercise}
	$D_B D_R D_B$ is NOT a reflection! In \textbf{Fig. 4} describe what Dehn twists do to the shaded regions and figure out what $D_B D_R D_B$ really is.
\end{exercise}

\end{document}

01 All things manifoldy/04 - Dehn Twists.tex


\title{FYS-STK4155 \\
Project 1}
\author{Lars Johan Brodtkorb}
\date{ }

\subsection*{Abstract}

In this exercise I have tried to figure out which method is to be preferred of ordinary least squares, Ridge and Lasso regressions. The results show a better prediction error for the ridge regression, but that may be due to a faulty implementation of the Lasso method.

\section{Introduction}

The main aim of this project is to study in more detail various
regression methods, including the Ordinary Least Squares (OLS) method,
Ridge regression and finally Lasso regression.
The methods are in turn combined with resampling techniques.

We will first study how
to fit polynomials to a specific two-dimensional function called
\href{{http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf}}{Franke's
	function}.  This
is a function which has been widely used when testing various  interpolation and fitting
algorithms. Furthermore, after having etsablished the model and the
method, we will employ resampling techniques such as the  cross-validation and/or
the bootstrap methods, in order to perform a proper assessment of our models.

The Franke function, which is a weighted sum of four exponentials  reads as follows
\begin{align*}
f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\
&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }.
\end{align*}

The function will be defined for $x,y\in [0,1]$.  Our first step will
be to perform an OLS regression analysis of this function, trying out
a polynomial fit with an $x$ and $y$ dependence of the form $[x, y,
x^2, y^2, xy, \dots]$. We will also include cross-validation and
bootstrap as resampling techniques.  As in homeworks 1 and 2, we
can use a uniform distribution to set up the arrays of values for $x$
and $y$, or as in the example below just a fix values for $x$ and $y$ with a given step size.
In this case we will have two predictors and need to fit a
function (for example a polynomial) of $x$ and $y$.  Thereafter we will
repeat much of the same procedure using the the Ridge and
Lasso regression methods, introducing thus a dependence on the bias
(penalty) $\lambda$.

Thereafter we are going to use (real) digital terrain data and try to
reproduce these data using the same methods. We will also try to go
beyond the second-order polynomials metioned above and explore
which polynomial fits the data best.

\section{Methods}

\subsection{Measures of prediction accuracy}

Find the confidence intervals of the $\beta$ parameters by computing their variances, evaluate the Mean Squared error (MSE)
\[ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
\]
and the $R^2$ score function.
If $\tilde{\hat{y}}_i$ is the predicted value of the $i-th$ sample and $y_i$ is the corresponding true value, then the score $R^2$ is defined as
\[
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
\]
where we have defined the mean value  of $\hat{y}$ as
\[
\bar{y} =  \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
\]

\subsection{Ordinary least square regression}

We have an input vector $X^T = (X_1,X_2,X_3,...,X_p)$ and want to predict a real-valued output Y. The linear regression model has the form: (\cite{elementsstat}, chapter 3.2)

\begin{equation}
f(X) = \beta_0 + \sum_{j=1}^{p}X_j\beta_j
\end{equation}

The linear model either assumes that the regression function $E(Y |X)$ is
linear, or that the linear model is a reasonable approximation. The $\beta_j$ ’s are unknown parameters or coefficients, and the variables $X_j$ can come
from many different sources. One of them is that of basis expansions such as

$X_2 = X_1^2, X_3 = X_1^3$, leading to a polynomial
representation;

We can use the basis expansions as a device to achieve more flexible representations for f(X). Polynomials are an example of this. Having great flexibility in approximation allows the basis expansions to work for many occurences, but their variability may increase heavily if they are applied outside their intended scope. The coefficients to achieve a functional form
in one region can cause the function to flap about madly in remote regions (\cite{elementsstat}, chapter 5.1, p. 140).

The polynomial method produces a dictionary $D$ consisting of typically a very large number of basis functions $|D|$, far more than we can afford to fit to our data. Along
with the dictionary we require a method for controlling the complexity
of our model, using basis functions from the dictionary. There are three
common approaches:
Restriction, selection and regularization methods, which we will revisit in the Ridge and Lasso segment.

No matter the source of the $X_j$, the model is linear in the parameters. Typically we have a set of training data $(x_1, y_1)...(x_N , y_N )$ from which
to estimate the parameters $\beta$. Each $x_i = (x_{i1}, x_{i2},...,x_{ip})^T$ is a vector of feature measurements for the i-th case.

Using linear least squared fittings for $X \in \R^2$ we can get the linear function of X that minimizes the sum of the sum of squared residuals of Y.

A unique solution for $\beta$ has the form:

\begin{align*}
\beta =(\textbf{X}^T\textbf{X})^{-1}\textbf{X}^T\textbf{y}
\end{align*}
\medskip

We have made minimal assumptions about the true distribution
of the data. In order to pin down the sampling properties of $\hat{\beta}$ , we now
assume that the observations $y_i$ are uncorrelated and have constant variance
$\sigma^2$, and that the $x_i$ are fixed (non random). The $\hyphenation{variance covariance}$
matrix of the least squares parameter estimates is then given by:

\begin{align*}
Var(\hat{\beta}) =(\textbf{X}^T\textbf{X})^{-1}\sigma^2
\end{align*}
\medskip

\subsection{K-fold cross-validation algorithm}

Cross-validation is probably the simplest and most widely used method for estimating prediction
error. This method directly estimates the expected
extra-sample error $Err = E[L(Y, \hat{f(X)})]$, the average generalization error when the method $\hat{f(X)}$ is applied to an independent test sample from the joint distribution of X and Y. As mentioned earlier, we might hope that cross-validation estimates the conditional error, with the training set T held fixed. Unfortunately, cross-validation typically estimates well only the expected prediction error (\cite{elementsstat}, chapter 7.10 - 7.12).

K-fold crossvalidation uses part of the available data to fit the model, and a different part to test it. We split the data into K roughly equal-sized parts. For the kth part, we fit the model to the other $K-1$ parts of the data, and calculate the prediction error of the fitted model when predicting the kth part of the data. We do this for k = 1, 2,...,K and
combine the K estimates of prediction error.

Given a set of models $f(x,\alpha)$ indexed by a tuning parameter $\alpha$, denote
$\hat{f}^{-\kappa(x,\alpha)}$ by the $\alpha$-th model fitted with the k-th part of the data removed. Then for this set of models we define:

\begin{equation}
CV(\hat{f},\alpha) = \frac{1}{N}\sum_{i=1}^{N}L(y_i,\hat{f}^{-\kappa(i)}(x_i,\alpha))
\end{equation}
\medskip

where L is the loss function, $\kappa$ is the removed data.

The function $CV(\hat{f},\alpha)$ provides an estimate of the test error curve, and we
find the tuning parameter $\hat{\alpha}$ that minimizes it. Our final chosen model is
$f(x,\hat{\alpha})$, which we then fit to all the data.
\medskip
\medskip

The steps of the cross-validation method: \newline
(a) Find a subset of “good” predictors that show fairly strong (univariate) correlation with the class labels, using all of the samples except those in fold k.\newline
(b) Using just this subset of predictors, build a multivariate classifier, using all of the samples except those in fold k.\newline
(c) Use the classifier to predict the class labels for the samples in
fold k.

\subsection{Bootstrap}

The bootstrap method has been used in this project to estimate the variance and bias.
As with cross-validation, the bootstrap seeks to estimate the conditional error, but typically estimates well only the expected prediction error.

Suppose we have a model fit to a set of training data. We denote the
training set by $Z = (z_1, z_2,...,z_N )$ where $z_i = (x_i, y_i).$ The basic idea is
to randomly draw datasets with replacement from the training data, each
sample the same size as the original training set. This is done B times
(B = 200 in my application), producing B bootstrap datasets.
Then we refit the model to each of the bootstrap datasets, and examine
the behavior of the fits over the B replications (\cite{elementsstat} p.249)

\subsection{Ridge and Lasso regression}

\subsubsection*{Ridge regression}
Ridge regression is a simple example of a regularization approach, while the lasso is both a regularization and selection method.

Selection methods adaptively scan the dictionary and include only those basis functions $h_m$ that contribute significantly to the fit of
the model.

Regularization methods use the entire dictionary but restrict the coefficients.  (\cite{elementsstat}, chapter 5.1 p.141)

The method used follows \cite{ridge} quite closely. In chapter 1.4.2 on page 8 in \cite{ridge} we find an estimator for $\beta$ and variance in the ridge regression:

\begin{equation*}
\beta=(\textbf{X}^T\textbf{X}+\lambda\textbf{I})^{-1}\textbf{X}^T\textbf{y}
\end{equation*}

\medskip
\begin{equation*}
Var(\hat{\beta}) =(\textbf{X}^T\textbf{X}+\lambda\textbf{I})^{-1}\sigma^2
\end{equation*}
\medskip

Ridge regression shrinks the regression coefficients by imposing a penalty on their size. The ridge coefficients minimize a penalized residual sum of squares.

\begin{equation}
\hat{\beta}_{ridge} =\argmin_{\beta}({\sum_{i=1}^{N}(y_i - \beta_0 -\sum_{j=1}^{p}x_{ij}\beta_j)^2 + \lambda \sum_{j=1}^{p}\beta_j^2})
\end{equation}
\medskip

Here $\lambda\ge 0 $ is a complexity parameter that controls the amount of shrinkage. The idea of penalizing by the sum-of-squares of the parameters is also used in neural networks,
where it is known as weight decay (\cite{elementsstat} p. 63)

\subsection*{Lasso regression}
The Lasso method is a shrinking method like Ridge. Just as in ridge regression, we can re-parametrize the constant $\beta_0$ by standardizing the predictors; the solution for $\hat{\beta_0}$ is $\hat{y}$, and thereafter we fit a model without an intercept. In the signal processing literature,
the Lasso is also known as basis pursuit (Chen et al., 1998).
We can also write the Lasso problem in the equivalent Lagrangian form:

\begin{equation}
\hat{\beta}_{lasso} =\argmin_{\beta}(\frac{1}{2}{\sum_{i=1}^{N}(y_i - \beta_0 -\sum_{j=1}^{p}x_{ij}\beta_j)^2 + \lambda \sum_{j=1}^{p}\abs{\beta_j}})
\end{equation}
\medskip

The difference from the ridge method is that the ridge penalty $\sum_{j=1}^{p}\beta_j^2$ is replaced with the Lasso penalty $\sum_{j=1}^{p}\abs{\beta_j}$

\section{Code implementation}
For most of the code implementation see github repository: \href{https://github.com/larsjbro/FYS-STK3155/tree/master/project1}{https://github.com/larsjbro/FYS-STK3155/tree/master/project1}. I will include the bootstrap method with comments in the report.

\subsection{Bootstrap}

The bootstrap method was implemented with the following code, which was used to generate the prediction error, bias and variance shown in the figures.\newline

\lstinputlisting[language=Python]{bootstrap.py}

\medskip

\section{Analysis}

\subsection{Results}

\subsubsection{OLS}

Table \ref{Table:1} shows the fitted coefficients for the best ordinary least squares method fitted to the Frankfunkction along with its standard deviation, z-score and 95 percent confidence interval. As we can se from the table, only a few of the coefficients are significantly different from 0. In total table \ref{Table:1} shows there are only 5 such coefficients. The other coefficients could have possibly been set to zero.

\FloatBarrier
\begin{table}

	\begin{tabular}{lrrrrr}
		\toprule
		{} &         coef &          std &   z\_score &        Confidence interval 0.025 &        Confidence interval 0.975 \\
		\midrule
		0  &    -1.648196 &     0.532137 & -3.097317 &    -2.691165 &    -0.605227 \\
		1  &    -1.919886 &     1.848958 & -1.038361 &    -5.543776 &     1.704005 \\
		2  &     6.699335 &     3.720084 &  1.800856 &    -0.591897 &    13.990566 \\
		3  &     1.577407 &    10.841403 &  0.145498 &   -19.671353 &    22.826166 \\
		4  &     0.248456 &     0.463744 &  0.535762 &    -0.660466 &     1.157378 \\
		5  &     3.048496 &     2.491046 &  1.223781 &    -1.833865 &     7.930857 \\
		6  &   -20.088662 &    12.870111 & -1.560877 &   -45.313616 &     5.136292 \\
		7  &   -17.914271 &    16.822026 & -1.064929 &   -50.884836 &    15.056294 \\
		8  &    81.253441 &    62.107653 &  1.308268 &   -40.475323 &   202.982204 \\
		9  &     0.006444 &     1.659116 &  0.003884 &    -3.245363 &     3.258252 \\
		10 &   -10.048598 &    13.844107 & -0.725839 &   -37.182548 &    17.085353 \\
		11 &   122.772857 &    58.256501 &  2.107453 &     8.592213 &   236.953501 \\
		12 &    18.202467 &    89.033985 &  0.204444 &  -156.300937 &   192.705871 \\
		13 &  -464.346238 &   295.293981 & -1.572488 & -1043.111806 &   114.419331 \\
		14 &    -5.577415 &     2.836466 & -1.966325 &   -11.136786 &    -0.018044 \\
		15 &   -17.666890 &    15.364312 & -1.149865 &   -47.780388 &    12.446608 \\
		16 &   188.349237 &    79.758452 &  2.361496 &    32.025544 &   344.672930 \\
		17 &   127.373969 &   100.448526 &  1.268052 &   -69.501525 &   324.249463 \\
		18 &  -781.831660 &   379.979896 & -2.057561 & -1526.578570 &   -37.084750 \\
		19 &     0.140168 &     8.835873 &  0.015864 &   -17.177825 &    17.458162 \\
		20 &    72.613196 &    63.561066 &  1.142416 &   -51.964203 &   197.190596 \\
		21 &  -603.170312 &   289.455482 & -2.083810 & -1170.492631 &   -35.847993 \\
		22 &  -176.329738 &   401.060283 & -0.439659 &  -962.393449 &   609.733972 \\
		23 &  2420.894666 &  1422.585633 &  1.701757 &  -367.321940 &  5209.111272 \\
		\bottomrule

	\end{tabular}
	\caption{Values for the best fit ordinary least squares method for the data from the Frankefunction with m=300 and sigma=0.5}
	\label{Table:1}
\end{table}
\FloatBarrier

Below are figures from the ordinary least squares method:
\FloatBarrier
\begin{figure}[!ht]
	\centering
	\FloatBarrier
    \includegraphics[width=1\textwidth]{plot_ols_without_r2/olsprediction_error_vs_degrees_m300_l300_s0.png}

	\caption{This shows the bias-variance tradeoff as function of the maximum polynomial degree used in the OLS method, with a sample size of 300.}
	\label{fig:1}
\end{figure}
\FloatBarrier

\medskip

\FloatBarrier
\begin{figure}[!ht]
	\centering
	\FloatBarrier
	\includegraphics[width=1\textwidth]{plot_ols_without_r2/olsprediction_error_vs_degrees_m300_l300_s10.png}

	\caption{This shows the bias-variance tradeoff as function of the maximum polynomial degree used in the OLS method, with a sample size of 300 and a random gaussian noise with standard deviation of 0.1.}
	\label{fig:1}
\end{figure}
\FloatBarrier

\medskip

\FloatBarrier
\begin{figure}[!ht]
	\centering
	\FloatBarrier
	\includegraphics[width=1\textwidth]{plot_ols_without_r2/olsprediction_error_vs_degrees_m300_l300_s50.png}

	\caption{This shows the bias-variance tradeoff as function of the maximum polynomial degree used in the OLS method, with a sample size of 300 and a random gaussian noise with standard deviation of 0.5.}
	\label{fig:1}
\end{figure}
\FloatBarrier

\medskip

\subsubsection{Ridge}

Below are the figures for varying sample sizes, degree of polynomials and penalty with Ridge.

\FloatBarrier
\begin{figure}[!ht]
	\centering
	\FloatBarrier
	\includegraphics[width=1\textwidth]{plot_ridge_without_r2/ridgeprediction_error_m300_d5_s0.png}

	\caption{This shows the bias-variance tradeoff as function of the regularization parameter, lambda,for Ridge regression with a sample size of 300 and a polynomial degree of 5}
	\label{fig:2}
\end{figure}
\FloatBarrier

\medskip

\FloatBarrier
\begin{figure}[!ht]
	\centering
	\FloatBarrier
	\includegraphics[width=1\textwidth]{plot_ridge_without_r2/ridgeprediction_error_m300_d5_s10.png}

	\caption{This shows the bias-variance tradeoff as function of the regularization parameter, lambda,for Ridge regression with a sample size of 300, polynomial degree of 5 and standard deviation of 0.1}
	\label{fig:2}
\end{figure}
\FloatBarrier

\medskip

\FloatBarrier
\begin{figure}[!ht]
	\centering
	\FloatBarrier
	\includegraphics[width=1\textwidth]{plot_ridge_without_r2/ridgeprediction_error_m300_d5_s50.png}

	\caption{This shows the bias-variance tradeoff as function of the regularization parameter, lambda,for Ridge regression with a sample size of 300, polynomial degree of 5 and standard deviation of 0.5}
	\label{fig:2}
\end{figure}
\FloatBarrier

\medskip

\subsubsection{Lasso}
Below are the figures for varying sample sizes, polynomials and penalty with Lasso.
\FloatBarrier
\begin{figure}[!ht]
	\centering
	\FloatBarrier
	\includegraphics[width=1\textwidth]{lasso_prediction_vs_lambda/lassoprediction_error_m300_d5_s0.png}

	\caption{This shows the bias-variance tradeoff as function of the regularization parameter, lambda, for Lasso regression method with a sample size of 300 and a polynomial degree of 5}
	\label{fig:3}
\end{figure}
\FloatBarrier

\medskip

\FloatBarrier
\begin{figure}[!ht]
	\centering
	\FloatBarrier
	\includegraphics[width=1\textwidth]{lasso_prediction_vs_lambda/lassoprediction_error_m300_d5_s10.png}

	\caption{This shows the bias-variance tradeoff as function of the regularization parameter, lambda, for Lasso regression method with a sample size of 300, polynomial degree of 5 and standard deviation of 0.1.}
	\label{fig:3}
\end{figure}
\FloatBarrier

\medskip

\FloatBarrier
\begin{figure}[!ht]
	\centering
	\FloatBarrier
	\includegraphics[width=1\textwidth]{lasso_prediction_vs_lambda/lassoprediction_error_m300_d5_s50.png}

	\caption{This shows the bias-variance tradeoff as function of the regularization parameter, lambda, for Lasso regression method with a sample size of 300, a polynomial degree of 5 and standard deviation of 0.5}
	\label{fig:3}
\end{figure}
\FloatBarrier

\medskip

\subsection{Discussion}
From all the figures I will now consider prediction error, bias and variance for the various methods.

\subsubsection{OLS}
For the OLS I can see that the minimum prediction error is for a polynomial of degree 4 for all standard deviations 0, 0.1 and 0.5. IN all cases the value of the prediction error is about 0.004. The bias is smallest at 5 degrees in all cases. The variance is smallest for a degree 4 polynomial for standard deviations of 0 and 0.1. For standard deviation of 0.5 the smallest variance is at 3. For a degree 1 polynomial, we do have a low variance, but the bias is really high.
\medskip

Table \ref{Table:1} shows that there were only 5 significant coefficients, the other ones could possibly have been set to 0, but that was not implemented. In principle I could have found a best subset model here.

\subsubsection{Ridge}
For the ridge regression I can see a low prediction error for lambda being $10^{-4}$ and $10^{-10}$ for the 0 and 0.1 standard deviation cases, with a value of the prediction error of about 0.0035 and 0.002. The lowest value being for 0.1 standard deviation and lambda $10^{-4}$. For the 0.5 standard deviation case, the prediction error is the smallest at about 0.25 for lambda $10^{-4}$.

\subsubsection{Lasso}
For the lasso regression at standard deviation of 0 I can see a minimum prediction error of about 0.005 for lambda $10^{-7}$. For standard deviation 0.1 the prediction error is up to 0.015 and for standard deviation 0.5 it is all the way up to 0.25. This tells me that something might have gone wrong with the implementation of lasso.

\section{Conclusion}

The results show that the Ridge regression gives a better prediction error than the other two methods. That does not agree with the expected, since the Lasso method should have been better. I tried to figure out the reason why that happened, and I came up with the following explanation:

The formulation of Lasso is the same as for Ridge, but for the extra $\beta$ term in Ridge. It seems clear from the results that the Ridge method was to be preferred, but on further inspection, it seemed that the Lasso method may have been worse because the standardization method was not done correctly for Lasso. I suspect this was the reason for Lasso being worse than the Ridge regression.

I then implemented this standardization, and still found that the Lasso regression was inferior to the ridge regression, which I can not fully explain.

\section*{Improvements}
I would have liked to complete the data analysis, and also figured out why Ridge regression showed superior results compared to the Lasso Regression. For the OLS method I could have found a best subset model after having found the significant coefficients.

\section{Bibliography}
\begin{thebibliography}{99}
	\bibitem{projecttext}$\href{https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/2018/Project1/pdf/Project1.pdf}{https://github.com/CompPhysics/MachineLearning/blob/master/doc/Projects/2018/Project1/pdf/Project1.pdf}$

	\bibitem{elementsstat}$\href{https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/elementsstat.pdf}{https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/elementsstat.pdf}$

	\bibitem{ridge}$\href{https://arxiv.org/pdf/1509.09169.pdf}{https://arxiv.org/pdf/1509.09169.pdf}$

	\bibitem{franke}$\href{http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf}{http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf}$

\end{thebibliography}