\chapter{Computer Science}
\section{Basics}
{\bf Definitions:}
$f \in O(g) \leftrightarrow
g \in \Omega (f) \leftrightarrow
L_{x \rightarrow \infty} {\frac {f(x)} {g(x)}} < \infty$.
$f \in o(g) \leftrightarrow
g \in \omega (f) \leftrightarrow
L_{x \rightarrow \infty} {\frac {f(x)} {g(x)}} = 0 $.  $G_1 \subset (G,E)$ is a
\emph{strongly connected component} iff $x,y \in G_1$ means there is a directed path
$x \rightarrow y$ and a directed path $y \rightarrow x$.
\\
\\
{\bf Recurrences:}  Suppose $T(n)= aT(n/b) + f(n)$.
If $f(n)= O(n^{log_b(a)-\epsilon})$ then $T(n)= \Theta(n^{log_b(a)})$.
If $f(n)= \Theta(n^{log_b(a)})$ then $T(n)= \Theta(n^{log_b(a)}lg(n))$.
If $f(n)= \Omega(n^{log_b(a)+\epsilon})$ and $af(n/b) \le c f(n), c<1$
then $T(n)= \Theta(f(n))$.\\
\\
{\bf Binary arithmetic:}
Adding an $m$ bit number and $n$ bit number takes $O(max(m,n))$ time and $O(m+n)$ space.
Multiplying an $m$ bit number and $n$ bit number takes $O(mn)$ time and $O(m+n)$ space.
The \emph{extended gcd} of an $m$ bit number and $n$ bit number takes
$O(mn)$ time and $O(m+n)$ space.
$GCD(u, v)$ average running time:
$O((1+{\frac {max(u,v)} {(u,v)}}) lg(min(u, v)))$.
$A^E \jmod{M}$ where $M$ is an $m$ bit number and $E$ is an $n$ bit number
takes $O(n m^2)$ time.
\\
\\
{\bf Theorem:}
Given $\epsilon>0$ there is a multiplication algorithm such that the number
of elementary operation $T(n)$ needed to multiply two $n$-bit numbers
satisfies
$T(n) <c(\epsilon)n^{1+\epsilon}$.  Strassen: $T(n)=O(n lg(n))$.
\\
\\
{\bf Floating Point Numbers:} $f \times b^{e-q}$ is represented as $(e, f)$.
\\
\\
{\bf Heapsort:} For a node, $t$, the \emph{heap property} is
$value(t) \geq value(LEFT(t))$ and $value(t) \geq value(RIGHT(t))$.
A heap list $A[1], A[2] , \ldots, A[n]$ can represent a heap with
$LEFT(A[i])= A[2i]$ and $RIGHT(A[i])= A[2i+1]$.  Note that a path
from a node to an edge is linearly ordered and
so largest element is at root.  Usually the
heap is arranged so that $value(LEFT(t)) \geq value(RIGHT(t))$.
\begin{multicols} {2} {
\begin{verbatim}
heapify(A,i,j,n)
{
    if(i is not a leaf) {
        k= child of i with largest elment;
        if(A[k]>A[i]) {
            swap (A[i], A[k]);
            heapify (A, k, j, n);
        }
    }
}
\end{verbatim}

\begin{verbatim}
buildheap(A,n)
{
    for(i=n;i>1;i--) {
        heapify(A,i,n,n);
    }
}

heapsort(A,n)
{
    buildheap(A,n);
    for(i=n;i>1;i--) {
        swap(A[1], A[i]);
        heapify(&A[1],i-1, i-1);
    }
}
\end{verbatim}
}
\end{multicols}

{\bf Shortest Path:} between x and y in $G=(V, E)$ where
$l(e)>0$ is the weight of $e \in E$ is $O(e lg(n))$. d(v) contains an overestimate
of the shortest path from s to v.
$prev(v)$ contains the previous element in the
shortest path from s to v.  (Ford-Bellman version works for negative weights.)
\begin{multicols} {2} {
\begin{verbatim}
shortestpath(V,E,s)  {
    for (v in V) {
        d(v):= infinity;
        prev(v):= empty-set;
        }
    H:= empty-set;
    d(s)= 0;
    mark(s);
    while (H is not empty) {
        h= deletemin(H);
        for e=(v, w) in E, w unmarked) {
            if(d(w)>(d(v)+l(e))) {
                d(w)=d(v)+l(e);
                prev(w)= v;
                insert(w, H);
                }
            }
        }
    }
\end{verbatim}
}
\end{multicols}

{\bf Union-find:}  Link(x, y): make x and y kids of a common parent.  Parent node
points to itself.  $m$ UNION-FIND operations on $n$ elements is $O((m+n) lg(n))$.

\begin{multicols} {2} {
\begin{verbatim}
makeset(x) {
    p(x)= x;
    rank(x)= 0;
    }

find(x) {
    if(x != p(x))
        p(x)=  find(p(x));
   return(p(x));
   }

link(x, y) {
    if(rank(x)>rank(y)) swap(x, y);
    if(rank(x)==rank(y)) rank(y)++;
    p(x)= y;
    return(y);
    }

union(x,y) {
    link(find(x), find(y));
    }
\end{verbatim}
}
\end{multicols}

{\bf Order statistics:} The algorithm below satisfies the recurrence
$T(n) \leq T({\frac n 5}) + T({\frac {3n} 4} + cn)$.
\\
\\
Select(k,S) \\
\{\\
\jt if($k<50$)
\jt \jt sort and return k'th element;\\
\jt Partition $S$ into $5$ sequences $S_1, \ldots , S_{\lfloor {\frac{|S|} 5} \rfloor}$, $T$,
where $T$ contains the up to $4$ leftovers;\\
\jt Sort each $5$ set;\\
\jt Let $M$ be the set of the medians for the $S_i$;\\
\jt m= Select($\lceil {\frac {|M|} 2} \rceil$;\\
\jt let $S_1$, $S_2$, $S_3$ be the sets of elements $<, =, > m$ respectively;\\
\jt if($|S_2|>k$)\\
\jt \jt return Select($k, S_1$);\\
\jt if($|S_1|+|S_2|>k$) \\
\jt \jt return $m$;\\
\jt return($k-|S_1|-|S_2|$, $S_3$);\\
\}
\\
\\
{\bf $2-3$ Trees:} Interior node has smallest key of 2nd and 3rd descendant.
Insert: Do membership test stop at terminal position; id 2 kids, add one, if
not, split into two, $(n, n')$.  Add $n'$ using insert.
Delete: If two kids left, done.
Otherwise, try to move node of a siblings under common parent; if you can't,
transfer this node to a sibling.  If this leaves a singleton, in the parent,
recurse the transfer on parent.
\\
\\
{\bf Minimal spanning trees:}  Consider a graph,
$G= (V,E)$, each edge $e \in E$ having weight $wt(e)$.
\\
\emph{Kruskal:}
\\
 Initialize a forest of trees consisting of $V$.\\
while( there is more than one component) \\
\jt 1. Remove edge from $E$ of minimum weight.\\
\jt 2. Add it if it unites two trees.  Discard it if it creates a cycle.\\
\\
\emph{Prim:}\\
1. Choose an arbitrary vertex $S= \{x\}$, $M=\emptyset$.\\
2. while($S \ne V$) \\
\jt 3. Choose an edge $e=(x, y), x \in S$ of minimum weight.\\
\jt 4. If $y \in S$, discard.  Otherwise $M= m \cup \{e\}$
\\
\\
{\bf NP Completeness:}
$P \subseteq N$.  If $A \le B$
\footnote{$A \le B$ means problem $A$ can be transformed to problem $B$ in polynomial time;
this is called a reduction \emph {from} $A$ \emph{to} $B$.}
and $B \in P$ then $A \in P$.  $L \in NPC$ if and only if
$L \in NP$, $A \in NP \rightarrow A \le L$.  Classical computation theory classifies
problems by a ``certain'' solution on all instances.  The class of problems which
can be solved in polynomial time
``up to an arbitrary error, $\epsilon$'' is called $RP$ for ``randomized
polynomial.''  $P \subseteq RP \subseteq NP$.
\\
\\
{\bf Problems in P and NP:}
P: MST.  Given a weighted graph, G, and a weight, K $\exists$ a tree,
NP: TSP.  Given a weighted graph, G, and a weight, K $\exists$ a cycle,
C, that connects all nodes  of G with weight $\leq K$.\\
P: Circuit value.  NP: Circuit SAT.\\
P: 2-SAT: Use $\phi= (a_1 \vee b_1  )  \wedge ... \wedge (a_n \vee b_n )$ to form graph with
nodes $a_i, b_i, {\overline {a_i}}, {\overline {b_i}}$ insert
edges ${\overline {a_i}} \rightarrow b_i$
and ${\overline {b_i}} \rightarrow a_i$.  Find strongly connected components.  If no
strongly connected component contains a variable and its negation, it is satisfiable;
otherwise not.  So 2-SAT is not NP hard.  NP: 3-SAT.  Note in disjunctive normal form
SAT is easy but translating is hard.
\\
P: matching.  NP: 3D matching.\\
P: Linear Programming.   NP: Integer Programming.\\
\\
{\bf Ford-Fulkerson:} Augmenting path p is a simple path from s to t that
increases the flow.
\begin{verbatim}
    Initialize flow, f to 0;
    while (there is an augmenting path, p)
        augment flow along p;
    return f;
\end{verbatim}
{\bf Undecidability:} Suppose $Term(P,X)$ is a boolean function which takes a program,
$P$, and an input $X$.
$Term(P,X)$ returns true iff $P$ terminates on $X$.
$Term(P,X)$ returns false iff $P$ does not terminate on $X$.
\\
\\
{\bf Theorem:}  $Term(P, X)$ does not exist.  Suppose it did. Set
\begin{verbatim}
diag(P,X) {
  if $Term(P,P)$==true
    loop
    }
\end{verbatim}
$diag(diag)$ terminates iff it doesn't terminate.  Contradiction.\\
\\
{\bf Stable Matching} (up to $n^2$ rounds). (1) Boy goes to favorite girl
on list.  (2) Girl tells highest choice ``maybe'', tells everyone else
No.  (3) Boy crosses off girls that have said no.
(4) terminate in the round when every girl has
told one boy ``maybe'', convert ``maybe'' to yes.\\
\\
{\bf Linear Programming Standard Form:}
Maximize $x= C^{T}X$, subject to $AX= B$, $X \geq 0$.
Problem:  There may be exponentially
many corners. (Reason: introduce to $n$ constraint inequalities $m$ slack variables;  the
corner points occur when $m$ variables are $0$.  There are ${{m+n} \choose m}$ ways to select
the variables to be set to $0$.)
Simplex idea: move along growing paths instead of trying all corners randomly.
Dual, minimize $x= B^{T}W$, subject to $A^{T}W= C$, $W \geq 0$.\\
\\
{\bf Notation:} \emph{basic variables} $\ne 0$, \emph{non basic variables} $=0$.
A is an $m \times n$
matrix, with $m$ variables (including slack) and $m$ constraints.
Tableau has basic variables and their values in two first columns.  Top row is
all variables as labels middle is matrix (A).
Rightmost column is constants (B).  Bottom row is $C-C^{T}X$ in terms of the
non-basic variables.  Basic algorithm is:
\begin{enumerate}
\item Locate most negative coefficient in bottom row, call column containing it $x_{j}$.
\item Compute ${\frac {B_{i}} {A_{ij}}}$.  The smallest one, denoted $k$, is the pivot.
\item Convert pivot to 1 and eliminate all coefficients in the same column.
\item Replace $x_{k}$ row by $x_{j}$.
\item repeat until no negative numbers in bottom row.
\end{enumerate}
{\bf SAT/$k$-sat reduction:}
$l_1 \vee l_2 \vee \ldots \vee l_n \rightarrow
l_1 \vee l_2 \vee x_1 \wedge
{\overline {x_1}} \vee l_3 \vee x_2 \wedge \ldots
{\overline {x_{n-3}}} \vee l_{n-1} \vee \ldots \vee l_n$.
Phase transition for SAT: ${\frac {clauses} {variables}} \approx 4.3$.
3-SAT $\rightarrow$ MQ.  Replace $+$ with $\vee$, $\cdot$ with $\wedge$ , 1 with true,
0 with false.  If $c_i= x_{i_1} \vee x_{i_2} \vee x_{i_3}$ add
$x_{i_1} + x_{i_2} + x_{i_3} x_{i_4}$ and
$x_{i_1} \cdot x_{i_2} + x_{i_2} \cdot x_{i_3} + x_{i_1} \cdot x_{i_3} = x_{i_5}$
and $x_{i_4} + x_{i_5} + x_{i_4} \cdot x_{i_5} =1$.
\\
\\
{\bf Theorem:} The following problems are
NP Complete: SAT, $k$-SAT ($k>2$), k-clique, Vertex Cover, Independent set,
Subset Sum, Partition, Bin Packing, Hamilton circuit.
\emph{Clique/SAT reduction:} Each occurrence of a variable is a vertex, edges
between vertices if their occurrence in the clauses have same complementarity.
$k$ is number of clauses.
\\
\\
{\bf Hard core bit:} Let $f$ be a one-way function from
$\{0,1\}^n$ to $\{0,1\}^n$,
$x \in \{0,1\}^n$ ,
$r \in \{0,1\}^n$ ,
and let $G$ be a function that takes
$\{0,1\}^n$ to $\{0,1\}^{n+1}$ by $G(x,r)= f(x), r, <x,r>$.  Let $P$ be a
prediction function.  Goldreich-Levin: If there is an algorithm $A$ such
that $| Prob_r [A(f(x), r) = <x,r>] - {\frac {1} {2}} | \geq \epsilon$ then
there is an algorithm $I$ that produces a list $L$ of size $\leq
{\frac {1} {{\epsilon^2}}}$ with $x$ in $L$, (2) $I$ runs in time
polynomial in $n$ and ${\frac {1} {\epsilon}}$ and doesn't compute $f$.
A function is \emph{negligible:} smaller that inverse of any polynomial.
Witness: $w:\Sigma^* \rightarrow P( \Gamma^* )$.  \emph{Decision problem:}
$A_w \subseteq \Sigma^*$, $A_w= \{ x \in
\Sigma * | w(x) \ne 0 \}$.
\emph{Example:} $x \in \Sigma^*$ is an encoding of a Boolean Form.
$y \in \Gamma *$
is an encoding of a truth assignment.
$\#P$ is class of witnesses, $w$, such that: (i) there is a P-time algorithm
to decide if $x \in w(x)$ and (ii)
$\exists k \in N$ such that $\forall y \in
w(x),
|y| \leq |x|^k$.  $w \in \#P \rightarrow A_w \in NP$ and
$A \in
NP \rightarrow \exists w, A=A_w$.
\\
\\
{\bf Theorem:}
Counting perfect matchings of a bipartite graph is $\#P$ complete.
\\
\\
{\bf Finite State Machine:} Finite alphabet, $A$, finite states, $S$, two functions:
$\delta: S \times A \rightarrow S$ and
$\gamma: S \times A \rightarrow A$.\\
Finite State Automata is FSM without output.\\
\\
{\bf Regular expressions:}
Language $L$ is a subset of $A^*$.
\emph{Regular expression}, $R$ over alphabet, $A$ with letters
$a \in A$: (1) $ \epsilon \in R$,
(2) $a \in A$, (3) $r^* \in R$ if $r \in T$,
(4) $r_1 r_2 \in R$ if $r_1 , r_2 \in R$,
(4) $r_1 \vee r_2 \in R$ if $r_1 , r_2 \in R$.
Language associated with a regular expression: (1) $ L(\epsilon)=\{ \epsilon \}$,
(2) $L(a)= \{a\}$, (3) $L(r^*) L(r)^*$,
(3) $L(r_1 r_2)= L(r_1 ) L(r_2)$,
(4) $L(r_1 \vee r_2) = L(r_1) \cup L(r_2)$.
$L$ is a \emph{regular language} if $\exists r \in R$ with $L=L(r)$.
\emph{Phrase structured Grammar},
$G$, consists of (1) Vocabulary $V$, (2) terminals (denoted by lower case letters)
$T \subseteq V$, (3) variables or non-terminals $V \setminus T$ (denoted
by upper case letters), (4) a designated non-terminal $S$, called the start symbol, (5)
a finite set $P$ of productions: $\alpha \rightarrow \beta$.
$w \Rightarrow w'$ iff $\exists u,v, w=u \alpha v$ and $w'= u \beta v$.
\\
\\
{\bf Grammars:}
Grammar types are defined by production rule limitation: (1) Type 0: no limitations,
(2) Type 1: production rules of the form
$\alpha \rightarrow \beta$, $|\alpha| \leq |\beta|$ or
$\alpha \rightarrow \epsilon$,
(3) Type 2: production rules of the form $A \rightarrow \beta$,
(4) Type 3: production rules of the form
$A \rightarrow a$ or $A \rightarrow aB$,
(5) \emph{context free:} production rules of the form $A \rightarrow \beta$,
(6) \emph{context sensitive:} production rules of the form
$\alpha A \alpha' \rightarrow \alpha \beta \alpha'$,
(7) regular: production rules of the form
$A \rightarrow a$, $A \rightarrow aB$ or $S \rightarrow \epsilon$.
Backus-Naur form for type 2 context free grammar:
(i) ::= replaces $\rightarrow$,
(ii) non-terminals enclosed in brackets $<>$ and
(iii) all productions with the same non-terminal
LHS are combined into a single RHS.
\emph{Example:}
$\langle sentence  \rangle$ ::=
$\langle noun$ $phrase \rangle\langle verb$ $phrase \rangle$,
$\langle noun$ $phrase \rangle$ ::=
$\langle noun \rangle|\langle article \rangle\langle noun \rangle$,
$\langle noun \rangle$::= boy.
\\
\\
{\bf Theorem:}
A language $L$ can be generated by a type 3 (regular) grammar iff there is a finite
automaton $M$ that accepts $L$.
Pushdown automata (with infinite stack) recognize $L$ iff $L$ is context free.
$L$ is recognized by a linear bounded automata  (tape linearly bounded in length of
input) iff $L$ is context sensitive.
\\
\\
{\bf Minimizing state machines:} Two states, $s_i$, $s_j$, are $0$ equivalent if
the states have the same output for every input.  States are $k+1$ equivalent if they
have the same outputs for any input and their successor states are $k$ equivalent.
Minimization procedure:  Define $\pi_0$ as all states that are $0$ equivalent.  Do until
no further refinement happens: sub-partition $\pi_{k}$ into $\pi_{k+1}$ into subblocks are
$k+1$ equivalent.  This terminates.  When it does, merge equivalent states.
\\
\\
{\bf Pumping Lemma:}  Let $L$ be a finite state grammar accepted by a finite state
machine, $M$, with $n$ states.
If $\alpha$ is a string accepted by $M$ of length at least $n$, then
$\alpha = u||v||w$ where $u||v^i||w$ is also in $L$.\\
\\
{\bf Definition:}
Turing machines are FSMs with a bi-directionally infinite tape with a finite
number of pre-marked squares and an additional transition function
$\sigma: S \times A \rightarrow \{L, R, HALT\}$.
\\
\\
{\bf Huffman algorithm:}  Label each node with frequency.  As long as more than one
node is present, take the two nodes with the lowest frequency and combine them
into a single node with the two combinants as children.
New node has combined frequency.  Left subnode has lower of two frequencies,
right the higher.  Read code by traversing from root.  Left traversal at parent is
0, right, 1.
\\
Resulting code is prefix free.  Further $H(X) \leq l(x) \leq H(X)+1$.
\subsection{Concurrency}
\begin{verbatim}
ECMA Consistency
    1. Reads and writes cannot move before volatile read.
    2. Reads and writes cannot move after volatile write.

CompareExchange(ref int loc, int value, int comp) {
    Monitor.Enter;
    ret= loc;
    if(ret==comp) loc= value;
    Monitor.Exit;
    return ret;
}

class SpinLock {
    volatile int isEntered=0;   // 1 if lock acquired
    int Enter() {
        while(CompareExchange(isEntered,1,0)!=0);
        }
    Exit() {
        isEntered= 0;
    }
}

Memory Consistency Rules
    1. Behavior of Thread in isolation is unaffected
    2. Reads cannot move before lock
    3. Writes cannot move after lock
\end{verbatim}

\begin{multicols} {2} {
\begin{verbatim}

DPLL(C,A) {
// C: clauses, A: literal assignments
//      Termination:
//              empty clause: unsatisfiable
//              empty set of clauses: satisfiable
    if(A is empty)
        return SATISFIED;
    if(A has an empty clause)
        return UNSATISFIABLE;
// unit clause is a clause with one literal
    if unit clause (l) occurs in A
        return DPLL (assign(l,C), A + l));
    if l occurs with same polarity throughout
        return DPLL (assign(l,C), A + l));
    l= choose-literal(A);
    return DPLL (assign(l,C), A + l)) OR
           DPLL (assign(not l,C), A + not l));
    }
Note: If A, B, C are p-free,
(A | p) & (B|!p) &C) is inconsistent iff (A|B)&C is.

Chase(C,x) {
    set x to t;
    delete all clauses containing x from C;
    delete all occurences of !x from clauses in C;
    if (empty clause)
        return UNSATISFIABLE;
    if (unit clause l)
        return Chase(l,t);
    if (C is empty)
        return SATISFIED;
\end{verbatim}
}
\end{multicols}
\begin{multicols} {2} {
\begin{verbatim}
Priority Queue (arrays start at 1 here)

ExtractMax(A) {
    if(heapsize(A)<1)
        return error;
    max= A[1];
    A[1]= A[heapsize(A)];
    heapsize(A)=heapsize(A)-1;
    Heapify(A,1);
    return max;
}

Insert(A,k) {
    heapsize(A)=heapsize(A)+1;
    i= heapsize(A);
    while(i>1 & A[parent(i)]<k) {
        A[i]= A[parent(i)];
        i= parent(i);
        }
    A[i]= k;
    }
\end{verbatim}
}
\end{multicols}
\begin{multicols} {2} {
\begin{verbatim}
Select(A,k) {
    // select kth element from A[1,...n-1]
    if(k==0) return min(A);
    // For randomized, choose x in A at random
    x= SideSelect(A);
    Set B= < y in A: y <=x>
    Set C= < y: y>x >
    if(k<|B|) return Select(B,k)
    return Select(C,|B|-k);
    }

SideSelect(A,k) {
    for(i<=0<=n=INT(size(A)/5))
        Sort successive 5 elements
        // A[5i]<=A[5i+1]<=A[5i+2]<=A[5i+3]<=A[5i+4]
    R= < A[5i+2] > , 0<=i<=n
    x= SideSelect(R,Size(R)/2);
    // note x <= 3*INT((n-5)/10) elements.
    }
// Note E(T(n))= E(T(sn))+n,  x ~ 3/4
\end{verbatim}
}
\end{multicols}
\begin{multicols} {2} {
\begin{verbatim}
struct semaphore {
    int count;
    ProcessQueue queue;
    };

void P(semaphore s) {
    if(s.count>0) {
         (s.count)--;
    else
         s.queue.Insert(); // block
    }
void V(semaphore s) {
    if(s.queue.empty())
        (s.count)++;
    else
        s.queue.remove(); //schedule process
    }

shared semaphore s= 1;
     P(s);
     //critical section
     V(s);
\end{verbatim}
}
\end{multicols}
\begin{multicols} {2} {
\begin{verbatim}
Map()
Reduce()
Scan()      // || prefix
Scatter()
Gather()

Readers/writers
linear sweep

\end{verbatim}
}
\end{multicols}
Architecture and current PCs:
$P= C \times V^2 \times f$.
\emph{Big endian} word: $0, 1, 2, 3$ (descending byte address).
\emph{Little endian} word: $3, 2, 1, 0$ (descending byte address).
\begin{center}
\begin{tabular} {|l|c|r|}
\hline
{\bf Optimization Level} & {\bf Description} & {\bf Level}\\
\hline
High & Procedure inlining & 3\\
Local & common subexpression & 1\\
Local & constant propagation & 1\\
Local & stack height reduction (expression tree) & 1\\
Global & global common subexpression & 2\\
Global & global constant propagation & 2\\
Global & code motion & 2\\
Global & induction variable elimination & 2\\
Global & loop unrolling & 4\\
Global & strip mining & 4\\
Arch specific & strength reduction & 1\\
Arch specific & pipeline scheduling & 1\\
Arch specific & branch offset & 1\\
\hline
\end{tabular}
\end{center}
Effect on performance of Bubblesort (100K items).  Base is 300MHz Sparc Ultra.
\begin{center}
\begin{tabular} {|r|r|r|r|r|}
\hline
{\bf Optimization level} & {\bf Relative performance} & {\bf Clocks} & {\bf Instructions} & {\bf CPI}\\
\hline
0 & 1.00 & 158,615 & 114,938 & 1.38 \\
1 & 2.37 & 66,990  & 37,470 & 1.79 \\
2 & 2.38 & 66,521 & 39,993 & 1.66 \\
3 & 2.41 & 65,747 & 44,993 & 1.46\\
\hline
\end{tabular}
\end{center}
SRAM: $.5-1 ns$, $4,000 \$/GB$.
DRAM: $50-70 ns$, $100 \$/GB$.
Disk: $10^7 ns$, $1 \$/GB$.
Dram address setup: 1 memory cycle, access time: 15 cycles, data transfer: 1 cycle.
4-way interleave plus multiword block gets time down to 20 cycles on average.
Miss penalty to main: 500 cycles, to L2: 25 cycles.  TLB: 512 entries.  Miss: 100 cycles.
Miss percentage; .5-1.
Disk seek latency: 10 ms, rotational latency: 5 ms, transfer rate: 50 MB/s, MTTF:
$10^6$ hours.  Bus speed: system (800 MHz), NB (266 MHz), SB (33 MHz).  Bandwidth:
\begin{center}
\begin{tabular} {|r|r|}
\hline
{\bf Device} & {\bf Bandwidth} \\
\hline
Memory & 3.2GB/sec \\
Disk & 150 MB/sec\\
AGP & 2.1 GB/sec\\
PCI& 132 MB/sec\\
NIC & 20 MB/sec\\
\hline
\end{tabular}
\end{center}
{\bf Dwarves:}
Finite state machines, combinatorics, graphs, Structured/unstructured grids, dense matrix, sparse
matrix, map-reduce, backtrack/branch-and-bound, $N$-body, FFT, Graphical models.\\
\\
$LU${\bf-factorization:}  Let $A \ne 0$ be an $m \times n$ matrix.  There are permutation matrices
$P,Q$ such that $P^T A Q= LU$ where $L$ is lower triangular and $U$ is upper triangular.
$QR$-factorization: Let $X \in {\mathbb C}^{n \times p}$ have rant $p$ then
$x= QR$ where $Q$ is an orthogonal matrix and $R$ is an upper triangular matrix.
$QR$-factorization via unitary operations is used in the least square approximation problem.
Spectral decomposition: $U^H A U= diag( \lambda_1 , \lambda_2 , \ldots , \lambda_n)$.
The eigenvalues of $X^H X$ are the sequence of singular values of $X$.
For a $p \times q$ matrix, row major storage is
$A[1,1]= a[1]$,
$A[1,2]= a[2]$, \ldots, $A[2,1]=a[q+1]$,
etc.,
and in general,
row major storage is $A[i,j]= a[(i-1)q+j]$,
column major storage is
$A[1,1]= a[1]$,
$A[1,2]= a[q+1]$, \ldots, $A[2,1]=a[2]$,  etc.,
and in general,
column major storage is $A[i,j]= a[(j-1)p+i]$.  One step of Gaussian Elimination:
$$
\left(
\begin{array}{cc}
\alpha_{11} & \alpha_{12}^T\\
\alpha_{21} & A{22}\\
\end{array}\right)=
\left(
\begin{array}{c}
\beta_1\\
b_2\\
\end{array}
\right)
\rightarrow
\left(
\begin{array}{cc}
\alpha_{11} & \alpha_{12}^T\\
0 & A{22}- \alpha_{11}^{-1}\alpha_{21}\alpha_{12}^T\\
\end{array}
\right)=
\left(
\begin{array}{c}
\beta_1\\
b_2- \alpha_{11}^{-1} \beta_1 \alpha_{21}\\
\end{array}
\right)
$$
{\bf Definition:}
A \emph{hard core predicate} of a one-way function, $f$ is easy to compute given $x$ but not
given $f(x)$.  If $f$ is a one-way function, $g(x,r)= (f(x),r)$ is a hard core predicate.
Let $B_x(y): \{0,1\}^n \rightarrow \{0,1\}$ be a probabilistic oracle with
$\epsilon$ advantage for $(x,y)$, that
is, $Pr(B_x(y)=(x,y))= {\frac {1+\epsilon} 2}$ and let $EQ_x(y)$ be an oracle for $x=y$.
Define
$Gd_B= \{ y: B_x(y)= (x,y) \}$ and
$Bd_B= \{ y: B_x(y) \ne (x,y) \}$ so
$|Gd_B|= {\frac {1 + \epsilon} 2} 2^n$ and
$|Bd_B|= {\frac {1 - \epsilon} 2} 2^n$.
\\
\\
{\bf Goldreich-Levin:}
Let  $R= \langle r_1, \ldots, r_m \rangle$ be a random selection of elements from
$[n]$, $z \in [n]$ and $b_j=(x,r_j)$.  Let $\langle S_j \rangle$ be a fixed
enumeration of the subsets of $[m]$.  $R[S]= \sum_{j \in S} r_j$.
Define the following algorithm:\\
\\
$STRONG-SC^{B_x}(z, r_1, \ldots, r_m, b_1, \ldots , b_m)$ \\
\jt $sum= 0;$ \\
\jt for($i=1; i \leq 2^m)$) \{\\
\jt \jt $R(S_i)= \sum_{j \in S_i} r_j;$ \\
\jt \jt $B[S_i]= \sum_{j \in S_i} b_j;$ \\
\jt \jt $c= B_x(z+R[S_i]) -b[S_i];$ \\
\jt \jt $sum+= c;$\\
\jt \jt \}\\
\jt return $sum > {\frac {2^m} 2}$;\\
\\
Define the following algorithm:\\
\\
$RECOVER^{B_x, EQ_x}(1^n)$ \\
\jt Pick $r_1, \ldots r_m \in [n]$ at random;\\
\jt for($i=1; i \leq 2^m)$) \{\\
\jt \jt $(b_1, \ldots, b_m)_2 = i-2;$\\
\jt \jt for($k=1; k \leq n$)\\
\jt \jt \jt $y^{(k)}= STRONG-SC^{B_x}(e_k, r_1, \ldots, r_m, b_1, \ldots, b_m);$\\
\jt \jt $y= y^{(1)}||y^{(2)} || \ldots || y^{(n)};$\\
\jt \jt if ($EQ_x(y)==1)$ \\
\jt \jt \jt return $y$;\\
\jt \jt \}\\
\\
{\bf Notation:}
Let
$q_B$ be the number of calls to $B_x$,
$q_E$ be the number of calls to $EQ_x$,
$\epsilon$ be the advantage for $B_x$,
$n$ the length of the strings and $t$  the running time of
$RECOVER^{B_x, EQ_x}(1^n)$. We have the following:
\\
\\
{\bf Theorem:} Let $m$ be a parameter and $M=2^m$.  There is an algorithm, $A$ which
makes $q_B= nM$ calls to $B_x$, $q_E= M$ calls to $EQ_x$ and runs in time
$t=O(nM^2)$ which determines $x$ with probability
at least $1-\delta, \delta= {\frac n {\epsilon^2 M}}$.\\
\\
First some lemmas and notation.  Let $R= \langle r_1, \ldots , r_m \rangle$ and
$X_1, X_2, \ldots , X_M: S \rightarrow R$ be a set of real valued random variables on
the sample space $S$.\\
\\
{\bf Lemma 1:} If
$X_1, X_2, \ldots , X_M: S \rightarrow R$ are pairwise independent and
$X= X_1 + X_2 + \ldots + X_M$ then
$Var(X)= Var(X_1) + Var(X_2 )+ \ldots + Var(X_M)$.
\begin{quote}
\emph{Proof:}
$Var(X)= E(X^2)-E(X)^2=
E(\sum_{i,j} X_i X_j) -
\sum_{i,j} E(X_i) E(X_j)
= \sum_i [E(X_i^2) - E(X_i)^2]- \sum_{i\ne j} [E(X_i X_j)- E(X_i)E(X_j)]$.  The
final bracketed sum is $0$ by pairwise independence.
\end{quote}
{\bf Lemma 2:} If
$X_1, X_2, \ldots , X_M: S \rightarrow R$ are pairwise independent,
$X= X_1 + X_2 + \ldots + X_M$  and $\mu= E(X)= \sum_{i=1}^M E(X_i)$ then
$Pr(|x-\mu| \geq A) \leq {\frac {\sum_{i=1}^M Var(X_i)} {A^2}}$.
\begin{quote}
\emph{Proof:}
By Chebychev, $Pr(|X-\mu| \geq A) \leq {\frac  {Var(X)} {A^2}}$ and by the previous lemma,
$Var(X)= Var(X_1) + Var(X_2) + \ldots + Var(X_M)$.
\end{quote}
{\bf Lemma 3:} Let $M=2^m$.
For any $z \in \{0, 1\}^n$,
$Pr(STRONG-SC^{B_x}(z, r_1, \ldots, r_m, b_1, \ldots , b_m)
\ne (z,x)) \leq {\frac 1 {M \epsilon^2}}$.
\begin{quote}
\emph{Proof:} For $R= \langle r_1, r_2, \ldots, r_m \rangle$ and $S_1, S_2, \ldots,
S_{2^m}$ a fixed enumeration of $[m]$, define a random variable $X_i(R)=1$ if
$B_x(z+R[S_i])= (x,z)+ b[S_i]$ and $0$ otherwise.  $X_i(R)$ and $X_j(R)$ are pairwise
independent since there is an $r_l \in R$ that is included in \emph{either} the sum
$R[S_i]$ or $R[S_j]$ but not both.  $E(X_i)= Pr(z+R[S_i] \in Gd_B)= {\frac {1 + \epsilon} 2}$.
Since $E(X_i^2)= E(X_i)$, $Var(x_i)= E(X_1^2)=E(X_i)^2= E(X_i)-E(X_1)^2= E(X_1)(1-E(X_i))=
{\frac {1+\epsilon} 2}
{\frac {1-\epsilon} 2} =
{\frac {1-\epsilon^2} 4} $.  Note that $\mu= {\frac {M(1+\epsilon)} 2}$ and so by the
previous lemma,  $P(X < {\frac {M} {2}}) =
P(|X-\mu| > {\frac {M \epsilon} {2}})  \leq
{\frac {M(1-\epsilon^2)/4} {(M \epsilon/2)^2}} \leq {\frac 1 {M \epsilon^2}}$.
\end{quote}
{\bf Proof of Theorem:} Let $M=2^m$.
For any $z \in \{0, 1\}^n$,
$Pr(RECOVER^{B_x, EQ_x}(1^n) \ne x) \leq {\frac n {M \epsilon^2}}$.
\begin{quote}
\emph{Proof:}
The loop in $RECOVER$ calls $STRONG-SC^{B_x}$ $n$ times.  Each call is wrong with
probability at most ${\frac 1 {M \epsilon^2}}$ so the probability that $RECOVER$ returns
the wrong answer is at most
${\frac n {M \epsilon^2}}$.
\end{quote}
Note that obtaining $x$ with $p= {\frac 1 2}$ occurs with $M= 2n\epsilon^{-2}$ and the
running time is $O(n^3 \epsilon^{-4})$ with
$q_B= O(n^2 \epsilon^{-2})$ and
$q_E= O(n \epsilon^{-2})$.


\subsection{Matrix Representation for Rational Bézier Surface}
\label{oct_sec:mrep}
\paragraph{}
The intersection between a straight line and a NURBS surface is calculated using matrix representation method.
The method in 2D (NURBS curve) has been introduced in Sec.~\ref{adap_sec:mrep}.
A tensor-product rational bézier surface of degree $(p_1,p_2)$ can be expressed as
\begin{equation}
	\phi(u,v)\in\mathbf{R}^2 \rightarrow \frac{\sum_{i=0}^{p_1}\sum_{j=0}^{p_2}w_{i,j}\mathbf{P}_{i,j}B_i^{p_1}(t)B_j^{p_2}(t) }{\sum_{i=0}^{p_1}\sum_{j=0}^{p_2}w_{i,j}B_i^{p_1}(t)B_j^{p_2}(t)}
\end{equation}
%
For the surface, $\mathbf{L}$ and $\mathbf{R}$ with order $(v_1,v_2)$
\begin{subequations}
\begin{align}
	\mathbf{L} =
	\begin{bmatrix}
		B_0^{v_1+p_1}(u)B_0^{v_2+p_2}(v) & B_1^{v_1+p_1}(u)B_0^{v_2+p_2}(v) & \dots & B_{v_1+p_1}^{v_1+p_1}(u)B_{v_2+p_2}^{v_2+p_2}(v)
	\end{bmatrix} \\
	\mathbf{R} =
	\begin{bmatrix}
		B_0^{v_1}(u)B_0^{v_2}(v)f_0(u,v) & B_0^{v_1}(u)B_1^{v_2}(v)f_0(u,v) & \dots & B_{v_1}^{v_1}(u)B_{v_2}^{v_2}(v)f_3(u,v)
	\end{bmatrix}
\end{align}
\end{subequations}
%
Following the same manner in Sec.~\ref{adap_sec:mrep}, it can be derived that
\begin{equation}
	\mathbf{S}_{\left( (i+k)(v_2+p_2+1)+j+l, l(v1+1)+k\right)} =
	\frac{\mathbf{C}_k^{v_1}\mathbf{C}_l^{v_2}\mathbf{C}_i^{p_1}\mathbf{C}_j^{p_2}}
		{\mathbf{C}_{i+k}^{v_1+d_2}\mathbf{C}_{j+l}^{v_2+d_2}}c_{(i,j)}
\end{equation}
%
\subsection{Properties of the $\mathbf{M_v}$ Matrix}
As described in the previous sections, the $\mathbf{M_v}$ matrix is defined so that
\begin{equation}
	\begin{bmatrix}
	\psi_1(t_0) \dots \psi_{m_v}(t_0)
	\end{bmatrix}
	\times
	\mathbf{M_v(\mathbf{P})}
	= \vec{0}
\end{equation}
where $\mathbf{P}$ is a point on the rational bézier curve/surface.
The order $v$ shall be no less than a critical value and it is proofed to be
\begin{itemize}
	\item $v>max(p-1,1)$ for rational bézier curve
	\item $(v_1,v_2) > (2p_1 -1, p_2 -1)$ or  $(v_1,v_2) > (p_1 -1, 2p_2 -1)$
\end{itemize}
The following properties are proofed in \citep{Laurent2014}
\begin{enumerate}
	\item For all degrees $geq$ critical degree and all point $\in\mathbf{R}^3$ , rank($\mathbf{M_v}(\mathbf{P})$) $<m_v$ if and only if $\mathbf{P}\in$ the closure of $\overline{Im}(\phi)$.
	\item If $\mathbf{P}\in\mathbf{R}^3$ is a point with a unique pre-image by $\phi$, the dimension of the null space of $\mathbf{M_v}(\mathbf{P})^T$ is one.
	\item $\delta\mathbf{M_v}(\mathbf{P}) = 0$ if $\mathbf{P} \in\overline{Im}(\phi)$
\end{enumerate}
where
\begin{equation}
	\delta\mathbf{M_v}(\mathbf{P}) = \prod_{i=1}^{m_v}\sigma_i(\mathbf{M_v}(\mathbf{P}))
\end{equation}
and $\sigma_i$ is the diagonal of $\Sigma$ in the SVD decomposition of $\mathbf{M_v}(\mathbf{P})=U\Sigma V^T$

\begin{enumerate}
	\setcounter{enumi}{3}
	\item $\forall\mathbf{P}\in\mathbf{R}^3$, $d(\mathbf{P},\overline{Im}(\phi))^{n_1}\leq c_1 \delta\mathbf{M_v}(\mathbf{P})$
	\item $\forall\mathbf{P}\in\mathbf{R}^3$, $\delta\mathbf{M_v}(\mathbf{P})^{n_2}\leq c_2 d(\mathbf{P},\overline{Im}(\phi))^{n_2} $
\end{enumerate}
where $c_1,c_2,n_1,n_2$ are constant.\\
These two properties give a distance function like function of the $M_v$ matrix. When the point get away to the surface, $\delta\mathbf{M_v}$ is getting larger and vice versa.
\paragraph{}
If the point is on the curve/surface, in which case $\delta\mathbf{M_v}(\mathbf{P}) = 0$, the corresponding parameter value on the curve/surface can be easily found by a SVD numerically.\\
The computation of the null space of $\mathbf{M_v}(\mathbf{P}) $ will give a single vector $V=[v_1,v_2,\dots,v_{m_v}]$based on 2 and $V$ will be proportional to
\begin{equation}
	\begin{bmatrix}
	\psi_1(t_0) \dots \psi_{m_v}(t_0)
	\end{bmatrix}
\end{equation}
More specifically, it will be proportional to
\begin{equation}
	\begin{bmatrix}
		B_0^v & B_1^v & \dots
	\end{bmatrix}
\end{equation}
for the rational bézier curves and
\begin{equation}
	\begin{bmatrix}
		B_0^{v1}B_0^{v2} & B_0^{v1}B_1^{v2} & \dots
	\end{bmatrix}
\end{equation}
for the rational bézier surfaces.
%
The calculation of the intersection is described in detail in \citep{Buse2010} and \citep{Ba2009}.
All intersections can be calculated at once by using matrix representation of the algebraic curve/surface.
\paragraph{}
Given a rational curve/surface $C1$
\begin{equation}
	\mathbf{P}^1 \xrightarrow{\phi_1} \mathbf{P}^n: (u,v) \rightarrow(f_0,f_1,f_2,f_3)(u,v)
\end{equation}
the aim is to find the intersection via matrix representation method between it with another ration curve $C2$
\begin{equation}
	\mathbf{P}^1 \xrightarrow{\phi_2} \mathbf{P}^n: (t) \rightarrow(g_0,g_1,g_2,g_3)(t)
\end{equation}
is to find
\begin{equation}
	\mathbf{M}_{v1}(\phi_2(t) = 0
\end{equation}
which leads to
\begin{equation}
	\mathbf{M_0}g_0 + \mathbf{M_1}g_1 + \mathbf{M_2}g_2 + \mathbf{M_3}g_3 = 0
	\label{mRep_intec_base}
\end{equation}
By knowing $g_n$ is a polynomial function with order $p$, Eq.~\eqref{mRep_intec_base} can be rearranged as
\begin{equation}
	\mathbf{M}(t) = \sum_{i=0}^p \mathbf{M_i}t^i
\end{equation}
After that, the generalized companion $q\times p$-matrices $A,B$ with rank $\rho$are introduced
\begin{equation}
	A =
	\begin{bmatrix}
		0		&I 		&\dots 		&\dots 		&0 		\\
		0 		&0 		&I 			&\dots 		&0 		\\
		\vdots 	&\vdots &\vdots 	&\vdots 	&\vdots \\
		0 		&0 		&\dots		&\dots 		&I 		\\
		M_0^t 	&M_1^t 	&\dots 		&\dots 		&M_{d-1}^t
	\end{bmatrix}
\end{equation}
\begin{equation}
	B =
	\begin{bmatrix}
		I 		&0 		&\dots 		&\dots 		&0 		\\
		0 		&I 		&0 			&\dots 		&0 		\\
		\vdots 	&\vdots &\vdots 	&\vdots 	&\vdots \\
		0 		&0 		&\dots 		&I 			&0 		\\
		0 		&0 		&\dots 		&\dots 		&-M_d^t \\
	\end{bmatrix}
\end{equation}
Before the eigenvalues are calculated, the regular part of a non-square pencil of the matrices shall be extracted first which is done by the following step
\paragraph{}
\textbf{Step 1}
	Transform B into its column echelon form by SVD-decomposition.
	\begin{equation}
	\begin{aligned}
		B_1 = BV_0 = [\underbrace{B_{1,1}}_{\rho} |\underbrace{0}_{q-\rho}]	\\
		A_1 = AV_0 = [\underbrace{A_{1,1}}_{\rho} |\underbrace{A_{1,2}}_{q-\rho}]
	\end{aligned}
	\end{equation}
\paragraph{}
\textbf{Step 2}
	Transform $A_{1,2}$ into its row echelon form
	\begin{equation}
		U_1A_{1,2} =
		\begin{bmatrix}
			\underline{A^\prime_{1,2}}\\
			0
		\end{bmatrix}
	\end{equation}
	where $A^\prime_{1,2}$ is in full row rank.\\
	At the end of step 2, matrix $A$ and $B$ can be represented as
	\begin{equation}
	\begin{aligned}
	A^\prime_1 &=
	\begin{bmatrix}
		A^\prime_{1,1} & A^\prime_{1,2} \\
		\cmidrule(lr){1-2}
		A_2 & 0
	\end{bmatrix}\\
	B^\prime_1 &=
	\begin{bmatrix}
		B^\prime_{1,1} & 0\\
		\cmidrule(lr){1-2}
		B_2 & 0
	\end{bmatrix}
	\end{aligned}
	\end{equation}
where $A^\prime_{1,2}$ has full row rank\\
$
\begin{bmatrix}
\underline{B^\prime_{1,1}}\\
B_2
\end{bmatrix}
$ has full column rank\\
$
\begin{bmatrix}
\underline{B^\prime_{1,1}}\\
B_2
\end{bmatrix}
$ and $B_2$ are in echelon form
\paragraph{}
$A_2$ and $B_2$ will be the new $A$ and $B$ matrices for next iteration until $B$ has full rank. If $B$ has full row rank but not full rank, $A=A^T$, $B=B^T$.
\paragraph{}
After these process, $A$ and $B$ become two square matrices and $B$ is invertible so that the solution for the intersection parameter $t$ can be determined from the eigenvalue of the matrix $AB^{-1}$


% jam 2004-09-10

\section{Maps between vector spaces}
\label{sec:maps}

In general, the functions discussed here map between real inner product spaces:
$\f:\Vspace \mapsto \Wspace$, where $\Vspace$ is the
{\it domain} and $\Wspace$ is the {\it codomain}.
The real inner product spaces are almost derived from some $\Reals^n$.

The {\it range} of $\f$, $\range(f)$, is the set $\f(\Vspace)$,
which may be a proper subset of its codomain $\Wspace$.
The {\it kernel} of $\f$, $\kernel(f)$, is the set
$\kernel(\f) = \{ \v \in \Vspace : \f(\v) = \0 \}$.

I often use the words `map' and `function' interchangeably.
When I want to distinguish between real- and vector-valued functions,
I may use 'map' for vector-valued functions and
'function' for real-valued ones.

I use $\Uspace$, $\Vspace$, $\Wspace$ for generic vector spaces,
$\u$, $\v$, $\w$, etc., for elements of vector spaces,
and
$\f$, $\g$, $\h$ for vector-valued maps/functions.
I generally do not distinguish $\Reals$, the real numbers,
and $\Reals^1$, or any other 1-dimensional real vector space.
I sometimes use $f$, $g$, $h$ for extra clarity in the special
case of real-valued functions.

The domains of many interesting functions,
such as those that depend on vertex positions,
are direct sum of inner product spaces.
The {\it direct sum} $\Vspace \oplus \Wspace$ is the inner product space
consisting of the ordered pairs $\{ (\v,\w) : \v \in \Vspace, \w \in \Wspace \}$
inheriting the inner product space operations in the obvious way:
$(\v_0,\w_0) \bullet (\v_1,\w_1) = (\v_0 \bullet \v_1) + (\w_0 \bullet \w_1).$
I will usually write an element of $\oplus^n \Vspace$ as
$(\v_0,\ldots,\v_{n-1})$
and use
$\f(\v_0,\v_1,\ldots,\v_{n-1})$
for a function that depends on $n$ vectors.

%-----------------------------------------------------------------

\subsection{Linear maps}
\label{sec:linear-maps}

A map $\Lmap(\v):\Vspace \mapsto \Wspace$
is {\it linear} iff
$\Lmap(a_0 \v_0 + a_1 \v_1) = a_0 \Lmap(\v_0) + a_1 \Lmap(\v_1)$.
I will often write $\Lmap\v \equiv \Lmap(\v)$.

Its not hard to see that, for a linear map,
the range and kernel are linear subspaces of the codomain and
domain, respectively.
Thus any linear map between inner product spaces
divides its domain and codomain each into 2 orthogonal subspaces.
The domain is divided into $\Vspace = \kernel(\Lmap) \oplus \kernel^{\perp}(\Lmap)$,
and the codomain is divided into $\Wspace = \range(\Lmap) \oplus \range^{\perp}(\Lmap)$.

The most common representation for linear maps is the {\it matrix:}
Let $\Lmap(\v):\Vspace \mapsto \Wspace$ be linear,
$\{ \e_0^{\Vspace} \ldots  \e_{m-1}^{\Vspace} \}$ an orthonormal basis for $\Vspace$,
and
$\{ \e_0^{\Wspace} \ldots \e_{n-1}^{\Wspace} \}$ an orthonormal  basis for $\Wspace$
Then $\Lmap$ can be expressed as
\begin{equation}
\Lmap
 =
\sum_{i=0}^{m-1} \sum_{j=0}^{n-1} L_{ij} ( \e_i^{\Wspace} \otimes \e_j^{\Vspace} )
\end{equation}
$(L_{ij})$ is the matrix representation of $\Lmap$ with respect to
the two bases\cite{halmos1958finite}.

It is important to note that there are many usful
representations for linear maps other than matrices \cite{mcdonald-1989b}.
Sometimes other representations are used for convenience,
or to enforce some constraint like symmetry.
In some cases, a non-matrix representation must be used,
because a particular linear transformation
cannot be accurately represented by a matrix of floating point numbers.

Examples:

\begin{itemize}

\item Column-wise:
$\Lmap = \sum_{j=0}^{n-1} ( \c_j^{\Lmap} \otimes \e_j^{\Vspace} )$

$\c_j^{\Lmap} \in \Wspace$ are the 'columns' of $\Lmap$.
$\linear_span\{ \c_0^{\Lmap} \ldots \c_{n-1}^{\Lmap} \} = \range(\Lmap)$
(see \autoref{sec:spans-and-projections}).

\item Row-wise:
$\Lmap = \sum_{i=0}^{m-1} ( \e_i^{\Wspace} \otimes  \r_i^{\Lmap} )$

$\r_i^{\Lmap} \in \Vspace$ are the 'rows' of $\Lmap$.
$\linear_span\{ \r_0^{\Lmap} \ldots \r_{m-1}^{\Lmap} \} =  \kernel(\Lmap)^{\perp}$
(see \autoref{sec:spans-and-projections}).

\item Householder:
$\h_{\v} = \Identity_{\Vspace} - \frac{2}{\| \v \|^2} (\v \otimes \v)$

Householder maps are usually chosen to zero the elements of
a vector, or a row or column of a matrix, for a contiguous range of
indices, say, $[i_0,\ldots,i_n)$.

\end {itemize}

%-----------------------------------------------------------------

\subsection{Affine maps}
\label{sec:affine-maps}

A function $\Amap(\v):\Vspace \mapsto \Wspace$
is {\it affine} if distributes over affine combinations:
$\Amap(\sum_{i=0}^{n-1} a_i \v_i) = \sum_{i=0}^{n-1} a_i \Amap(\v_i) $
for all $\{a_i\}$ such that $1 = \sum_{i=0}^{n-1} a_i$.
(Note that I am describing affine maps on vector (linear) spaces,
rather than the slightly more general notion of affine maps on affine spaces.)
Any linear map between vector spaces is automatically affine.
The other major class of affine maps on vector spaces are the translations.
A {\it translation,} $\Tmap_{\t}$, $\Vspace \mapsto \Vspace$,
simply adds a vector ($\t$) to its argument:
$\Tmap_{\t} \v = \v + \t$.
It's not hard to see that any affine map between two vector spaces
can be represented as the sum of a linear map and a translation.
A typical representation for a general affine map $\Amap : \Vspace \mapsto \Wspace$
is as a pair $(\Lmap,\t)$ where $\Lmap : \Vspace \mapsto \Wspace$ is linear,
$\t \in \Wspace$, and $\Amap(\v) = \Lmap(\v) + \t$.

%-----------------------------------------------------------------

\subsection{Spans and projections}
\label{sec:spans-and-projections}

Let $\Vspace$ be an $n$-dimensional inner product space.

The {\it linear span} of a set of $m$ vectors in $\Vspace$
is the set of linear combinations of those vectors:
\begin{equation}
\linear_span\{ \v_0 \ldots \v_{m-1} \} = \{\v \in \Vspace : \v = \sum_{i=0}^{m-1} a_i \v_i\}
\end{equation}
$\linear_span\{ \v_0 \ldots \v_{m-1} \}$ is a linear subspace of $\Vspace$.

The {\it projection} $\Projection_{\Sset} \v$ of a vector $\v \in \Vspace$
onto an arbitrary subset $\Sset \subset \Vspace$
is the closest point in $\Sset$ to $\v$.
Projection onto a linear subspace is a linear map and
can be computed by summing
elementary orthogonal projections onto an orthonormal basis for the subspace.

An orthonormal basis for $\linear_span\{ \v_0 \ldots \v_{m-1} \}$
(and $\linear_span\{ \v_0 \ldots \v_{m-1} \}^\perp$)
can be computed using the QR decomposition
of the map $\Vmap = \sum_{i=0}^{m-1} \v_i \otimes \e_i$,
(the $n \times m$ matrix whose columns are the $\v_i$).
See Golub and Van Loan~\cite[sec.~5.2]{golub-vanloan-1996}.

The {\it affine span} of a set of $m+1$ vectors in $\Vspace$
is the set of affine combinations of those vectors:
\begin{equation}
\affine_span\{ \p_0 \ldots \p_{m} \} = \{\v \in \Vspace : \v = \sum_{i=0}^{m} b_i \p_i;
1 = \sum_{i=0}^{m} b_i \}.
\end{equation}
$\affine_span\{ \p_0 \ldots \p_{m} \}$ is an affine subspace of $\Vspace$.
$\b = ( b_0 \ldots b_m )$ are {\it barycentric coordinates}
for $\v$ with respect to $\{ \p_0 \ldots \p_{m} \}$.
The barycentric coordinates are unique if $\{ \p_0 \ldots \p_{m} \}$
are affinely independent.

Any affine subspace, $\Aspace$, of a vector space, $\Vspace$ can be represented as
as a translation of a linear subspace of $\Vspace$:
$\Aspace = \Tspace(\Aspace) + \t$,
$\Tspace(\Aspace)$ is the set of differences of elements of $\Aspace$,
a linear subspace of $\Vspace$.
If $\t$ is any element of $\Aspace$.
then projection onto $\Aspace$
can be computed as a translation of an orthogonal projection onto $\Tspace(\Aspace)$:
$\Projection_{\Aspace} (\p) = \t + \Projection_{\Tspace(\Aspace)} (\p - \t)$.
Typically, we pick $\t$ to be the smallest element of $\Aspace$.
Projection onto an affine space is clearly an affine map.

We can represent the affine span of a set of $m+1$ vectors
as a translation of a linear span:
\begin{equation}
\affine_span\{ \p_0 \ldots \p_{m} \} = \p_m + \linear_span\{\v_0 \ldots \v_{m-1}\}
\end{equation}
where $\v_i = \p_i - \p_m$,
which allows us to compute the projection onto
$\affine_span\{ \p_0 \ldots \p_{m} \}$
again using the QR decomposition
of $\Vmap = \sum_{i=0}^{m-1} \v_i \otimes \e_i$.

%-----------------------------------------------------------------

\subsection{Inverses and pseudo-inverses}
\label{sec:Inverses-and-pseudo-inverses}

A convenient definition for the {\it true inverse}
of a map $\f(\v):\Vspace \mapsto \Wspace$ is
$\f^{-1}(\w) = \{ \v : \f(\v) = \w \}$.
The usual definition of inverse treats $\f^{-1}$
as a map from $\Wspace \mapsto \Vspace$,
which is undefined where the value of the true
inverse is not a set containing a single point.

For maps between inner product spaces,
the {\it pseudo-inverse}, $f^{-}$, is a map $\Wspace \mapsto \Vspace$
defined everywhere on $\Wspace$.
Let $\hat{\w}$ be an element of $\Wspace$ closest to $\w$
such that $\f^{-1}(\w)$ is not empty.
Let $\hat{\v}$ be a minimum norm element of $\f^{-1}(\hat{\w})$.
Then $\f^{-}(\w) = \hat{\v}$.

If $\Lmap$ is linear, then it's not hard to see that
$\hat{\w} = \pi_{\range(\Lmap)} \w$, the projection of $\w$
on the range of $\Lmap$
and
$\hat{\v}$ is the unique element of $\kernel^{\perp}(\Lmap)$
such that $\Lmap(\hat{\v}) = \hat{\w}$.

The pseudo-inverse of a linear map can be characterized
by the four Moore-Penrose conditions
(see Golub and Van Loan\cite[sec.~5.5.4]{golub-vanloan-1996}):
\begin{enumerate}
\item $\Lmap \Lmap^{-} \Lmap = \Lmap$
\item $\Lmap^{-} \Lmap \Lmap^{-} = \Lmap^{-}$
\item $\left( \Lmap \Lmap^{-} \right)^{\dagger} = \Lmap \Lmap^{-}$
\item $\left( \Lmap^{-} \Lmap \right)^{\dagger} = \Lmap^{-} \Lmap$
\end{enumerate}

When the 'columns' of $\Lmap$, $\r_j^{\Lmap}$
($\Lmap = \sum_{j=0}^{n-1} ( \Lmap_j^{\Wspace} \otimes \e_j^{\Vspace} )$)
are linearly independent,
then a useful identity is:
\begin{equation}
\label{eq:full-rank-pseudo-inverse}
\Lmap^{-} = \left( \Lmap^{\dagger} \Lmap \right)^{-1} \Lmap^{\dagger}
\end{equation}

The pseudoinverse can be computed
using standard matrix decompositions such as
the QR and SVD \cite{golub-vanloan-1996}.
The pseudoinverse is an example of a linear transformation
which should {\em not} be represented by a matrix
\cite{mcdonald-1989b}.

If $\Amap$ is affine,
let $\Amap = \Lmap + \t$,
where $\Lmap$ is linear,
and $\t$ is an element of $\range(\Amap)$.
Then $\Amap^{-}(\w) = \Lmap^{-}( \w - \t )$.

%-----------------------------------------------------------------


\lab{Line Search Algorithms}{Line Search Algorithms}
\label{lab:line_search}
\objective{Investigate various Line-Search algorithms for numerical optimization.}

\section*{Overview of Line Search Algorithms}
Imagine you are out hiking on a mountain, and you lose track of the trail. Thick fog
gathers around, reducing visibility to just a couple of feet. You decide it is time
to head back home, which is located in the valley located near the base of the mountain.
How can you find your way back with such limited visibility? The obvious way might be to
pick a direction that leads downhill, and follow that direction as far as you can, or
until it starts leading upward again. Then you might choose another downhill direction,
and take that as far as you can, repeating the process. By always choosing a downhill
direction, you hope to eventually make it back to the bottom of the valley, where you live.

This is the basic approach of line search algorithms for numerical optimization.
Suppose we have a real-valued function $f$ that we wish to minimize. Our goal is to find the
point $x^*$ in the domain of $f$ such that $f(x^*)$ is the smallest value in the range of
$f$. For some functions, we can use techniques from calculus to analytically obtain this
minimizer. However, in practical applications, this is often impossible, especially when
we need a system that works for a wide class of functions. A line search algorithm starts with
an initial guess at the minimizer, call it $x_0$, and iteratively produces a sequence of
points $x_1, x_2, x_3, \ldots$ that hopefully converge to the minimizer $x^*$. The basic
iteration to move from $x_k$ to $x_{k+1}$ involves two steps: first, choosing a search direction $p_k$
in which to proceed from the current point, and second, specifying a step size $\alpha_k$ to travel
in this direction. The next point is determined by the formula
$$
x_{k+1} = x_k + \alpha_kp_k.
$$
This procedure is called a line search because at each iteration, we are simply examining the
function in a particular linear direction. The choice of the step size $\alpha_k$ is often
chosen by solving a one-dimensional optimization problem in the given direction. In this lab,
we will discuss approaches to choosing the step size and the search direction.

%Line search procedures are an integral part of many nonlinear optimization techniques.
%In some sense, they represent the simplest nontrivial case in general optimization, as
%we only have to worry about one parameter. And yet far more sophisticated optimization
%algorithms really crucially on the effectiveness and efficiency of line searches, since
%higher-dimensional problems are often broken down into one-dimensional optimizations.
%There are many different line search methods, and their effectiveness depends very much
%on the nature of the optimization problem. Although the line search procedure is often
%only a subroutine of the optimization algorithm at hand, understanding the basics of
%the line search is necessary for understanding the robustness of the entire algorithm.
%\section*{Optimizing Functions on the Real Numbers}
%\subsection*{Derivative versus Derivative-Free Methods}
%As you have seen in calculus classes, the derivative of a function gives information
%about how the value of the function changes at each point, and can be used to determine
%local optima. However, not all objective functions are differentiable, so we need other
%techniques at our disposal. Line search methods may be broadly separated into two groups
%based on whether they use the derivative of the objective function. We discuss two
%simple examples to illustrate this distinction.
%
%\subsection*{Golden Section Search}
%This method is appropriate when minimizing a real-valued function on the reals over a
%closed interval. The function must further satisfy the \emph{unimodal} property, i.e.
%it has just one local minimum, and is monotonic to the left and right of the minimum.
%The goal, of course, is to find the
%global minimum. We do this by making a sequence of guesses that we hope will converge
%quickly to the minimum. Although we may not end up with the exact minimum, this method
%will allow us to pin down the true minimum within an interval of any given width in a
%finite number of steps.
%
%For the Golden Section Search, each step consists of evaluating the function at two
%points within the current interval, comparing these values, and then reducing the size
%of the interval for the next step. Let us consider a typical step in the algorithm. At
%the outset, we have our function $f$ and a closed interval $[a, b]$ over which we seek
%to minimize $f$. Choose two points $a'$ and $b'$ within the interval, and assume that
%$a' < b'$. Now calculate $f(a')$ and $f(b')$, and assume that $f(a') \geq f(b')$.
%Because of the unimodal condition, we now know that the minimizer must be in the
%interval $[a', b]$, for otherwise the function $f$ would have a local minimum in
%both $[a, a']$ and $[a', b]$. In the next step, we repeat the process over the interval
%$[a', b]$. If instead we had $f(b') \geq f(a')$, then we choose the interval $[a, b']$
%for the next step, and if the two values are equal, then it does not matter which
%interval is chosen.
%
%We now have the basic description of the algorithm, but how do we choose the two test
%points $a'$ and $b'$? There is in fact an optimal choice, which reduces the amount of
%work we have to do. Given an interval $[a, b]$, choose $a'$ and $b'$ satisfying
%\begin{align*}
%a' &= a + \rho(b - a) \\
%b' &= a + (1 - \rho)(b - a),
%\end{align*}
%where $\rho = \frac{1}{2}(3 - \sqrt{5}) \approx 0.382$. By choosing these particular
%points, we need to only evaluate the function at one additional point in the next step.
%To demonstrate this fact, the reader may verify that, within the interval $[a, b']$,
%the point $a'$ already satisfies the equation
%\begin{equation*}
%a' = a + (1 - \rho)(b' - a),
%\end{equation*}
%and so we need only evaluate the function at the point $c$ satisfying
%\begin{equation*}
%c = a + \rho(b' - a).
%\end{equation*}
%(The constant $\rho$ is not difficult to derive, and is related to the famous Golden Ratio, hence the name of this algorithm.)
%
%At each step, the interval is reduced by a factor of $1-\rho$, which means that after
%$n$ steps, we have pinned down the minimizer to within an interval approximately
%$(0.61803)^n$ times the length of the original interval. Note that this convergence is
%independent of the objective function.
%
%\begin{problem}
%Implement Golden Section Search as described above. Use this to minimize $e^x - 4x$
%on the interval $\lbrack 0, 3 \rbrack$. How many steps do you need to take to get
%within $.001$ of the true minimizer? Check that with the sentence preceding this
%problem.
%\end{problem}

\subsection*{One-Dimensional Newton's Method}
Let us first start out with a basic task: minimizing a function of one variable.
We will use a popular approach known as Newton's Method, which is a basic line search
algorithm that uses the derivatives of the function to select a direction and
step size.

To use this method, we need a real-valued function of a real variable that is twice
differentiable. The idea is to approximate the function with a quadratic polynomial and
then solve the trivial problem of minimizing the polynomial. Doing so in an iterative
manner can lead us to the actual minimizer. Let $f$ be a function satisfying the
appropriate conditions, and let us make an initial guess, $x_0$. The relevant quadratic
approximation to $f$ is
\begin{equation*}
q(x) = f(x_0) + f'(x_0)(x-x_0) + \frac{1}{2}f''(x_0)(x-x_0)^2,
\end{equation*}
or just the second-degree Taylor polynomial for $f$ centered at $x_0$. The minimum
for this quadratic function is easily found by solving $q'(x) = 0$, and we take the
obtained $x$-value as our new approximation. The formula for the $(n+1)$-th
approximation, which the reader can verify, is
\begin{equation*}
x_{n+1} = x_n - \frac{f'(x_n)}{f''(x_n)}.
\end{equation*}
In the one dimensional case, there are only two search directions: to the right ($+$)
or to the left ($-$). Newton's method chooses the search direction
$\text{sign}(-f'(x_n)/f''(x_n))$ and the step size $|f'(x_n)/f''(x_n)|$.

As is typical with optimization algorithms, Newton's Method generates a sequence of
points or successive approximations to the minimizer. However, the convergence
properties of this sequence depend heavily on the initial guess $x_0$ and the function
$f$. Roughly speaking, if $x_0$ is sufficiently close to the actual minimizer, and if
$f$ is well-approximated by parabolas, then one can expect the sequence to converge
quickly. However, there are cases when the sequence converges slowly or not at all.
See Figure \ref{linesearch:newton}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{newton.pdf}
\caption{The results of Newton's Method using two
different initial guess. The global minimizer was
correctly found with initial guess of 1. However,
an initial guess of 4 led to only a local minimum.}
\label{linesearch:newton}
\end{figure}

\begin{problem}
Implement Newton's Method as described using the following function declaration.
\begin{lstlisting}
def newton1d(f, df, ddf, x, niter=10):
    '''
    Perform Newton's method to minimize a function from R to R.

    Parameters
    ----------
    f : callable function object
        The objective function (twice differentiable)
    df : callable function object
        The first derivative
    ddf : callable function object
        The second derivative
    x : float
        The initial guess
    niter : integer
        The number of iterations

    Returns
    ------
        min : float
            The approximated minimizer
    '''
    pass
\end{lstlisting}

Use this function to minimize $x^2 + \sin(5x)$ with an initial guess of $x_0 = 0$.
Now try other initial guesses farther away from the true minimizer, and note when the
method fails to obtain the correct answer.
\end{problem}

\section*{General Line Search Methods}
\subsection*{Step Size Calculation}
We now examine Line Search methods in more generality. Given a differentiable function
$f : \mathbb{R}^n \rightarrow \mathbb{R}$ that we wish to minimize, and assuming that
we already have a current point $x_k$ and direction $p_k$ in which to search, how do we
choose our step size $\alpha_k$? If our step size is too small, we will not make good progress
toward the minimizer, and convergence will be slow. If the step size is too large, however,
we may overshoot and produce points that are far away from the solution.
A common approach to pick an appropriate step size involves the \emph{Wolfe conditions}:

\begin{align*}
&f(x_k + \alpha_kp_k) \leq f(x_k) + c_1\alpha_k\nabla f_k^Tp_k, &(0 < c_1 < 1),
\\ &\nabla f(x_k + \alpha_kp_k)^Tp_k \geq c_2\nabla f_k^Tp_k, &(c_1 < c_2 < 1).
\end{align*}

Here, we use the shorthand notation $\nabla f_k$ to
mean the gradient of $f$ evaluated at the point $x_k$. The search direction $p_k$ is
often required to satisfy $p_k^T \nabla f_k < 0$, in which case it is called a
\emph{descent direction}, since the function is guaranteed to decrease in
this direction. Generally speaking, choosing a step size $\alpha_k$ satisfying these conditions
ensures that we achieve sufficient decrease in the function and also that we do not
terminate the search at a point of steep decrease (since then we could achieve even
better results by choosing a slightly larger step size). The first condition is known
as the \emph{Armijo} condition.

Finding such a step size satisfying these conditions is not always an easy task, however.
One simple approach, known as \emph{backtracking}, starts with an initial step size
$\alpha$, and repeatedly scales it down until the Armijo condition is satisfied.
That is, choose $\alpha >0, \rho \in (0, 1), c\in (0, 1)$, and while
$$
f(x_k + \alpha p_k) > f(x_k) + c\alpha\nabla f_k^Tp_k,
$$
re-scale $\alpha := \rho\alpha$. Once the loop terminates, set $\alpha_k = \alpha$. Note that the value
$\nabla f_k^Tp_k$ remains fixed for the duration of the backtracking algorithm, and hence need only
be calculated once at the beginning.
%The second of the two conditions can be replaced by
%\begin{equation*}
%| \nabla f(x_k + \alpha_kp_k)^Tp_k| \leq c_2 | \nabla f_k^Tp_k|,
%\end{equation*}
%and in this case we have the \emph{strong Wolfe conditions}.
%
%The \emph{Goldstein conditions} for choosing step size:
%\begin{equation*}
%f(x_k) + (1-c)\alpha_k\nabla f_k^Tp_k \leq f(x_k + \alpha_kp_k) \leq f(x_k) +
%c\alpha_k\nabla f_k^Tp_k,
%\end{equation*}
%where $0 < c < 0.5$. Similar to the Wolfe conditions, the Goldstein conditions ensure
%sufficient decrease in the function and prevent the step size from being too small. In
%the area of quasi-Newton optimization methods, however, the Wolfe conditions are
%preferred.

\begin{problem}
Implement this backtracking algorithm using the following function declaration.
\begin{lstlisting}
def backtracking(f, slope, x, p, a=1, rho=.9, c=10e-4):
    '''
    Perform a backtracking line search to satisfy the Armijo condition.

    Parameters
    ----------
    f : callable function object
        The objective function
    slope : float
        The value of grad(f)^T p
    x : ndarray of shape (n,)
        The current iterate
    p : ndarray of shape (n,)
        The current search direction
    a : float
        The intial step length (set to 1 in Newton and quasi-Newton methods)
    rho : float
        A number in (0,1)
    c : float
        A number in (0,1)

    Returns
    -------
        alpha : float
            The computed step size satisfying the Armijo condition.
    '''
    pass
\end{lstlisting}
\end{problem}

\subsection*{Choosing a Search Direction}
There are many different ways to choose a search direction $p_k$. As noted earlier, it is usually
a requirement to choose a descent direction. We will compare two methods, both using derivative
information about the function.

\emph{Gradient Descent.} Recall that the gradient of a function at a given point gives the direction
in which the function is increasing fastest. Thus, the negative of the gradient points in the direction
of fastest decrease. In the method of Gradient Descent, we choose our search direction to be
this direction of steepest descent, that is,
$$
p_k = -\nabla f_k.
$$
This is a very natural choice, since we seem to be approaching the minimum value as fast as possible.
However, depending on the nature of the objective function, convergence may be slow. See Figure
\ref{linesearch:comparison}.

\begin{figure}
\centering
\includegraphics[width=\textwidth]{comparison.pdf}
\caption{Paths generated by Gradient Descent (green) and Newton's Method (blue).
Note that the Newton path takes a more direct route toward the minimizer (located
at the origin).
}
\label{linesearch:comparison}
\end{figure}

\emph{Newton's Method.} We now generalize the one-dimensional Newton's method presented above.
We use both the gradient and the Hessian matrix (which gives information on the curvature of the
function at a given point) to choose a search direction. This is more computationally intensive,
but it leads to very fast convergence in many cases. See Figure
\ref{linesearch:comparison}. Our search direction is
$$
p_k = -\nabla^2 f_k^{-1} \nabla f_k,
$$
where $\nabla^2 f_k^{-1}$ is the inverse of the Hessian matrix of $f$ at the point $x_k$.
In other words, $p_k$ is the solution to the linear system
$$
\nabla^2 f_k p_k = -\nabla f_k.
$$

\begin{problem}
Implement the Gradient Descent algorithm and Newton's Method using the following function declarations.
In each function, you should call your backtracking function with
values $\alpha = 1, \rho = .9$, and $c = 10^{-4}$. The \li{scipy.linalg} module
may be useful when computing the search direction in Newton's Method.
\begin{lstlisting}
def gradientDescent(f, df, x, niter=10):
    '''
    Minimize a function using gradient descent.

    Parameters
    ----------
    f : callable function object
        A differentiable real-valued function
    df : callable function object
        The gradient of the function
    x : ndarray of shape (n,)
        The initial point
    niter : integer
        The number of iterations to run.

    Returns
    -------
        pts: list of ndarrays
            The sequence of points generated
    '''
    pass

def newtonsMethod(f, df, ddf, x, niter=10):
    '''
    Minimize a function using Newton's method.

    Parameters
    ----------
    f : callable function object
        Real-valued, twice-differentiable function
    df : callable function object
        The gradient of the function
    ddf : callable function object
        The Hessian of the function
    x : ndarray of shape (n,)
        The initial point
    niter : integer
        The number of iterations

    Returns
    -------
    pts : list of ndarrays
        The sequence of points generated
    '''
    pass
\end{lstlisting}

\end{problem}

\subsection*{Line Search in SciPy}
The SciPy module \li{scipy.optimize} contains implementations of various optimization algorithms,
including several line search methods. In particular, the module provides a useful routine for
calculating a step size satisfying the Wolfe Conditions described above, which is more robust
and efficient than our simple backtracking approach. We recommend its use for the remainder of
this lab. The function is called \li{line_search}, and accepts several arguments. We can typically
leave the keyword arguments at their default values, but we do need to pass in the objective
function, its gradient, the current point, and the search direction. The following code gives
an example of its usage, using the objective function $f(x, y) = x^2+4y^2$.
\begin{lstlisting}
>>> import numpy as np
>>> from scipy.optimize import line_search
>>>
>>> def objective(x):
>>>     return x[0]**2 + 4*x[1]**2
>>>
>>> def grad(x):
>>>     return 2*x*np.array([1, 4])
>>>
>>> x = np.array([1., 3.]) #current point
>>> p = -grad(x)           #current search direction
>>> a = line_search(objective, grad, x, p)[0]
>>> print a
0.125649913345
\end{lstlisting}
Note that the function returns a tuple of values, the first of which is the step size. We have illustrated
the very basic use of this function. See the documentation for further uses.

\section*{Non-linear Least Squares Problems}
We now discuss a very important class of problems known as Least Squares problems. These
are unconstrained optimization problems that seek to minimize an objective function of the form
$$
f(x) = \frac{1}{2}\displaystyle\sum_{j=1}^m r_j^2(x),
$$
where each $r_i : \mathbb{R}^n \rightarrow \mathbb{R}$ is smooth, and $m \geq n$. Such problems
arise in many scientific fields, including economics, physics, and statistics. Linear Least
Squares problems form an important subclass, and can be solved directly without the need for an
iterative method. At present we will focus on the non-linear case, which can be solved with a
line search method.

To motivate the problem further, suppose you are given a set of data points, and you have some kind of model for the data.
You need to choose particular values for the parameters in your model, and you wish to do so in a way that ``best fits"
the observed data. What do we mean by ``best fit"? We need some way to measure the error between our model and the data set,
and then minimize this error. The best fit will correspond to the choice of parameters that minimize the error function.

More formally, suppose we are given the data points $(t_1, y_1), (t_2, y_2), \ldots, (t_m, y_m)$, where $y_i \in \mathbb{R}$
and $t_i \in \mathbb{R}^n$ for $i = 1, \ldots, m$. Let $\phi(x, t)$ be our model for this data set, where $x$ is
a vector of parameters of the model, and $t \in \mathbb{R}^n$. We can measure the error at the $i$-th data point by the value
$$r_i(x) := \phi(x, t_i) - y_i,$$ and by summing the squares of these errors, we obtain our non-linear least squares objective
function:
$$
f(x) = \frac{1}{2} \displaystyle \sum_{j=1}^m  r_j^2(x).
$$

The individual functions $r_i$ that measure the error between the model and the data point are known as \emph{residuals},
and we can aggregate these functions into a \emph{residual vector}
$$
r(x) := (r_1(x), r_2(x), \ldots, r_m(x))^T.
$$
The Jacobian of $r(x)$ can be expressed in terms of the gradients of each $r_i$ as follows:
$$
J(x) = \begin{bmatrix} \nabla r_1(x)^T \\ \nabla r_2(x)^T \\ \vdots \\ \nabla r_m(x)^T \end{bmatrix}
$$
You can further verify that
\begin{align*}
\nabla f(x) &= J(x)^T r(x), \\
\nabla^2 f(x) &= J(x)^TJ(x) + \displaystyle \sum_{j=1}^m r_j(x) \nabla^2r_j(x).
\end{align*}
That second term in the formula for $\nabla^2 f$ involves second derivatives and can be problematic to compute. Often in practice,
this term is small, either because the residuals themselves are small, or are nearly affine in a neighborhood of the solution and
hence the second derivatives are small.
The simplest method for solving the nonlinear least squares problem, known as the \emph{Gauss-Newton Method}, exploits this
observation, simply ignoring the second term and making the approximation
$$
\nabla^2 f(x) \approx J(x)^TJ(x).
$$
The method then proceeds in a manner similar to Newton's Method. In particular, at the $k$-th iteration, we choose a search
direction $p_k$ that solves the linear system
$$
J_k^TJ_kp_k = -J_k^Tr_k.
$$

For convenience, we summarize these steps in Algorithm \ref{alg:guassnewton}.
\begin{algorithm}
\begin{algorithmic}[1]
\Procedure{Gauss-Newton}{}
    \State \textrm{Choose initial parameter vector } $x_0$
    \State $k \gets 0$
    \While{$J_k^Tr_k \neq 0$}
        \State \textrm{solve } $J_k^TJ_kp_k = -J_k^Tr_k$
        \State \textrm{choose step size } $\alpha_k$ \textrm{ satisfying Wolfe Conditions.}
        \State $x_{k+1} \gets x_k + \alpha_kp_k$
        \State $k \gets k+1$
    \EndWhile
\EndProcedure
\end{algorithmic}
\caption{Gauss-Newton Method}
\label{alg:guassnewton}
\end{algorithm}

\begin{problem}
Implement the Gauss-Newton method using the following function declaration.
\begin{lstlisting}
def gaussNewton(f, df, jac, r, x, niter=10):
    '''
    Solve a nonlinear least squares problem with Gauss-Newton method.

    Parameters
    ----------
    f : callable function object
        The objective function
    df : callable function object
        The gradient of f
    jac : callable function object
        The jacobian of residual vector
    r : callable function object
        The residual vector
    x : ndarray of shape (n,)
        The initial point
    niter : integer
        The number of iterations

    Returns
    -------
    min : ndarray of shape (n,)
        The minimizer
    '''
    pass
\end{lstlisting}

Feel free to use SciPy functions to solve linear systems and calculate step sizes in your algorithm.
\end{problem}

Let us work through an example of a nonlinear least squares problem. Suppose we have data points
generated from a sine function and slightly perturbed by gaussian noise. In Python we can generate such
data as follows:
\begin{lstlisting}
>>> t = np.arange(10)
>>> y = 3*np.sin(0.5*t)+ 0.5*np.random.randn(10)
\end{lstlisting}
Now we write Python functions for our model, the residual vector, the Jacobian, the objective function,
and the gradient. The calculations for all of these are straight forward.
\begin{lstlisting}
>>> def model(x, t):
>>>     return x[0]*np.sin(x[1]*t)
>>> def residual(x):
>>>     return model(x, t) - y
>>> def jac(x):
>>>     ans = np.empty((10,2))
>>>     ans[:,0] = np.sin(x[1]*t)
>>>     ans[:,1] = x[0]*t*np.cos(x[1]*t)
>>>     return ans
>>> def objective(x):
>>>     return .5*(residual(x)**2).sum()
>>> def grad(x):
>>>     return jac(x).T.dot(residual(x))
\end{lstlisting}
By inspecting our data, we might make an initial guess for the parameters $x_0 = (2.5, 0.6)$.
We are now ready to use our \li{gaussNewton} function to find the least squares solution.
\begin{lstlisting}
>>> x0 = np.array([2.5,.6])
>>> x = gaussNewton(objective, grad, jac, residual, x0, niter=10)
\end{lstlisting}
We can plot everything together to compare our fitted model with the data and the original sine
curve from which the data were generated.
\begin{lstlisting}
dom = np.linspace(0,10,100)
plt.plot(t, y, '*')
plt.plot(dom, 3*np.sin(.5*dom), '--')
plt.plot(dom, x[0]*np.sin(x[1]*dom))
plt.show()
\end{lstlisting}
The results are shown in Figure \ref{linesearch:gaussNewton}. As you can see, after just 10
iterations, we have found a very good fit.
\begin{figure}
\centering
\includegraphics[width=\textwidth]{gaussNewton.pdf}
\caption{Perturbed data (stars) generated from a sine curve (dashed line),
together with the fitted sine curve (solid line).
}
\label{linesearch:gaussNewton}
\end{figure}

\subsection*{Non-linear Least Squares in Python}
The module \li{scipy.optimize} also has a method to solve non-linear least squares problem, and it
is quite convenient. The function is called \li{leastsq}, and in its most basic use, you only need
to pass in the residual function and starting point as arguments. In the example above, we simply
need to execute the following code:
\begin{lstlisting}
>>> from scipy.optimize import leastsq
>>> x2 = leastsq(residual, x0)[0]
\end{lstlisting}
This should give us the same answer, but much faster.
\begin{problem}
We have census data giving the population of the United States every ten years since 1790.
For convenience, we have entered the data in Python below, so that you may simply copy and
paste.
\begin{lstlisting}
>>> #Start with the first 8 decades of data
>>> years1 = np.arange(8)
>>> pop1 = np.array([3.929, 5.308, 7.240, 9.638, 12.866,
>>>                 17.069, 23.192, 31.443])
>>>
>>> #Now consider the first 16 decades
>>> years2 = np.arange(16)
>>> pop2 = np.array([3.929, 5.308, 7.240, 9.638, 12.866,
>>>                 17.069, 23.192, 31.443, 38.558, 50.156,
>>>                 62.948, 75.996, 91.972, 105.711, 122.775,
>>>                 131.669])
\end{lstlisting}
Consider just the first 8 decades of population data. By plotting the data and having
an inclination that population growth tends to be exponential, it is reasonable to
hypothesize an exponential model for the population, that is,
$$
\phi(x_1,x_2,x_3,t) = x_1\exp(x_2(t+x_3)).
$$
By inspection, find a reasonable
initial guess for the parameters $(x_1, x_2, x_3)$ (i.e. $(150, .4, 2.5)$).
Write a function for this model in Python, along with the corresponding residual vector,
and fit the model using the \li{leastsq} function.  Plot the data against the fitted curve,
to see how close you are.

Now consider all 16 decades of data. If you plot your curve from above with this more complete
data, you will see that the model is no longer a good fit. Instead, the data suggest
a logistic model, which also arises from a differential equations treatment of population growth.
Thus, your new model is
$$
\phi(x_1,x_2,x_3,t) = \frac{x_1}{1+\exp(-x_2(t+x_3))}.
$$
By inspection, find a reasonable
initial guess for the parameters $(x_1, x_2, x_3)$ (i.e. $(150, .4, -15)$).
Again, write Python functions for the model and the corresponding residual vector,
and fit the model. Plot the data against the fitted curve. It should be a good fit.
\end{problem}


\chapter*{RSA Algorithm comments}\label{ch:rsa-algorithm-comments}

\section*{One way functions}\label{sec:one-way-functions}
One way function -- is a function that is easy to compute on every input, but
hard to invert given the image of a random input.
For instance, the function
\begin{equation*}
    f(m) = m^e \bmod N \equiv C
\end{equation*}
where $e, N$ are public constants is one-awy function,
because it is easy to compute $C$ given $m$, however it is hard to compute $m$ given $C$.

\section*{Euler's totient theorem}\label{sec:euler's-totient-theorem}
Given a number $N$ and its prime factorization $p_1^{e_1}\cdot p_2^{e_2} \cdots p_k^{e_k}$, then Euler's totient function
$\phi(N)$ is defined as

\[
    \phi(N) = (p_1^{e_1} - p_1^{e_1 - 1}) \cdot (p_2^{e_2} - p_2^{e_2 - 1}) \cdots (p_k^{e_k} - p_k^{e_k - 1})
\]

In particular, for positive number $M$ such that its factorization is $p1 \cdot p2$, the $\phi(M)$ is

\[
    \phi(M) = (p_1 -1) \cdot (p_2 - 1)
\]

Euler's theorem relates the modular division and exponent as follows, given number $m$, then
\[
    m^{\phi(N)} = 1 \bmod N
\]
It means that reminder of division $m^{\phi(N)}$ by $N$ is always 1.
By the equality $1^K = 1$
\[
    M^{K \cdot \phi(N)} = 1 \bmod N
\]
If we multiply both parts by $M$, we get
\[
    M \cdot M^{K \cdot \phi(N)} = M^{K \cdot \phi(N) + 1} = M \bmod N
\]

%! suppress = TooLargeSection

\section*{RSA Encryption algorithm}\label{sec:rsa-encryption-algorithm}
The RSA algorithm is named after Ron Rivest, Adi Shamir and Len Adleman, who invented it in 1977 [\cite{rivest1978method}].
The basic technique was first discovered in 1973 by Clifford Cocks [\cite{cocks1973note}] of CESG (part of the British GCHQ)
but this was a secret until 1997.
The patent taken out by RSA Labs has expired.

Historically, the process of encryption is considered to be symmetric one.
That means that prior the communication, the sides conclude on the common key to be used in encryption.
This process is similar to the first sharing keys and only after that the locked chest with the message.
Such approach is highly cost since it requires to share the defined keys between each actor if the number of actors
is greater than 2.
Much more simpler is to think about secured communication channel that in terms of asymmetric encryption.
The real life example would be if Alice shares with all actors an opened lock having key.
So that Bob receives an opened lock, writes letter to Alice, puts letter to the chest, locks this chest with received
from Alice lock.
This way, only Alice will be able to open the chest and to read the letter.
This is an idea of the asymmetric encryption.
However, such a simple from first glance idea requires complex number theory approach.
A concept of opened lock may be interpreted in terms of one-way functions.
One way function -- is a function that is easy to compute on every input, but hard to invert given the image of
a random input.
Thus, it is much simpler to close the lock without key, but very difficult to open lock trying the combinations
of the key.
For instance, the function
\begin{equation*}
    f(m) = m^e \bmod N = C
\end{equation*}
where $e, N$ are public constants is one-awy function,
because it is easy to compute $C$ given $m$, however it is hard to compute $m$ given $C$.
So, assume that Alice defines two positive integer constants $e, N$ and sends it to Bob.
Bob encrypts the secret message $m$ using $f(m)$
\[
    f(m) = m^e \bmod N = C
\]
Then Bob sends encrypted message $C$ to the Alice.
Given $C$ Alice must fetch the Bob's message $m$.
In order to decrypt $C$, Alice has to compute
\[
    C^d \bmod N = m^{ed} \bmod N \equiv m,
\]
where $e$ for encryption and $d$ for decryption.
Now the problem is to define such $d$ that it is hard to the listener to fetch it.
In order to define the secret $d$, Alice chooses two enough big prime numbers: $P, \; Q$, let's say around 150 digits
both.
Then Alice multiplies these two prime numbers in order to get $N$
\[
    N = P \cdot Q
\]
The $N$ is around 300 digits.
Now Alice can share $N$ with anyone, since it takes decades to find its prime factorization by the fundamental problem
of prime factorization.
Next, it is very important to know such a function, which depends on the knowledge of factorization of $N$.
Such function is an Euler's totient function.
Given a number $N$ and its prime factorization $p_1^{e_1}\cdot p_2^{e_2} \cdots p_k^{e_k}$, the Euler's totient function
$\phi(N)$ is defined as
\[
    \phi(N) = (p_1^{e_1} - p_1^{e_1 - 1}) \cdot (p_2^{e_2} - p_2^{e_2 - 1}) \cdots (p_k^{e_k} - p_k^{e_k - 1})
\]
In particular, for positive number $M$ such that its factorization is $p1 \cdot p2$, the $\phi(M)$ is
\[
    \phi(M) = (p_1 -1) \cdot (p_2 - 1)
\]
Euler's theorem relates the modular division and exponent as follows, given number $m$, then
\[
    m^{\phi(N)} = 1 \bmod N
\]
It means that reminder of division $m^{\phi(N)}$ by $N$ is always 1.
By the equality $1^K = 1$
\[
    M^{K \cdot \phi(N)} = 1 \bmod N
\]
If we multiply both parts by $M$, we get
\[
    M \cdot M^{K \cdot \phi(N)} = M^{K \cdot \phi(N) + 1} = M \bmod N
\]
It follows that Alice is able to define the secret $d$ as follows
\begin{gather*}
    e \cdot d = K \cdot \phi(N) + 1\\
    d = \frac{K \cdot \phi(N) + 1}{e}\\
\end{gather*}
The following image demonstrates the concept of RSA approach
\begin{figure}[H]
    \centering
    \includegraphics[width=1\textwidth]{Pictures/12_RSA_encryption_concept_diagram}
    ~\caption{RSA algorithm concept diagram. Source: [\cite{mango2021figma}].}\label{fig:figure8}
\end{figure}
To summarize, the process by the steps is as follows
\begin{itemize}
    \item Alice defines the large secret prime numbers $P, \; Q$.
    \item Alice computes $N = P \cdot Q$ and $\phi = (P-1)(Q-1)$
    \item Alice chooses an integer $e$, $1<e< \phi$ such that $\gcd(e, \phi) = 1$.
    \item Alice computes secret exponent $d$, $1<d< \phi$ such that $ed \equiv 1 \bmod \phi$.
    \item Alice shares public key $(N,e)$ with Bob and keeps private key $(d, p, q)$ is secret.
    \item Bob defines the message $m$, encrypts it as $C = m^{e} \bmod N$.
    \item Bob sends $C$ to Alice.
    \item Alice decrypts $C$ using her secret $d$, so she gets $m$
    \[
        m = C^d \bmod N
    \]
\end{itemize}
Security of the RSA approach is based on the complexity of fundamental problem of prime factorization,
which takes decades to solve having enough large number.


\subsection{IID}

\subsubsection{Identically distributed}

\(x\) is identically distributed to \(y\) if:

\(\forall i (\exists x_i \rightarrow P(x_i)=P(y_i))\)

\subsubsection{Covariance matrix of IID variables}

For IID varaibles, the covariance  matrix is:

\(\Sigma = \sigma^2 I\)


% !TEX encoding = UTF-8 Unicode
% !TEX spellcheck = en_US
% !TEX root = ../../../ICMA2020.tex

\subsection{Optimal Excitation}
\label{subsec:OptimalExcitation}

For precise and robust parameter estimation an optimized trajectory needs to be generated. In this paper an optimized \textsc{Fourier} series trajectory is used as a baseline for comparison. It is optimized by minimizing the condition number $\kappa(\boldsymbol{C})$ of the design matrix $\boldsymbol{C}$ \cite{Bona.2005,Kostic.2004,Olsen.2002,Swevers.1997}.
The \textsc{Fourier} series for each joint $j$ is given by:
\begin{equation}
\label{eq:fourier_series}
q_j(t) = \alpha_{0,j} + \sum \limits_{k=1}^{n_\mathrm{h}} \left\{ \frac{\alpha_{k,j}}{k \omega_\mathrm{b}} \sin(k \omega_\mathrm{b} t) + \frac{\beta_{k,j}}{k \omega_\mathrm{b}} \cos(k \omega_\mathrm{b} t) \right\}.
\end{equation}
Based on the suggestions of \cite{Swevers.1997,Bona.2005}, the order is set to $n_\mathrm{h}=5$ and the base frequency is set to $\omega_\mathrm{b}=2\pi \cdot 0.1\,\mathrm{Hz}$. The amplitudes $\alpha_{k,j}$ and $\beta_{k,j}$ are varied by a genetic algorithm under consideration of the given boundary conditions, see below.
% For the optimization a genetic algorithm from the \textit{Global Optimisation Toolbox} in \textsc{Matlab} was used.
To take into account the different magnitudes and units of the individual parameters, $\boldsymbol{C}$ is normalized during the optimization according to \cite{Sun.2008}.

%\textbf{(Aufgrund der begrenzten Seitenzahl würde ich hier bei dem Verweis auf die Quelle bleiben.)}


\chapter{Fundamental Properties of $q$-series and Theta functions}

\section{$q$-analogue of the Binomial Theorem}

The $q$-analogue of the binomial theorem is a generalization involving the parameter $q$ that returns the binomial theorem is the limit as $q \to 1$ and we replace $a$ by $q^a$.
\begin{theorem}
For  $|q|, |z| < 1$
\begin{equation}
    \sum_{n = 0}^{\infty} \frac{(a)_n}{(q)_n}z^n = \frac{(az)_\infty}{(z)_\infty}.
\end{equation}
\end{theorem}

\begin{proof}
Note that the product on the RHS of (2.1) converges uniformly on compact subsets of $|z| < 1$ and so represents an analytic function on $|z| < 1$. Thus we may write
\begin{equation}
    F(z) = \frac{(az)_\infty}{(z)_\infty} = \sum_{n = 0}^\infty A_n z^n
\end{equation}
From the product representation in (2.2), we can readily verify that
\begin{equation}
    (1-z)F(z) = (1-az)F(qz).
\end{equation}
Equating coefficients of $z^n$ on both sides of (2.3), we get
\begin{align*}
    A_n - A_{n-1} = q^n A_n - aq^{n-1}A_{n-1}
\end{align*}
or
\begin{equation}
    A_n = \frac{1-aq^{n-1}}{1-q^n} A_{n-1}
\end{equation}
Using $A_0 = 1$ in (2.2), we deduce that
\begin{equation}
    A_n = \frac{(a)_n}{(q)_n}
\end{equation}
Using (2.5) in (2.2), we complete the proof for (2.1)
\end{proof}

\subsection{Euler's Corollaries}
\begin{equation}
    \sum_{n = 0}^{\infty} \frac{z^n}{(q)_n} = \frac{1}{(z)_\infty}
\end{equation}
\begin{equation}
    \sum_{n = 0}^{\infty} \frac{(-z)^nq^{n(n-1)/2}}{(q)_n} = (z)_\infty
\end{equation}

Euler discovered these corollaries by application of (2.1)
\newpage
\section{Jacobi Triple Product Identity}
We present Jacobi's Triple Product Identity, which is immensely useful in the study of partitions in simplification of $q$-series products.
\begin{theorem}
For $z \neq 0$ and $|q| < 1$
\begin{equation}
    \sum_{n = -\infty}^{\infty} z^n q^{n^2} = (-zq;q^2)_\infty (-q/z;q^2)_\infty (q^2;q^2)_\infty
\end{equation}
\end{theorem}
\begin{proof}
    In (2.7), replace $q$ by $q^2$ and $z$ by $-zq$ to get
\begin{equation}
\begin{split}
      (-zq;q^2)_\infty &= \sum_{n = 0}^{\infty} \frac{z^n q^{n^2}}{(q^2;q^2)_n} = \frac{1}{(q^2;q^2)}\sum_{n = 0}^{\infty} z^n q^{n^2}(q^{2n+2};q^2)_\infty \\
    &= \frac{1}{(q^2;q^2)}\sum_{n-\infty}^{\infty} z^n q^{n^2}(q^{2n+2};q^2)_\infty,
\end{split}
\end{equation}
since $(q^{2n+2};q^2)_\infty = 0$ when $n$ is a negative integer.

Now apply (2.7), again with replacing $q$ with $q^2$ and $z$ with $q^{2n+2}$. Thus from (2.9),
\begin{align*}
(-zq;q^2)_\infty &= \frac{1}{(q^2;q^2)_\infty}\sum_{n = -\infty}^{\infty} z^nq^{n^2} \sum_{r=0}^\infty \frac{(-1)^rq^{(2n+2)r+r^2-r}}{(q^2;q^2)_r}\\
&= \frac{1}{((q^2;q^2)_\infty} \sum_{r = 0}^{\infty} \frac{(-1)^rz^{-r}q^r}{(q^2;q^2)_r} \sum_{n= -\infty}^\infty z^{n+r}q^{(n+r)^2}
\\&= \frac{1}{(q^2;q^2)_\infty} \sum_{r = 0}^{\infty} \frac{(-q/z)^r}{(q^2;q^2)_r} \sum_{n= -\infty}^\infty z^mq^{m^2}
\\&= \frac{1}{(q^2;q^2)_\infty(-q/z;q^2)_\infty} \sum_{n= -\infty}^\infty z^mq^{m^2}
\end{align*}
\newpage
by (2.6) with $z$ replaced by $-q/z$ and $q$ replaced by $q^2$, and therefore the restriction $|q/z| < 1$. Rearranging the final equation completes the proof of (2.8) for $|q/z| < 1$. However by analytic continuation, it hold's for all complex $z \neq 0$ completing the proof.
\end{proof}

\subsection{Corollory: Euler's Pentagonal Number Theorem}
Euler's Pentagonal Number Theorem occurs as a special case of Jacobi's Triple Product Identity, which we state due to its combinatorial importance.
\begin{equation}
    \sum_{n = -\infty}^{\infty} (-1)^nq^{n(3n-1)/2} = \sum_{n = -\infty}^{\infty} (-1)^nq^{n(3n+1)/2} = (q;q)_\infty
\end{equation}


% !TEX root = /Users/hzd88688126com/Desktop/USFD_Academic-_Report_LaTeX-Template/main.tex
\section{Adaptive Step Size}
\subsection{GASS}
Based on the previous analysis, the step size should be large for fast convergence and getting small to reduce steady state errors. The gradient adaptive step-size (GASS) is implemented with time-varying step $\mu(n)$. The MA(1) process $x(n)=0.9\eta(n-1)+\eta(n)$ with white noise $\eta \sim \mathcal{N}(0, 0.5)$ is simulated. For the GASS, the gradient step will be controlled by a constant $\rho$ and $\psi(n)$. In addition, there are three algorithms which will update the $\psi(n)$. As introduced in guidelines, the Benveniste applies a time-varying adaptive filter, which provides low pass filtering of the instantaneous gradient \cite{mandic2009complex}. Thus, the Benveniste's algorithm is robust to the noise and should be more accurate. The Ang \& Farhang's algorithm replaces the low-pass filter term with a constant $\alpha$. And the Matthews \& Xie's algorithm simplifies the the algorithm by Ang \& Farhang by setting $\alpha$ to zero, which only use the instantaneous gradient to update. Hence, the performance for this algorithm should be relative poor.
\begin{figure}[htbp]
     \centering{}
     \begin{subfigure}[b]{0.4\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/22/22a1.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.4\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/22/22a2.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.4\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/22/22a3.eps}
     \end{subfigure}
      ~
     \begin{subfigure}[b]{0.4\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/22/22a4.eps}
     \end{subfigure}
        \caption{GASS estimated weights error and squared error}
        \label{fig:2_2_a}
\end{figure}\\
Fig.\ref{fig:2_2_a} depicts the weight error and learning curves of three algorithms and standard LMS. When setting the initial $\mu$ =0.01, the standard LMS with fixed step has the slowest convergence speed and large steady state error. The Benveniste's algorithm converges before 100 samples, whereas the Ang \& Farhang's converges at 200 samples and Matthews is after 250 samples. However, even if the Matthews's algorithm perform worst among GASS algorithm, it is still slightly better than the standard LMS. When increasing the initial step $\mu=0.1$, all of algorithms converge rapidly with squared error lower than -300$dB$. However, the GASS algorithm requires more computational complexity than the standard LMS.
\subsection{NLMS}
Given a the update weight $\mathbf{w}(n+1)=\mathbf{w}(n)+\mu e_p(n)\mathbf{x}(n)$, the relationship between posteriori error $e_p(n)$ and priori error $e(n)$can be derived by
\begin{align}
e_p(n)&=d(n)-\mathbf{x}^T(n)\mathbf{w}(n+1)\notag\\
	  &=d(n)-\mathbf{x}^T(n)\mathbf{w}(n)-\mu e_p(n)\mathbf{x}^T(n)\mathbf{x}(n) \notag\\
	  &=e(n)-\mu e_p(n)\|\mathbf{x}(n)\|^2\notag\\
	  &=\frac{e(n)}{1+\mu \|\mathbf{x}(n)\|^2} \label{eq:ep}
\end{align}
Substituting Eq.\ref{eq:ep} into update equation,
\begin{align}
	\mathbf{w}(n+1)&=\mathbf{w}(n)+\mu e_p(n)\mathbf{x}(n)\notag\\
				   &=\mathbf{w}(n)+ \frac{\mu}{1+\mu \|\mathbf{x}(n)\|^2} e(n)\mathbf{x}(n)\notag\\
				   &=\mathbf{w}(n)+ \frac{1}{\frac{1}{\mu}+\|\mathbf{x}(n)\|^2} e(n)\mathbf{x}(n)\notag\\
				   &=\mathbf{w}(n)+ \frac{\beta}{\epsilon+\mathbf{x}^T(n)\mathbf{x}(n) } e(n)\mathbf{x}(n)\label{eq:nllms}
\end{align}
Thus, the update equation based on a posteriori error is equivalent to the NLMS algorithm, where $\epsilon=\frac{1}{\mu}$ and $\beta=1$.
\subsection{GNGD vs GASS}
Based on the NLMS algorithm, the generalized normalized gradient descent (GNGD) applies a time-varying $\epsilon(n)$. Compared with the Benveniste's algorithm, the GNGD algorithm converges rapidly of weight estimation approximately at 40 samples when the initial step $\mu=0.1$. The squared error
for the GNGD is smaller as well. However, when increasing the initial step $\mu=1$, the Benveniste's algorithm converges faster than the GNGD. Thus, the performance of the GASS algorithm is significantly affected by initial step $\mu$, while it is not a problem for the GNGD algorithm.
\begin{figure}[htbp]
     \centering
     \begin{subfigure}[b]{0.4\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/22/22c1.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.4\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/22/22c2.eps}
     \end{subfigure}
     ~
      \begin{subfigure}[b]{0.4\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/22/22c3.eps}
     \end{subfigure}
     ~
     \begin{subfigure}[b]{0.4\textwidth}
         \centering
         \includegraphics[width=\textwidth]{fig/22/22c4.eps}
     \end{subfigure}
        \caption{GAGN and Benveniste's GASS estimated weights and squared error}
        \label{fig:2_2_c}
\end{figure}\\
As to the complexity of these two algorithms, the computation complexity is caused by the matrix product in step update process. The Benveniste's algorithm, as shown below,
\begin{equation}
	\mathbf{\psi}(n)=[\mathbf I\overbrace{-\underbrace{\mu(n-1)\mathbf x(n-1)\mathbf {x}^T(n-1)}_{N^2}]\mathbf{\psi}(n-1)+}^{2N}\underbrace{e(n-1)\mathbf x(n-1)}_{N}
\end{equation}
has $O(N^2)$ multiplications, if the model order is $N$. The reason is that the Benveniste's algorithm applies matrix outer production which introduces a matrix result. Therefore, the computational complexity of the Benveniste grows quadratically. For the GNGD, as shown below,
\begin{equation}
	\epsilon(n+1)=\epsilon{n}-\rho\mu\frac{e(n)e(n-1)\overbrace{\mathbf x^T(n)\mathbf x(n-1)}^{N}}{(\epsilon(n-1)+\underbrace{\|\mathbf x(n-1)\|^2}_{N})^2}
\end{equation}
It only has inner product of $\mathbf x(n)$, resulting in a numerical value. Hence, the computational complexity is reduced over order $M$ which is $O(N)$.


%!TEX root = problems.tex

%\printanswers

\noindent
The following problem comes from Douglas Hofstadter's book \emph{Gödel, Escher, Bach}~\cite{hofstadter79}.
Take the time to try to solve this problem yourself.
The problem concerns strings of characters.
The only characters allowed/available are M, U and I.

\subsubsection*{Question}
You start with the string MI and the goal is to use the four rules below to convert it into the string MU.
In the rules, the letter x denotes any string.
You can use the rules any number of times and in any order, just so long as you get to MU~\cite{muwikipedia}.

\begin{center}
\begin{tabular}{rlll}

 & Rule & Example & Explanation  \\
\midrule
1 &	xI to xIU & MI	to	MIU & Append U to the end of a string ending in I. \\
2 &	Mx to Mxx & MIU	to	MIUIU & Double the string after the M. \\
3 &	xIIIy to xUy & MUIIIU	to	MUUU & Replace any III with a U. \\
4 &	xUUy to xy & MUUU	to	MU & Remove any UU. \\
\bottomrule
\end{tabular}
\end{center}