\textbf{Consider the numerical solution of the wave equation with Dirichlet boundary condition:
\begin{align*}
u_{tt} = u_{xx},~~~ x\in (−1, 1),~~~ u(t, −1) = u(t, 1) = 0,~~~ u(0, x) = e^{-32x^2},~~~ u_t(0, x) = 0.
\end{align*}
Assuming sufficient smoothness in u and appropriate initial conditions, show that solving (1) is equivalent to solving the acoustic equations
\begin{align*}
u_t = p_x,\\
p_t = u_x,
\end{align*}
with boundary conditions $u(t, −1) = u(t, 1) = 0$. Find the appropriate initial condition for p.}

\textbf{Solve this problem using second order centered finite differences in space and $ode45$ in time. Use a staggered equispaced grid. That is, discretize $u$ at the gridpoints $x_j = −1 + jh$ and $p$ at the nodes $x_{\frac{j+1}{2}}= −1 + (j+\frac{1}{2})h$. Notice that boundary conditions for $p$ are not required in this case. Plot your solution ($u$ and $p$ on the same figure) for $t = 0.5, 1, 1.5, 2$.}

\textbf{Estimate the accuracy of your answer at $t = 4$, notice that $u(4, x) = u(0, x)$.}

\vspace{0.3in}

For the first part note that
\begin{align*}
u_{tt}=\frac{\partial}{\partial t}p_x=p_{xt},
\end{align*}
and
\begin{align*}
u_{xx}=\frac{\partial}{\partial x}p_t=p_{tx}.
\end{align*}

Since $u$ is sufficiently smooth, both $p_{xt}$ and $p_{tx}$ are continuous. Therefore they must be equal and
\begin{align*}
u_{tt}=p_{xt}=p_{tx}=u_{xx}.
\end{align*}
To find the initial condition for $p$ we simply use the initial conditions for $u$,
\begin{align*}
p_t(0,x)=u_x(0,x)=-64xe^{-32x^2}.
\end{align*}
and
\begin{align*}
p_x(0,x)=u_t(0,x)=0.
\end{align*}

For the second part of the problem, let us express the acoustic equations in matrix form
\begin{align*}
\frac{d}{dt}\begin{bmatrix} u \\ p \end{bmatrix}=\begin{bmatrix} 0 & \frac{\partial}{\partial x}\\ \frac{\partial}{\partial x} & 0\end{bmatrix}\begin{bmatrix} u \\ p \end{bmatrix}.
\end{align*}
We are using staggered meshes so the space derivative operator for $u$ and $p$ are going to be different, letting the previous equation in matrix form be like
\begin{align*}
\frac{d}{dt}\begin{bmatrix} u \\ p \end{bmatrix}=\begin{bmatrix} 0 & D_p\\ D_u & 0\end{bmatrix}\begin{bmatrix} u \\ p \end{bmatrix},
\end{align*}
or
\begin{align*}
\frac{d}{dt}\textbf{V}=M\textbf{V},
\end{align*}
where $\textbf{V}=\begin{bmatrix} u \\ p \end{bmatrix}$ and $M$ is the matrix composed by $D_u$, $D_p$ and zeros. Since we have $N+1$ nodes of $u$ and we only solve for the interior, our matrix  $D_u$ will only have $N-1$ columns. However, we want to evaluate $u_x$ in the $N$ nodes for $p$, therefore $D_u$ will have $N$ rows. Thus, $D_u$ is $N\times N-1$. We are going to use central finite differences to evaluate the first derivative of $u$, but since we are going to evaluate the result in the staggered $p$ nodes we have
\begin{align*}
\frac{\partial}{\partial t}p_j=\frac{\partial}{\partial x}u_{j+\frac{1}{2}}=\frac{u_{j+1}-u_{j}}{h},
\end{align*}
where $j=1,...N$, since we are using \textsl{Matlab} indices. Since we are only solving the interior for $u$ there is going to be a shift in the matrix and the real system we are going to solve is the following
\begin{align*}
\frac{\partial}{\partial t}\begin{bmatrix} u \\ p \end{bmatrix}=\begin{bmatrix} 0 & D_p\\ \tilde{D}_u & 0\end{bmatrix}\begin{bmatrix} u \\ p \end{bmatrix},
\end{align*}
or
\begin{align*}
\frac{\partial}{\partial t}\textbf{V}=\tilde{M}\textbf{V}.
\end{align*}
Therefore, the discretization yields
\begin{align*}
\frac{\partial}{\partial t}p_j=\frac{\partial}{\partial x}\tilde{u}_{j-\frac{1}{2}}=\frac{\tilde{u}_{j}-\tilde{u}_{j-1}}{h}.
\end{align*}
Note that $\tilde{u}_j=u_{j+1}$. For the first and last points we include the boundary conditions,
\begin{align*}
\frac{\partial}{\partial t}p_1=\frac{\partial}{\partial x}\tilde{u}_{-\frac{1}{2}}=\frac{\tilde{u}_{1}-\tilde{u}_{0}}{h}=\frac{u_{2}-u_{1}}{h}=\frac{u_2}{h}=\frac{\tilde{u_1}}{h},
\end{align*}
and
\begin{align*}
\frac{\partial}{\partial t}p_N=\frac{\partial}{\partial x}\tilde{u}_{N-\frac{1}{2}}=\frac{\tilde{u}_{N}-\tilde{u}_{N-1}}{h}=\frac{u_{N+1}-u_{N}}{h}=\frac{-u_{N}}{h}=-\frac{\tilde{u}_{N-1}}{h}.
\end{align*}
From the previous equations we obtain
\begin{align*}
\frac{\partial}{\partial t}\begin{bmatrix} p_1 \\ p_2 \\p_3\\ \vdots \\p_{N-1}\\p_N \end{bmatrix}=\frac{1}{h}\begin{bmatrix}
1 & 0 & 0 & \cdots & 0\\
-1 & 1 & 0 & \cdots & 0\\
0 & -1 & 1 & \cdots & 0\\
\vdots & \ddots & \ddots & \ddots  & \vdots\\
0 & & 0 & -1 & 1 \\
0 & \cdots & 0 & 0 & 1\\
\end{bmatrix} \begin{bmatrix} \tilde{u}_1 \\ \tilde{u}_2 \\\tilde{u}_3\\ \vdots \\\tilde{u}_{N-2}\\\tilde{u}_{N-1} \end{bmatrix}.
\end{align*}
Therefore,
\begin{align*}
D_u=\frac{1}{h}\begin{bmatrix}
1 & 0 & 0 & \cdots & 0\\
-1 & 1 & 0 & \cdots & 0\\
0 & -1 & 1 & \cdots & 0\\
\vdots & \ddots & \ddots & \ddots  & \vdots\\
0 & \cdots & 0 & -1 & 1 \\
0 & \cdots & 0 & 0 & 1\\
\end{bmatrix}.
\end{align*}
We repeat the process now for $D_p$,
\begin{align*}
\frac{\partial}{\partial t}\tilde{u}_j=\frac{\partial}{\partial x}p_{j+\frac{1}{2}}=\frac{p_{j+1}-p_{j}}{h},
\end{align*}
where we are only solving for the interior of $u$ which involves all the nodes of $p$ and we don't have to impose any boundary conditions. Thus, according to the previous discretization, the matrix
\begin{align*}
D_p=\frac{1}{h}\begin{bmatrix}
-1 & 1 & 0 & \cdots & 0\\
0 & -1 & 1 & \cdots & 0\\
\vdots & \ddots & \ddots & \ddots  & \vdots\\
0 & \cdots & 0 & -1 & 1
\end{bmatrix},
\end{align*}
which has dimension $N-1\times N$. With this two matices and we can complete $M$ and solve the acoustic equations.

The solution is shown in the next figure, where we can see the velocity and pressure waves at different values of time.

\begin{figure}[H]
\centering     %%% not \center
\hspace*{\fill}
\subfigure[$t=0.5$ s.]{\includegraphics[scale=0.6]{P4_05}}
\hfill
\subfigure[$t=1.0$ s.]{\includegraphics[scale=0.6]{P4_1}}
\hspace*{\fill}

\hspace*{\fill}
\subfigure[$t=1.5$ s.]{\includegraphics[scale=0.6]{P4_15}}
\hfill
\subfigure[$t=2.0$ s.]{\includegraphics[scale=0.6]{P4_2}}
\hspace*{\fill}
\caption{Solution of acoustic equations at different times.}
\end{figure}

In the next figure we can see the contour plot of the solutions. Note that in fact after four seconds the velocity profile is the same, allowing us to assume a period of four seconds. In the movies generated in \textsl{Matlab} but not shown in this document we could see how the velocity and pressure waves would travel together sometimes in phase and sometimes with a phase shift of $\pi$ radians. We can also observe this in the contours by "superposing" the images.

\begin{figure}[H]
\centering     %%% not \center
\hspace*{\fill}
\subfigure[Velocity contours.]{\includegraphics[scale=0.5]{P4_contour1.eps}}
\hfill
\subfigure[Pressure contours.]{\includegraphics[scale=0.5]{P4_contour2.eps}}
\hspace*{\fill}
\caption{Solution of the acoustic equations.}
\end{figure}

For the last part of the problem we estimate the accuracy of the solution. In the following figure we can see the initial condition and the solution after one period. Using $N=80$ the accuracy seems to be good enough. The $L_1$-norm of the difference between the two is
\begin{align*}
e_{L1}=1.839686\cdot 10^{-2}.
\end{align*}

\begin{figure}[H]
\centering     %%% not \center
{\includegraphics[scale=0.75]{P4_periodic.eps}}
\caption{Comparison of the solution after 1 period, $N=80$.}
\end{figure}

\subsection*{Matlab code for this problem}
\begin{verbatim}
%% Problem 4
clear all;close all;format long;clc
legendfontsize=14;
axisfontsize=14;
labelfontsize=16;
N = 80;
L=2;
h=L/N;
xu=linspace(-1,1,N+1);
xp=linspace(-1+h/2,1-h/2,N);

Du = gallery('tridiag',N,-1,1,0); % In sparse form.
Du(:,end)=[];
Du=Du/h;
Dp = gallery('tridiag',N,0,-1,1); % In sparse form.
Dp(end,:)=[];
Dp=Dp/h;
Z1 = zeros(N-1,N-1);
Z2 = zeros(N,N);
M = [Z1 Dp ; Du Z2];

% initial condition
u0 = exp(-32*xu.^2)';
p0 = zeros(N,1);
v0 = [u0(2:end-1) ; p0];

t = 0:0.01:4;

[T,V] = ode45(@(t,v) M*v, t, v0);
for k = 2:length(t)
    plot(xu(2:end-1),V(k,1:N-1)','b*-',xp,V(k,N:2*N-1)','r*-')
    grid on
    ylim([-1 1])
    shg
    drawnow
end
%%

[T,X] = meshgrid(t,xu(2:end-1));
figure
contourf(T,X,round(V(:,2:N)',3))
xlabel('$t$','fontsize',labelfontsize,...
            'interpreter','latex')
ylabel('$x$','fontsize',labelfontsize,...
            'interpreter','latex')
set(gca,'fontsize',axisfontsize)
txt='Latex/FIGURES/P4_contour1';
saveas(gcf,txt,'epsc')

[T,X] = meshgrid(t,xp);
figure
contourf(T,X,round(V(:,N:end)',3))
xlabel('$t$','fontsize',labelfontsize,...
            'interpreter','latex')
ylabel('$x$','fontsize',labelfontsize,...
            'interpreter','latex')
set(gca,'fontsize',axisfontsize)
txt='Latex/FIGURES/P4_contour2';
saveas(gcf,txt,'epsc')
%%
figure
plot(xu(2:end-1),V(find(t==0.5),1:N-1),'b*')
hold on
plot(xp,V(find(t==0.5),N:2*N-1),'r*')
grid on
axis([-1 1 -1 1])
xlabel('$x$','fontsize',labelfontsize,...
            'interpreter','latex')
legend({'$u(0.5,x)$','$p(0.5,x)$'},...
        'Interpreter','latex','fontsize',legendfontsize)
set(gca,'fontsize',axisfontsize)
txt='Latex/FIGURES/P4_05';
saveas(gcf,txt,'epsc')

figure
plot(xu(2:end-1),V(find(t==1),1:N-1),'b*')
hold on
plot(xp,V(find(t==1),N:2*N-1),'r*')
grid on
axis([-1 1 -1 1])
xlabel('$x$','fontsize',labelfontsize,...
            'interpreter','latex')
legend({'$u(1,x)$','$p(1,x)$'},...
        'Interpreter','latex','fontsize',legendfontsize)
set(gca,'fontsize',axisfontsize)
txt='Latex/FIGURES/P4_1';
saveas(gcf,txt,'epsc')

figure
plot(xu(2:end-1),V(find(t==1.5),1:N-1),'b*')
hold on
plot(xp,V(find(t==1.5),N:2*N-1),'r*')
grid on
axis([-1 1 -1 1])
xlabel('$x$','fontsize',labelfontsize,...
            'interpreter','latex')
legend({'$u(1.5,x)$','$p(1.5,x)$'},...
        'Interpreter','latex','fontsize',legendfontsize)
set(gca,'fontsize',axisfontsize)
txt='Latex/FIGURES/P4_15';
saveas(gcf,txt,'epsc')

figure
plot(xu(2:end-1),V(find(t==2),1:N-1),'b*')
hold on
plot(xp,V(find(t==2),N:2*N-1),'r*')
grid on
axis([-1 1 -1 1])
xlabel('$x$','fontsize',labelfontsize,...
            'interpreter','latex')
legend({'$u(2,x)$','$p(2,x)$'},...
        'Interpreter','latex','fontsize',legendfontsize)
set(gca,'fontsize',axisfontsize)
txt='Latex/FIGURES/P4_2';
saveas(gcf,txt,'epsc')

u4=V(length(t),1:N-1)';
figure
plot(xu(2:end-1),u0(2:end-1),'b',xu(2:end-1),u4,'r*')
grid on
axis([-1 1 -0.3 1.3])
xlabel('$x$','fontsize',labelfontsize,...
            'interpreter','latex')
legend({'$u(0,x)$','$u(4,x)$'},...
        'Interpreter','latex','fontsize',legendfontsize)
set(gca,'fontsize',axisfontsize)
txt='Latex/FIGURES/P4_periodic';
saveas(gcf,txt,'epsc')
err=norm(u0(2:end-1)-u4,inf)
\end{verbatim}


\section{A very simple example - polynomial curve fitting}\label{sec:simple}
\subsection{The first program in C++}
{\em Example file polyfit0.cpp in the directory doc/tutorial/code/polyfit.}\\
Let us start with the very simple inverse problem of fitting a polynomial curve of degree $P$
\[ f(x) = p_0 +p_1 x + \ldots + p_P x^P= \sum\limits_{i=0}^{P} p_i x^i \]
to some existing data $y$. The unknown model is the coefficient vector $\m=[p_0,\ldots,p_P]$.
The vectorized function for a vector $\x=[x_1,\ldots,x_N]^T$ can be written as matrix-vector product
\begin{equation}\label{eq:yAx}
  \f(\x) = \A \x \quad\mbox{with}\quad \A=\left[ \begin{array}{cccc}
  1 & x_1 & \ldots & x_1^P \\ \vdots & \vdots & \ddots & \vdots \\ 1 & x_N & \ldots & x_N^P
  \end{array} \right] = [ \textbf{1}\quad \x \quad \x^2 \ldots \x^P ] \quad.
\end{equation}

We set up the modelling operator, i.e. to return $\f(\x)$ for given $p_i$, as a class derived from the modelling base class. The latter holds the main mimic of generating jacobian, gradients by brute force. The only function to overwrite is \cw{response()}.

\begin{lstlisting}[language=C++,morekeywords={RVector,ModellingBase}]
class FunctionModelling : public ModellingBase {
public:
    /*! constructor, nc: number of coefficients, xvec: abscissae */
    FunctionModelling( int nc, const RVector & xvec, bool verbose=false )
        : ModellingBase( verbose ), x_( xvec ), nc_( nc ){
        regionManager_->setParameterCount( nc ); //! instead of a mesh
    }
    /*! the main thing - the forward operator: returns f(x) */
    RVector response( const RVector & par ){
        RVector y( x_.size(), par[ 0 ] );   //!constant vector of p0
        for ( size_t i = 1; i < nc_; i ++ ) //! p1 to pP
            y += pow( x_, i ) * par[ i ];   //! add pi*x^i
        return y;                           //! return sum
    }
    /*! define the startmodel */
    RVector startModel( ){ return RVector( nc_, 0.5 ); }
protected:
    RVector x_; //! abscissa vector x
    int nc_; //! number of coefficients
};
\end{lstlisting}

In the constructor the $\x$ vector and the number of coefficients are saved as protected variables\footnote{Usually all variables are denoted with an underscore and declared as protected. Instead of accessing the values directly, we use set and get functions that control the validity of the arguments.}.
The function \cw{setParameterCount} setups the parameterisation as a 0d mesh of \cw{nc} unknowns.
The main type used is \cw{RVector}, a vector of real (\cw{double}) values.

We now want to apply the function to the real inversion of data and write a main program.
\begin{lstlisting}[language=C++,morekeywords={RVector,RMatrix,RInversion,EXIT_SUCCESS}]
int main( int argc, char *argv [] ){
    int np = 1; /*! maximum polynomial degree fixed to 1 */
    RMatrix xy; //! two-column matrix from file holding x and y
    loadMatrixCol( xy, "datafile.dat" );
    /*! initialise modelling operator */
    FunctionModelling f( np + 1, xy[ 0 ] ); //! first data column
    /*! initialise inversion with data and forward operator */
    RInversion inv( xy[ 1 ], f );
     /*! the problem is well-posed and does not need regularization */
    inv.setLambda( 0 );
    /*! actual inversion run yielding coefficient model */
    RVector coeff( inv.run() );
    /*! save coefficient vector to file */
    save( coeff, "out.vec" );
    /*! exit programm legally */
    return EXIT_SUCCESS;
}
\end{lstlisting}

The data in the two-column data file is read into a real matrix (\lstinline|RMatrix xy;|) whose columns can be assessed by \lstinline|xy[i]|. We initialise the forward class as defined and the inversion by specifying data and forward operator.
Then any options of the inversion can be set, such as the regularization parameter being zero.

Instead of using a fixed polynomial degree and a pre-defined file name we might specify this by the command line in order to have a user-friendly tool.
For this, an option map is applied reading the last argument and the optionally defined -n switch.
\begin{lstlisting}[language=C++,morekeywords={OptionMap,RMatrix,loadMatrixCol}]
    int np = 1;
    std::string datafile;
    OptionMap oMap;
    oMap.setDescription("Polyfit - fits two-column data with polynomials");
    oMap.addLastArg( datafile, "Datafile" );
    oMap.add( np, "n:", "np", "Number of polynomials" );
    oMap.parse( argc, argv );
    RMatrix xy;
    loadMatrixCol( xy, datafile );
\end{lstlisting}

The provided datafile \verb|y_2.1x+1.1.dat| holds synthetic noisified data for a linear function.
A call \verb|curvefit -n1 y_2.1x+1.1.dat| yields values close to the synthetic ones.

\subsection{A first Python program}
{\em Example file polyfit.py in the directory doc/tutorial/code/polyfit.}\\
Python is a very flexible language for programming and scripting and has many packages for numerical computing and graphical visualization.
For this reason, we built Python bindings and compiled the library pygimli.
As a main advantage, all classes can be used and derived.
This makes the use of GIMLi very easy for non-programmers.
All existing modelling classes can be used, but it is also easy to create new modelling classes.

We exemplify this by the preceding example.
First, the library must be imported.
To avoid name clashes with other libraries we suggest to import it to an easy name, e.g. by using \lstinline|import pygimli as pg|.
As a result, all gimli objects (classes and functions) can be referred to with a preceding \lstinline|pg.|, e.g. \lstinline|pg.RVector| is the real vector \lstinline|RVector|.
Next, the modelling class is derived from ModellingBase, a constructor is defined and the response function is defined.
\begin{lstlisting}[language=python]
import pygimli as pg
class FunctionModelling( pg.ModellingBase ):
    # constructor
    def __init__( self, nc, xvec, verbose = False  ):
        pg.ModellingBase.__init__( self, verbose )
        self.x_ = xvec
        self.nc_ = nc
        self.regionManager().setParameterCount( nc )
    # response function
    def response( self, par ):
        y = pg.RVector( self.x_.size(), par[ 0 ] )
        for i in range( 1, self.nc_ + 1 ):
            y += pg.pow( self.x_, i ) * par[ i ];
        return y;
    # start model
    def startModel( self ):
        return pg.RVector( self.nc_, 0.5 )
\end{lstlisting}

The pygimli library must once be imported (in this case under the name g) and all classes (e.g. modelling operators) can be used by pg.classname, e.g. pg.RVector is the already known vector of real (double) values.

The main program is very easy then and the code is very similar to C++.
Data are loaded, both forward operator and inversion are created.
Inversion options are set and it the result of run is save to a file.
That's it.

\begin{lstlisting}[language=python]
    xy = pg.RMatrix()
    pg.loadMatrixCol( xy, datafile );
    # two coefficients and x-vector (first data column)
    f = FunctionModelling( options.np + 1, xy[ 0 ] )
    # initialize inversion with data and forward operator and set options
    inv = pg.RInversion( xy[ 1 ], f );
    # constant absolute error of 0.01 (not necessary, only for chi^2)
    inv.setAbsoluteError( 0.01 );
    # the problem is well-posed and does not need regularization
    inv.setLambda( 0 );
    # actual inversion run yielding coefficient model
    coeff = inv.run();
    pg.save( coeff, "out.vec" );
\end{lstlisting}

As a main advantage of Python, the actual computations can be easily combined with post-processing or visualization, even building graphical user-interfaces.
In this code example we use matplotlib, a plotting library inside of pylab, a compound of different routines for numerics and plotting, very much comparable to MatLab.

\begin{lstlisting}[language=python]
import pylab as P
P.plot( xy[0], xy[1], 'rx', xy[0], inv.response(), 'b-' )
P.show()
\end{lstlisting}

Similar to C++, command line options can be parsed using the class OptionParser, see the code file.
The output is illustrated for two a synthetic function $y=2.1x+1.1$ noisified with Gaussian noise for two different orders in Figure \ref{fig:polyfit}.

\begin{figure}[hbt]%
\includegraphics[width=0.5\columnwidth]{polyfit-n1}\hfill
\includegraphics[width=0.5\columnwidth]{polyfit-n3}%
\caption{Polynomial fit for noisified synthetic data using first order (left) and third order (right) polynomials.}%
\label{fig:polyfit}%
\end{figure}

In the following we continue the description with C++ but all are provided as well in Python without significant code changes.
Main exception is the non-existence of declarations, e.g. the C++ declaration \lstinline|GIMLi::RTransLogLU mytrans( lower, upper );| in Python would be written \lstinline|mytrans = pg.RTransLogLU( lower, upper)|.
Whereas it is common to work in name spaces in C++, in python function conflicts easily appear due to lack of declaration.
To avoid that, we recommend importing modules as such, e.g. \lstinline|import pylab as P| and \lstinline|import pygimli as g|.

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\subsection{An own Jacobian}
{\em Example file polyfit1.cpp in the directory doc/tutorial/code/polyfit.}\\
For the latter example, the underlying Gauss-Newton scheme creates a Jacobian matrix by brute force (perturbation).
If we want to apply an own algorithm we overwrite the function createJacobian() in the modelling class by using the matrix $\A$ from (\ref{eq:yAx}):
\begin{lstlisting}[language=C++,morekeywords={RMatrix,RVector,size_t}]
    void createJacobian( RMatrix & jacobian, const RVector & model ) {
        jacobian.resize( x_.size(), nc_ );
        for ( size_t i = 0 ; i < nc_ ; i++ )
            for ( size_t j = 0 ; j < x_.size() ; j++ )
                jacobian[ j ][ i ] = pow( x_[ j ], i );
    }
\end{lstlisting}

The result of the inversion is of course the same as before.
Note that the resize function checks for the right size and allocates space if necessary.

Alternatively we might to use other minimisation methods even though it is not necessary for this example.
\sperre (Steepest descent, NLCG, Quasi-Newton)

Note that \lstinline|RInversion| is an instance of the template class \lstinline|Inversion< ValueType >| with the value type \lstinline|double| and the matrix type \lstinline|RMatrix|\footnote{RMatrix is a full matrix of real (double) values.}.
One can, of course, use other types, e.g. float-precision or a complex vector \lstinline|CVector|.
In the latter case, the appropriate matrix type \lstinline|CMatrix| should be used in the forward class.
For other problems sparse Jacobian matrices or block matrices are more efficient.
See also the appendix \ref{app:matrix} for matrix types.
%For other problems the Jacobian matrix has only few entries and can be approximated by a sparse matrix.
%Therefore the matrix type \lstinline|RSparseMapMatrix| exists, which is itself an instance of a template type with \lstinline|long int| index and double values.
%See also the appendix \ref{app:matrix} for matrix types.


\subsection{The counting measure}

\(\mu (E)\)

This provides the number of elements in \(E\).


%
% Chapter 4.3
%

\section*{4.3 Fundamental Theorem of Calculus}

\subsection*{FTC Part 1}

If \(f\) is continuous on \([a, b]\), then the function \(g\) defined by
$$ g(x) = \int_a^x f(t)dt \quad a \leq x \leq b $$
is continuous on \([a, b]\) and differentiable on \((a, b)\), and \(g'(x)=f(x)\).

\subsection*{FTC Part 2}

If \(f\) is continuous on \([a, b]\), then
$$ \int_a^b f(x)dx = F(b)- F(a) $$
where \(F\) is any antiderivative of \(f\), that is, a function \(F\) such that \(F'=f\).


\subsubsection{Electric Fields}
\noindent
Gauss's Law of Electricity is an important application of Divergence Theorem in physics. It says that $\Phi_{E} = \frac{Q_{in}}{\epsilon_{0}}$. That is, the electric flux through a closed surface (real or hypothetical) is proportional to the charge contained within that surface. Rewritten more formally,
\begin{equation*}
	\oint\limits_{S}\oint{\vec{E} \cdot \mathrm{d}\vec{s}} = \iiint\limits_{V}{\frac{Q_{in}}{\epsilon_{0}}\mathrm{d}V}.
\end{equation*}
So,
\begin{equation*}
	\iiint\limits_{V}{\nabla\cdot\vec{E}\mathrm{d}V} = \iiint\limits_{V}{\frac{\sigma}{\epsilon_{0}}\mathrm{d}V}
\end{equation*}
where $\sigma$ is the charge density.
If we let $V \to 0$,
\begin{equation*}
	\nabla \cdot \vec{E} = \frac{\sigma}{\epsilon_{0}}.
\end{equation*}
This is the differential form of Gauss's Law and the 1st of Maxwell's Equations.


\section{Geometric view of systems of equations}
\label{sec:systems-geometric}

\begin{outcome}
  \begin{enumerate}
  \item Relate the types of solution sets of a system of two (three)
    variables to the intersections of lines in a plane (the
    intersection of planes in $3$-dimensional space)
  \end{enumerate}
\end{outcome}

As you may remember, linear equations like $2x+3y=6$ can be graphed as
straight lines in the coordinate plane. We say that this equation is
in two variables, in this case $x$ and $y$.  Suppose you have two such
equations, each of which can be graphed as a straight line, and consider
the resulting graph of two lines. What would it mean if there exists a
point of intersection between the two lines? This point, which lies on
{\em both} graphs, gives $x$ and $y$ values for which both
equations are true. In other words, this point gives the ordered pair
($x,y$) that satisfies both equations.  If the point $(x, y
)$ is a point of intersection, we say that $(x, y)$
is a \textbf{solution} to the two equations. In linear algebra, we
often are concerned with finding the solution(s) to a system of
equations, if such solutions exist.  First, we consider graphical
representations of solutions and later we will consider the algebraic
methods for finding solutions.

When looking for the intersection of two lines in the plane, several
situations may arise. The following picture demonstrates the possible
situations when considering two equations (two lines in the plane)
involving two variables.
\begin{equation*}
  \begin{tikzpicture}[scale=0.9]
    \draw[thick, <->](0,2)--(0,0)--(2,0);
    \draw[thick, red](0,1.5)--(1.5,0);
    \draw[thick, blue](0,0)--(1.5,1.5);
    \draw[fill, purple](0.75,0.75) circle [radius=2pt];
    \node[below right] at (2,0){$x$};
    \node[left] at (0,2){$y$};
    \node[below] at (1,-0.5){One solution};

    \draw[thick, <->](5,2)--(5,0)--(7,0);
    \draw[thick, red](5,0.5)--(6.5, 1.5);
    \draw[thick, blue](5,0)--(6.5,1);
    \node[below right] at (7,0){$x$};
    \node[left] at (5,2){$y$};
    \node[below] at (6,-0.5){No solutions};

    \draw[thick, <->](10,2)--(10,0)--(12,0);
    \draw[thick, purple](10,0)--(11.5, 1.5);
    \node[below right] at (12,0){$x$};
    \node[left] at (10,2){$y$};
    \node[below] at (12,-0.5){Infinitely many solutions};
  \end{tikzpicture}
\end{equation*}
In the first diagram, there is a unique point of intersection, which
means that there is only one (unique) solution to the two equations.
In the second, there are no points of intersection and no
solution. There is no solution because the two lines are parallel and
they never intersect.  The third situation that can occur, as
demonstrated in diagram three, is that the two lines are really the
same line. For example, $x+y=1$ and $2x+2y=2$ are two equations that
yield the same line when graphed. In this case there are infinitely
many points that are solutions of these two equations, as every
ordered pair which is on the graph of the line satisfies both
equations.

When considering linear systems of equations, there are always three
possibilities for the number of solutions: there is exactly one
solution, there are infinitely many solutions, or there is no
solution.  When we speak of {\em solving} a system of equations, we
usually mean finding {\em all} of its solutions. This can mean finding
one solution (if the solution is unique), finding infinitely many
solutions, or finding that there is no solution.

\begin{example}{A graphical solution}{graphical-solution}
  Use a graph to solve the following system of equations:
  \begin{equation*}
    \begin{array}{r@{~}c@{~}l}
      x+y&=&3 \\
      y-x&=&5.
    \end{array}
  \end{equation*}
\end{example}

\begin{solution}
  Through graphing the above equations and identifying the point of
  intersection, we can find the solution(s). Remember that we must
  have either one solution, infinitely many, or no solutions at all.
  The following graph shows the two equations, as well as the
  intersection. Remember, the point of intersection represents the
  solution of the two equations, or the $(x,y)$ which satisfy both
  equations. In this case, there is one point of intersection at
  $(-1, 4)$ which means we have one unique solution, $x = -1$,
  $y = 4$.
  \begin{center}
    \begin{tikzpicture}[scale=0.75]
      \draw[thick, ->](-4.5,0) -- (2.5,0);
      \draw[thick, ->](0,-1) -- (0,7);
      \draw(-4,-0.2) -- (-4,0.2);
      \draw(-3,-0.2) -- (-3,0.2);
      \draw(-2,-0.2) -- (-2,0.2);
      \draw(-1,-0.2) -- (-1,0.2);
      \draw(1,-0.2) -- (1,0.2);
      \draw(-0.2,1) -- (0.2,1);
      \draw(-0.2,2) -- (0.2,2);
      \draw(-0.2,4) -- (0.2,4);
      \draw(-0.2,6) -- (0.2,6);
      \draw[green!60!black, ultra thick, domain=-3.5:2] plot (\x, {3-\x});
      \draw[blue, ultra thick, domain=-3.5:2] plot (\x, {\x+5});
      \node[right] at (1.2,2){$x+y=3$};
      \node[right] at (1.2,6){$y-x=5$};
      \draw[help lines,dashed](-1,0)--(-1,4)--(0,4);
      \node[below] at (-4,-0.5){$-4$};
      \node[below] at (-3,-0.5){$-3$};
      \node[below] at (-2,-0.5){$-2$};
      \node[below] at (-1,-0.5){$-1$};
      \node[below] at (1,-0.5){$1$};
      \node[left] at (-0.25,1){$1$};
      \node[left] at (-0.25,2){$2$};
      \node[left] at (-0.25,3){$3$};
      \node[left] at (-0.25,4){$4$};
      \node[left] at (-0.25,5){$5$};
      \node[left] at (-0.25,6){$6$};
      \node[below right] at (2,0){$x$};
      \node[left] at (0,7){$y$};
      \node[left=1ex] at (-1,4){$(x,y) = (-1,4)$};
    \end{tikzpicture}
    \vspace{-2ex}
  \end{center}
\end{solution}

In the above example, we investigated the intersection point of two
equations in two variables, $x$ and $y$. Now we will consider the
graphical solutions of three equations in two variables.

Consider a system of three equations in two variables. Again, these
equations can be graphed as straight lines in the plane, so that the
resulting graph contains three straight lines. Recall the three
possibilities for the number of solutions: no solution, one solution,
and infinitely many solutions.  With three lines, there are more
complex ways of achieving these situations. For example, you can
imagine the case of three intersecting lines having no common point of
intersection. Perhaps you can also imagine three intersecting lines
which do intersect at a single point. These two situations are
illustrated below.

\begin{center}
  \begin{tikzpicture}
    \draw[thick, <->](0,2)--(0,0)--(2,0);
    \draw[thick, green!60!black](0,1.5)--(1.5,0);
    \draw[thick, blue](0,0)--(1.5,1.5);
    \draw[thick, red](0.5,0)--(0.5,1.5);
    \node[below right] at (2,0){$x$};
    \node[left] at (0,2){$y$};
    \node[below] at (1,-0.5){No solution};

    \draw[thick, <->](5,2)--(5,0)--(7,0);
    \draw[thick, green!60!black](5,1.5)--(6.5,0);
    \draw[thick, blue](5,0)--(6.5,1.5);
    \draw[thick, red](5.75,0)--(5.75,1.5);
    \draw[fill](5.75,0.75) circle [radius=2pt];
    \node[below right] at (7,0){$x$};
    \node[left] at (5,2){$y$};
    \node[below] at (6,-0.5){One solution};
  \end{tikzpicture}
\end{center}

Consider the first picture above. While all three lines intersect with
one another, there is no common point of intersection where all three
lines meet at one point. Hence, there is no solution to the three
equations. Remember, a solution is a point $(x, y)$ which satisfies
\textbf{all} three equations.  In the case of the second picture, the
lines intersect at a common point. This means that there is one
solution to the three equations whose graphs are the given lines.  You
should take a moment now to draw the graph of a system which results
in three parallel lines. Next, try the graph of three identical lines.
Which type of solution is represented in each of these graphs?

We have now considered the graphical solutions of systems of two
equations in two variables, as well as three equations in two
variables. However, there is no reason to limit our investigation to
equations in two variables. We will now consider equations in three
variables.

You may recall that equations in three variables, such as
$2x+4y-5z=8$, form a plane. Above, we were looking for intersections
of lines in order to identify any possible solutions. When graphically
solving systems of equations in three variables, we look for
intersections of planes. These points of intersection give the
$(x, y, z)$ that satisfy all the equations in the system.  What types
of solutions are possible when working with three variables?  Consider
the following picture involving two planes, which are given by two
equations in three variables.

\begin{center}
  \begin{tikzpicture}[scale=0.6,z={(-0.55,-0.45)}]
    \draw[fill=blue!10] (0,0,-4) -- (0,0,2) -- (-3,0,2) -- (-3,0,-4) -- cycle;
    \draw[fill=blue!25] (0,0,-4) -- (1.5,-2,-4) -- (1.5,-2,2) -- (0,0,2) -- cycle;
    \draw[fill=blue!10] (0,0,-4) -- (0,0,2) -- (3,0,2) -- (3,0,-4) -- cycle;
    \draw[fill=blue!25] (0,0,-4) -- (-1.5,2,-4) -- (-1.5,2,2) -- (0,0,2) -- cycle;
  \end{tikzpicture}
\end{center}

Notice how these two planes intersect in a line. This means that the
points $(x,y,z)$ on this line satisfy both equations in the
system. Since the line contains infinitely many points, this system
has infinitely many solutions.

It could also happen that the two planes fail to intersect. However,
is it possible to have two planes intersect at a single point? Take a
moment to attempt drawing this situation, and convince yourself that
it is not possible! This means that when we have only two equations in
three variables, there is no way to have a unique solution! Hence, the
only possibilities for the number of solutions of two equations in
three variables are no solution or infinitely many solutions.

Now imagine adding a third plane. In other words, consider three equations in three variables. What types of solutions are now possible? Consider the following diagram.

\begin{center}
  \begin{tikzpicture}[scale=0.6,z={(-0.55,-0.45)}]
    \draw[fill=blue!10] (0,0,-4) -- (0,0,2) -- (-3,0,2) -- (-3,0,-4) -- cycle;
    \draw[fill=red!25] (-0.75,1,-4) -- (-0.75,1,2) -- (-2.5,1,2) -- (-2.5,1,-4) -- cycle;
    \draw[fill=blue!30] (0,0,-4) -- (1.5,-2,-4) -- (1.5,-2,2) -- (0,0,2) -- cycle;
    \draw[fill=blue!10] (0,0,-4) -- (0,0,2) -- (3,0,2) -- (3,0,-4) -- cycle;
    \draw[fill=blue!30] (0,0,-4) -- (-1.5,2,-4) -- (-1.5,2,2) -- (0,0,2) -- cycle;
    \draw[fill=red!25] (0.75,1,-4) -- (0.75,1,2) -- (-0.75,1,2) -- (-0.75,1,-4) -- cycle;
  \end{tikzpicture}
\end{center}

In this diagram, there is no point which lies in all three
planes. There is no intersection between \textbf{all} three planes so
there is no solution. The picture illustrates the situation in which
the line of intersection of the new plane with one of the original
planes forms a line parallel to the line of intersection of the first
two planes. However, in three dimensions, it is possible for two lines
to fail to intersect even though they are not parallel. Such lines are
called \textbf{skew lines}%
\index{skew lines}.

Recall that when working with two equations in three variables, it was
not possible to have a unique solution. Is it possible when
considering three equations in three variables? In fact, it is
possible, and we demonstrate this situation in the following picture.

\begin{center}
  \begin{tikzpicture}[scale=0.6,z={(-0.55,-0.45)}]
    \draw[fill=blue!10] (0,0,0) -- (0,0,-4) -- (-3,0,-4) -- (-3,0,0) -- cycle;
    \draw[fill=blue!30] (0,0,0) -- (1.5,-2,0) -- (1.5,-2,-4) -- (0,0,-4) -- cycle;
    \draw[fill=blue!10] (0,0,0) -- (0,0,-4) -- (3,0,-4) -- (3,0,0) -- cycle;
    \draw[fill=blue!30] (0,0,0) -- (-1.5,2,0) -- (-1.5,2,-4) -- (0,0,-4) -- cycle;
    \draw[fill=red!25] (-3,0,0) -- (3,0,0) -- (3,-2,0) -- (-3,-2,0) -- cycle;
    \draw[fill=blue!10] (0,0,0) -- (0,0,2) -- (-3,0,2) -- (-3,0,0) -- cycle;
    \draw[fill=blue!30] (0,0,0) -- (1.5,-2,0) -- (1.5,-2,2) -- (0,0,2) -- cycle;
    \draw[fill=blue!10] (0,0,0) -- (0,0,2) -- (3,0,2) -- (3,0,0) -- cycle;
    \draw[fill=red!25] (-3,0,0) -- (3,0,0) -- (3,2,0) -- (-3,2,0) -- cycle;
    \draw[fill=blue!30] (0,0,0) -- (-1.5,2,0) -- (-1.5,2,2) -- (0,0,2) -- cycle;
  \end{tikzpicture}
\end{center}

In this case, the three planes have a single point of intersection.
Can you think of other possibilities? Another is that the three planes
could intersect in a line, resulting in infinitely many solutions, as
in the following diagram.

\begin{center}
  \begin{tikzpicture}[scale=0.6,z={(-0.55,-0.45)}]
    \draw[fill=blue!10] (0,0,-4) -- (0,0,2) -- (-3,0,2) -- (-3,0,-4) -- cycle;
    \draw[fill=red!25] (-1.5,-2,-4) -- (-1.5,-2,2) -- (0,0,2) -- (0,0,-4) -- cycle;
    \draw[fill=blue!30] (0,0,-4) -- (1.5,-2,-4) -- (1.5,-2,2) -- (0,0,2) -- cycle;
    \draw[fill=blue!10] (0,0,-4) -- (0,0,2) -- (3,0,2) -- (3,0,-4) -- cycle;
    \draw[fill=blue!30] (0,0,-4) -- (-1.5,2,-4) -- (-1.5,2,2) -- (0,0,2) -- cycle;
    \draw[fill=red!25] (1.5,2,-4) -- (1.5,2,2) -- (0,0,2) -- (0,0,-4) -- cycle;
  \end{tikzpicture}
\end{center}

We have now seen how three equations in three variables can have no
solution, a unique solution, or intersect in a line resulting in
infinitely many solutions.  It is also possible that all three
equations describe the same plane, which also leads to infinitely many
solutions.

You can see that when working with equations in three variables, there
are many more possibilities for achieving solutions (or no solutions)
than when working with two variables. It may prove enlightening to
spend time imagining (and drawing) many possible scenarios, and you
should take some time to try a few.

You should also take some time to imagine (and draw) graphs of systems
in more than three variables.  Equations like $x+y-2z+4w=8$ with more
than three variables are often called \textbf{hyperplanes}%
\index{hyperplane}%
\index{plane!hyperplane}.  You may soon realize that it is tricky to
draw the graphs of hyperplanes! In fact, most people cannot visualize
more than three dimensions. Fortunately, through the tools of linear
algebra, we can examine systems of equations in four variables, five
variables, or even hundreds or thousands of variables, without ever
needing to graph them. Instead we will use {\em algebra} to manipulate
and solve these systems of equations. We will introduce these
algebraic tools in the following sections.


\subsection{Disks}

We defined an open disk at \((a,b)\) of radius \(r\) as:

\(\{(x,y)\in \mathbb R^2:(x-a)^2+(y-b)^2 < r^2\}\)

For a closed disk it is:

\(\{(x,y)\in \mathbb R^2:(x-a)^2+(y-b)^2 \le r^2\}\)

\subsection{Annulus}

An annulus is a disk, which excludes a smaller disk inside the disk

\subsection{Punctured disk}

If the interior disk is just a point, it is a punctured disk.


\subsection{Elasticnet}

Regularisation of LLS. Combines lasso and ridge regression.

\(L_1\) and \(L_2\) regularisation


\section*{Problem 3: Ridge Regression [40 pts] (Hyun-Ah \& Petar)}

\subsection* {3.1 MLE (Petar)}

% \subsection*{Problem 1: MLE [10 pts]}

% \textbf{NOTE: Do not submit your code for this portion of the assignment, submit the plots only.  Plots should be submitted with the rest of the assignment in Gradescope.}

We are given covariates $\mathbf{X} \in \mathbb{R}^{n \times d}$ and responses $\mathbf{y} \in \mathbb{R}^{n \times 1}$, such that:
\begin{align}
        \mathbf{y} = \mathbf{X}\mathbf{\beta} + \mathbf{\epsilon}
\end{align}
where $\mathbf{\beta} \in \mathbb{R}^{{\color{red}d} \times 1}$ is a vector of predictors and $\mathbf{\epsilon}$ a vector of Gaussian noise components given by: $\epsilon_i \sim \mathcal{N}(0, \sigma^2)$. \\ \\
a) Derive the conditional probability $p(y_i | \mathbf{x_i}, \mathbf{\beta}, {\color{red}\sigma})$, where $y_i$ is the $i$-th component of $\mathbf{y}$,  and $\mathbf{x_i}$ is the $i$-th row of $\mathbf{X}$. \\ \\
b) Assume we have $n$ i.i.d samples. Derive the log-likelihood of the data $\ell(\mathbf{y} | \mathbf{\beta})$. \\ \\
c) Show that maximizing the log-likelihood is equivalent to the following criterion:
\begin{align}
    \min_{\mathbf{\beta}} ||\mathbf{y} - \mathbf{X}\mathbf{\beta}||_2^2
\end{align}
d) Derive the MLE $\hat{\mathbf{\beta}}$.

We have worked through the linear regression problem. Now, we will explore linear regression problem with regularization.

\subsection*{3.2 Ridge regression}
Continued from Problem 1, we are given a matrix of $n$ training samples with $p$ dimensions, $\mathbf{X}\in \mathbb{R}^{n \times p}$, a parameter vector $\mathbf{\beta}\in\mathbb{R}^{p}$, and a output vector $\mathbf{y}\in\mathbb{R}^{n}$.
L2 penalized linear regression problem shown below penalizes the L2 norm of the parameter vector. This is also known as the ridge regression problem.
\begin{align*}
    \min_{\mathbf{\beta}} \frac{1}{2}||\mathbf{y} - \mathbf{X}\mathbf{\beta}||_2^2 + \frac{\lambda}{2} ||\mathbf{\beta}||_2^2
\end{align*}

a) Show that the solution to the ridge regression problem is $\beta^*= (\mathbf{X}^T\mathbf{X} +\lambda \mathbf{I})^{-1}\mathbf{X}^T\mathbf{y}$.

b) How does the solution differ from the solution to the ordinary linear regression problem? Explain how the solution changes as $\lambda \rightarrow 0$, and $\lambda \rightarrow \infty$.

\newpage


\subsection{Hausdorf space}

In a Hausdorf (or T\(_2\)) space, any two different points have neighbourhoods which are disjoint.


\subsection{Hausdorff distance}\label{subsec:hausdorff_distance}

Let \( (X, \mu) \) be a \hyperref[def:complete_metric_space]{complete metric space}.

\begin{definition}\label{def:hausdorff_distance}\mcite[144]{DontchevRockafellar2014}
  Fix two sets \( E \subseteq X \) and \( F \subseteq X \).

  The \term{excess} of \( E \) beyond \( F \) is defined as
  \begin{balign*}
     & e: \pow X \times \pow X \to \BbbR \cup \{ \infty \} \\
     & e(E, F) \coloneqq \begin{cases}
      +\infty,                                                                                & E = \varnothing, D = \varnothing                      \\
      0,                                                                                & E = \varnothing, D \neq \varnothing                   \\
      \sup_{x \in E} \op{dist}(x, F) \reloset{*}{=} \inf \{\delta > 0 \colon E \subseteq F_\delta \}, & E \neq \varnothing \nonumber\refstepcounter{equation}
    \end{cases}
  \end{balign*}
  where \( F_\delta \coloneqq \{ y \in X \colon \op{dist}(y, F) \leq \delta \} \).

  The \term{Pompeiu-Hausdorff distance} or simply \term{Hausdorff} distance between them is then defined as
  \begin{equation*}
    h(E, F) \coloneqq \max\{ e(E, F), e(F, E) \} = \inf \{\delta > 0 \colon E \subseteq F_\delta, F \subseteq E_\delta \}.
  \end{equation*}
\end{definition}
\begin{proof}(of the equality \( * \))
  Note that the set
  \begin{equation*}
    F_{e(E, F)} = \{ x \in X \colon \op{dist}(x, F) \leq \sup_{x \in E} \op{dist}(x, F) \}
  \end{equation*}
  obviously includes \( E \).

  Now let \( \delta > 0 \) be any real number that satisfies \( E \subseteq F_\delta \), i.e.
  \begin{equation*}
    E \subseteq F_\delta = \{ x \in X \colon \op{dist}(x, F) \leq \delta \},
  \end{equation*}
  which implies that
  \begin{equation*}
    e(E, F) = \sup_{x \in E} \op{dist}(x, F) \leq \delta.
  \end{equation*}
\end{proof}

\begin{proposition}\label{thm:hausdorff_distance_is_metric}
  The Hausdorff distance is a metric on the nonempty compact subsets of \( X \).
\end{proposition}
\begin{proof}
  Let \( E \), \( F \) and \( G \) be nonempty compact subsets of \( X \).

  The function \( h \) is nonnegative. Since we exclude empty and unbounded sets, We do not care about infinite values.

  \SubProofOf{def:metric_space/M1} Obviously \( h(E, E) = 0 \). If \( h(E, F) = 0 \), then there exists no point of \( E \) outside \( F \) and vice versa, hence \( E = F \).

  \SubProofOf{def:metric_space/M2} This follows from the symmetry of the \( \max \) function.

  \SubProofOf{def:metric_space/M3} For any point \( y \in X \), we have
  \begin{balign*}
    \op{dist}(x, G)
    =
    \inf_{z \in G} \mu(x, z)
    \leq
    \mu(x, y) + \inf_{y \in G} \mu(y, z)
    =
    \mu(x, y) + \op{dist}(y, G).
  \end{balign*}

  Select \( y \in F \) that minimizes the distance \( \mu(x, y) \) over \( F \) (compactness allows us), so that \todo{Prove \hyperref[thm:weierstrass_extreme_value_theorem]{Weierstrass' theorem}}
  \begin{balign*}
    \op{dist}(x, G)
    \leq
    \mu(x, y) + \op{dist}(y, G)
    =
    \op{dist}(x, F) + \op{dist}(y, G)
    \leq
    \op{dist}(x, F) + e(F, G).
  \end{balign*}

  It now follows that
  \begin{balign*}
    e(E, G)
     & =
    \inf \{\delta > 0 \colon E \subseteq G_\delta \}
    =    \\ &=
    \inf \{\delta > 0 \colon E \subseteq \{ x \in X \colon \op{dist}(x, G) \leq \delta \}
    \leq \\ &\leq
    \inf \{\delta > 0 \colon E \subseteq \{ x \in X \colon \op{dist}(x, F) + e(F, G) \leq \delta, y \in X \}
    =    \\ &=
    e(F, G) + \inf \{\delta > 0 \colon E \subseteq F_\delta \}
    =    \\ &=
    e(F, G) + e(E, F).
  \end{balign*}
\end{proof}


%!TEX root = ../thesis.tex

\chapter{The title of chapter two}

The integers, along with the two operations of addition and
multiplication, form the prototypical example of a ring.  In
mathematics, a ring is one of the fundamental algebraic structures
used in abstract algebra. It consists of a set equipped with two
binary operations that generalize the arithmetic operations of
addition and multiplication. Through this generalization, theorems
from arithmetic are extended to non-numerical objects such as
polynomials, series, matrices and functions.

A ring is an abelian group with a second binary operation that is
associative, is distributive over the abelian group operation, and has
an identity element (this last property is not required by some
authors, see § Notes on the definition). By extension from the
integers, the abelian group operation is called addition and the
second binary operation is called multiplication.

Whether a ring is commutative or not (that is, whether the order in which two elements are multiplied changes the result or not) has profound implications on its behavior as an abstract object. As a result, commutative ring theory, commonly known as commutative algebra, is a key topic in ring theory. Its development has been greatly influenced by problems and ideas occurring naturally in algebraic number theory and algebraic geometry. Examples of commutative rings include the set of integers equipped with the addition and multiplication operations, the set of polynomials equipped with their addition and multiplication, the coordinate ring of an affine algebraic variety, and the ring of integers of a number field. Examples of noncommutative rings include the ring of \(n \times n\) real square matrices with \(n \geq 2\), group rings in representation theory, operator algebras in functional analysis, rings of differential operators in the theory of differential operators, and the cohomology ring of a topological space in topology.


In this problem we start by plotting the lagrange polynomials
\begin{align*}
l(x)=\prod_{j=0}^N(x-x_j),
\end{align*}
for equispaced nodes (figure 1), for Chebishev's nodes of the first kind (figure 2) and for Chebishev's nodes of the second kind (figure 3). The nodes are placed on the zeroes of the functions and represented by the red asterisk. We see how the Chebishev's nodes are more concentrated towards the boundaries, being very useful for boundary conditions.

\begin{figure}[H]
\centering     %%% not \center
{\includegraphics[scale=0.6]{Lagrange.eps}}
\caption{Equispaced nodes.}
\end{figure}

\begin{figure}[H]
\centering     %%% not \center
{\includegraphics[scale=0.6]{Chebishev1.eps}}
\caption{First kind Chebishev's nodes.}
\end{figure}

\begin{figure}[H]
\centering     %%% not \center
{\includegraphics[scale=0.6]{Chebishev2.eps}}
\caption{Second kind Chebishev's nodes.}
\end{figure}

Using those nodes we compute approximations to the function $f(x)=\cos(3x)$. In the following plots we show the error and the upper bound of the error given by the Cauchy interpolation error formula
\begin{align*}
f(x)-p_n(x)=\frac{f^{(n+1)}(\xi)}{(n+1)!}l(x),
\end{align*}
where $p_n(x)$ represents the approximation. Given the function to approximate, we can then find the upper bound
\begin{align*}
f(x)-p_n(x)=\frac{f^{(n+1)}(\xi)}{(n+1)!}l(x)\leq\frac{3^{n+1}}{(n+1)!}l(x):=e_{Cauchy}.
\end{align*}
In the following figures we can see how Cauchy error formula gives us the upper bound for equispaced nodes (figure 7), Chebishev nodes of the first kind (figure 8), Chebishev nodes of the second kind (figure 9). As in problem 2, the drops in the error are due to the fact that the error must be zero at those points used to run the interpolation and, since we are using logarithmic scale, are seen as these sudden drops in the error plot.
\begin{figure}[H]
\centering     %%% not \center
{\includegraphics[scale=0.75]{Cauchy_equi.eps}}
\caption{Error of equispaced nodes.}
\end{figure}

\begin{figure}[H]
\centering     %%% not \center
{\includegraphics[scale=0.75]{Cauchy_cheb1.eps}}
\caption{Error of Chebishev's nodes of first kind.}
\end{figure}

\begin{figure}[H]
\centering     %%% not \center
{\includegraphics[scale=0.75]{Cauchy_cheb2.eps}}
\caption{Error of Chebishev's nodes of second kind.}
\end{figure}

To finish, we also compute the $L_2$ norm of the error and check that in fact we obtain an over all upper bound for the different methdos (see table 1). As we can check, Cauchy's interpolation formula gives us a good upper bound of the error for each distribution of the nodes. Note that the error of the Chebishev's methods is smaller. As we saw in class, equispaced nodes have high errors at the boundaries.

\begin{table}[H]
\centering
\begin{tabular}{r|c c c }
%\hline
%\multicolumn{3}{|c|}{Datos}\\
  & Equispaced & First kind Chebishev & Second kind Chebishev\\
\hline
$e_{Cauchy}$ & $     3.54887848499\cdot 10^{-4}$ & $9.68648228641\cdot 10^{-5}$ & $1.58167667439\cdot 10^{-4}$\\
$e_{Interpolation}$ & $7.05156144371\cdot 10^{-5}$ & $1.18371532261\cdot 10^{-5}$ & $1.49100947533\cdot 10^{-5}$\\
\end{tabular}
\caption{$L_2$ error norm of different methods.}
\end{table}

\subsection*{Matlab code for this problem}
\begin{verbatim}
%% Problem 4
N=10;
f = @(x) cos(3*x);
xx=linspace(-1,1,1000);
xk0=linspace(-1,1,N+1)';
xk1=chebpts(N+1,1);
xk2=chebpts(N+1,2);
l0 = @(x) prod(x-xk0);
l1 = @(x) prod(x-xk1);
l2 = @(x) prod(x-xk2);
%
figure
plot(xk0,l0(xk0),'r*')
hold on
plot(xx,l0(xx))
grid on
xlabel('$x$','fontsize',labelfontsize,'interpreter','latex')
saveas(gcf,'Latex/FIGURES/Lagrange','epsc')
saveas(gcf,'Latex/FIGURES/Lagrange','fig')

%
figure
plot(xk1,l1(xk1),'r*')
hold on
plot(xx,l1(xx))
grid on
xlabel('$x$','fontsize',labelfontsize,'interpreter','latex')
saveas(gcf,'Latex/FIGURES/Chebishev1','epsc')
saveas(gcf,'Latex/FIGURES/Chebishev1','fig')

%
figure
plot(xx,l2(xx))
hold on
plot(xk2,l2(xk2),'r*')
grid on
xlabel('$x$','fontsize',labelfontsize,'interpreter','latex')
saveas(gcf,'Latex/FIGURES/Chebishev2','epsc')
saveas(gcf,'Latex/FIGURES/Chebishev2','fig')

w0 = baryWeights(xk0);
p0 = @(x) bary(x,f(xk0),xk0,w0);
err_bound0 = 3^(N+1)*l0(xx)/factorial(N+1);
figure
semilogy(xx,abs(err_bound0))
hold on
semilogy(xx,abs(f(xx)-p0(xx)))
grid on
axis([-1 1 1e-10 1e-4])
legend({'$e_{Cauchy}$','$e_{equi}$'},...
'fontsize',legendfontsize,'interpreter','latex','Location','north')
xlabel('$x$','fontsize',labelfontsize,'interpreter','latex')
ylabel('$e(x)$','fontsize',labelfontsize,'interpreter','latex')
saveas(gcf,'Latex/FIGURES/Cauchy_equi','epsc')
saveas(gcf,'Latex/FIGURES/Cauchy_equi','fig')

w1 = baryWeights(xk1);
p1 = @(x) bary(x,f(xk1),xk1,w1);
err_bound1 = 3^(N+1)*l1(xx)/factorial(N+1);
figure
semilogy(xx,abs(err_bound1))
hold on
semilogy(xx,abs(f(xx)-p1(xx)))
grid on
axis([-1 1 1e-10 1e-4])
legend({'$e_{Cauchy}$','$e_{Cheb1}$'},...
'fontsize',legendfontsize,'interpreter','latex','location','north')
xlabel('$x$','fontsize',labelfontsize,'interpreter','latex')
ylabel('$e(x)$','fontsize',labelfontsize,'interpreter','latex')
saveas(gcf,'Latex/FIGURES/Cauchy_cheb1','epsc')
saveas(gcf,'Latex/FIGURES/Cauchy_cheb1','fig')

w2 = baryWeights(xk2);
p2 = @(x) bary(x,f(xk2),xk2,w2);
err_bound2 = 3^(N+1)*l2(xx)/factorial(N+1);
figure
semilogy(xx,abs(err_bound2))
hold on
semilogy(xx,abs(f(xx)-p2(xx)))
grid on
axis([-1 1 1e-10 1e-4])
legend({'$e_{Cauchy}$','$e_{Cheb2}$'},...
'fontsize',legendfontsize,'interpreter','latex','location','north')
xlabel('$x$','fontsize',labelfontsize,'interpreter','latex')
ylabel('$e(x)$','fontsize',labelfontsize,'interpreter','latex')
saveas(gcf,'Latex/FIGURES/Cauchy_cheb2','epsc')
saveas(gcf,'Latex/FIGURES/Cauchy_cheb2','fig')

E_equi=norm(f(xx)-p0(xx),2)
E_Cauchy_equi=norm(err_bound0,2)
E_Chebishev1=norm(f(xx)-p1(xx),2)
E_Cauchy_cheb1=norm(err_bound1,2)
E_Chebishev2=norm(f(xx)-p2(xx),2)
E_Cauchy_cheb2=norm(err_bound2,2)

\end{verbatim}


\chapter{Finding the $\mathbf{k}$-th Smallest Number}
The objective of this challenge is to find the $k$-th smallest number in an
unsorted array. Some elements may be of equal value.

\section{Sort-and-Find}
This solution sorts the unsorted array and returning the value at index $k$. As
seen before, time complexity analysis indicates that this approach requires more
than $O(n\log n)$ amount of time. Since the sorting algorithm has already been
presented before, it will not be features again.

\section{Quick Select}
This solution is modeled after the Quicksort algorithm. Unlike Quicksort, this
algorithm only focuses on one side of the partition, and therefore its expected
running time is $T(n)=\Theta(n)$.
\imb[3][15]{\alg/kthsmall.cpp}
\imb[17][24]{\alg/kthsmall.cpp}