\section{Code}

\begin{minted}{python}
# -*- coding: utf-8 -*-
"""
TRABAJO 1.
Autor Blanca Cano Camarero
Grupo 2
"""

import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D # to display 3d function

np.random.seed(1)
def STOP_EXECUTION_TO_SEE_RESULT():
        input('\n--- End of a section, press any enter to continue ---\n')

print('GRADIENT DESCENDENT\n')
print('Exercise 1\n')

## 1

def gradient_descent(initial_point, loss_function, gradient_function,  eta, max_iter, target_error):
    '''
    initicial point: w_0
    E: error function
    gradient_function
    eta:  step size

    ### stop conditions ###
    max_iter
    target_error

    #### return ####
    (w,iterations)
    w: the coordenates that minimize E
    it: the numbers of iterations needed to obtain w

    '''

    iterations = 0
    error = E( initial_point[0], initial_point[1])
    w = initial_point

    while ( (iterations < max_iter) and(error > target_error)):

        w = w - eta * gradient_function(w[0], w[1])

        iterations += 1
        error = loss_function(w[0], w[1])

    return w, iterations

############  2 ###################################
def E(u,v):
    '''
    Function to minimize
    '''
    return np.float64(
        ( u**3 * np.e**(v-2) - 2*v**2 * np.e**(-u) )**2
    )

def dEu(u,v):
    '''
    Partial derivate of E with respect to the variable u
    '''
    return np.float64(
        2
        *( u**3 * np.e**(v-2) - 2*v**2 * np.e**(-u))
        *( 3* u**2 * np.e**(v-2) + 2*v**2 * np.e**(-u))
    )

def dEv(u,v):
    '''
    Partial derivate of E with respect to the variable v
    '''
    return np.float64(
        2
        *( u**3 * np.e**(v-2) - 2*v**2 * np.e**(-u) )
        *( u**3 * np.e**(v-2) - 4*v * np.e**(-u))
    )

def gradE(u,v):
    '''
        gradient of E
    '''
    return np.array([dEu(u,v), dEv(u,v)])

######  conditions
eta = 0.1
max_iter = 10000000000
target_error = 1e-14
initial_point = np.array([1.0,1.0])
w, it = gradient_descent( initial_point,E, gradE, eta, max_iter, target_error )

# DISPLAY FIGURE

x = np.linspace(-30, 30, 50)
y = np.linspace(-30, 30, 50)
X, Y = np.meshgrid(x, y)
Z = E(X, Y) #E_w([X, Y])
fig = plt.figure()
ax = Axes3D(fig)
surf = ax.plot_surface(X, Y, Z, edgecolor='none', rstride=1,
                        cstride=1, cmap='jet')
min_point = np.array([w[0],w[1]])
min_point_ = min_point[:, np.newaxis]
ax.plot(min_point_[0], min_point_[1], E(min_point_[0], min_point_[1]), 'r*', markersize=10)
ax.set(title='Ejercicio 1.2. Función sobre la que se calcula el descenso de gradiente')
ax.set_xlabel('u')
ax.set_ylabel('v')
ax.set_zlabel('E(u,v)')
plt.show()

######### Exercise 1, part 2 answers  ######
print('2 a) Function:  E(u,v) = (u^3 e^{(v-s)} - 2* v^2 e^{-u})^2')

print('dE_u = 2(u^3 e^{(v-s)} - 2* v^2 e^{-u})(3u^2e^{(v-2)} + 2 v^2 e^{-u} ), ')

print( 'dE_v =  2(u^3 e^{(v-s)} - 2* v^2 e^{-u})(u^3 e^{(v-2)} - 4 v e^{-u}')

print ('So de gradient is: \n nabla E(u,v) =(2(u^3 e^{(v-s)} - 2* v^2 e^{-u})(3u^2e^{(v-2)} + 2 v^2 e^{-u} ), 2(u^3 e^{(v-s)} - 2* v^2 e^{-u})(u^3 e^{(v-2)} - 4 v e^{-u}) )')

print ('2b) Numbers of iterations : ', it)
print ('2c) Final coodinates: (', w[0], ', ', w[1],')')

############################### EXERCISE 1 PART 3 #########

def f(x,y):
    '''
    Function to minimize
    '''
    return np.float64(
        (x+2)**2 + 2*(y-2)**2 + 2* np.sin( 2* np.pi * x)* np.sin( 2* np.pi * y)
    )

def dfx(x,y):
    '''
    Partial derivate of f with respect to the variable x
    '''
    return np.float64(
        2*( x+2 ) + 4* np.pi* np.cos(  2* np.pi * x )* np.sin(2* np.pi * y)
    )

def dfy(x,y):
    '''
    Partial derivate of f with respect to the variable y
    '''
    return np.float64(
        4*( y-2 ) + 4* np.pi* np.cos(  2* np.pi * y )* np.sin(2* np.pi * x)

    )

def gradF(x,y):
    '''
        gradient of E
    '''
    return np.array([dfx(x,y), dfy(x,y)])

######################## gradien descendent with trace  ############

def gradient_descent_trace(initial_point, loss_function, gradient_function,  eta, max_iter):
    '''
    initicial point: w_0
    loss_function: error function
    gradient_function
    eta:  step size

    ### stop conditions ###
    max_iter

    #### return ####
    (w,iterations)
    w: the coordenates that minimize loss_function
    it: the numbers of iterations needed to obtain w

    '''

    iterations = 0
    error = loss_function( initial_point[0], initial_point[1])
    w = [initial_point]

    while iterations < max_iter:

        new_w = w[-1] - eta * gradient_function(w[-1][0], w[-1][1])

        iterations += 1
        error = loss_function(new_w[0], new_w[1])
        w.append( new_w )

    return w, iterations

######  conditions
smaller_eta = 0.01
bigger_eta = 0.1
max_iter = 50
initial_point = np.array([-1.0,1.0])

#### run
smaller_w, smaller_it = gradient_descent_trace( initial_point,f, gradF, smaller_eta, max_iter)
bigger_w, bigger_it = gradient_descent_trace( initial_point,f, gradF, bigger_eta, max_iter)

images_smaller_eta = [ f(x[0],x[1]) for x in smaller_w ]
images_bigger_eta = [ f(x[0],x[1]) for x in bigger_w]

print(f'With eta = {smaller_eta}, coordenates (x,y)= {(smaller_w[-1][0],smaller_w[-1][1])}, the number of iterations: {smaller_it} and the image is f(x,y) = {images_smaller_eta[-1]}')

print(f'With eta = {bigger_eta}, coordenates (x,y)={(bigger_w[-1][0],bigger_w[-1][1])}, the number of iterations: {bigger_it} and the image is f(x,y) = {images_bigger_eta[-1]}')

########### PLOTTING
x_label = 'Number of iterations'
y_label = 'f(x,y)'
bigger_eta_label = '$\eta$ = '+str(bigger_eta)
smaller_eta_label = '$\eta$ = '+str(smaller_eta)

## bigger eta ####
plt.clf()
plt.plot(images_bigger_eta, label=bigger_eta_label)
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.title(f'Gradient descent of f with $\eta =$ {bigger_eta}')

plt.legend()
plt.show()
STOP_EXECUTION_TO_SEE_RESULT()
## smaller eta ###

plt.clf()
plt.plot(images_smaller_eta, label=smaller_eta_label)
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.title(f'Gradient descent of f with $\eta =$ {smaller_eta}')

plt.legend()
plt.show()

STOP_EXECUTION_TO_SEE_RESULT()
## comparation ###
plt.clf()
plt.plot(images_bigger_eta, label=r"$\eta$ = 0.1")
plt.plot( images_smaller_eta, label=r"$\eta$ = 0.01")

plt.xlabel('Iterations')
plt.ylabel('f(x,y)')

plt.title("Comparation of the gradient descendent for $f$ changing eta value  ")

plt.legend()
plt.show()
STOP_EXECUTION_TO_SEE_RESULT()

### exta experimental ####

epsilon_eta = 1e-14
max_iter = 50
initial_point = np.array([-1.0,1.0])

epsilon_eta_label = '$\eta$ = '+str(epsilon_eta)
#### run
epsilon_w, epsilon_it = gradient_descent_trace( initial_point,f, gradF, epsilon_eta, max_iter)
images_epsilon_eta = [ f(x[0],x[1]) for x in epsilon_w ]

print(f'With eta = {epsilon_eta}, coordenates (x,y)={(epsilon_w[-1][0],epsilon_w[-1][1])}, the number of iterations: {epsilon_it} and the image is f(x,y) = {images_epsilon_eta[-1]}')
STOP_EXECUTION_TO_SEE_RESULT()

plt.clf()
plt.plot(images_epsilon_eta, label=epsilon_eta_label)
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.title(f'Gradient descent of f with $\eta =$ {epsilon_eta}')

plt.legend()
plt.show()

STOP_EXECUTION_TO_SEE_RESULT()
## smaller eta and tiny one ###

plt.clf()

plt.plot(images_epsilon_eta, label=epsilon_eta_label)
plt.plot( images_smaller_eta, label=r"$\eta$ = 0.01")

plt.xlabel('Iterations')
plt.ylabel('f(x,y)')

plt.title("Comparation of the gradient descendent for $f$ changing eta value  ")

plt.legend()
plt.show()
STOP_EXECUTION_TO_SEE_RESULT()
######################################################################

#### exercise 3.b

initial_points = map(np.array, [[-.5, -.5],[1, 1], [2.1, -2.1], [-3,3],  [-2, 2]])
minimum_value = np.Infinity

print('{:^17}  {:^17}  {:^9}'.format('Initial', 'Final', 'Value'))

for initial in initial_points:

  w, _ = gradient_descent_trace(initial, f, gradF, 0.01, 50)
  local_minimum = w[-1]
  local_value = f(local_minimum[0], local_minimum[1])

  print('{}  {}  {: 1.5f}'.format(initial, local_minimum, local_value))

STOP_EXECUTION_TO_SEE_RESULT()

"""
Exercise 2
Author: Blanca Cano Camarero
"""

def STOP_EXECUTION_TO_SEE_RESULT():
        input('\n--- End of a section, press any enter to continue ---\n')

print('_______LINEAR REGRESSION EXERCISE _______\n')
print('Exercise 1')
input('\n Enter to start\n')

label5 = 1
label1 = -1

def readData(file_x, file_y):
        '''
        function for read data
        '''
	# reads files
        datax = np.load(file_x)
        datay = np.load(file_y)
        y = []
        x = []
        # Solo guardamos los datos cuya clase sea la 1 o la 5
        for i in range(0,datay.size):
                if datay[i] == 5 or datay[i] == 1:
                        if datay[i] == 5:
                                y.append(label5)
                        else:
                                y.append(label1)
                        x.append(np.array([1, datax[i][0], datax[i][1]]))

        x = np.array(x, np.float64)
        y = np.array(y, np.float64)

        return x, y

def Error(x,y,w):
    '''quadratic error
    INPUT
    x: input data matrix
    y: target vector
    w:  vector to

    OUTPUT
    quadratic error >= 0
    '''
    error_times_n = np.float64(np.linalg.norm(x.dot(w) - y.reshape(-1,1))**2)

    return np.float64(error_times_n/len(x))

def dError(x,y,w):
    ''' gradient
    OUTPUT
    column vector
    '''
    return  2/len(x)*(x.T.dot(x.dot(w) - y.reshape(-1,1)))

def sgd(x,y, eta = 0.01, max_iter = 1000, batch_size = 32, error=10**(-10)):
        '''
        Stochastic gradient descent
        INPUT
        x: data set
        y: target vector
        eta: learning rate
        max_iter

        OUTPUT
        w: weight vector
        '''

        #initialize data
        w = np.zeros((x.shape[1], 1), np.float64)
        n_iterations = 0

        len_x = len(x)
        x_index = np.arange( len_x )
        batch_start = 0
        w_error = Error(x,y,w)

        while n_iterations < max_iter and w_error > error :

                #shuffle and split the same into a sequence of mini-batches
                np.random.shuffle(x_index)
                for batch_start in range(0,  len_x, batch_size):
                        iter_index = x_index[ batch_start : batch_start + batch_size]

                        w = w - eta* dError(x[iter_index, :], y[iter_index], w)

                n_iterations += 1
                w_error = Error(x,y,w)

        return w

def sgd_exact_number_iter(x,y, eta = 0.01, max_iter = 1000, batch_size = 32, error = 10**(-10)):
        '''
        Stochastic gradient descent
        INPUT
        x: data set
        y: target vector
        eta: learning rate
        max_iter
        OUTPUT
        w: weight vector
        '''
        #initialize data
        w = np.zeros((x.shape[1], 1), np.float64)

        n_iterations = 0
        batch_start = 0
        len_x = len(x)

        x_index = np.arange( len_x )
        w_error = Error(x,y,w)

        while n_iterations < max_iter and w_error > error:
                #shuffle and split the same into a sequence of mini-batches
                if batch_start == 0:
                        x_index = np.random.permutation(x_index)
                iter_index = x_index[ batch_start : batch_start + batch_size]

                w = w - eta* dError(x[iter_index, :], y[iter_index], w)

                n_iterations += 1

                batch_start += batch_size
                if batch_start >= len_x: # if end, restart
                        batch_start = 0

                w_error = Error(x,y,w)

        return w

def pseudoInverseMatrix ( X ):
    '''
    INPUT
    X: is a matrix (must be a np.array) to use transpose and dot method
    OUTPUT
    hat matrix
    '''

    '''
    #S =( X^TX ) ^{-1}
    simetric_inverse = np.linalg.inv( X.T.dot(X) )

    # S X^T = ( X^TX ) ^{-1} X^T
    return simetric_inverse.dot(X.T)
    '''
    return np.linalg.pinv(X)

# Pseudoinverse
def pseudoInverse(X, Y):
    '''
    INPUT
    X is the feature matrix
    Y is the target vector (y_1, ..., y_m)

    OUTPUT:
    w: weight vector
    '''
    X_pseudo_inverse = pseudoInverseMatrix ( X )
    Y_transposed = Y.reshape(-1, 1)

    w = X_pseudo_inverse.dot( Y_transposed)

    return w

# Evaluating the autput

def performanceMeasurement(x,y,w):
    '''Evaluating the output binary case

    INPUT
    X is the feature matrix
    Y is the target vector (y_1, ..., y_m)

    OUTPUT:
    w: weight vector
    OUTPUT:
    bad_negative, bad_positives, input_size
    '''

    # defference between the sign of the regression and the target vector
    sign_column = np.sign(x.dot(w)) - y.reshape(-1,1)

    bad_positives = 0
    bad_negatives = 0

    for sign in sign_column[:,0]:
        if sign > 0 :
                bad_positives += 1
        elif sign < 0 :
                bad_negatives += 1

    input_size = len(y)

    return bad_negatives, bad_positives, input_size

def evaluationMetrics (x,y,w, label = None):
    '''PRINT THE PERFORMANCE MEASUREMENT
    '''
    bad_negatives, bad_positives, input_size = performanceMeasurement(x,y,w)

    accuracy = ( input_size-(bad_negatives +bad_positives))*100 / input_size

    if label :
        print(label)
    print (f'For w^T = {w.reshape(1,-1)}')
    print ( 'Input size: ', input_size )
    print( 'Bad negatives :', bad_negatives)
    print( 'Bad positives :', bad_positives)
    print( 'Accuracy rate :', accuracy, '%')

## Draw de result

### scatter plot
def plotResults (x,y,w, title = None):
        label_5 = 1
        label_1 = -1

        labels = (label_5, label_1)
        colors = {label_5: 'b', label_1: 'r'}
        values = {label_5: 'Number 5', label_1: 'Number 1'}

        plt.clf()

        # data set plot
        for number_label in labels:
                index = np.where(y == number_label)
                plt.scatter(x[index, 1], x[index, 2], c=colors[number_label], label=values[number_label])

        # regression line
        # x = 0
        symmetry_for_cero_intensity = -w[0]/w[2]

        #  x = 1, 0 = w0 + w1 * w2 * x2
        # then y = (-w0 - w1) /w2
        symmetry_for_one_intensity= (-w[0] - w[1])/w[2]

        #plotting order
        plt.plot([0, 1], [symmetry_for_cero_intensity, symmetry_for_one_intensity], 'k-', label=(title+ ' regression'))

        if title :
                plt.title(title)
        plt.xlabel('Average intensity')
        plt.ylabel('Simmetry')
        plt.legend()
        plt.show()

def plotResultMultiplesLines(x,y,multiple_w, main_title = None, multiples_title = None):
        '''
        INPUT
        x featue matrx
        y labels vector
        multiple_w vector of different weight vector
        multiple_titles
        '''
        label_5 = 1
        label_1 = -1

        labels = (label_5, label_1)
        colors = {label_5: 'b', label_1: 'r'}
        values = {label_5: 'Number 5', label_1: 'Number 1'}

        plt.clf()

        # data set plot
        for number_label in labels:
                index = np.where(y == number_label)
                plt.scatter(x[index, 1], x[index, 2], c=colors[number_label], label=values[number_label])

        for i in range(len(multiple_w)):
                w = multiple_w[i]
                title = multiple_title[i]
                # regression line
                # x = 0
                symmetry_for_cero_intensity = -w[0]/w[2]

                #  x = 1, 0 = w0 + w1 * w2 * x2
                # then y = (-w0 - w1) /w2
                symmetry_for_one_intensity= (-w[0] - w[1])/w[2]

                #plotting order
                plt.plot([0, 1],
                         [symmetry_for_cero_intensity, symmetry_for_one_intensity],
                         #'k-',
                         label=(title+ ' regression'))

        if main_title :
                plt.title(main_title)
        plt.xlabel('Average intensity')
        plt.ylabel('Simmetry')
        plt.legend()
        plt.show()

### _____________ DATA ____________________

# Reading training data set
x, y = readData('datos/X_train.npy', 'datos/y_train.npy')
# Reading test data set
x_test, y_test = readData('datos/X_test.npy', 'datos/y_test.npy')

w_pseudoinverse = pseudoInverse(x, y)
print("\n___ Goodness of the Pseudo-inverse fit ___\n")
print("  Ein:  ", Error(x, y, w_pseudoinverse))
print("  Eout: ", Error(x_test, y_test, w_pseudoinverse))

evaluationMetrics (x,y,w_pseudoinverse, '\nEvaluating output training data set')
evaluationMetrics (x_test, y_test, w_pseudoinverse, '\nEvaluating output test data set')
plotResults(x,y,w_pseudoinverse, title = 'Pseudo-inverse')

print(f'\nThe weight vector is {w_pseudoinverse}')

STOP_EXECUTION_TO_SEE_RESULT()

print("\n___ Goodness of the Stochastic Gradient Descendt (SGD) fit ___\n")

batch_sizes =[1,32,200,len(y)] #batch sizes compared in the experiment

n_iterations = [50,300]
for iteration in n_iterations:
        multiple_w = [w_pseudoinverse]
        multiple_title = ['pseudo-inverse']

        for _batch_size in batch_sizes:
                w = sgd(x,y, eta = 0.01, max_iter = iteration, batch_size = _batch_size)

                _title = f'SGD, batch size {_batch_size}'
                print( '\n\t'+_title)
                print ("Ein: ", Error(x,y,w))
                print ("Eout: ", Error(x_test, y_test, w))
                evaluationMetrics (x,y,w, '\nEvaluating output training data set')
                evaluationMetrics (x_test, y_test, w, '\nEvaluating output test data set')
                #plotResults(x,y,w, title = _title)
                multiple_w.append(w)
                multiple_title.append(_title)

                STOP_EXECUTION_TO_SEE_RESULT()

        plotResultMultiplesLines(x,y,multiple_w,f'Comparative batch sizes, {iteration} iterations',multiple_title)

STOP_EXECUTION_TO_SEE_RESULT()

print ('Exercise 2\n')

#### EXPERIMIENTS ###########

## a)
print('\nEXPERIMENT (a) \n')
def simula_unif(N, d, size):
        ''' generate a trining sample of N  points
in the square [-size,size]x[-size,size]
'''
        return np.random.uniform(-size,size,(N,d))

### data

size_training_example = 1000
dimension = 2
square_half_size = 1

training_sample = simula_unif( size_training_example,
                               dimension,
                               square_half_size)

STOP_EXECUTION_TO_SEE_RESULT()

plt.clf()
plt.scatter(training_sample[:, 0], training_sample[:, 1], c='b')
plt.title('Muestra de entrenamiento generada por una distribución uniforme')
plt.title('Training sample generated by a uniform distribution')
plt.xlabel('$x_1$ value')
plt.ylabel('$x_2$ value')

plt.show()

STOP_EXECUTION_TO_SEE_RESULT()

## b)

print('\nEXPERIMENT (b) \n')
def f(x1, x2):
	return np.sign(
            (x1 -0.2)**2
            +
            x2**2 -0.6
        )

def noisyVector(y, percent_noisy_data):
        '''
        y target vector to introduce noise
        size_training_example: number of point generated in each experiment,
        percent_noisy_data
        '''
        len_y = len(y)

        index = list(range(len_y))
        np.random.shuffle(index)

        size_noisy_data = int((len_y*percent_noisy_data)/ 100 )

        noisy_y = np.copy(y)
        for i in index[:size_noisy_data]:
                noisy_y[i] *= -1
        return noisy_y

#labels
y = np.array( [f(x[0],x[1]) for x in training_sample ])

percent_noisy_data = 10.0

y = noisyVector(y,  percent_noisy_data)

## draw
labels = (1, -1)
colors = {1: 'blue', -1: 'red'}

plt.clf()

for l in labels:

	index = np.where(y == l)
	plt.scatter(training_sample[index, 0],
                    training_sample[index,1],
                    c=colors[l],
                    label=str(l))

plt.title('Labelled training sample before noise')
plt.xlabel('$x_1$ value')
plt.ylabel('$x_2$ value')
plt.legend()
plt.show()

STOP_EXECUTION_TO_SEE_RESULT()

plt.clf()

for l in labels:

	index = np.where(y == l)
	plt.scatter(training_sample[index, 0],
                    training_sample[index,1],
                    c=colors[l],
                    label=str(l))

plt.title('Labelled training sample after noise')
plt.xlabel('$x_1$ value')
plt.ylabel('$x_2$ value')
plt.legend()
plt.show()

STOP_EXECUTION_TO_SEE_RESULT()

#### C
print('\nEXPERIMENT (c) \n')
eta = 0.01
batch_size = 5
maximum_number_iterations = 1000

x = np.array( [
        np.array([ 1, x_n[0], x_n[1] ])
        for x_n in training_sample
])

y = np.array( [f(x_n[0],x_n[1]) for x_n in training_sample ])
y = noisyVector(y,  percent_noisy_data)
w = sgd_exact_number_iter(x, y, eta, maximum_number_iterations, batch_size )

_title = f'SGD, batch size {batch_size}'
print( '\n\t'+_title)
print ("Ein: ", Error(x,y,w))
evaluationMetrics (x,y,w, '\nEvaluating output training data set')

STOP_EXECUTION_TO_SEE_RESULT()

plt.clf()

for l in labels:

	index = np.where(y == l)
	plt.scatter(training_sample[index, 0],
                    training_sample[index,1],
                    c=colors[l],
                    label=str(l))

plt.title('Linear regression fit')
plt.xlabel('$x_1$ value')
plt.ylabel('$x_2$ value')

# regression line
# x_0 = -1
y_0 = (w[1] - w[0]) /w[2]
#x_1 = 1
y_1 = -( w[1]+w[0]) /w[2]

plt.plot([-1, 1], [y_0, y_1], 'k-', label=('SGD regression'))
plt.xlim([-1,1])
plt.ylim([-1,1])
plt.show()

STOP_EXECUTION_TO_SEE_RESULT()

## d
print('\n EXPERIMENT (d), lineal regression\n')

def experiment(featureVector,
               number_of_repetitions = 1000,
               size_training_example = 1000,
               percent_noisy_data = 10.0
               ):
        '''
        INPUT
        featureVector: function that return  np.array
        number_of_repetitions: experiment repetitions ,
        size_training_example: number of point generated in each experiment,
        percent_noisy_data:

        OUTPUT
        (error_in, error_out)
        '''
        total_in_error = 0
        total_out_error = 0

        for _ in range( number_of_repetitions):
        ## data generation
                training_sample = simula_unif( size_training_example,
                                               dimension,
                                               square_half_size)

                test_sample = simula_unif( size_training_example,
                                           dimension,
                                           square_half_size)
                test_y = np.array( [f(x[0],x[1]) for x in test_sample ])
                test_y = noisyVector(test_y, percent_noisy_data)

                y = np.array( [f(x[0],x[1]) for x in training_sample ])
                y = noisyVector(y,  percent_noisy_data)

                # fit
                x = np.array( [
                        featureVector(x_n)
                        for x_n in training_sample
                ])

                x_test = np.array( [
                        featureVector(x_n)
                        for x_n in test_sample
                ])

                w = sgd_exact_number_iter(x, y, eta, maximum_number_iterations, batch_size = 32)

                total_in_error += Error(x,y,w)
                total_out_error += Error(x_test, test_y, w)

        error_in = float(total_in_error / number_of_repetitions)
        error_out = float(total_out_error / number_of_repetitions)

        return error_in, error_out

def linearFeatureVector(x_n):
        return np.array( [
                1,
                x_n[0],
                x_n[1]
               ] )

_number_of_repetitions = 1000
error_in, error_out = experiment( linearFeatureVector,
                                  number_of_repetitions = _number_of_repetitions,
                                  size_training_example = 1000,
                                  percent_noisy_data = 10.0
               )
print(f'The mean value of E_in in all {_number_of_repetitions} experiments is: {error_in}')
print(f'The mean value of E_out in all {_number_of_repetitions} experiments is: {error_out}')

STOP_EXECUTION_TO_SEE_RESULT()

# e)

print('\nEXPERIMENT (e)\n' )
eta = 0.01
batch_size = 5
maximum_number_iterations = 1000

def quadraticFeatureVector(x_n):
        '''
        INPUT
         xn = (x1,x2) vector of coordinates

        '''
        return np.array([ 1,
                   x_n[0],
                   x_n[1],
                   x_n[0]*x_n[1],
                   x_n[0]* x_n[0],
                   x_n[1]* x_n[1]  ])

x = np.array( [
        quadraticFeatureVector(x_n)
        for x_n in training_sample
])
y = np.array( [f(x[0],x[1]) for x in training_sample ])
y = noisyVector(y,  percent_noisy_data)

for i in [10, 50, 100, 200, 500, 700, 1000]:
        maximum_number_iterations = i
        w = sgd_exact_number_iter(x, y, eta, maximum_number_iterations , batch_size = 17)

        print('\nFor one experiment:')
        _title = f'SGD, batch size {batch_size}, number iterations {maximum_number_iterations}'
        print( '\n',_title)
        print ("Ein: ", Error(x,y,w))
        evaluationMetrics (x,y,w, '\nEvaluating output training data set')
print('')

STOP_EXECUTION_TO_SEE_RESULT()

## plotting

def equation (x,y,w):
        '''
        INPUT
        x coordinate
        y coodinate
        w weights vector

        OUTPUT
        Real number, the scalar product of features vector dot weights vector
        '''
        return ( w[0,0]
                 + w[1,0] * x
                 + w[2,0] * y
                 + w[3,0] * x * y
                 + w[4,0] * x**2
                 + w[5,0] * y**2
                )
'''
PLOTING LINEAR REGRESSION

We are going to plot the (x,y) \in [-1,-1]^2 that their value after
the linear regression for classification  is near to 0.

That means that they are in the limit area.
'''
error = 10**(-2.1)
space = np.linspace(-1,1,100)

z = [[ equation(i,j,w) for i in space] for j in space ]

plt.contour(space,space, z, 0, colors=['black'],linewidths=2 )

for l in labels:

	index = np.where(y == l)
	plt.scatter(training_sample[index, 0],
                    training_sample[index,1],
                    c=colors[l],
                    label=str(l))

plt.title('Quadratic regression fit')
plt.xlabel('$x_1$ value')
plt.ylabel('$x_2$ value')
plt.legend( loc = 'lower left')
plt.show()

STOP_EXECUTION_TO_SEE_RESULT()

## EXPERIMENT
error_in, error_out = experiment( quadraticFeatureVector,
                                  number_of_repetitions = _number_of_repetitions,
                                  size_training_example = 1000,
                                  percent_noisy_data = 10.0
                                 )
print(f'The mean value of E_in in all {_number_of_repetitions} experiments is: {error_in}')
print(f'The mean value of E_out in all {_number_of_repetitions} experiments is: {error_out}')
print('==========================================')
STOP_EXECUTION_TO_SEE_RESULT()

"""
Exercise 3
Author: Blanca Cano Camarero
"""

def f(x,y):
    '''
    Function to minimize
    '''
    return np.float64(
        (x+2)**2 + 2*(y-2)**2 + 2* np.sin( 2* np.pi * x)* np.sin( 2* np.pi * y)
    )

def dfx(x,y):
    '''
    Partial derivate of f with respect to the variable x
    '''
    return np.float64(
        2*( x+2 ) + 4* np.pi* np.cos(  2* np.pi * x )* np.sin(2* np.pi * y)
    )

def dfy(x,y):
    '''
    Partial derivate of f with respect to the variable y
    '''
    return np.float64(
        4*( y-2 ) + 4* np.pi* np.cos(  2* np.pi * y )* np.sin(2* np.pi * x)

    )

def gradf(x,y):
    '''
        gradient of E
    '''
    return np.array([dfx(x,y), dfy(x,y)])

def ddfxx(x,y):
    return 2 - 8*np.pi**2*np.sin(2*np.pi*x)*np.sin(2*np.pi*y)

def ddfyy(x,y):

    return 4 - 8*np.pi**2*np.sin(2*np.pi*x)*np.sin(2*np.pi*y)

def ddfxy(x,y):
    return 8*np.pi**2*np.cos(2*np.pi*x)*np.cos(2*np.pi*y)

def hessianf(x, y):
    return np.array([
        ddfxx(x,y),
        ddfxy(x,y),
        ddfxy(x,y),
        ddfyy(x,y),
    ]).reshape((2, 2))

def newton_trace(initial_point, fun, grad_fun, hessian, eta, max_iter):
    """ Newton method
    INPUT
    - initial_point:
    - f: differential function
    - grad_fun: Gradient
    - hessian: hessian
    - eta: learning rate
    - max_iter: number of iterations

    OUTPUT
    w trace
    """

    w = initial_point
    w_list = [initial_point]
    iterations = 0

    while iterations < max_iter:
        w = w - eta *np.linalg.inv(hessian(w[0],w[1])).dot(grad_fun(w[0], w[1]))
        w_list.append(w)
        iterations += 1

    return np.array(w_list)

# Dicrease with the x-axis

## Exercise 1

######  conditions
smaller_eta = 0.01
bigger_eta = 0.1
max_iter = 50
initial_point = np.array([-1.0,1.0])

#### run
smaller_w = newton_trace( initial_point,f, gradf, hessianf, smaller_eta, max_iter)
bigger_w = newton_trace( initial_point,f, gradf, hessianf, bigger_eta, max_iter)

images_smaller_eta = [ f(x[0],x[1]) for x in smaller_w ]
images_bigger_eta = [ f(x[0],x[1]) for x in bigger_w]

print(f'With eta = {smaller_eta}, coordenates (x,y)= {(smaller_w[-1][0],smaller_w[-1][1])}, the number of iterations: {max_iter} and the image is f(x,y) = {images_smaller_eta[-1]}')

print(f'With eta = {bigger_eta}, coordenates (x,y)={(bigger_w[-1][0],bigger_w[-1][1])}, the number of iterations: {max_iter} and the image is f(x,y) = {images_bigger_eta[-1]}')

########### PLOTTING
x_label = 'Number of iterations'
y_label = 'f(x,y)'
bigger_eta_label = '$\eta$ = '+str(bigger_eta)
smaller_eta_label = '$\eta$ = '+str(smaller_eta)

## bigger eta ####
plt.clf()
plt.plot(images_bigger_eta, label=bigger_eta_label)
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.title(f"Newton's adjust of f with $\eta =$ {bigger_eta}")

plt.legend()
plt.show()

## smaller eta ###

plt.clf()
plt.plot(images_smaller_eta, label=smaller_eta_label)
plt.xlabel(x_label)
plt.ylabel(y_label)
plt.title(f"Newton's adjust of f with $\eta =$ {smaller_eta}")

plt.legend()
plt.show()

## comparation ###
plt.clf()
plt.plot(images_bigger_eta, label=r"$\eta$ = 0.1")
plt.plot( images_smaller_eta, label=r"$\eta$ = 0.01")

plt.xlabel('Iterations')
plt.ylabel('f(x,y)')

plt.title("Comparation of the newton's method for $f$ changing eta value  ")

plt.legend()
plt.show()

STOP_EXECUTION_TO_SEE_RESULT()

#### exercise 3.b
eta = 0.1
max_iter = 50
initial_points = map(np.array, [[-.5, -.5],[1, 1], [2.1, -2.1], [-3,3],  [-2, 2]])
minimum_value = np.Infinity

plt.clf()
print('{:^17}  {:^17}  {:^9}'.format('Initial', 'Final', 'Value'))

for initial in initial_points:

  w = newton_trace( initial,f, gradf, hessianf, eta, max_iter)
  local_minimum = w[-1]
  local_value = f(local_minimum[0], local_minimum[1])
  #plot
  images = [ f(x[0],x[1]) for x in w ]
  plt.plot(images, label=f'Initial point {initial}')

  print('{}  {}  {: 1.5f}'.format(initial, local_minimum, local_value))

plt.xlabel('Iterations')
plt.ylabel('f(x,y)')

plt.title("Comparation of the newton's method for $f$ changing initial point ")

plt.legend()
plt.show()

## without the biggest

initial_points = map(np.array, [[-.5, -.5],[1, 1], [-3,3],  [-2, 2]])
minimum_value = np.Infinity

plt.clf()
print('{:^17}  {:^17}  {:^9}'.format('Initial', 'Final', 'Value'))

for initial in initial_points:

  w = newton_trace( initial,f, gradf, hessianf, eta, max_iter)
  local_minimum = w[-1]
  local_value = f(local_minimum[0], local_minimum[1])
  #plot
  images = [ f(x[0],x[1]) for x in w ]
  plt.plot(images, label=f'Initial point {initial}')

  print('{}  {}  {: 1.5f}'.format(initial, local_minimum, local_value))

plt.xlabel('Iterations')
plt.ylabel('f(x,y)')

plt.title("Comparation of the newton's method for $f$ changing initial point  ")

plt.legend()
plt.show()

STOP_EXECUTION_TO_SEE_RESULT()

# let see gradient values
points =  [[-0.38067878, -0.52778221],
           [1.06677195, 0.91078249],
           [ 3.26077803, -3.11750721] ,
           [-2.,  2.]]

for x,y in points :

    print(f'For {(x,y)}')
    print(f'\tThe gradient value is {gradf(x,y)}')
    print(f'\tThe inverse hessian is {np.linalg.inv(hessianf(x,y))}\n')

STOP_EXECUTION_TO_SEE_RESULT()

\end{minted}


\section{Linear Algebra}

\begin{definition}[Basis]
    Given a set of vectors in $\mathbf{R}^{n}, V$, which is linearly
    independent, the set $V$ is a \textit{basis} if you can span
    $\mathbf{R}^{n}$ using $V$.
\end{definition}

\begin{definition}[Backward substituion]\label{backwardsubstituion}
    Opposite of~\nameref{forwardsubstituion}.
\end{definition}

\begin{definition}[Condition number]\label{conditionnumber}
    A measure of how the output value for a function changes respective to
    input variables.
\end{definition}

\begin{definition}[Column space]
    All linear combinations of the columns of a matrix $A$.
\end{definition}

\begin{definition}[Consistent]
    $\iff$ the rightmost column of an augmented matrix is not a pivot column.
    I.e.\ in a 3x4 matrix, if the last row is zero and 2nd column has pivot
    column, then the system is still consistent.
\end{definition}

\begin{definition}[Diagonal Dominance]
    $\forall{i}, \exists{i} A_{i, i}, \forall i, \iff A_{i,i}
    \geq \sum\limits_{j = 0, j\neq i}^{m}$,
    then matrix $A$ is DD.
\end{definition}

\begin{definition}[Diagonlizable matrix]
    Given a matrix $A$, we can express it as $A = PDP^{-1}$.
    Then, we can compute:
    \begin{align}
        A^{2} = (PDP^{-1})(PDP^{-1}) = PD(P^{-1}P)DP^{-1} = PDDP^{-1} \\
        \text{Because} P \cdot P{-1} = I \\
        \dots A^{k} = PD^{k}P^{-1}
    \end{align}

    So, we use diagonal matrices to easily raise matrices to power $k$.

    To diagonalize a matrix $A^{n\times{n}}$, it is required that it has $n$ linearly
    independent eigenvectors.
\end{definition}

\begin{definition}[Divergence]
    In vector calculus, divergence measures the magnitude to the
    gradient of~\nameref{vectorfield}s at a given point.

\end{definition}

\begin{definition}[Dotting matrices]
    \includegraphics[scale=0.3]{mm.png}
\end{definition}

\begin{definition}[Echelon form]
    A matrix where:
    \begin{enumerate}
        \item All nonzero rows are above zero-rows
        \item Each pivot column is placed from left to right
    \end{enumerate}
\end{definition}

\begin{definition}[Eigenvector]\label{eigen}
    Given a square matrix A, when A is multiplied with an eigenvector $v$,
    the resulting matrix A${^\prime}$ is a multiple of $v$.
    The multipe is denoted by $\lambda$ and is called an eigenvalue.
    So, $Av = \lambda v$

\end{definition}

\begin{definition}[Forward substituion]\label{forwardsubstituion}
    In an iterative method, if we first solve an element $A_{i,j}$,
    then when we caluclate some $A_{i+a, j+b}$ in the same iteration, then we
    use the updated value of $A_{i,j}$. Opposite
    of~\nameref{backwardsubstituion}

\end{definition}

\begin{definition}[Gaussian Elimination]
    AKA ``Row reduction''. Add row $x$ to row $y, y \neq x$, to reduce $y$ to
    zeroes.

\end{definition}

\begin{definition}[Ill-conditioned matrix]
    A matrix is ill-conditioned if the~\nameref{conditionnumber} is large.
    If you make a small change in an ill conditioned matrix, there are usually
    large differences in results from calculations with this maxtrix.

\end{definition}

\begin{definition}[Inner product]\label{innerprod}
    The product $u^{T}v$. Given that $u, v \in R^{i\times{j}}$, then we need to transpose
    to perform normal dot products.

    Inner products are commutative: $u \dot v \equiv v \dot u$.
\end{definition}

\begin{definition}[Inverse]
    For a matrix $A \in \mathbf{R}^{2x2}$:
    $$
    A^{-1} = \frac{1}{|A|}
    \begin{bmatrix}
    d & -b \\
    -c & a
    \end{bmatrix}
    = \frac{1}{ad - bc}
    \begin{bmatrix}
    d & -b \\
    -c & a
    \end{bmatrix}
    $$
    For a matrix $A \in \mathbf{R}^{3x3}$: bcacab

    Properties:
    \begin{itemize}
    \item $(A^{-1})^{-1} = A$
    \end{itemize}
\end{definition}

\begin{definition}[Kernel]
    For a vector space given by a~\nameref{lintrans}, a kernel is the set
    of vectors $v \in V$, s.t. $T(u) = 0$.
\end{definition}

\begin{definition}[Length of Vector]\label{vectorlength}
    For a vector
    \begin{align*}
        v = [a_{1}, a_{2}, \dots , a_{n}] \\
        |v| = \sqrt{a^{2}_{1} + a^{2}_{2} + \dots + a^{2}_{n}}{}
    \end{align*}
\end{definition}

\begin{definition}[Linear dependence]
    For $V = {v_{1}, \dots, v_{n}}$, and $\forall v, v is vector$,
    if none of the vectors in V can be written as a linear combination
    from the other vectors in V, the set is linearly independent.
\end{definition}

\begin{definition}[Linear transformation]\label{lintrans}
    Take a vector space into another, s.t.\
    \begin{align}
        T(u + v) = T(u) + T(v) &\forall u,v \in v \\
        T(cu) = cT(u) &\forall u \in V
    \end{align}
\end{definition}

\begin{definition}[Matrix of observations]
    Collect a sample, subject it to tests, and for each test, you give a value.
    For $n$ tests and $m$ samples, you will result in $\Re^{n\times{m}}$
    \textit{matrix of observations}.
\end{definition}

\begin{definition}[Norm]
    The ``length'' of a vector, just remember to sum up over all dimensions.
    Each number is raised to $n$, and the total sum is then raised to

    \begin{description}
        \item[L1]
        \item[L2] Can be defined as:
            \begin{align}
                \phi^{2} = \phi \times \phi =
                \dots = \int{\phi (x)^{2}  dx} \\
                |x| = \sqrt{ \sum\limits_{k = 1}^{n}{x_{k}|^{2}}}
            \end{align}
    \end{description}

    $\frac{1}{p}$.
\end{definition}

\begin{definition}[Normalization of vectors]
    $ \hat{X} \equiv \frac{X}{|X|} $, where $|x|$ is the~\nameref{vectorlength}
    $X$ is, in this case, evaluated as the additive sum of it's entries.
\end{definition}

\begin{definition}[Null space]
    Given a matrix $A$, if you solve for that each row = 0,
    all possible values for each $x$ makes out the Null space.

    Note that here it is apossible to get free variables for
    some x, and bound to others.

\end{definition}

\begin{definition}[Numerical stability]
    This concept describes how changes in the input should not affect the result.
    An example could be that sorting a list should not change the outcome from
    applying a function to the input.
\end{definition}

\begin{definition}[Orthogonal]\label{orthogonal}
    Two lines that intersect each other at 90 degrees.\\
    \begin{itemize}
        \item Orthogonal matrices preserve dot products:
        given two vectors $u$, and $v$, and an orthogonal matrix Q,
        the following is true:
        $u \times v = Qu \times Qv$
        \item The determinant of an orthogonal matrix is always 1 or -1
        \item The transpose of $Q$ is equal to it's inverse, hence:
            $Q \times Q^{T} = I$
    \end{itemize}
\end{definition}

\begin{definition}[Orthonormal]
    In~\nameref{innerprod} space are orthonormal if they are
    orthogonal and unit vectors.
\end{definition}

\begin{definition}[Perpendicular]
    Similar to~\nameref{orthogonal}, but with lines.
\end{definition}

\begin{definition}[Pivoting]
    Finding the first non-zero element in an algorithm on linear systems.
    To do so, you can do things like gaussian elimination, etc.
    \begin{description}
        \item[Complete pivoting] find the largest absolute value of a pivot,
            considering all elements.
        \item[Partial pivoting] finds the largest absolute value in the pivot
            column.
        \item[Scaled pivoting] finds the largest absolute value of a pivot
            column relative to it's entries in the same row.
    \end{description}
\end{definition}

\begin{definition}[Plane]
    A flat, two-dimensional surface
\end{definition}

\begin{definition}[Positive semidefninite matrix]
    Properties:
    \begin{itemize}
        \item Nonnegative~\nameref{eigen}s
        \item $X = V^{T}V$ for some $V \in \mathbf{R}^{mxn}$
        \item $X = \sum\limits_{i=1}^{m}\lambda_{i}w_{i}w^{t}_{i}$
            for some $\lambda_{i} \geq 0$ and vectors $w_{i} \in \mathbf{R}^{n}$
            such that $w^{T}_{i}w = 1$ and $w^{T}_{i}w_{j} = 0$

    \end{itemize}
\end{definition}

\begin{definition}[Preconditoner]
    A matrix $P$ such that $P^{-1}A$ has a smaller~\nameref{conditionnumber}
    than $A$.

\end{definition}

\begin{definition}[Projection]
    define a vector $v$ and $u$.
    \begin{align*}
        L &= \left\{cv | c \in \mathbf{R} \right\}  \\
        proj(v) &= l \in L \text{\ such that\ } u - proj(v)
        \text{\ is~\nameref{orthogonal} to l}  \\
        \textit{I.e., } proj(v) &= cv, c \in \mathbf{R}
    \end{align*}

    Properties:
    \begin{itemize}
        \item $proj(v) = proj(v)^{2}$
        \item Linear independence on $u, v$ also relates for $v - proj(v), u$
        \item adding $proj(v)$ to $v$ gives you $u$
    \end{itemize}

\end{definition}

\begin{definition}[Relaxation methods]
     relaxation methods are iterative methods for solving systems of equations,
     including nonlinear systems. Examples are Gauss-Seidel, Jacboi, etc.

\end{definition}

\begin{definition}[Singular matrix]
    Any square matrix without an inverse.
    A matrix is singular $\iff$ its determinant is 0.

\end{definition}

\begin{definition}[Span of vectors]\label{vectorspan}
    All linear combinations of a set of vectors.
    \begin{align*}
        V &= \left\{v_{1}, \dots, v_n\right\} \\
        C &= \left\{c \in C | \mathbf{R}\right\} \\
        span &= c_{1}v_{1} + \dots + c_{n}v_{n}
    \end{align*}
\end{definition}

\begin{definition}[Spectrum of a matrix]
    The multiset of its eigenvalues
\end{definition}

\begin{definition}[Spectral Radius]
    The largest absolute eigenvalue of a matrix.
\end{definition}

\begin{definition}[Successive Over Relaxation]
    Gauss-Seidel is the same as SOR (successive over-relaxation) with $\omega=1$
\end{definition}

\begin{definition}[Symmetric]
    \begin{itemize}
        \item Length of rows is equal
        \item The transpose is equal to the originl
    \end{itemize}
\end{definition}

\begin{definition}[Tensor]
    Geometric objects that describe linear relations between vectors or scalars.
\end{definition}

\begin{definition}[Toeplitz matrix]
    $$
    \begin{bmatrix}
    a & b & c & d \\
    b & a & b & c \\
    c & b & a & b \\
    d & c & b & a
    \end{bmatrix}
    $$

    Noteworthingly, this kind of matrix can be inverted in $O(n \times \log{n})$
\end{definition}

\begin{definition}[Trace]
    Sum of all diagonal entries in a matrix.
    $A_{11} + A_{22} + A_{nn}$
\end{definition}

\begin{definition}[Transpose]
    Take column $i$ and make it into a column. Repeat.
\end{definition}

\begin{definition}[Unit vector]
    A vector who's length is 1.
\end{definition}

\begin{definition}[vector length]
    number of "steps" in a vector
\end{definition}


\documentclass{article}

\include{stddefs}
\include{imodefs}

\begin{document}

\chapterno{8}
\chapter{The Hessian}

In Chapter \ref{chapter:convexfunctions} we exploited the second derivative
$f''(x)$ of a one variable real function $f:(a, b) \rightarrow \RR$ to analyze
convexity along with local minima and maxima.

In this chapter we introduce an analogue of the second derivative for real functions $f:\RR^n\rightarrow \RR$ of
several variables. This will be an $n\times n$ matrix. The important notion of a matrix
being positive (semi-) definite introduced in Section \ref{Sectionsymmat} will now
make its appearance.

\section{Introduction}

In Section \ref{section:Taylor} the Taylor expansion for a one variable differentiable
function $f:\RR\rightarrow \RR$ centered a $x_0$ with step size
$h  = x - x_0$ was introduced as
\begin{equation}\label{taylor}
f(x_0 + h) = f(x_0) + f'(x_0) h + \frac{1}{2} f''(x_0) h^2 + \cdots
\end{equation}

Recall that the second derivative $f''(x_0)$ contains a
wealth of information about the function. Especially if $f'(x_0) = 0$,
then we might glean from $f''(x_0)$ if $x_0$ is a local maximum or
minimum or none of these (see Theorem \ref{derconv} and review Exercise \ref{exderconv}).

We also noticed that gradient descent did not work so well only
descending along the gradient. We need to take the second
derivative into account to get a more detailed picture of the
function.

\section{Several variables}

Our main character is a differentiable function $F:\RR^n\rightarrow \RR$ in
several variables. We already know that
$$
F(x_0 + h) = F(x_0) + \nabla F(x_0) h + \epsilon(h) \abs{h},
$$
where $x_0$ and $h$ are vectors in $\RR^n$ (as opposed to the good old
numbers in \eqref{taylor}). Take a look back at Definition \ref{diffdef} for
the general definition of differentiability.

We wish to have an analogue of the Taylor expansion in \eqref{taylor} for
such a function of several variables. To this end we introduce the function
$g:\RR\rightarrow \RR$ given by
\begin{equation}\label{onedimg}
g(t) = F(x_0 + t h).
\end{equation}
Notice that
$$
g(t) = (F\circ A)(t),
$$
where $A: \RR\rightarrow \RR^n$ is the function given by $A(t) = x_0 + t h$. In particular
we get
\begin{equation}\label{chrule1}
g'(t) = F'(x_0 + t h) h = \nabla F(x_0 + t h) h
\end{equation}
by using the chain rule (see Theorem \ref{chainrule}).

\beginshex
Explain how the chain rule is applied to get \eqref{chrule1}.
\endshex

The derivative $g'(t)$ is also composed of several functions and again we may
compute $g''(t)$ by using the chain rule:
\begin{align}
  g''(t) &= (C\circ B \circ A)'(t)\\
        &= (C\circ B)'(A(t)) A'(t) \\
        &= C'(B(A(t))) B'(A(t)) A'(t),
\end{align}
where $B: \RR^n\rightarrow \RR^n$ is defined by
$$
B(v) = \nabla F(v)^T
$$
and $C:\RR^n \rightarrow \RR$ by
$$
C(v) = v^T h.
$$

\begin{definition}[emph]

The \emph{Hessian matrix} of $F$ at the point
$x\in \RR^n$ is defined by

\begin{equation*}
  \nabla^2 F(x) :=
  \begin{pmatrix}
    \dfrac{ \partial^2 F}{ \partial x_1 \partial x_1}(x) &
    \cdots & \dfrac{ \partial^2 F}{ \partial x_1 \partial
      x_n}(x)
    \\
    \vdots & \ddots & \vdots
    \\
    \dfrac{ \partial^2 F}{ \partial x_n \partial x_1}(x) &
    \cdots & \dfrac{\partial^2 F}{ \partial x_n\partial
      x_n}(x)
  \end{pmatrix}
  .

\end{equation*}
\end{definition}

A very important observation is that $\nabla^2 F(x)$ above is a
symmetric matrix. Again, you should review what this means by clicking
back to Section \ref{Sectionsymmat}.

\beginshex
Why is the Hessian matrix symmetric?
\endshex

\begin{example}\label{sagegradhess}
  Suppose that $f: \RR^2\rightarrow \RR$ is given by
  $$
  f(x, y) = \sin(x y) + x^2 y^2 + y.
  $$
  Then the gradient
  $$
  \nabla f = \left(\frac{\partial f}{\partial x}, \frac{\partial f}{\partial y} \right)
  $$
  and the Hessian

  $$
  \nabla^2 f =
  \begin{pmatrix}
    \dfrac{\partial^2 f}{\partial x^2} & \dfrac{\partial^2 f}{\partial x \partial y} \\
    \\
    \dfrac{\partial^2 f}{\partial y \partial x} & \dfrac{\partial^2 f}{\partial y^2}
  \end{pmatrix}
  $$
  of $f$ are computed in the Sage window below.

  \begin{sage}
x, y = var('x, y')
f = sin(x*y) + x^2*y^2 + y
print("gradient = ", f.gradient())
print("Hessian = ", f.hessian())
  \end{sage}

See the \url{further documentation}{http://doc.sagemath.org/html/en/reference/calculus/sage/calculus/functions.html} for Calculus functions in Sage.
\end{example}

\beginshex
Verify (just this once) by hand the computations done by Sage in Example \ref{sagegradhess}.

Also, experiment with a few other functions in the Sage window and compute their
Hessians.
\endshex

By applying Proposition \ref{proppd} it is not too hard to see that the Hessian
matrix fits nicely into the framework above, since
\begin{equation}\label{exhess}
B'(v) = \nabla^2 F(v).
\end{equation}

The full application of the chain rule then gives
\begin{equation}\label{chrule2}
g''(t) = h^T \nabla^2 F(x_0 + t h) h.
\end{equation}

\beginshex
Give a detailed explanation as to why \eqref{exhess} holds.
\endshex

\section{Newton's method for finding critical points}

We may use Newton's method for computing critical points for a function
$F:\RR^n\rightarrow \RR$ of several variables. Recall that a
critical point is a point $x_0\in \RR^n$ with $\nabla F(x_0) = 0$.
By \eqref{onestepmultnewt} and \eqref{exhess} the computation in Newton's method becomes
\begin{equation}\label{newthess}
x_1 = x_0 - \left(\nabla^2 F(x_0)\right)^{-1} \nabla F(x_0).
\end{equation}
In practice
the (inverse) Hessian appearing in \eqref{newthess} is often a heavy
computational burden. This leads to the socalled
\url{quasi-Newton methods}{https://en.wikipedia.org/wiki/Quasi-Newton_method}, where
the inverse Hessian in \eqref{newthess} is replaced by other matrices.

\begin{example}\label{Exbadlogist}
We will return to the logistic regression in Example \ref{challengerexample} about the
Challenger disaster. Here we sought to maximize the function
  \begin{equation}\label{logtwo}
  \ell(\alpha, \beta) = \sum_{i=1}^m E_i (\alpha + \beta x_i) - \log(1 + e^{\alpha + \beta x_i}).
  \end{equation}

In order to employ Newton's method we compute the gradient and the Hessian of \eqref{logtwo}
\begin{align}\label{hesslogist}
  \frac{\partial \ell}{\partial \alpha} &= \sum_{i=1}^m E_i - \sigma(\alpha + \beta x_i)\\
  \frac{\partial \ell}{\partial \beta} &= \sum_{i=1}^m E_i x_i - x_ i\sigma(\alpha + \beta x_i)\\
  \frac{\partial^2 \ell}{\partial \alpha^2} &= \sum_{i=1}^m - \sigma'(\alpha + \beta x_i)\\
  \frac{\partial^2 \ell}{\partial \beta \partial \alpha} &= \sum_{i=1}^m -\sigma'(\alpha + \beta x_i) x_i\\
  \frac{\partial^2 \ell}{\partial \beta^2} &= \sum_{i=1}^m - \sigma'(\alpha + \beta x_i) x_i^2,
\end{align}
where
$$
\sigma(t) = \frac{1}{1 + e^{-t}}
$$
is the sigmoid function.

Notice the potential problem in using Newton's method here: the formula for
the second order derivatives in \eqref{hesslogist} show that if the $\alpha +\beta x_i$ are just mildly big, say $\geq 50$, then
the Hessian is extremely close to the zero matrix and therefore Sage considers it non-invertible and
\eqref{newthess} fails.

In the code below we have nudged the initial vector so that it works, but you
can easily set other values and see its failure. Optimization is not just mathematics, it also calls
for some good (engineering) implementation skills (see for example details on the
\url{quasi Newton algorithms}{https://en.wikipedia.org/wiki/Quasi-Newton_method}).

In the instance below we do, however, get a gradient that is pratically $(0, 0)$.

\begin{hideinbutton}{Code for Newton's method}
\begin{sage}
x0 = [11,-0.2]
noofits = 10

x = [53.0, 56.0, 57.0, 63.0, 66.0, 67.0, 67.0, 67.0, 68.0, 69.0, 70.0, 70.0, 70.0, 70.0, 72.0, 73.0, 75.0, 75.0, 76.0, 76.0, 78.0, 79.0, 80.0, 81.0]
E = [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]

Es = sum(E)
Exs = sum(u*v for (u, v) in zip(x, E))

def sigmoid(a, b, x):
  s = 1/(1 + exp(-a - b*x))
  return s.n()

def sigmoidm(a, b, x):
  s = sigmoid(a, b, x)
  return s*(1-s)

def gradient(v):
  a = v[0]
  b = v[1]
  return vector((Es - sum(sigmoid(a, b, t) for t in x), Exs - sum(t*sigmoid(a, b, t) for t in x)))

def hessian(v):
  a = v[0]
  b = v[1]
  h11 = - sum(sigmoidm(a, b, t) for t in x)
  h12 = - sum(sigmoidm(a, b, t)*t for t in x)
  h22 = - sum(sigmoidm(a, b, t)*t*t for t in x)
  return matrix([[h11, h12], [h12, h22]])

def newtonstep(v0):
  d = gradient(v0)
  h = hessian(v0)
  return v0 - h.inverse()*d

v = vector(x0)
for k in range(noofits):
  print(v)
  v = newtonstep(v)

print("x0 = ", x0)
print("Number of Newton iterations =  ", noofits)
print("Predicted maximal point =  ", v)

alpha = v[0]
beta = v[1]

print("Gradient at predicted maximal point = ", gradient(v))
print("Predicted probability of failure at 31F = ", sigmoid(alpha, beta, 31))
\end{sage}
\end{hideinbutton}
\end{example}

\subsection{Transforming data for better numerical performance}

The numerical problems with Newton's method in Example \ref{Exbadlogist} can be prevented by transforming the input data.
It makes sense to transform data from large numbers to
smaller numbers around $0$. There is a rather standard way of doing this.

Suppose in logistic regression we have a set of data
\begin{equation}\label{datapts}
x_1, x_2, \dots, x_n
\end{equation}
associated with outcomes $E_1, \dots, E_n$. Then the function
  $$
  \ell(\alpha, \beta) = \sum_{i=1}^m E_i (\alpha + \beta x_i) - \log(1 + e^{\alpha + \beta x_i}).
  $$
  from Example \ref{exlogist} becomes much more manageable if we first
  transform the data according to
  $$
  x'_i = \frac{x_i - \overline{x}}{\sigma}
  $$
  and instead optimize the function
  $$
  \ell'(\alpha, \beta) = \sum_{i=1}^m E_i (\alpha + \beta x'_i) - \log(1 + e^{\alpha + \beta x'_i}).
  $$
  Here
  $$
  \overline{x} = \frac{x_1 + x_2 + \cdots + x_n}{n}
  $$
  is the mean value and
  $$
  \sigma^2 = \frac{(x_1 - \overline{x})^2 + (x_2 - \overline{x})^2 + \cdots + (x_n - \overline{x})^2}{n}
  $$
  the variance of the data in \eqref{datapts}.

  Now if $\alpha'$ and $\beta'$ is an optimum for $\ell'$, then
  \begin{align}\label{transformtrick}
    \alpha &= \alpha' - \frac{\overline{x}}{\sigma} \beta'\\
    \beta &= \frac{\beta'}{\sigma}
  \end{align}
  is an optimum for $\ell$, since
  $$
  \ell'(\alpha, \beta) = \ell\left(\alpha - \beta \frac{\overline{x}}{\sigma}, \frac{\beta}{\sigma}\right).
  $$

\beginshex
Why is the claim/trick alluded to in \eqref{transformtrick} true?

Below is a snippet of Sage code implementing the trick in \eqref{transformtrick}.
The function \texttt{test} takes as input \texttt{x0} (an initial vector like \texttt{[0,0]}) and
\texttt{noofits} (the number of iterations of Newton's method). You execute this in the Sage
window by adding for example
\begin{code}
  test([0,0], 10)
\end{code}
and then pressing \texttt{Compute}.

Experiment and compare with the official output from Example \ref{challengerexample}. Also, compute
the gradient of the output below for the original non-transformed problem.

\begin{hideinbutton}{Transformed code}
\begin{sage}
x = [53.0, 56.0, 57.0, 63.0, 66.0, 67.0, 67.0, 67.0, 68.0, 69.0, 70.0, 70.0, 70.0, 70.0, 72.0, 73.0, 75.0, 75.0, 76.0, 76.0, 78.0, 79.0, 80.0, 81.0]
E = [1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
mean = sum(x)/len(x)
std = sqrt(sum(map(lambda t: (t-mean)^2, x))/len(x))

x = list(map(lambda t: (t-mean)/std, x))

Es = sum(E)
Exs = sum(u*v for (u, v) in zip(x, E))

def sigmoid(a, b, x):
  s = 1/(1 + exp(-a - b*x))
  return s.n()

def sigmoidm(a, b, x):
  s = sigmoid(a, b, x)
  return s*(1-s)

def gradient(v):
  a = v[0]
  b = v[1]
  return vector((Es - sum(sigmoid(a, b, t) for t in x), Exs - sum(t*sigmoid(a, b, t) for t in x)))

def hessian(v):
  a = v[0]
  b = v[1]
  h11 = - sum(sigmoidm(a, b, t) for t in x)
  h12 = - sum(sigmoidm(a, b, t)*t for t in x)
  h22 = - sum(sigmoidm(a, b, t)*t*t for t in x)
  return matrix([[h11, h12], [h12, h22]])

def newtonstep(v0):
  d = gradient(v0)
  h = hessian(v0)
  return v0 - h.inverse()*d

def test(x0, noofits):
  v = vector(x0)
  for k in range(noofits):
    print(v)
    v = newtonstep(v)
  print("Number of Newton iterations =  ", noofits)
  print("Predicted maximal point in transformed problem =  ", v)
  tv =   matrix([[1, -mean/std], [0, 1/std]])*v
  print("Predicted maximal point in original problem =  ", tv)

\end{sage}
\end{hideinbutton}
\endshex

\section{The Taylor series in several variables}

Now we are in a position to state at least the first terms in the
Taylor expansion for a differentiable function $F:\RR^n\rightarrow \RR$.
The angle of the proof is to reduce to the one-dimensional case through
the function $g(t)$ defined in \eqref{onedimg}. Here
one may prove that

\begin{equation}\label{taylorg}
g(t) = g(0) + g'(0) t + \frac{1}{2} g''(0) t^2 + \epsilon(t)t^2,
\end{equation}
where $\epsilon(0) = 0$ with $\epsilon$ continuous at $0$, much like
in the definition of differentiability except that we also include
the second derivative.

Now \eqref{taylorg} translates into

\begin{equation}\label{critminmax}
  F(x_0 + t h) = F(x_0) + \left(\nabla F(x_0) h \right) t + \frac{1}{2} \left( h^T \nabla^2 F(x_0) h \right) t^2 + \epsilon(t) t^2
\end{equation}
by using \eqref{chrule1} and \eqref{chrule2}.

From \eqref{critminmax} one reads the following nice criterion, which may be viewed as a several variable generalization of Theorem \ref{derconv}.

\begin{theorem}[emph]\label{thmcritminmax}
  Let $x_0$ be a critical point for $F:\RR^n\rightarrow \RR$. Then
  \begin{enumerate}[(i)]
  \item
    $x_0$ is a local minimum if $\nabla^2 F(x_0)$ is positive definite.
  \item
    $x_0$ is a local maximum if $-\nabla^2 F(x_0)$ is positive definite (here we call
    $\nabla^2 F(x_0)$ negative definite).
  \item\label{thmcritminsaddle}
    $x_0$ is a \url{saddle point}{https://en.wikipedia.org/wiki/Saddle_point} if $\nabla^2 F(x_0)$ is indefinite.
  \end{enumerate}
\end{theorem}

\begin{remark}
  We need to clarify two things concerning the above theorem.

  \begin{enumerate}[(i)]
  \item
    An $n\times n$ indefinite matrix is a symmetric matrix $A$ with the property that there exists
    $u, v\in \RR^n$ with
    \begin{align*}
      u^T A u &>0\quad\text{and}\\
      v^T A v &<0
    \end{align*}
  \item
    A saddle point $x_0$ for $F$ is defined by the existence of two vectors $u, v\in \RR^n$, such that
    \begin{align*}
      &t=0\quad \text{is a local minimum for the function}\quad f(t) = F(x_0 + t u)\\
      &t=0\quad \text{is a local maximum for the function}\quad  g(t) = F(x_0 + t v)
    \end{align*}
    as illustrated in the graphics below.

    \includegraphics{saddle.png}
    \end{enumerate}
\end{remark}

\begin{example}\label{examplehesscrit}
Consider, with our new technology in Theorem \ref{thmcritminmax}, Exercise \ref{exsaddlept} once again.
Here we analyzed the point $v_0 = (0, 0)$ for the function
$$
f(x, y) = x^3 + x y + y^3
$$
and showed (by a trick) that $v_0$ is neither a local maximum nor a local minimum for $f$. The Hessian
matrix for $f(x, y)$ at $v_0$ is
$$
H = \begin{pmatrix} 0 & 1 \\ 1 & 0 \end{pmatrix}.
$$

Now
Theorem \ref{thmcritminmax}\ref{thmcritminsaddle} shows that $v_0$ is a saddle point, since
$$
\begin{pmatrix} x & y \end{pmatrix} H \begin{pmatrix} x \\ y \end{pmatrix} = 2 x y
$$
and
\begin{align*}
u^T H u &> 0\qquad\text{for } u = \begin{pmatrix} 1 \\ 1 \end{pmatrix}\\
v H v &< 0\qquad\text{for } v =  \begin{pmatrix} 1 \\ -1 \end{pmatrix}.
\end{align*}

\begin{sage}
x, y = var('x, y')
a = 1
plot3d(x^3 + a*x*y+ y^3, (x, -0.4, 0.4), (y, -0.4, 0.4),
adaptive=True, color=rainbow(60, 'rgbtuple'))
\end{sage}
\end{example}

\beginshex
Try plotting the graph for \footnote{different values of \texttt{a}}{\texttt{a=4} shows the saddle point clearly.} in the Sage window in
Example \ref{examplehesscrit}. What do you observe for the point $v_0$ with
respect to the function? Does \texttt{a} have to be a number? Could it be a symbolic
expression in the variables \texttt{x} and \texttt{y} like \texttt{a = -10*cos(x)*sin(y)}?
\endshex

\beginshex
Check the computation of the Hessian matrix $H$ in Example \ref{examplehesscrit} by showing
that the Hessian matrix for $f$  at the point $(x, y)$ is
$$
\begin{pmatrix}
6 x & 1\\
1 & 6 y
\end{pmatrix}.
$$
\endshex

\beginshex
What about $u$ and $v$ in Example \ref{examplehesscrit}? How do they relate to the hint
given in Exercise \ref{exsaddlept}?
\endshex

\beginshex
  Give an example of a function $F:\RR^2\rightarrow \RR$ having a local minimum at
  $x_0$, where $\nabla^2 F(x_0)$ is not positive definite.
\endshex

\beginshex

The following exercise is a \texttt{sci2u} exercise from the Calculus book.

\begin{enumerate}[(i)]
\item
  The point $\left(0, \frac{\sqrt{3}}{3}\right)$ is a critical point for
  $$
  f(x, y) = x^3 + y^3 - y.
  $$
  What does Theorem \ref{thmcritminmax} say about this point?
\item
  The point $\left(\frac{1}{3}, \frac{1}{3}\right)$ is a critical point for
  $$
  f(x, y) = -x^3 -x^2 + x - y^3 + 2 y^2 - y.
  $$
  What does Theorem \ref{thmcritminmax} say about this point?
\item
  The point $(0, 1)$ is a critical point for
  $$
  f(x, y) = x^3 - x^2 + y^3 - y^2 - y.
  $$
  What does Theorem \ref{thmcritminmax} say about this point?
\end{enumerate}
\endshex

\beginshex
Consider the function
$$
f(x, y) = x^4 y^2 + x^2 y^4 - 3 x^2 y^2.
$$
Compute its critical points and decide on their types according to Theorem \ref{thmcritminmax}.
Try to convince yourself that
$$
f(x, y) \geq -1
$$
for every $x, y\in \RR$.
\endshex

\beginshex
Give an example of a function $f:\RR\rightarrow \RR$ that has a local maximum, but where
there exists $x\in \RR$ with $f(x) > M$ for any given (large) number $M$.
\endshex

\section{Convex functions of several variables}

Below is the generalization of Theorem \ref{fp} to several
variables. You have already seen this in an exercise in the
previous chapter, right?

\begin{theorem}[emph]\label{thmdiffsubdif}
  Let $f: U\rightarrow \RR$ be a differentiable function, where
  $U\subseteq \RR^n$ is an open convex subset. Then $f$ is convex if
  and only if
  \begin{equation}
    f(x) \geq f(x_0) + \nabla f(x_0) (x-x_0)\label{subgr}
  \end{equation}
  for every $x, x_0\in U$.
\end{theorem}
\begin{hideinbutton}{Proof}
  Suppose that \eqref{subgr} holds and let $x_t = (1-t)x_0 + t x$ with
  $0 \leq t \leq 1$, where $x_0, x\in U$.  To prove that $f$ is convex
  we must verify the inequality
  \begin{equation}
    f(x_t) \leq (1-t) f(x_0) + t f(x).\label{subdineq}
  \end{equation}
  Let $\xi = \nabla f(x_t)$. Then
  \begin{align*}
    f(x) &\geq f(x_t) + \xi (1-t) (x-x_0)\\
    f(x_0) &\geq f(x_t) - \xi t (x-x_0)
  \end{align*}
  by \eqref{subgr}.  If you multiply the first inequality by $t$, the
  second by $1-t$ and then add the two, you get \eqref{subdineq}.

  Suppose on the other hand that $f$ is a convex function. Let $x_0,
  x\in U$. Since $U$ is an open subset, it follows that $(1-t)x_0 + t
  x\in U$ for $t\in I=(-\delta, 1 + \delta)$, where $\delta>0$ is
  sufficiently small. Now define the function $g:I\rightarrow \RR$ by
  \begin{equation*}
    g(t) = f((1-t) x_0 + t x) = f(x_0 + t (x-x_0)).
  \end{equation*}
  Being the composition of two differentiable functions, $g$ is
  differentiable.  Suppose that $0\leq \alpha \leq 1$ and $t_1, t_2\in
  I$. Then
  \begin{align*}
   g((1- \alpha) t_1 + \alpha t_2) &= f(x_0 + ((1-\alpha)
    t_1 + \alpha t_2)(x-x_0))
    \\
    &=f((1-\alpha)(x_0 + t_1(x-x_0)) + \alpha (x_0+t_2(x-x_0)))
    \\
    &\leq(1-\alpha) f(x_0 + t_1 (x-x_0)) + \alpha f(x_0 + t_2(x-x_0))
    \\
    &=(1-\alpha) g(t_1) + \alpha g(t_2)
  \end{align*}
  showing that $g$ is a convex function.  By Theorem \ref{fp},
  \begin{equation*}
    g(1) \geq g(0) + g'(0),
  \end{equation*}
  which translates into
  \begin{equation*}
    f(x) \geq f(x_0) + \nabla f(x_0) (x-x_0)
  \end{equation*}
  by using the chain rule in computing $g'(0)$.
\end{hideinbutton}

\beginshex
  Prove that a bounded convex differentiable function $f:\RR^n\rightarrow \RR$ is
  constant.
\endshex

The following is the generalization of  Corollary \ref{corfdp}.

\newcommand{\Hess}[1]{\nabla^2 #1}

\begin{theorem}[emph]\label{thmhessconv}
  Let $f: U\rightarrow \RR$ be a differentiable function with continuous
  second order partial derivatives, where $U\subseteq \RR^n$ is a
  convex open subset. Then $f$ is convex if and only if the Hessian
  $\Hess{f}(x)$ is positive semidefinite for every $x\in U$. If
  $\Hess{f}(x)$ is positive definite for every $x\in U$, then $f$ is
  strictly convex.
\end{theorem}
\begin{hideinbutton}{Proof}
  We have done all the work for a convenient reduction to the one
  variable case. Suppose that $f$ is convex. Then the same reasoning
  as in the proof of Theorem \ref{thmdiffsubdif} shows that
  \begin{equation*}
    g(t) = f(x + t v)
  \end{equation*}
  is a convex function for every $x\in U$ and every $v\in \RR^n$ from
  an open interval $(-\delta, \delta)$ to $\RR$ for suitable
  $\delta>0$. Therefore $g''(0) = v^t \Hess{f}(x) v \geq 0$ by
  Theorem \ref{fdp}. This proves that the matrix $\Hess{f}(x)$ is
  positive semidefinite for every $x\in U$.  Suppose on the other hand
  that $\Hess{f}(x)$ is positive semidefinite for every $x\in U$.
  Then Theorem \ref{fdp} shows that $g(t) = f(x + t(y-x))$ is a convex
  function from $(-\delta, 1+\delta)$ to $\RR$ for $\delta>0$ small
  and $x, y\in U$, since
  \begin{equation*}
    g''(\alpha) = (y-x)^t \Hess{f}(x + \alpha(y-x)) (y-x) \geq 0
  \end{equation*}
  for $0\leq \alpha \leq 1$. Therefore $f$ is a convex function, since
  \begin{align*}
    f((1-t) x + t y) = &g((1-t)\cdot 0 + t\cdot 1) \\
    &\leq(1-t) g(0) + t g(1) = (1-t) f(x) + t f(y).
  \end{align*}
  The same argument (using the last part of Theorem \ref{fdp} on
  strict convexity), shows that $g$ is strictly convex if
  $\Hess{f}(x)$ is positive definite. It follows that $f$ is strictly
  convex if $\Hess{f}(x)$ is positive definite for every $x\in U$.
\end{hideinbutton}

\beginshex
  Prove that
  \begin{equation*}
    f(x, y) = x^2 + y^2
  \end{equation*}
  is a strictly convex function from $\RR^2$ to $\RR$. Also, prove that
  $$
  \{(x, y)\in \RR^2 \mid x^2 + y^2 \leq 1\}
  $$
  is a convex subset of $\RR^2$.
  \endshex

\beginshex
  Is $f(x, y) = \cos(x) + \sin(y)$ strictly convex on some non-empty
  open convex subset of the plane?
\endshex

\beginshex
 Show that $f:\RR^2 \rightarrow \RR$ given by
  \begin{equation*}
    f(x, y) = \log(e^x + e^y)
  \end{equation*}
  is a convex function. Is $f$ strictly convex?
\endshex

\beginshex
Let $f:\RR^2\rightarrow \RR$ be given by
  \begin{equation*}
    f(x, y) = a x^2 + b y^2 + c x y,
  \end{equation*}
  where $a, b, c\in \RR$.

  \begin{enumerate}[(i)]
  \item Show that $f$ is a strictly convex function if and only if $a > 0$
    and $4 a b - c^2 > 0$.
  \item Suppose now that $a > 0$ and $4 a b - c^2>0$.  Show that $g(x,
    y) = f(x, y) + x + y$ has a unique global minimum and give a
    formula for this minimum in terms of $a, b$ and $c$.
\end{enumerate}
\endshex

\section{How to decide the definiteness of a matrix}

In this section we will outline a straightforward method for
deciding if a matrix is positive definite, positive semidefinite,
negative definite or indefinite.

Before proceeding it is a must that you do the following exercise.

\beginshex
Show that a diagonal matrix
$$
\begin{pmatrix}
  \lambda_1 & 0 & \dots & 0\\
  0 &\lambda_2 & \dots &0\\
  \vdots & \vdots & \ddots & \vdots\\
  0 & 0 &\dots & \lambda_n
\end{pmatrix}
$$
is positive definite if and only if $\lambda_1 > 0, \dots, \lambda_n > 0$
and positive semidefinite
if and only if $\lambda_1 \geq  0, \dots, \lambda_n \geq 0$.

Also, show that if $A$ is a symmetric matrix, then $A$ is
positive definite if and only if
$$
B^t A B
$$
is positive definite, where $B$ is an invertible matrix.
\endshex

The crucial ingredient is the following result.

\begin{theorem}[emph]\label{thmsymred}
  Let $A$ be a real symmetric $n\times n$ matrix. Then there exists an
  invertible matrix $B$, such that $B^T A B$ is a diagonal matrix.
\end{theorem}

The proof contains an algorithm for building $B$ by different steps.
We will supply examples afterwards illustrating these. Incidentally,
how does this help you in deciding for example that a given matrix
is positive definite? If you cannot answer this question, please
do the exercise above.

\begin{hideinbutton}{Proof}
   Suppose that $A=(a_{ij})$. If $A$ has a non-zero entry in the upper
  left hand corner i.e., $a_{11}\neq 0$, then
  \begin{equation*}
    B_1^T A B_1 =
    \begin{pmatrix}
      a_{11} & 0 & \cdots & 0\\
      0 & c_{11} & \cdots & c_{1, n-1}\\
      \vdots & \vdots & \ddots & \vdots \\
      0 & c_{n-1, 1} & \cdots & c_{n-1, n-1},
    \end{pmatrix}
  \end{equation*}
  where $C = (c_{ij})$ is a real symmetric matrix and $B_1$ is the
  invertible $n\times n$ matrix
  \begin{equation*}
    % \def\frac#1#2{{#1\strut \over #2}\,}
    \begin{pmatrix}
      1 & -\frac{ a_{12}}{a_{11}} & \cdots & -\frac{
        a_{1n}}{a_{11}}\\
      0 & 1 & \cdots & 0\\
      \vdots & \vdots & \ddots & \vdots\\
      0 & 0 & \cdots & 1
    \end{pmatrix}
    .
  \end{equation*}
  By induction on $n$ we may find an invertible matrix $(n-1)\times
  (n-1)$ matrix $B_2$ such that
  \begin{equation*}
    B_2^t C B_2 =
    \begin{pmatrix}
      a_1 & 0 &\cdots & 0\\
      0 & a_2 & \cdots & 0\\
      \vdots & \vdots & \ddots &\vdots\\
      0 & 0 & \cdots & a_{n-1}
    \end{pmatrix}
    .
  \end{equation*}

  Putting
  \begin{equation*}
B = B_1
    \begin{pmatrix}
      1 & 0\\
      0 & B_2
    \end{pmatrix},
    \end{equation*}
    it follows that
\begin{equation*}
    B^t A B =
    \begin{pmatrix}
      a_{11} & 0 &\cdots & 0\\
      0 & a_1 & \cdots & 0\\
      \vdots & \vdots & \ddots &\vdots\\
      0 & 0 & \cdots & a_{n-1}
    \end{pmatrix}
    .
  \end{equation*}

We now treat the case of a zero entry in the upper left hand corner
  i.e., $a_{11}=\nobreak 0$.  Suppose first that $a_{jj} \neq 0$ for
  some $j > 1$. Let $P$ denote the identity matrix with the first and
  $j$-th rows interchanged.  The operation $A\mapsto A P$ amounts to
  interchanging the first and $j$-th columns in $A$.  Similarly
  $A\mapsto P^t A$ is interchanging that first and $j$-th rows in $A$.
  The matrix $P$ is invertible and $P^t A P$ is a symmetric matrix
  with $(P^t A P)_{11} = a_{jj}\neq 0$ and we have reduced to the case
  of a non-zero entry in the upper left hand corner.

  If $a_{ii} = 0$ for every $i = 1, \dots, n$ we may assume that $a_{1
    j}\neq 0$ for some $j>1$. Let $B$ denote the identity matrix where
  the entry in the first column and $j$-th row is $1$. The operation
  $A\mapsto A B$ amounts to adding the $j$-th column to the first
  column in $A$. Similarly $A\mapsto B^t A$ is adding the $j$-th row
  to the first row in $A$. All in all we get $(B^t A B)_{11} = 2 a_{1
    j} \neq 0$, where we have used that $a_{ii} = 0$ for $i=1, \dots,
  n$. Again we have reduced to the case of a non-zero entry in the
  upper left hand corner.
\end{hideinbutton}

\begin{example}\label{eksempel1}
  Consider the $3\times 3$ real symmetric matrix.
  \begin{equation*}
    A = (a_{ij}) =
    \begin{pmatrix}
      1 & 5 & 2\\
      5 & 3 & 0\\
      2 & 0 & 5
    \end{pmatrix}
    .
  \end{equation*}
  Here $a_{11} = 1 \neq 0$. Therefore the fundamental step in the
  proof of Theorem \ref{thmsymred} applies and
  \begin{equation*}
    \begin{pmatrix}
      1 & 0 & 0 \\
      -5 & 1 & 0\\
      -2 & 0 & 1
    \end{pmatrix}
    A
    \begin{pmatrix}
      1 & -5 & -2\\
      0 & 1 & 0\\
      0 & 0 & 1
    \end{pmatrix}
    =
    \begin{pmatrix}
      1 & 0 & 0\\
      0 & -22 & -10\\
      0 & -10 & 1
    \end{pmatrix}
  \end{equation*}
  and again
  \begin{equation*}
    \begin{pmatrix}
      1 & 0 & 0 \\
      0 & 1 & 0\\
      0 & -\frac{5}{11} & 1
    \end{pmatrix}
    \begin{pmatrix}
      1 & 0 & 0\\
      0 & -22 & -10\\
      0 & -10 & 1
    \end{pmatrix}
    \begin{pmatrix}
      1 & 0 & 0\\
      0 & 1 & -\frac{5}{11}\\
      0 & 0 & 1
    \end{pmatrix}
    =
    \begin{pmatrix}
      1 & 0 & 0\\
      0 & -22 & 0\\
      0 & 0 & \frac{61}{11}
    \end{pmatrix}
    .
  \end{equation*}

Summing up we get
  \begin{equation*}
    B =
    \begin{pmatrix}
      1 & -5 & -2\\
      0 & 1 & 0\\
      0 & 0 & 1
    \end{pmatrix}
    \begin{pmatrix}
      1 & 0 & 0\\
      0 & 1 & -\frac{5}{11}\\
      0 & 0 & 1
    \end{pmatrix}
    =
    \begin{pmatrix}
      1 & -5 & \frac{3}{11}\\
      0 & 1 & -\frac{5}{11}\\
      0 & 0 & 1
    \end{pmatrix}
    .
  \end{equation*}
  You are invited to check that
  \begin{equation*}
    B^t A B =
    \begin{pmatrix}
      1 & 0 & 0\\
      0 & -22 & 0\\
      0 & 0 & \frac{61}{11}
    \end{pmatrix}
    .
  \end{equation*}
\end{example}

\begin{example}\label{eksempel2}
  Let
  \begin{equation*}
    A =
    \begin{pmatrix}
      0 & 0 & 1 & 1\\
      0 & 0 & 2 & 3\\
      1 & 2 & 1 & 4\\
      1 & 3 & 4 & 0
    \end{pmatrix}.
  \end{equation*}
  Here $a_{11} = a_{22} = 0$, but the diagonal element $a_{33}\neq
  0$. So we are in the second step of the proof of
  Theorem \ref{thmsymred}.  Using the matrix
  \begin{equation*}
    P =
    \begin{pmatrix}
      0 & 0 & 1 & 0\\
      0 & 1 & 0 & 0\\
      1 & 0 & 0 & 0\\
      0 & 0 & 0 & 1
    \end{pmatrix}
  \end{equation*}
  we get
\begin{equation*}
  P^t A P =
    \begin{pmatrix}
      1 & 2 & 1 & 4\\
      2 & 0 & 0 & 3\\
      1 & 0 & 0 & 1\\
      4 & 3 & 1 & 0
    \end{pmatrix}
    .
  \end{equation*}

  As argued in the proof, this corresponds to interchanging the first
  and third columns and then interchanging the first and third
  rows. In total you move the non-zero $a_{33}$ to the upper left
  corner in the matrix.
\end{example}

\begin{example}\label{eksempel3}
  Consider the symmetric matrix
  \begin{equation*}
    A =
    \begin{pmatrix}
      0 & 1 & 1 & 1\\
      1 & 0 & 1 & 1\\
      1 & 1 & 0 & 1\\
      1 & 1 & 1 & 0
    \end{pmatrix}
    .
  \end{equation*}
  We have zero entries in the diagonal. As in the third step in the
  proof of Theorem \ref{thmsymred} we must find an invertible matrix
  $B_1$, such that the upper left corner in $B_1^t A B_1$ is
  non-zero. In the proof it is used that every diagonal element is
  zero: if we locate a non-zero element in the $j$-th column in the
  first row, we can add the $j$-th column to the first column and then
  the $j$-th row to the first row obtaining a non-zero element in the
  upper left corner. For $A$ above we choose $j=2$ and the matrix
  $B_1$ becomes
  \begin{equation*}
    B_1 =
    \begin{pmatrix}
      1 & 0 & 0 & 0\\
      1 & 1 & 0 & 0\\
      0 & 0 & 1 & 0\\
      0 & 0 & 0 & 1
    \end{pmatrix}
  \end{equation*}
  so that
\begin{equation*}
  B_1^t A B_1 =
    \begin{pmatrix}
      2 & 1 & 2 & 2\\
      1 & 0 & 1 & 1\\
      2 & 1 & 0 & 1\\
      2 & 1 & 1 & 0
    \end{pmatrix}
    .
  \end{equation*}

\end{example}

\beginshex
Let $A$ be any matrix. Show that
$$
A^T A
$$
is positive semidefinite.
\endshex

\beginshex
 Find inequalities defining the set
  \begin{equation*}
    \left\{(a, b)\in \RR^2 \middle\vert
      \begin{pmatrix}
        2 & 1 & a\\ 1 & 1 & 1 \\ a & 1 & b
      \end{pmatrix}
      \,\,\text{is positive definite}
    \right\}.
  \end{equation*}
  Same question with positive semidefinite. Sketch and compare the two
  subsets of the plane $\{(a, b) \mid a, b\in \RR\}$.
\endshex

\beginshex
Let $f:\RR^3\rightarrow \RR$ denote the function given by
$$
f(x, y, z) = x^2 + y^2 + z^2 +  a x y +  x z +  y z,
$$
where $a\in \RR$. Let $H$ denote the Hessian of
$f$ in a point $(x, y, z)\in \RR^3$.

\begin{enumerate}[(i)]
\item
  Compute $H$.
\item
  Show that $f(v) = v^T A v$ for $v=(x, y, z)\in \RR^3$ and $A = \frac{1}{2} H$.
\item
  Compute a non-zero vector $v\in \RR^3$, such that $H v = 0$ in
  the case, where $a=2$. Is $H$ invertible in this case?
\item
  Show that $f$ is strictly convex if $-1 < a < 2$.
\item
  Is $f$ strictly convex if $a=2$?

  \begin{hideinbutton}{Hint}
    Consider the line segment between $0$ and a suitable vector
    $u\neq 0$, where $f(u) = 0$.

  \end{hideinbutton}
\end{enumerate}
\endshex

\beginshex
Why is the subset given by the inequalities
  \begin{align*}
    x &\geq 0\\
    y &\geq 0\\
    x y - z^2 &\geq 0
  \end{align*}
  a convex subset of $\RR^3$?
\endshex

\end{document}


\documentclass[letterpaper]{article}

\usepackage{fullpage}
\usepackage{nopageno}
\usepackage{amsmath}
\usepackage{amssymb}
\allowdisplaybreaks

\newcommand{\abs}[1]{\left\lvert #1 \right\rvert}

\begin{document}
\title{Notes}
\date{October 1, 2014}
\maketitle
if $G$ is a group, and $A\subseteq G$ and $B\subseteq G$ then $AB=\{ab|a\in A, b\in B\}\subseteq G$.

\section*{proposition}

let $G$ be a group, then $H,K$ subgroups of $G$. Assume that $h^{-1}kh\in K$ for all $h\in H$, $k\in K$ then $HK$ is a subgroup of $G$ that contain s both $H$ and $K$, in fact, $HK$ is the smallest subgroup of $G$ that contains both $H$ and $K$. Assumption only important if we are not dealing with abelian groups.

\subsubsection*{proof}
$a,b\in HK$. Write $a=h_1k_1,b=h_2k_2$ with $h_i\in H,k_i\in K$ then $a\cdot b=h_1k_1h_2k_2=h_1h_2(h_2^{-1}k_1h_2)k_2\in HK$

$a=hk, a^{-1}=(hk)^{-1}=k^{-1}h^{-1}=h^{-1}(hk^{-1}h^{-1})\in HK$

\subsection*{examples}
$S_3, H=\{(1),(12)\}, K=\{(1),(123),(132)\}, (12)(123)=(23)\in HK, (12)(132)=(13)\in HK$ so $HK=G$ and is therefore contained by G

$(\mathbb{Z},+)$, $H=a\mathbb{Z}, k=b\mathbb{Z}$, let $d=(a,b)$

claim: $a\mathbb{Z}+b\mathbb{Z}=d\mathbb{Z}$. clearly $a\mathbb{Z}\subseteq d\mathbb{Z}$, $b\mathbb{Z}\subseteq d\mathbb{Z}$.

$a\mathbb{Z}+b\mathbb{Z}$ is the smallest subgroup that contains both $a\mathbb{Z}$ and $b\mathbb{Z}$. so $a\mathbb{Z}+b\mathbb{Z}\subseteq d\mathbb{Z}$.

$d=\gcd(a,b)$ so we can write $d=ma+nb$. let $\alpha\in d\mathbb{Z}$ and write $\alpha=dt, t\in \mathbb{Z}$ then $\alpha=dt=mat+nbt\in a\mathbb{Z}+b\mathbb{Z}$. so $d\mathbb{Z}\subseteq a\mathbb{Z}+b\mathbb{Z}$
\section*{thm subgroup gen by a subset}
$G$ is a group, if $a\in G$ $<a>=\{a^i|i\in \mathbb{Z}\}$ is the smallest subgroupthat contains $a$.

\subsubsection*{proof}
let $S\subseteq G$, let $<S>=\{\underbrace{a_1a_2\dots a_k}_{\text{word}}|a_i\in S\text{ or }{a_i}^{-1}\in S, k\in \mathbb{N}\}$ then $<S>$ is a subgroup, $<S>=\cap \forall H$ where $S\subseteq H\subseteq G$, and $H$ is a subgroup of $G$, $<S>$ is the smallest subgroup of $G$that contains $S$.

so it is closed under multiplication, identity is in it, and the inverse of all words are in it.

show containment both ways, one is clear because we have words of length 1 that span $S$ and so $S$ is one of the elements of our $H$ intersection.

\section*{example}
$a,b\in G, S=\{a,b\}\subseteq G, <S>=\{a_1a_2\dots a_k|a_i\in\{a,a^{-1},b,b^{-1}\}\}$

if $ab=ba$ then $<S>=\{a^{i}b^{j}|i\in \mathbb{Z}, j\in \mathbb{Z}\}$

\section*{maps}
studied groups, subgroups. now we are going to talk about maps

if we have groups $G_1,G_2$ and $\varphi:G_1\to G_2$ is a group homomorphism provided $x\to\varphi(x), y\to\varphi$ means that $\varphi(x*y)=\varphi(x)*\varphi(y)$ for all $x,y\in G_1$.
\section*{examples}
identity: $x\to x$

$(\mathbb{R},+)=G_1, (\mathbb{R}^+,\cdot)=G_2$. $\varphi(x)=e^x$. ie $\varphi(x+y)=e^{x+y}=e^xe^y=\varphi(x)\varphi(y)$.

\section*{notation}
let $\varphi:G_1\to G_2$ be a group homomorphism, then $\ker\varphi=\{x\in G_1|\varphi(x)=e\}$

homomorphism always takes the identity in $G_1$ to $G_2$.

$\varphi(e_1)\varphi(e_1)^{-1}=\varphi(e_1e_1)\varphi(e_1)^{-1}=\varphi(e_1)\varphi(e_1)\varphi(e_1)^{-1}=e_2=\varphi(e_1)$

prove that $\ker \varphi$ is a subgroup

now we say that $\varphi$ is an isomorphism if $\varphi$ is a group homomorphism and $\varphi$ is bijective.

both of the previous examples are isomorphisms.

so from an algebraic point of view, there is no difference between addition on the reals and multiplication on the positive reals.

\section*{proposition}
let $\varphi$ be an isomorphism. the following are true
\begin{enumerate}
\item
$\varphi^{-1}$ which is the map from $G_2$ to $G_1$ is also an isomorphism.
\item
if $G_1$ is abelian, then $G_2$ is abelian.
\item
if $G_1$ is cyclic then so is $G_2$
\item
if $a\in G_1$ then $\text{ord}(a)=\text{ord}(\varphi(a))$
\end{enumerate}

\begin{enumerate}
\item
need to prove $\varphi^{-1}(\alpha\beta)=\varphi^{-1}(\alpha)\varphi^{-1}(\beta)$ for all $\beta\in G_2$, but $\varphi$ is injective so it is enough to prove that $\varphi(\varphi^{-1}(\alpha\beta))=\varphi(\varphi^{-1}(\alpha)\varphi^{-1}(\beta)=\varphi(\varphi^{-1}(\alpha))\varphi(\varphi^{-1}(\beta))=\alpha\beta$
\item
assume $G_1$ is abelian
\begin{align*}
  \alpha\beta=\varphi(\varphi^{-1}\left(\alpha\right)\varphi^{-1}(\beta))
\end{align*}
\item
hint: assume that $G_1=<a>$ for some $a\in G_1$ and then prove that $G_2=<\varphi(a)$
\item
no hint
\end{enumerate}
\section*{example}
\begin{align*}
  \mathbb{Z}_4\not\equiv\mathbb{Z}_2\times\mathbb{Z}_2
\end{align*}
by contradiction, assume that there exists an isomorphism $\varphi$ from z4 to z2+z2. $[1]\in \mathbb{Z}_4$ and $\text{ord}[1]=4$ so then $\text{ord}\varphi([1])=4$. But all elements of $\mathbb{Z}_2\times\mathbb{Z}_2$ has no elements of order 4, so there is no isomorphisms. however, if $\gcd(m,n)=1$ then $\mathbb{Z}_{mn}\equiv\mathbb{Z}_m\times\mathbb{Z}_n$
\end{document}


\chapter{Supplemental Algorithms}\label{chap:appendix-algo}

Find here concrete implementation details on the EFTs described
in Theorem~\ref{thm:eft}. They do not use branches, nor access to the
mantissa that can be time-consuming.

\begin{breakablealgorithm}
  \caption{\textit{EFT of the sum of two floating point numbers.}}

  \begin{algorithmic}
    \Function{\(\left[S, \sigma\right] = \mathtt{TwoSum}\)}{$a, b$}
      \State \(S = a \oplus b\)
      \State \(z = S \ominus a\)
      \State \(\sigma = (a \ominus (S \ominus z)) \oplus (b \ominus z)\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

\noindent In order to avoid branching to check which among
\(\left|a\right|, \left|b\right|\) is largest, \texttt{TwoSum} uses 6 flops
rather than 3.

\begin{breakablealgorithm}
  \caption{\textit{Splitting of a floating point number into two parts.}}

  \begin{algorithmic}
    \Function{\(\left[h, \ell\right] = \mathtt{Split}\)}{$a$}
      \State \(z = a \otimes (2^r + 1)\)
      \State \(h = z \ominus (z \ominus a)\)
      \State \(\ell = a \ominus h\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

\noindent For IEEE-754 double precision floating point number, \(r = 27\)
so \(2^r + 1\) will be known before \texttt{Split} is called. In all,
\texttt{Split} uses 4 flops.

\begin{breakablealgorithm}
  \caption{\textit{EFT of the product of two floating point numbers.}}

  \begin{algorithmic}
    \Function{\(\left[P, \pi\right] = \mathtt{TwoProd}\)}{$a, b$}
      \State \(P = a \otimes b\)
      \State \(\left[a_h, a_{\ell}\right] = \mathtt{Split}(a)\)
      \State \(\left[b_h, b_{\ell}\right] = \mathtt{Split}(b)\)
      \State \(\pi = a_{\ell} \otimes b_{\ell} \ominus (((P \ominus
          a_h \otimes b_h)
          \ominus a_{\ell} \otimes b_h) \ominus a_h \otimes b_{\ell})\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

\noindent This implementation of \texttt{TwoProd} requires 17 flops.
For processors that provide a fused-multipy-add operator (\texttt{FMA}),
\texttt{TwoProd} can be rewritten to use only 2 flops:

\begin{breakablealgorithm}
  \caption{\textit{EFT of the sum of two floating point numbers with a FMA.}}
  \label{alg:two-prod-fma}

  \begin{algorithmic}
    \Function{\(\left[P, \pi\right] = \mathtt{TwoProdFMA}\)}{$a, b$}
      \State \(P = a \otimes b\)
      \State \(\pi = \mathtt{FMA}(a, b, -P)\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

\noindent The following algorithms from \cite{Ogita2005} can be used as a
compensated method for computing a sum of numbers. The first is a vector
transformation that is used as a helper:

\begin{breakablealgorithm}
  \caption{\textit{Error-free vector transformation for summation.}}
  \label{alg:vec-sum}

  \begin{algorithmic}
    \Function{\(\mathtt{VecSum}\)}{$p$}
      \State \(n = \texttt{length}(p)\)
      \For{\(j = 2, \ldots, n\)}
        \State \(\left[p_j, p_{j - 1}\right] = \mathtt{TwoSum}\left(
            p_j, p_{j - 1}\right)\)
      \EndFor
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

\noindent The second (\texttt{SumK}) computes a sum with results that are as
accurate as if computed in \(K\) times the working precision. It requires
\((6K - 5)(n - 1)\) floating point operations.

\begin{breakablealgorithm}
  \caption{\textit{Summation as in K-fold precision
      by \((K - 1)\)-fold error-free vector transformation.}}
  \label{alg:sum-k}

  \begin{algorithmic}
    \Function{\(\mathtt{result} = \mathtt{SumK}\)}{$p, K$}
      \For{\(j = 1, \ldots, K - 1\)}
        \State \(p = \mathtt{VecSum}(p)\)
      \EndFor
      \State \(\mathtt{result} = p_1 \oplus p_2 \oplus \cdots \oplus p_n\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

\noindent Since the final error \(\cdb{K - 1}\) will not track the errors
during computation, we have a non-EFT version of
Algorithm~\ref{alg:local-error-eft}:

\begin{breakablealgorithm}
  \caption{\textit{Compute the local error (non-EFT).}}
  \label{alg:local-error}

  \begin{algorithmic}
    \Function{\(\widehat{\ell} =
        \mathtt{LocalError}\)}{$e, \rho, \delta b$}
      \State \(L = \texttt{length}(e)\)
      \\
      \State \(\widehat{\ell} = e_1 \oplus e_2\)
      \For{\(j = 3, \ldots, L\)}
        \State \(\widehat{\ell} = \widehat{\ell} \oplus e_j\)
      \EndFor
      \\
      \State \(\widehat{\ell} = \widehat{\ell} \oplus \left(
          \rho \otimes \delta b\right)\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

In order to discuss varied implementations of Newton's method in
Chapter~\ref{chap:compensated-newton}, we define a generic algorithm
that takes an ``update function'' (\(\mathtt{update\_fn}\)), i.e.
a callable that will produce the next Newton update \(p(s) / p'(s)\)
computed in a problem-specific way.

\begin{breakablealgorithm}
  \caption{\textit{Generic Newton's method for scalar functions.}}
  \label{alg:generic-newton}

  \begin{algorithmic}
    \Function{\(x_{\ast} = \mathtt{NewtonGeneric}\)}
             {$\mathtt{update\_fn}, x_0, \mathtt{tol}, \mathtt{max\_iter}$}
      \State \(x = x_0\)
      \\
      \For{\(j = 1, \ldots, \mathtt{max\_iter}\)}
        \State \(\mathtt{update} = \mathtt{update\_fn}(x)\)
        \State \(x = x \ominus \mathtt{update}\)
        \If{\(\left|\mathtt{update}\right| < \mathtt{tol}\)}
          \State \textbf{break}
        \EndIf
      \EndFor
      \\
      \State \(x_{\ast} = x\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

In \cite[Algorithm~3]{Jiang2010}, the \texttt{CompDeCasteljau} (
Algorithm~\ref{alg:comp-de-casteljau}) is modified for the computation
of the derivative \(p'(s)\). Since \(p'(s)\) has coefficients
\(n \Delta b_j\) where \(c_j = \Delta b_j = b_{j + 1} - b_j\),
we can begin the computation with nonzero
\(\widehat{\partial c}_j\) (as opposed to
\(\cdb{1}_j^{(n)} = 0\) in \texttt{CompDeCasteljau}).

\begin{breakablealgorithm}
  \caption{\textit{Compensated de Casteljau
      algorithm for polynomial first derivative evaluation.}}
  \label{alg:comp-de-casteljau-derivative}

  \begin{algorithmic}
    \Function{\(\mathtt{result} = \mathtt{CompDeCasteljauDer}\)}{$b, s$}
      \State \(n = \texttt{length}(b) - 1\)
      \State \(\left[\widehat{r}, \rho\right] = \mathtt{TwoSum}(1, -s)\)
      \\
      \For{\(j = 0, \ldots, n - 1\)}
        \State \(\left[\widehat{c}_j^{(n - 1)},
          \widehat{\partial c}_j^{(n - 1)}\right] =
          \mathtt{TwoSum}(b_{j + 1}, -b_j)\)
      \EndFor
      \\
      \For{\(k = n - 2, \ldots, 0\)}
        \For{\(j = 0, \ldots, k\)}
          \State \(\left[P_1, \pi_1\right] = \mathtt{TwoProd}\left(
              \widehat{r}, \widehat{c}_j^{(k + 1)}\right)\)
          \State \(\left[P_2, \pi_2\right] = \mathtt{TwoProd}\left(
              s, \widehat{c}_{j + 1}^{(k + 1)}\right)\)
          \State \(\left[\widehat{c}_j^{(k)}, \sigma_3\right] =
              \mathtt{TwoSum}(P_1, P_2)\)
          \State \(\widehat{\ell}_{1, j}^{(k)} = \pi_1 \oplus \pi_2 \oplus
              \sigma_3 \oplus \left(\rho \otimes
              \widehat{c}_j^{(k + 1)}\right)\)
          \State \(\widehat{\partial c}_j^{(k)} =
              \widehat{\ell}_{1, j}^{(k)} \oplus
              \left(s \otimes \widehat{\partial c}_{j + 1}^{(k + 1)}
              \right) \oplus
              \left(\widehat{r} \otimes
              \widehat{\partial c}_j^{(k + 1)}\right)\)
        \EndFor
      \EndFor
      \\
      \State \(\mathtt{result} = n \otimes \left[\widehat{c}_0^{(0)} \oplus
          \widehat{\partial c}_0^{(0)}\right]\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}

For an applied usage of \texttt{CompDeCasteljau}
(Algorithm~\ref{alg:comp-de-casteljau}) in
Chapter~\ref{chap:compensated-newton}, both the non-compensated
value \(\widehat{b}\) and the compensation term \(\widehat{\partial b}\) are
needed. So we define a partial EFT. We say \emph{partial} because
the compensation term is rounded.

\begin{breakablealgorithm}
  \caption{\textit{EFT for de Casteljau algorithm for polynomial evaluation.}}
  \label{alg:eft-de-casteljau}

  \begin{algorithmic}
    \Function{\(\left[\widehat{b}, \widehat{\partial b}\right] =
      \mathtt{DeCasteljauEFT}\)}{$b, s$}
      \State \(n = \texttt{length}(b) - 1\)
      \State \(\left[\widehat{r}, \rho\right] = \mathtt{TwoSum}(1, -s)\)
      \\
      \For{\(j = 0, \ldots, n\)}
        \State \(\widehat{b}_j^{(n)} = b_j\)
        \State \(\cdb{1}_j^{(n)} = 0\)
      \EndFor
      \\
      \For{\(k = n - 1, \ldots, 0\)}
        \For{\(j = 0, \ldots, k\)}
          \State \(\left[P_1, \pi_1\right] = \mathtt{TwoProd}\left(
              \widehat{r}, \widehat{b}_j^{(k + 1)}\right)\)
          \State \(\left[P_2, \pi_2\right] = \mathtt{TwoProd}\left(
              s, \widehat{b}_{j + 1}^{(k + 1)}\right)\)
          \State \(\left[\widehat{b}_j^{(k)}, \sigma_3\right] =
              \mathtt{TwoSum}(P_1, P_2)\)
          \State \(\widehat{\ell}_{1, j}^{(k)} = \pi_1 \oplus \pi_2 \oplus
              \sigma_3 \oplus \left(\rho \otimes
              \widehat{b}_j^{(k + 1)}\right)\)
          \State \(\cdb{1}_j^{(k)} =
              \widehat{\ell}_{1, j}^{(k)} \oplus
              \left(s \otimes \cdb{1}_{j + 1}^{(k + 1)}
              \right) \oplus
              \left(\widehat{r} \otimes
              \cdb{1}_j^{(k + 1)}\right)\)
        \EndFor
      \EndFor
      \\
      \State \(\widehat{b} = \widehat{b}_0^{(0)}\)
      \State \(\widehat{\partial b} = \cdb{1}_0^{(0)}\)
    \EndFunction
  \end{algorithmic}
\end{breakablealgorithm}