# -*- coding: utf-8 -*-

import copy
import numpy as np

class Objective(object):

    def forward(self, outputs, targets):

        raise NotImplementedError()

    def backward(self, outputs, targets):
        raise NotImplementedError()

    def __str__(self):
        return self.__class__.__name__

class MeanSquaredError(Objective):

    def forward(self, outputs, targets):
        return 0.5 * np.mean(np.sum(np.power(outputs - targets, 2), axis=1))

    def backward(self, outputs, targets):
        return outputs - targets

MSE = MeanSquaredError

class HellingerDistance(Objective):

    def forward(self, outputs, targets):
        root_difference = np.sqrt(outputs) - np.sqrt(targets)
        return np.mean(np.sum(np.power(root_difference, 2), axis=1) / np.sqrt(2))

    def backward(self, outputs, targets):
        root_difference = np.sqrt(outputs) - np.sqrt(targets)
        return root_difference / (np.sqrt(2) * np.sqrt(outputs))

HeD = HellingerDistance

class BinaryCrossEntropy(Objective):

    def __init__(self, epsilon=1e-11):
        self.epsilon = epsilon

    def forward(self, outputs, targets):
        outputs = np.clip(outputs, self.epsilon, 1 - self.epsilon)
        return np.mean(-np.sum(targets * np.log(outputs) + (1 - targets) * np.log(1 - outputs), axis=1))

    def backward(self, outputs, targets):
        outputs = np.clip(outputs, self.epsilon, 1 - self.epsilon)
        divisor = np.maximum(outputs * (1 - outputs), self.epsilon)
        return (outputs - targets) / divisor

BCE = BinaryCrossEntropy

class SoftmaxCategoricalCrossEntropy(Objective):

    def __init__(self, epsilon=1e-11):
        self.epsilon = epsilon

    def forward(self, outputs, targets):
        outputs = np.clip(outputs, self.epsilon, 1 - self.epsilon)
        return np.mean(-np.sum(targets * np.log(outputs), axis=1))

    def backward(self, outputs, targets):
        outputs = np.clip(outputs, self.epsilon, 1 - self.epsilon)
        return outputs - targets

SCCE = SoftmaxCategoricalCrossEntropy

def get(objective):
    if objective.__class__.__name__ == 'str':
        if objective in ['mse', 'MSE']:
            return MSE()
        if objective in ['mean_squared_error', 'MeanSquaredError']:
            return MeanSquaredError()
        if objective in ['hellinger_distance', 'HellingerDistance']:
            return HellingerDistance()
        if objective in ['hed', 'HeD']:
            return HeD()
        if objective in ['binary_cross_entropy', 'BinaryCrossEntropy']:
            return BinaryCrossEntropy()
        if objective in ['bce', 'BCE']:
            return BCE()
        if objective in ['softmax_categorical_cross_entropy', 'SoftmaxCategoricalCrossEntropy']:
            return SoftmaxCategoricalCrossEntropy()
        if objective in ['scce', 'SCCE']:
            return SCCE()
        raise ValueError('Unknown objective name: {}.'.format(objective))

    elif isinstance(objective, Objective):
        return copy.deepcopy(objective)

    else:
        raise ValueError("Unknown type: {}.".format(objective.__class__.__name__))

Machine Learning Projects/Xeno-Deep Learning library from scratch/xeno/objectives.py


#/usr/bin/python
#coding:utf8

import numpy as np
def Cosine(x, y):
  return 1.0*np.dot(x, y)/np.sqrt(np.dot(x,x))/np.sqrt(np.dot(y,y))
def Sigmoid(x):
  return 1.0 / (1.0 + np.exp(-1.0 * x))
def SigmoidGradient(x):
  return Sigmoid(x) * (1 - Sigmoid(x))

#平方误差,2范式的正则化项
def SquareErrorFunction2F(target_matrix_y, output_matrix_y, weights, lamda):
  m = len(target_matrix_y)
  err = target_matrix_y - output_matrix_y
  errors = 1.0/(2.0*m) * np.sum(err * err) + lamda*1.0/(2.0*m)*np.sum(weights * weights)
  return errors

def LogisticErrorFunction2F(target_matrix_y, output_matrix_y, weights, lamda):
  m = len(target_matrix_y)
  vector_j = target_matrix_y * np.log(output_matrix_y) + (1.0 - target_matrix_y) * np.log(1.0 - output_matrix_y)
  errors = -1.0/m*np.sum(vector_j) + lamda/(2.0*m) * np.sum(weights * weights)
  return errors

def RandomSample(size, a, b):
  epsilon_init = 0.12;
  arr = (b - a) * np.random.random_sample(size) + a
  return arr * epsilon_init * 2 - epsilon_init

import math
def GaussFunc(x, dimension, mu, sigma):
  inv = np.linalg.inv(sigma)
  e = -0.5 * np.dot(x - mu, inv)
  e = np.dot(e, (x-mu).T)
  re = 1./np.sqrt((2.*math.pi)**dimension) * 1./np.sqrt(np.linalg.det(sigma)) * np.exp(e)
  if x.ndim <= 1:return re
  else:return np.diag(re)

if __name__ == '__main__':
  print( Cosine(np.array([-1,-2]), np.array([1,2])))
  print( SigmoidGradient(np.array([-1,-2])))
  print( SquareErrorFunction2F(np.array([[0,1], [1,0]]), np.array([[0.5,0.5], [0.5,0.5]]), np.array([1,2,3]), 1))
  print( LogisticErrorFunction2F(np.array([[0,1], [1,0]]), np.array([[0.5,0.5], [0.5,0.5]]), np.array([1,2,3]), 1))
  print( RandomSample((2,3), 0, 1))
  print( GaussFunc(np.array([0,0]), 2, [0, 0], np.array([[1.,0],[0.,1]])))
  print( GaussFunc(np.array([[0,0.],[1,2.]]), 2, [0, 0], np.array([[1.,0],[0.,1]])))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon May 20 14:40:10 2019

@author: kenneth
"""

import numpy as np
import pandas as pd

class Regression(object):
    def __init__(self):
        return

    def fit_predict(self, X, Y):
        self.X = X
        self.Y = Y
        #--either beta syntax below would work
        #beta = np.linalg.solve(np.dot(self.X.T, self.X), np.dot(self.X.T, self.Y))
        beta = np.linalg.inv(self.X.T.dot(self.X)).dot(self.X.T).dot(self.Y)
        Y_hat = self.X.dot(beta)
        return Y_hat

    #-Mean Square Error
    def RMSE(self, yh, y):
        return np.sqrt(np.square(yh - y).mean())
    #-Mean Square Error
    def MSE(self, yh, y):
        return np.square(yh - y).mean()
    #-Mean Absolute Error
    def MAE(self, yh, y):
        return np.abs(yh - y).mean()
    #-R-squared Error
    def R_squared(self, yh, y):
        #-- R_square = 1 - (SS[reg]/SS[total])
        # 1 - (y-yh).dot(y-yh)/(y - y.mean()).dot(y - y.mean()) OR
        return (1 -(np.sum(np.square(y - yh))/np.sum(np.square(y - y.mean()))))

    def summary(self, X, y, y_hat):
        #y_hat = self.fit_predict(self.X, self.Y)
        print('*'*40)
        print('\t\tSummary')
        print('*'*40)
        print('RMSE: %s'%(self.RMSE(y_hat,  Y)))
        print('*'*40)
        print('MSE: %s'%(self.MSE(y_hat,  Y)))
        print('*'*40)
        print('MAE: %s'%(self.MAE(y_hat,  Y)))
        print('*'*40)
        print('R_squared = %s'%(self.R_squared(y_hat,  Y)))
        print('*'*40)

    def plot(self, X, Y, y_hat):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(X.shape[0]), Y)
        plt.plot(np.arange(X.shape[0]), y_hat)
        plt.legend(loc = 2)
        plt.title('True vlaue vs Predicted value')
        plt.xlabel('Data point')
        plt.ylabel('True vlaue vs Predicted value')

class GradientDescent(Regression):
    '''
    Inherits Regression class
    '''
    def __init__(self):
        super().__init__()
        return

    def cost(self, X, Y, beta):
        '''
        param: X = training examples/data. column vector <x1, x2, ...., xn | x E R^D>
        param: Y = target. vector  <y | y E R^DX1>
        param: beta = coefficients, e.g b0, b1
        Return: cost
        '''
        return (1/2*len(Y)) * np.sum(np.square(X.dot(beta) - Y))

    def GD(self, X, Y, beta, alpha, iterations, early_stopping = None):
        '''
        param: X = NxD feature matrix
        param: Y = Dx1 column vector
        param: beta = Dx1 beta vector coefficients
        param: alpha = learning rate. Default 1e-2
        param: iterations = Number of times to run. Default 1000

        Return type; final beta/coefficients, cost and bata iterations
        '''
        cost_rec = np.zeros(iterations)
        beta_rec = np.zeros((iterations, X.shape[1]))
        if early_stopping:
            for ii in range(iterations):
                #compute gradient
                beta = beta - (1/len(Y)) *(alpha) * (np.dot(X.T, (np.dot(X,beta) - Y)))
                beta_rec[ii, :] = beta.T
                cost_rec[ii] = self.cost(X, Y, beta)
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
                #--compare last and previous value. stop if they are the same
                if not cost_rec[ii] == cost_rec[ii -1]:
                    continue
                else:
                    break
            y_hat = X.dot(beta)
            return beta, cost_rec[:ii], beta_rec, y_hat, ii
        else:
            for ii in range(iterations):
                #compute gradient
                beta = beta - (1/len(Y)) *(alpha) * (np.dot(X.T, (np.dot(X,beta) - Y)))
                beta_rec[ii, :] = beta.T
                cost_rec[ii] = self.cost(X, Y, beta)
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec, beta_rec, y_hat, ii

    def plot_cost(self, cost, iter_):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(iter_), cost)
        plt.title('Cost vs Number of iteration')
        plt.xlabel('Iteration')
        plt.ylabel('Cost')
        plt.show()

class StochasticGradientDescent(Regression):
    '''
    Inherits Regression class
    '''
    def __init__(self):
        super().__init__()
        return

    def cost(self, X, Y, beta):
        '''
        param: X = training examples/data. column vector <x1, x2, ...., xn | x E R^D>
        param: Y = target. vector  <y | y E R^DX1>
        param: beta = coefficients, e.g b0, b1
        Return: cost
        '''
        return (1/2*len(Y)) * np.sum(np.square(X.dot(beta) - Y))

    def StochGD(self, X, Y, beta, alpha, iterations, early_stopping = None):
        '''
        param: X = NxD feature matrix
        param: Y = Dx1 column vector
        param: beta = Dx1 beta vector coefficients
        param: alpha = learning rate. Default 1e-2
        param: iterations = Number of times to run. Default 1000

        Return type; final beta/coefficients, cost and bata iterations
        '''
        cost_rec = np.zeros(iterations)
        len_y = len(Y)
        if early_stopping:
            for ii in range(iterations):
                #compute gradient
                cost_val = []
                for ij in range(len_y):
                    random_samples = np.random.randint(1, len_y)
                    X_samp = X[:random_samples]
                    Y_samp = Y[:random_samples]
                    beta = beta - (1/len(Y_samp)) *(alpha) * (np.dot(X_samp.T, (np.dot(X_samp,beta) - Y_samp)))
                    cost_val.append(self.cost(X_samp, Y_samp, beta))
                    if cost_val[ij] == cost_val[ij -1]:
                        break
                    else:
                        continue
                cost_rec[ii] = np.average(cost_val)
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
                #--compare last and previous value. stop if they are the same
                if not cost_rec[ii] == cost_rec[ii -1]:
                    continue
                else:
                    break
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec[:ii], y_hat, ii
        else:
            for ii in range(iterations):
                #compute gradient
                cost_val = 0.0
                for ij in range(len_y):
                    random_samples = np.random.randint(1, len_y)
                    X_samp = X[:random_samples]
                    Y_samp = Y[:random_samples]
                    beta = beta - (1/len(Y_samp)) *(alpha) * (np.dot(X_samp.T, (np.dot(X_samp,beta) - Y_samp)))
                    cost_val += self.cost(X_samp, Y_samp, beta)
                cost_rec[ii] = cost_val
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec, y_hat, ii

    def plot_cost(self, cost, iter_):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(iter_), cost)
        plt.title('Cost vs Number of iteration')
        plt.xlabel('Iteration')
        plt.ylabel('Cost')
        plt.show()

class MinibatchGradientDescent(Regression):
    '''
    Inherits Regression class
    '''
    def __init__(self):
        super().__init__()
        return

    def cost(self, X, Y, beta):
        '''
        param: X = training examples/data. column vector <x1, x2, ...., xn | x E R^D>
        param: Y = target. vector  <y | y E R^DX1>
        param: beta = coefficients, e.g b0, b1
        Return: cost
        '''
        return (1/2*len(Y)) * np.sum(np.square(X.dot(beta) - Y))

    def minbatchGD(self, X, Y, beta, alpha, iterations, batch_size = None, early_stopping = None):
        '''
        param: X = NxD feature matrix
        param: Y = Dx1 column vector
        param: beta = Dx1 beta vector coefficients
        param: alpha = learning rate. Default 1e-2
        param: iterations = Number of times to run. Default 1000

        Return type; final beta/coefficients, cost and bata iterations
        '''
        cost_rec = np.zeros(iterations)
        len_y = len(Y)
        number_batches = int(len_y/batch_size)
        if early_stopping:
            for ii in range(iterations):
                cost_val = 0
                #randomize dataset using permutation
                random_samples = np.random.permutation(len_y)
                X_random = X[random_samples]
                Y_random = Y[random_samples]
                for ij in range(0, len_y, number_batches):
                    #split into batches
                    X_samp = X_random[ij:ij+batch_size]
                    Y_samp = Y_random[ij:ij+batch_size]
                    beta = beta - (1/len(Y_samp)) *(alpha) * (np.dot(X_samp.T, (np.dot(X_samp,beta) - Y_samp)))
                    cost_val += self.cost(X_samp, Y_samp, beta)
#                    if cost_val[ij] == cost_val[ij -1]:
#                        break
#                    else:
#                        continue
                cost_rec[ii] = cost_val #np.average(cost_val)
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
                #--compare last and previous value. stop if they are the same
                if not cost_rec[ii] == cost_rec[ii -1]:
                    continue
                else:
                    break
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec[:ii], y_hat, ii
        else:
            for ii in range(iterations):
                cost_val = 0
                #randomize dataset using permutation
                random_samples = np.random.permutation(len_y)
                X_random = X[random_samples]
                Y_random = Y[random_samples]
                for ij in range(0, len_y, number_batches):
                    #split into batches
                    X_samp = X_random[ij:ij+batch_size]
                    Y_samp = Y_random[ij:ij+batch_size]
                    beta = beta - (1/len(Y_samp)) *(alpha) * (np.dot(X_samp.T, (np.dot(X_samp,beta) - Y_samp)))
                    cost_val += self.cost(X_samp, Y_samp, beta)
                cost_rec[ii] = cost_val
                print('*'*40)
                print('%s iteratiion, cost = %s'%(ii, cost_rec[ii]))
            print('*'*40)
            y_hat = X.dot(beta)
            return beta, cost_rec, y_hat, ii

    def plot_cost(self, cost, iter_):
        import matplotlib.pyplot as plt
        plt.plot(np.arange(iter_), cost)
        plt.title('Cost vs Number of iteration')
        plt.xlabel('Iteration')
        plt.ylabel('Cost')
        plt.show()

#%%

from collections import Counter
def extractFeatures(model, k_features, fscore = None):
    features = Counter(model.get_booster().get_score())
    features = features.most_common(k_features)
    features = [x[0] for x in features if x[1] >= fscore]
    return features

features = extractFeatures(estimator, 20, 1)
features = ['Area m2',
             'District_id',
             'Street Width',
             'Driver Room',
             'Extra Unit',
             'Apartments',
             'Bed Rooms',
             'WC',
             'With Stairs',
             'Living Rooms',
             'Servant Room']

import statsmodels.api as sm
X = df_standard_no_out[features]
X = np.c_[np.ones((X.shape[0], 1)), X]
Y = df_standard_no_out[['Price']].values

#--Multivariant regression
lm = Regression()
yhat = lm.fit_predict(X, Y)
lm.summary(X, Y, yhat)
gd.plot(X[:200], Y[:200], y_hat[:200])

#--Gradient descent
iterations = 1000
gd = GradientDescent()
beta,cost_rec,theta_rec, yhat, stopping = gd.GD(X, Y, beta = np.zeros(X.shape[1]).reshape(-1, 1), alpha = 0.1, iterations = iterations, early_stopping=True)
gd.summary(X, Y, yhat)
gd.plot_cost(cost_rec, stopping)

#--stochastic gradient descent
stgrad = StochasticGradientDescent()
beta,cost_rec, yhat, stopping = stgrad.StochGD(X, Y, beta = np.zeros(X.shape[1]).reshape(-1, 1), alpha = 0.8, iterations = iterations, early_stopping=True)
stgrad.summary(X, Y, yhat)
stgrad.plot_cost(cost_rec, stopping)

#--minibatch gradient descent
minibatch = MinibatchGradientDescent()
beta,cost_rec, yhat, stopping = minibatch.minbatchGD(X, Y, beta = np.zeros(X.shape[1]).reshape(-1, 1), alpha = 0.01, iterations = iterations, batch_size = 20, early_stopping=True)
minibatch.summary(X, Y, yhat)
minibatch.plot_cost(cost_rec, stopping)

#%% ADDISTIONAL FEATURES FOR REGRESSION


from __future__ import division
import numpy as np
import scipy as sp
import matplotlib.pyplot as plt
import numpy.linalg as npla
import scipy.linalg as spla
import scipy.special
import pandas as pd

# # Part (a)

# Code might send out RunTime warnings. These are benign, caused due to Python 2.7.
# Can verify here: https://stackoverflow.com/questions/40845304/runtimewarning-numpy-dtype-size-changed-may-indicate-binary-incompatibility

def get_approx_jv(n, z):

    # returns approximate value of J_{n+1}

    if (n < 1).any() == True:
        return ValueError

    return ( ( ( ( 2 * n ) / z ) * scipy.special.jv(n, z) ) - scipy.special.jv(n - 1, z) )

z = 20

true_jv = scipy.special.jv(np.arange(2, 51), z)
approx_jv = get_approx_jv(np.arange(1, 50), z)

# I have printed the magnitude of the Relative Error

relative_err = (approx_jv - true_jv) / true_jv
relative_err = np.abs(relative_err)

df = pd.DataFrame({"n" : [i for i in range(2, 51)], "LHS" : true_jv, "RHS" : approx_jv, "Relative Error" : relative_err}, columns=["n", "LHS", "RHS", "Relative Error"])
print(df.to_string(index_names=False))

plt.xlabel("Value of n")
plt.ylabel("Absolute Value of Relative Error")
plt.title("Problem 3(a)")
plt.plot( np.arange(2, 51), relative_err)
plt.savefig("problem_3a.png")

# Uncomment to see figures
plt.show()


import numpy as np
from matplotlib import pyplot as plt

from numpy import sin, cos, pi
from scipy.optimize import fsolve

def _Kronig_Penney(K, k, a, U0b):
    """
    Kronig-Penney model.
    """
    return U0b / (2*K) * sin(K * a) + cos(K * a) - cos(k * a)

@np.vectorize
def E(k, n=1, a=1, U0b=4, **kwargs):
    """
    Get energy from k in Kronig-Penney model.
    ---
    Arguments:
        k: wavenumber, a number or an numpy.array.
        n: the n-th band, starting from 0.
        a: distance between atoms.
        U0b: the area of the potential wall.
        **kwargs: something else to pass to scipy.optimize.fsolve.
    Returns:
        The energies corresponding to k, in the unit of hbar^2/2m.
    """
    K = fsolve(_Kronig_Penney, (n + 1/2)*pi, args=(k, a, U0b))
    return K**2

def getBands(n=3, a=1, U0b=4, **kwargs):
    """
    Get n bands in Kronig-Penney model.
    ---
    Arguments:
        k: wavenumber, a number or an numpy.array.
        n: the n-th band, starting from 1.
        a: distance between atoms.
        U0b: the area of the potential wall.
        **kwargs: something else to pass to scipy.optimize.fsolve.
    Returns:
        bands: An array of the energies, in the unit of hbar^2/2m.
            `bands[i]` is the i-th band.
    """
    k = np.linspace(-pi/a, pi/a, 100)
    bands = [E(k, i, a, U0b=4, **kwargs) for i in range(n)]
    return bands

def plotBands(n=3, a=1, U0b=4,
    title=None, **kwargs):
    """
    Plot n bands in Kronig-Penney model.
    ---
    Arguments:
        k: wavenumber, a number or an numpy.array.
        n: the n-th band, starting from 1.
        a: distance between atoms.
        U0b: the area of the potential wall.
        title: the title of the ax.
            If title == None then it is "Plotting of n bands".
        **kwargs: something else to pass to scipy.optimize.fsolve.
    Returns:
        (fig, ax)
    """
    if title == None:
        title = "Plotting of {} bands".format(n)
    k = np.linspace(-pi/a, pi/a, 100)
    bands = getBands(n, a, U0b, **kwargs)

    fig, ax = plt.subplots(figsize=(5, 15), dpi=200)

    for i, band in enumerate(bands):
        ax.plot(k, band, label="${}$-th band".format(i))

    ax.set_xlabel("$k$")
    ax.set_ylabel("$2m E/\hbar^2$")

    ax.set_title(title)
    ax.minorticks_on()
    ax.grid(True)

    ax.legend()

    return fig, ax

if __name__ == "__main__":
    plotBands()


"""Contains the quadratic cost function."""

from typing import cast

import numpy as np

def quadratic_loss(x: np.ndarray, y: np.ndarray) -> np.ndarray:
    """
    Calculate the vectorized quadratic loss for a single example.

    Parameters
    ----------
    x : np.ndarray
        A vector of the predicted values.
    y : np.ndarray
        A vector of the expected values. Must match x's dimensions.

    """
    if x.ndim != 1:
        raise ValueError("Can only compute the quadratic cost for vectors")

    if x.shape != y.shape:
        raise ValueError(
            "Can only compute the quadratic cost of two vectors of equal shape"
        )

    result = 0.5 * ((y - x) ** 2)
    return cast(np.ndarray, result)

def delta_quadratic_loss(x: np.ndarray, y: np.ndarray) -> np.ndarray:
    """
    Calculate the vectorization of the derivative of quadratic loss.

    Partial derivative taken with respect to x.
    """
    if x.ndim != 1:
        raise ValueError("Can only compute the quadratic cost for vectors")

    if x.shape != y.shape:
        raise ValueError(
            "Can only compute the quadratic cost of two vectors of equal shape"
        )

    result = x - y
    return cast(np.ndarray, result)


import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from sklearn import linear_model
import matplotlib.animation as animation
import os
import subprocess

# apply gradient descent algorithm to optimiza cost function of several variables

def cost(x):
	"""Cost function of linear regression (LR) model with coefficient x"""
	m = A.shape[0]
	return .5/m *np.linalg.norm(A.dot(x)-b,2)**2

def grad(x):
	"""Gradient function of cost function"""
	m = A.shape[0]
	return 1/m *A.T.dot(A.dot(x)-b)

def numerical_grad(x, cost):
	"""Calculating numerical gradient with coefficient x to check whether our grad function is correct or not"""
	eps = 1e-4
	g = np.zeros_like(x)
	for i in range(len(x)):
		x_p = x.copy()
		x_n = x.copy()
		x_p[i] += eps
		x_n[i] -= eps
		g[i] = (cost(x_p) - cost(x_n))/(2*eps)
	return g

def check_grad(x, cost, grad):
	"""Check grad function by comparing actual gradient value and numerical gradient (estimation)"""
	x = np.random.rand(x.shape[0], x.shape[1])
	grad1 = grad(x)
	grad2 = numerical_grad(x, cost)
	return True if np.linalg.norm(grad1 - grad2) < 1e-6 else False

def gradient_descent(x_init, grad, alpha):
	"""
	Running gradient descent with initial value x_init, grad function, learning rate alpha
	This function return x: list of solution after each iteration and iteration: last iteration it has gone through
	"""
	m = len(x_init)
	x = [x_init]
	for iteration in range(itr):
		x_new = x[-1] - alpha*grad(x[-1])
		if np.linalg.norm(grad(x_new))/m < 1e-3: #Stop gradient descent when grad value is too small.
			break
		x.append(x_new)
	return (x, iteration)

############### Main ####################

# create random data
A = np.array([[2,9,7,9,11,16,25,23,22,29,29,35,37,40,46]]).T
b = np.array([[2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]]).T

fig1 = plt.figure('GD for Linear Regression')
ax = plt.axes(xlim=(-10, 60), ylim=(-1, 20)) #restrict the figure showing only values in specified range
plt.plot(A,b,'ro', label='_nolegend_') # plot data points
line, = ax.plot([], [], color = 'blue') # plot function used in animation

# plot solution found by scikit learn
lr = linear_model.LinearRegression()
lr.fit(A,b)
x0_gd = np.linspace(1,46,2)
y0_sklearn = lr.intercept_[0] + lr.coef_[0][0]*x0_gd
plt.plot(x0_gd, y0_sklearn, color='green')

# apply gradient descent
itr = 90
learning_rate = .0001 #.0001
x_init = np.array([[1], [2]])

ones = np.ones((A.shape[0],1), dtype=np.int8) #Append bias to A
A = np.concatenate((ones,A), axis=1)

print('Checking gradient...', check_grad(np.random.rand(2, 1), cost, grad))

x_init = np.array([[1], [2]])

# running gradient descent
myGD = gradient_descent(x_init, grad, learning_rate)

# plot x_init (black line)
x0_gd = np.linspace(1,46,2)
y0_init = x_init[0][0] + x_init[1][0]*x0_gd
plt.plot(x0_gd, y0_init, color = 'black')

# plot lines in each iteration
def update(i):
	x_gd = myGD[0][i]
	y0_gd = x_gd[0][0] + x_gd[1][0]*x0_gd
	line.set_data(x0_gd,y0_gd)
	plt.xlabel('Iteration: {}/{}, learning rate: {}'.format(i+1, myGD[1], learning_rate))
	return line,

# legend for graph
plt.legend(('Value in each GD iteration', 'Solution by formular', 'Inital value for GD'), loc=(0.52, 0.01))
ltext = plt.gca().get_legend().get_texts()
plt.setp(ltext[0], fontsize=10)
plt.setp(ltext[1], fontsize=10)
plt.setp(ltext[2], fontsize=10)

# save animation to mp4 file using ffmpeg
Writer = animation.writers['ffmpeg']
writer = Writer(fps=15, metadata=dict(artist='Me'), bitrate=1800)

line_ani = animation.FuncAnimation(fig1, update, myGD[1]+1, interval=50, blit=True)
# # Save mp4 file for animation
# line_ani.save('lines.mp4', writer='ffmpeg')

# # Convert mp4 to gif
# # https://stackoverflow.com/questions/11269575/how-to-hide-output-of-subprocess-in-python-2-7
# FNULL = open(os.devnull, 'w')
# subprocess.call(['ffmpeg', '-i', 'lines.mp4', 'lines.gif'], stdout=FNULL, stderr=subprocess.STDOUT) # Execute command line to convert .mp4 to .gif using ffmpeg and hide output of command line to terminal
plt.show()

plt.figure('Iter and cost function')

cost_list = []
iter_list = []

for i in range(itr):
	iter_list.append(i+1)

for i in range(itr):
	cost_list.append(cost(myGD[0][i]))

plt.plot(iter_list, cost_list)
plt.xlabel('Iteration')
plt.ylabel('Cost value')

plt.show()


#!/usr/bin/env python
from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
from scipy import optimize
from numpy import newaxis, r_, c_, mat, e
from numpy.linalg import *

def plotData(X, y):
    #pos = (y.ravel() == 1).nonzero()
    #neg = (y.ravel() == 0).nonzero()
    pos = (y == 1).nonzero()[:1]
    neg = (y == 0).nonzero()[:1]

    plt.plot(X[pos, 0].T, X[pos, 1].T, 'k+', markeredgewidth=2, markersize=7)
    plt.plot(X[neg, 0].T, X[neg, 1].T, 'ko', markerfacecolor='r', markersize=7)

def sigmoid(z):
    g = 1. / (1 + e**(-z.A))
    return g

def costFunction(theta, X, y):
    m = X.shape[0]

    predictions = sigmoid(X * c_[theta])
    J = 1./m * (-y.T.dot(np.log(predictions)) - (1-y).T.dot(np.log(1 - predictions)))

    #grad = 1./m * X.T * (predictions - y)
    return J[0][0]##, grad.A

def predict(theta, X):
    p = sigmoid(X * c_[theta]) >= 0.5
    return p

def plotDecisionBoundary(theta, X, y):
    plotData(X[:, 1:3], y)

    if X.shape[1] <= 3:
        plot_x = r_[X[:,2].min()-2,  X[:,2].max()+2]
        plot_y = (-1./theta[2]) * (theta[1]*plot_x + theta[0])

        plt.plot(plot_x, plot_y)
        plt.legend(['Admitted', 'Not admitted', 'Decision Boundary'])
        plt.axis([30, 100, 30, 100])
    else:
        pass

if __name__ == '__main__':
    data = np.loadtxt('ex2data1.txt', delimiter=',')
    X = mat(c_[data[:, :2]])
    y = c_[data[:, 2]]

    # ============= Part 1: Plotting

    print 'Plotting data with + indicating (y = 1) examples and o ' \
          'indicating (y = 0) examples.'

    plotData(X, y)
    plt.ylabel('Exam 1 score')
    plt.xlabel('Exam 2 score')
    plt.legend(['Admitted', 'Not admitted'])
    plt.show()

    raw_input('Press any key to continue\n')

    # ============= Part 2: Compute cost and gradient

    m, n = X.shape

    X = c_[np.ones(m), X]

    initial_theta = np.zeros(n+1)

    cost, grad = costFunction(initial_theta, X, y), None

    print 'Cost at initial theta (zeros): %f' % cost
    print 'Gradient at initial theta (zeros):\n%s' % grad

    raw_input('Press any key to continue\n')

    # ============= Part 3: Optimizing using fminunc

    options = {'full_output': True, 'maxiter': 400}

    theta, cost, _, _, _ = \
        optimize.fmin(lambda t: costFunction(t, X, y), initial_theta, **options)

    print 'Cost at theta found by fminunc: %f' % cost
    print 'theta: %s' % theta

    plotDecisionBoundary(theta, X, y)
    plt.show()

    raw_input('Press any key to continue\n')

    # ============== Part 4: Predict and Accuracies

    prob = sigmoid(mat('1 45 85') * c_[theta])
    print 'For a student with scores 45 and 85, we predict an admission ' \
          'probability of %f' % prob

    p = predict(theta, X)
    print 'Train Accuracy:', (p == y).mean() * 100

    raw_input('Press any key to continue\n')


import numpy as np
import matplotlib.pyplot as plt

class DescidaGradienteRosenbrock():

    MAX_PASSOS = 20000
    TOLERANCIA_MINIMA = 1.0 * (10 ** -4)

    def __init__(self, learning_rate):
        self.learning_rate = learning_rate

    def executar(self):
        passo = 1
        ponto = np.array([0, 0, 0])
        ponto_anterior = None
        gradiente = None
        tolerancia = 1000000
        valor = self.func_rosenbrock(ponto[0], ponto[1], ponto[2])
        valor_anterior = None
        self.todos_valores = [valor]

        while passo <= self.MAX_PASSOS and tolerancia >= self.TOLERANCIA_MINIMA:
            # Calculando gradiente
            gradiente = self.calc_gradiente(ponto[0], ponto[1], ponto[2])

            # Calculando novo ponto
            ponto_anterior = ponto
            ponto = ponto - (self.learning_rate * gradiente)

            # Calculando novo valor
            valor_anterior = valor
            valor = self.func_rosenbrock(ponto[0], ponto[1], ponto[2])
            self.todos_valores.append(valor)

            # Calculando tolerância
            tolerancia = self.calc_tolerancia(ponto_anterior, ponto)

            print("(%d) x1=%.16f x2=%.16f x3=%.16f tolerancia=%.16f valor=%.16f" % (passo, ponto[0], ponto[1], ponto[2], tolerancia, valor))
            passo += 1

        print("=============================================================")
        print("Passos = %d" % (passo - 1))
        print("x1 = %.16f" % (ponto[0]))
        print("x2 = %.16f" % (ponto[1]))
        print("x3 = %.16f" % (ponto[2]))
        print("valor = %.16f" % (valor))

    def plotar_grafico(self, titulo):
        # Mostrando gráfico
        plt.plot(self.todos_valores)
        plt.title(titulo)
        plt.xlabel("Número de atualizações de $x$")
        plt.ylabel("Valor da função $f(x)$")
        plt.show()

    def func_rosenbrock(self, x1, x2, x3):
        resp = 100 * ((x2 - (x1 ** 2)) ** 2)
        resp += (1 - x1) ** 2
        resp += 100 * ((x3 - (x2 ** 2)) ** 2)
        resp += (1 - x2) ** 2

        return resp

    def calc_gradiente(self, x1, x2, x3):
        grad_x1 = 2 * ((200 * (x1 ** 3)) - (200 * x1 * x2) + x1 - 1)

        grad_x2 = (-200 * (x1 ** 2)) + (400 * (x2 ** 3)) + (x2 * (202 - (400 * x3))) -2

        grad_x3 = 200 * (x3 - (x2 ** 2))

        return np.array([grad_x1, grad_x2, grad_x3])

    def calc_tolerancia(self, valor_anterior, valor):
        return np.linalg.norm(valor - valor_anterior) / np.linalg.norm(valor_anterior)

# CÓDIGO PRINCIPAL
if __name__ == "__main__":
    descida = DescidaGradienteRosenbrock(10 ** -3)
    try:
        descida.executar()
    except:
        pass

    descida.plotar_grafico("Evolução do valor da função $f(x)$ com $lr = 10^{-3}$")


from __future__ import division
import numpy as np

from activation_functions import Sigmoid

def accuracy_score(y_true, y_pred):
    accuracy = np.sum(y_true == y_pred) /len(y_true)
    return accuracy

class Loss(object):
    def loss(self, y_true, y_pred):
        return NotImplementedError()

    def gradient(self, y, y_pred):
        raise NotImplementedError()

    def acc(self, y, y_pred):
        return NotImplementedError()

class SquareLoss(Loss):
    def __init__(self):
        self.loss_name = 'SquareLoss'

    def loss(self, y, y_pred):
        return 0.5 * np.power((y - y_pred), 2)

    def gradient(self, y, y_pred):
        return -(y - y_pred)

    def acc(self, y, y_pred):
        return 0.5 * np.mean(np.power((y - y_pred), 2))

class CrossEntropy(Loss):
    def __init__(self):
        self.loss_name = 'CrossEntropy'

    def loss(self, y, AL):
        # Avoid division by zero
        AL = np.clip(AL, 1e-15, 1 - 1e-15)
        return (- y * np.log(AL) - (1 - y) * np.log(1 - AL)) #/(y.shape[-1])#(1)normalize in main

    def acc(self, y, AL):
        return accuracy_score(y[0], AL[0]>=0.5)

    def gradient(self, y, AL):
        # Avoid division by zero
        AL = np.clip(AL, 1e-15, 1 - 1e-15)
        #print  ((- (y / AL) + (1 - y) / (1 - AL) ).shape,'cross-function output dA')
        return (- (y / AL) + (1 - y) / (1 - AL)) #/(y.shape[-1])#(2)normalize when updating grad

class MultiClassCrossEntropy(Loss):
    def __init__(self):
        self.loss_name = 'MultiClassCrossEntropy'

    def loss(self, y, AL):
        return -np.sum(y * np.log(AL) ,axis= 0, keepdims = True) #/(y.shape[-1])#(1)normalize in main

    def gradient(self, y, AL):
        # Avoid division by zero
        AL = np.clip(AL, 1e-15, 1 - 1e-15)

        assert(AL.shape == (-(y / AL)).shape)

        return - (y / AL) # /(y.shape[-1])#(2)normalize when updating grad

class SoftmaxCrossEntropy(Loss):
    def __init__(self):
        self.loss_name = 'SoftmaxCrossEntropy'

    def loss(self, y, Z):
        # Avoid division by zero
        Z_aux = Z.shape
        Z = Z - np.max(Z, axis = 0, keepdims=True)
        assert Z_aux ==Z.shape

        log_e_Z = Z- np.log(np.sum( np.exp(Z), axis=0, keepdims=True))
        return (-np.sum( y * log_e_Z ,axis= 0))  # (1)normalize in main

    def gradient(self, y, Z):
        Z -= np.max(Z, axis = 0, keepdims=True)
        p = np.exp(Z)/ np.sum( np.exp(Z), axis=0, keepdims=True)

        # Avoid division by zero
        return  (-y + p)  #(2)Normalize when updating grad


"""
Activation functions and their derivatives
"""
import numpy as np

def sigmoidp(x):
    return 1.0 / (1.0 + np.exp(-x))

def sigmoidn(x):
    return np.exp(x) / (1.0 + np.exp(x))

# The sigmoid functions for positive and negative valus
# are done differently to avoid overvflow erros while
# calculating the exponent
def sigmoid(x):
    xp = np.multiply(x, (x>0))
    xn = np.multiply(x, (x<0))
    return sigmoidp(xp) + sigmoidn(xn) - 0.5

def sigmoid_prime(x):
    return sigmoid(x) * (1 - sigmoid(x))

# softmax neumerator and denominators are both devided by the largest component of
# the denominator to avoid overflow errors
def softmax(x):
    return np.exp(x-np.max(x, axis=0)) / np.sum(np.exp(x-np.max(x, axis=0)), axis=0)

def tanh(x):
    return np.tanh(x)

def tanh_prime(x):
    return 1 - tanh(x) * tanh(x)

def relu(x):
    return np.maximum(x, 0)

def relu_prime(x):
    return (x > 0).astype(float)


# 5.2 Learning is just parameter estimation
import torch
import numpy as np
import csv
import matplotlib.pyplot as plt

# data
t_c = [0.5, 14.0, 15.0, 28.0, 11.0, 8.0, 3.0, -4.0, 6.0, 13.0, 21.0]
t_u = [35.7, 55.9, 58.2, 81.9, 56.3, 48.9, 33.9, 21.8, 48.4, 60.4, 68.4]
t_c = torch.tensor(t_c)
t_u = torch.tensor(t_u)

# Visualizing the data
plt.figure(figsize=(12, 6))
# plt.plot(np.arange(1, 12, step=1), t_c.numpy())
plt.scatter(t_u.numpy(), t_c.numpy(), s=120, c="Orange")
plt.xlabel("MEASUREMENT")
plt.ylabel("TEMPERATURE (Celsius)")
plt.grid()
plt.show()

# Define model
def model(t_u, w, b):
    out = w * t_u + b
    return out

# Define loss function
def loss_fn(t_p, t_c):
    squared_diffs = (t_p - t_c)**2
    return squared_diffs.mean()

# Initializer the parameters
w = torch.ones(())
b = torch.zeros(())

t_p = model(t_u, w, b)
t_p

# check the value of the loss
loss = loss_fn(t_p, t_c)
loss

delta = 0.1
loss_rate_of_change_w = \
(loss_fn(model(t_u, w + delta, b), t_c) -
loss_fn(model(t_u, w - delta, b), t_c)) / (2.0 * delta)

learning_rate = 1e-2
w = w - learning_rate * loss_rate_of_change_w

loss_rate_of_change_b = \
(loss_fn(model(t_u, w, b + delta), t_c) -
loss_fn(model(t_u, w, b - delta), t_c)) / (2.0 * delta)

b = b - learning_rate * loss_rate_of_change_b

# d loss_fn / d w = (d loss_fn / d t_p) * (d t_p / d w)

def loss_fn(t_p, t_c):
    squared_diffs = (t_p - t_c)**2
    return squared_diffs.mean()

def dloss_fn(t_p, t_c):
    dsq_diffs = 2 * (t_p - t_c) / t_p.size(0)
    return dsq_diffs

def model(t_u, w, b):
    return w * t_u + b

def dmodel_dw(t_u, w, b):
    return t_u

def dmodel_db(t_u, w, b):
    return 1.0

def grad_fn(t_u, t_c, t_p, w, b):
    dloss_dtp = dloss_fn(t_p, t_c)
    dloss_dw = dloss_dtp * dmodel_dw(t_u, w, b)
    dloss_db = dloss_dtp * dmodel_db(t_u, w, b)
    return torch.stack([dloss_dw.sum(), dloss_db.sum()])

def training_loop(n_epochs, learning_rate, params, t_u, t_c):
    for epoch in range(1, n_epochs + 1):
        w, b = params
        t_p = model(t_u, w, b)
        loss = loss_fn(t_p, t_c)
        grad = grad_fn(t_u, t_c, t_p, w, b)
        params = params - learning_rate * grad
        print('Epoch %d, Loss %f' % (epoch, float(loss)))
    return params

training_loop(
    n_epochs = 100,
    learning_rate = 1e-2,
    params = torch.tensor([1.0, 0.0]),
    t_u = t_u,
    t_c = t_c)

training_loop(
    n_epochs = 100,
    learning_rate = 1e-4,
    params = torch.tensor([1.0, 0.0]),
    t_u = t_u,
    t_c = t_c)

t_un = 0.1 * t_u

training_loop(
    n_epochs = 100,
    learning_rate = 1e-2,
    params = torch.tensor([1.0, 0.0]),
    t_u = t_un,
    t_c = t_c)

params = training_loop(
    n_epochs = 5000,
    learning_rate = 1e-2,
    params = torch.tensor([1.0, 0.0]),
    t_u = t_un,
    t_c = t_c,
    )

t_p = model(t_un, *params)
fig = plt.figure(dpi=600)
plt.xlabel("Temperature (°Fahrenheit)")
plt.ylabel("Temperature (°Celsius)")
plt.plot(t_u.numpy(), t_p.detach().numpy())
plt.plot(t_u.numpy(), t_c.numpy(), 'o')
plt.grid()
plt.show()

5_The mechanics of learning/5.2 Learning is just parameter estimation.py


import numpy as np
import copy

testing_matrix = np.matrix('1 7 -1 5; 4 -1 1 1; 5 -3 1 -1')

def gauss(matrix: np.matrix, show=False) -> tuple:
    '''
    Gaussian elimination can solve linear equation systems of any size.
    Input should be a numpy matrix. There is a variable called
    'testing_matrix' already defined for demo purpose :)
    '''
    matrix_copy = copy.deepcopy(matrix)
    variables_amount = len(np.asarray(matrix_copy[0]).reshape(-1)) - 1

    if (variables_amount > len(matrix_copy)):
        raise Exception('Too much variables!')
        return None

    if show:
        print('starting matrix:')
        print(matrix_copy)

    for subtraction_index in range(variables_amount - 1):

        subtraction_matrix = matrix_copy[subtraction_index]

        for line in range(1 + subtraction_index, variables_amount):
            matrix_to_subtract = matrix_copy[line]
            matrix_copy[line] = matrix_to_subtract * subtraction_matrix[0, subtraction_index] - subtraction_matrix * matrix_to_subtract[0, subtraction_index]

        if show:
            print('iteration {}:'.format(subtraction_index))
            print(matrix_copy)

    solutions = np.array([])

    for equation in range(variables_amount - 1, -1, -1):
        addition = 0
        for variable, solution in zip(range(variables_amount - 1, equation, -1), solutions):
            addition += matrix_copy[equation, variable] * solution

        solutions = np.append(solutions, (matrix_copy[equation, variables_amount] - addition) / matrix_copy[equation, equation])

    solutions = np.flip(solutions)

    if show:
        print('\nsolutions set:\n{}'.format(solutions))

        print('\nsolutions:')
        for solution, index in zip(solutions, range(len(solutions))):
            print('x{i}: {s}'.format(s = solution, i = index))

    return tuple(solutions)


def price_options(S=100.0, K=100.0, sigma=0.25, r=0.05, days=260, paths=10000):
    """
    Price European and Asian options using a Monte Carlo method.

    Parameters
    ----------
    S : float
        The initial price of the stock.
    K : float
        The strike price of the option.
    sigma : float
        The volatility of the stock.
    r : float
        The risk free interest rate.
    days : int
        The number of days until the option expires.
    paths : int
        The number of Monte Carlo paths used to price the option.

    Returns
    -------
    A tuple of (E. call, E. put, A. call, A. put) option prices.
    """
    import numpy as np
    from math import exp,sqrt

    h = 1.0/days
    const1 = exp((r-0.5*sigma**2)*h)
    const2 = sigma*sqrt(h)
    stock_price = S*np.ones(paths, dtype='float64')
    stock_price_sum = np.zeros(paths, dtype='float64')
    for j in range(days):
        growth_factor = const1*np.exp(const2*np.random.standard_normal(paths))
        stock_price = stock_price*growth_factor
        stock_price_sum = stock_price_sum + stock_price
    stock_price_avg = stock_price_sum/days
    zeros = np.zeros(paths, dtype='float64')
    r_factor = exp(-r*h*days)
    euro_put = r_factor*np.mean(np.maximum(zeros, K-stock_price))
    asian_put = r_factor*np.mean(np.maximum(zeros, K-stock_price_avg))
    euro_call = r_factor*np.mean(np.maximum(zeros, stock_price-K))
    asian_call = r_factor*np.mean(np.maximum(zeros, stock_price_avg-K))
    return (euro_call, euro_put, asian_call, asian_put)


import numpy as np
from .Model import Model
from src.si.data.dataset import Dataset
from src.si.util.metrics import mse

class LinearRegression(Model):
    def __init__(self, gd=False, epochs=1000, lr=0.001):
        super(LinearRegression, self).__init__()
        self.gd = gd
        self.theta = None
        self.num_iterations = epochs
        self.lr = lr

    def fit(self, dataset):
        X, Y = dataset.getXy()
        X = np.hstack((np.ones((X.shape[0], 1)), X))
        self.X = X
        self.Y = Y
        if self.gd:
            self.train_gd(X, Y)
        else:
            self.train_closed(X, Y)
        self.is_fited = True

    def train_gd(self, X, Y):
        m = X.shape[0]
        n = X.shape[1]
        self.history = {}
        self.theta = np.zeros(n)
        for epoch in range(self.num_iterations):
            grad = 1/m*(X.dot(self.theta)-Y).dot(X)
            self.theta -= self.lr*grad
            self.history[epoch] = [self.theta[:], self.cost()]

    def train_closed(self, X, Y):
        self.theta = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(Y)

    def predict(self, x):
        assert self.is_fited
        _X = np.hstack(([1], x))
        return np.dot(self.theta, _X)

    def cost(self):
        y_pred = np.dot(self.X, self.theta)
        return mse(y_pred, self.Y)

class LinearRegressionReg(LinearRegression):
    def __init__(self, gd = False, epochs = 1000, lr=0.001,lbd = 1):
        super(LinearRegressionReg, self).__init__(gd=gd, epochs=epochs, lr=lr)
        self.lbd = lbd
        self.gd = gd
        self.num_iterations = epochs

    def train_closed(self, X, Y):
        n = X.shape[1]
        identity = np.eye(n)
        identity[0, 0] = 0
        self.theta = np.linalg.inv(X.T.dot(X)+self.lbd*identity).dot(X.T).dot(Y)
        self.is_fited = True

    def train_gd(self, X, Y):
        m = X.shape[0]
        n = X.shape[1]
        self.history = {}
        self.theta = np.zeros(n)
        lbds = np.full(m, self.lbd)
        lbds[0] = 0
        for epoch in range(self.num_iterations):
            grad = (X.dot(self.theta)-Y).dot(X)
            self.theta -= (self.lr/m)*(lbds+grad)
            self.history[epoch] = [self.theta[:], self.cost()]

    def predict(self, X):
        assert self.is_fited
        _x = np.hstack(([1], X))
        return np.dot(self.theta, _x)

    def cost(self):
        y_pred = np.dot(self.X, self.theta)
        return mse(self.Y, y_pred)/2


# gradient descent optimization with adam for a two-dimensional test function
from math import sqrt
from numpy import asarray
from numpy.random import rand
from numpy.random import seed

# objective function
def objective(x, y):
	return x**2.0 + y**2.0

# derivative of objective function
def derivative(x, y):
	return asarray([x * 2.0, y * 2.0])

# gradient descent algorithm with adam
def adam(objective, derivative, bounds, n_iter, alpha, beta1, beta2, eps=1e-8):
	# generate an initial point
	x = bounds[:, 0] + rand(len(bounds)) * (bounds[:, 1] - bounds[:, 0])
	score = objective(x[0], x[1])
	# initialize first and second moments
	m = [0.0 for _ in range(bounds.shape[0])]
	v = [0.0 for _ in range(bounds.shape[0])]
	# run the gradient descent updates
	for t in range(n_iter):
		# calculate gradient g(t)
		g = derivative(x[0], x[1])
		# build a solution one variable at a time
		for i in range(x.shape[0]):
			# m(t) = beta1 * m(t-1) + (1 - beta1) * g(t)
			m[i] = beta1 * m[i] + (1.0 - beta1) * g[i]
			# v(t) = beta2 * v(t-1) + (1 - beta2) * g(t)^2
			v[i] = beta2 * v[i] + (1.0 - beta2) * g[i]**2
			# mhat(t) = m(t) / (1 - beta1(t))
			mhat = m[i] / (1.0 - beta1**(t+1))
			# vhat(t) = v(t) / (1 - beta2(t))
			vhat = v[i] / (1.0 - beta2**(t+1))
			# x(t) = x(t-1) - alpha * mhat(t) / (sqrt(vhat(t)) + eps)
			x[i] = x[i] - alpha * mhat / (sqrt(vhat) + eps)
		# evaluate candidate point
		score = objective(x[0], x[1])
		# report progress
		print('>%d f(%s) = %.5f' % (t, x, score))
	return [x, score]

# seed the pseudo random number generator
seed(1)
# define range for input
bounds = asarray([[-1.0, 1.0], [-1.0, 1.0]])
# define the total iterations
n_iter = 60
# steps size
alpha = 0.02
# factor for average gradient
beta1 = 0.8
# factor for average squared gradient
beta2 = 0.999
# perform the gradient descent search with adam
best, score = adam(objective, derivative, bounds, n_iter, alpha, beta1, beta2)
print('Done!')
print('f(%s) = %f' % (best, score))


import matplotlib.pyplot as plt
import numpy as np
#
x_0 = int(input("Enter the initial value of x (try 0): "))
y_0 = int(input("Enter the initial value of y (try 0): "))
t=float(input("Enter the value of x at which we want the value of y(try 1.2): "))
h= float(input("Enter the value of interval h(0.1 or 0.01 or 0.001): "))
def euler_method(x_0, y_0, h, t):  ## t is final point where we want the value of function
 x=x_0
 y=y_0
 def fun(x,y):
     return x+2*y                                                   # return 3*np.exp(-x)-0.4*y
                                                        #   #return (x + y + x * y)

## make lists to store values
 lsty=[]  # y data points which we get numerically
 lstx=[]  # x data points which we get numerically
 while x < t:  # here t is final point
   y+=fun(x,y)*h
   lsty.append(y)
   x=x+h
   lstx.append(x)

 print("Approximate solution at x = ", int(x), " is ", "%.6f"% y)
 print("The number of iterations are n = ",int((x-x_0)/h+1))
   # print(list[y], end=" ")
 # print (lstx)
 # print (lsty)

 ## --------------Plotting------------------------
 ## ----------Numerical plot----------------------

 ## for interval h=0.1-->
 if h==0.1:
  # plt.subplot(1,3,1)
  plt.plot(lstx,lsty,'-r',label ="Numerical curve")
  ##---------- functional plot---------------------
  xp= np.linspace(0,1.2,12)
  f=0.25*np.exp(2*xp) - 0.5*xp-0.25

  plt.plot(xp,f, 'c', label="Functional curve")
  plt.savefig("Euler_Method_h_0.1.png")
  plt.grid()
  plt.legend()
  plt.title("h=0.1")
  plt.show()

 ### for interval h=0.01--->
 elif h==0.01:
  # plt.subplot(1,3,2)
  plt.plot(lstx,lsty,'-r',label ="Numerical curve")
  xp= np.linspace(0,1.2,121)
  f=0.25*np.exp(2*xp) - 0.5*xp-0.25
  plt.plot(xp,f, 'c', label="Functional curve")
  plt.savefig("Euler_Method_h_0.01.png")
  plt.grid()
  plt.legend()
  plt.title("h=0.01")
  plt.show()

 # for interval h=0.001--->
 elif h==0.001:
  # plt.subplot(1,3,3)
  plt.plot(lstx,lsty,'-r',label ="Numerical Euler method curve")
  xp= np.linspace(0,1.2,1201)
  f=0.25*np.exp(2*xp) - 0.5*xp-0.25
  plt.plot(xp,f, 'c', label="Functional curve")
  plt.title("h=0.001")
  plt.savefig("Euler_Method_h_0.001.png")
  plt.grid()
  plt.legend()
  plt.show()

euler_method(x_0,y_0,h,t)
#euler_method(0,1,0.025,.1)

##----------plotting -------------

def f(x):
    return 0.25*np.exp(2*x) - 0.5*x-0.25
print ("The exact value is ", f(1.2))

#x= np.linspace(0,1.2,121)
# f=0.25*np.exp(2*x) - 0.5*x-0.25
# plt.plot(x,f,lstx,lsty)
# plt.xlabel('x')  # naming the x axis
# plt.ylabel('f(x)')  # naming the y axis
# plt.show()


import numpy as np

def distances(x, distance = "E", p = None):
    """
    Take a data matrix, return the distances matrix.

    Args:
        x: numpy array, data matrix.
        distance: {"E": Euclidean Distance, "Minkowski": Minkowski Distance, "Manhattan": Manhattan Distance,
                "C": Chebyshev Distance, "Mahalanobis": Mahalanobis Distance, "Corr": Correlation Coefficient,
                "cos": cosine}, default "E".
        p: If Minkowski distance is chosen, p should be set, default None.

    Return:
        Distances matrix.
    """
    if x.ndim == 1: x = np.reshape(x, [1, x.shape[0]])
    if x.ndim == 0: x = np.reshape(x, [1, 1])

    x = x.astype(np.float64)

    if distance == "Minkowski" and p == None:
        raise TypeError("Missing p for Minkowski distance.")

    if distance in  {"E", "Minkowski", "Manhattan", "C"}:
        if distance == "Manhattan": p = 1
        if distance == "E": p = 2
        res = np.zeros([1, x.shape[0]])
        for i in x:
            dis = np.sum(np.abs(x - i) ** p, axis = 1) ** (1/p) if distance != "C" else np.max(np.abs(x - i), axis = 1)
            dis = np.reshape(dis, [1, x.shape[0]])
            res = np.vstack([res, dis])
        return res[1:]

    if distance == "Mahalanobis":
        cov = np.cov(x.T)
        try:
            inv = np.linalg.inv(cov)
        except np.linalg.LinAlgError:
            raise ValueError("Cov matrix is singular matrix")
        res = np.zeros([1, x.shape[0]])
        for i in x:
            diff = x - i
            dis = np.diag(np.dot(np.dot(diff, inv), diff.T))
            res = np.vstack([res, dis])
        return res[1:]

    if distance == "Corr":
        return np.corrcoef(x)

    if distance == "cos":
        res = np.dot(x, x.T)
        for i in range(x.shape[0]):
            factor = res[i,i] ** 0.5
            res[:,i] = res[:,i] / factor
            res[i,:] = res[i,:] / factor
        return res


import numpy as np
import matplotlib.pyplot as plt
# from numpy import zeros

t0 = 0
tn = 100
h = 0.01
square_h = h**2/2
n = (tn - t0)/h

# first initial condition
x0 = -1.0
v0 = 0.1
f0 = -4 *x0*(x0**2 -1)

x = np.zeros(int(n))
v = np.zeros(int(n))
f = np.zeros(int(n))

x[0] = x0
v[0] = v0
f[0] = f0

# second initial condition
x01 = 1.0
v01 = 0.1
f01 = -4 *x01*(x01**2 -1)

x1 = np.zeros(int(n))
v1 = np.zeros(int(n))
f1 = np.zeros(int(n))

x1[0] = x01
v1[0] = v01
f1[0] = f01

# for loop for first initial conditions
for i in range(int(n-1)):
  # Velocity Verlet algorithm
  x[i+1] = x[i] + h * v[i] + square_h * f[i]
  f[i+1] = -4 * x[i] * (x[i]**2 -1)
  v[i+1] = v[i] + h * (f[i+1] + f[i])/2

# for loop for second initial conditions
for i in range(int(n-1)):
  x1[i+1] = x1[i] + h * v1[i] + square_h * f1[i]
  f1[i+1] = -4 * x1[i] * (x1[i]**2 -1)
  v1[i+1] = v1[i] + h * (f1[i+1] + f1[i])/2

# --------------plotting--------------
fig, ax = plt.subplots(1, 2)
fig.set_size_inches(10.5, 5.5)
fig.suptitle('Phase trajectory using Verlet Algorithm')

ax[0].plot(x, v, 'k')
ax[0].set(xlabel = 'x', ylabel='velocity (v)')
ax[0].set_title("(x0, p0)=" +"("+str(x0) + ',' + str(v0)+")")

ax[1].plot(x1, v1, 'b')
ax[1].set(xlabel = 'x')
ax[1].set_title(" (x0, p0)=" +"("+str(x01) + ',' + str(v01)+")")

# plt.savefig('Phase-Space-Velocity-Verlet.png')
plt.show()


import numpy as np
from numpy import linalg

def gcd(a, b):
    return gcd(b, a % b) if b != 0 else a

def extended_gcd(a, b):
    s, old_s = 1, 0
    t, old_t = 0, 1

    while b != 0:
        q = a // b
        a, b = b, a % b
        s, old_s = old_s, s - q * old_s
        t, old_t = old_t, t - q * old_t
    return a, s, t

def diophantine_sol_exists(a, b, c):
    hcf = gcd(a, b)
    return c % hcf == 0

# returns solution of diophantine equation in form (x, y) if exists
# returns the particular solution otherwise returns nothing
def diophantine(a, b, c):
    hcf = gcd(a, b)
    # solution exits
    if c % hcf == 0:
        _, s, t = extended_gcd(a / hcf, b / hcf)
        return int((c / hcf) * s), int((c / hcf) * t)

def diophantine_general(a, b, c, iters=10):
    if diophantine_sol_exists(a, b, c):
        x, y = diophantine(a, b, c)
        hcf = gcd(a, b)
        for i in range(iters):
            yield x + i * (b // hcf), y - i * (a // hcf)

def additive_inverse(a, n):
    return (n - a) % n

def multiplicative_inverse_exists(a, n):
    return gcd(a, n) == 1

def multiplicative_inverse(b, n):
    if multiplicative_inverse_exists(b, n):
        return (extended_gcd(n, b)[2] + n) % n

def sol_single_var_le(a, b, n, c=0):
    if c != 0:
        b = (b - c) % n

    hcf = gcd(a, n)
    sols = []
    if b % hcf == 0:
        a, b, n = a // hcf, b // hcf, n // hcf
        x_0 = (b * multiplicative_inverse(a, n)) % n
        for k in range(hcf):
            sols.append(x_0 + k * n)
    return sols

def modMatInv(A, p):  # Finds the inverse of matrix A mod p
    n = len(A)
    adj = np.zeros(shape=(n, n))
    for i in range(0, n):
        for j in range(0, n):
            adj[i][j] = ((-1) ** (i + j) * int(round(linalg.det(minor(A, j, i))))) % p
    return (modInv(int(round(linalg.det(A))), p) * adj) % p

def modInv(a, p):  # Finds the inverse of a mod p, if it exists
    for i in range(1, p):
        if (i * a) % p == 1:
            return i
    raise ValueError(str(a) + " has no inverse mod " + str(p))

def minor(A, i, j):  # Return matrix A with the ith row and jth column deleted
    A = np.array(A)
    sub_matrix = np.zeros(shape=(len(A) - 1, len(A) - 1))
    p = 0
    for s in range(0, len(sub_matrix)):
        if p == i:
            p = p + 1
        q = 0
        for t in range(0, len(sub_matrix)):
            if q == j:
                q = q + 1
            sub_matrix[s][t] = A[p][q]
            q = q + 1
        p = p + 1
    return sub_matrix

def is_prime(number: int) -> bool:
    for i in range(2, number):
        if number % i == 0:
            return False
    return True

def product(numbers: list) -> int:
    result = 1
    for number in numbers:
        result *= number
    return result

def relatively_prime(a: int, b: int) -> bool:
    return gcd(a, b) == 1

def exp_func(x, y):
    exp = bin(y)
    value = x

    for i in range(3, len(exp)):
        value = value * value
        if exp[i:i + 1] == '1':
            value = value * x
    return value

def mod_exponentiation(X, E, m):
    Y = 1
    while E > 0:
        if E % 2 == 0:
            X = (X * X) % m
            E /= 2
        else:
            Y = (X * Y) % m
            E -= 1
    return Y

def totient(number: int) -> int:
    count = 0
    for i in range(1, number):
        if gcd(number, i) == 1:
            count += 1
    return count

def relatively_prime_numbers(n):
    numbers = []
    for i in range(1, n):
        if relatively_prime(i, n):
            numbers.append(i)
    return numbers

def crt_representation(number: int, crt: tuple) -> list:
    result = []
    for mod in crt:
        result.append(number % mod)
    return result

def are_pairwise_relatively_prime(numbers) -> bool:
    for i, number in enumerate(numbers):
        for j in range(i + 1, len(numbers)):
            if not relatively_prime(number, numbers[j]):
                return False
    return True

# noinspection PyTypeChecker
def valid_crc(crc: tuple, number: int) -> bool:
    return are_pairwise_relatively_prime(crc) and product(crc) == number

def is_product_of_2_primes(number: int) -> bool:
    for i in range(2, number):
        if number % i == 0:
            quotient = number // i
            if is_prime(i) and is_prime(quotient):
                return True
    return False

def discrete_log_table(number: int) -> list:
    table = []
    for i in range(1, number):
        row = []
        for j in range(1, number):
            row.append(pow(i, j, number))
        table.append(row)
    return table

def discrete_log(n, b, m):
    for i in range(m):
        if pow(b, i, m) == n:
            return i


import numpy as np

def reg(u):
    u_reg = 1 / (np.linalg.norm(u)) * u
    u_reg_norm = np.linalg.norm(u_reg)

    print(f"{u_reg} , norm {u_reg_norm}")

reg(np.array([2, -7]))
reg(np.array([-9, 3, 7]))
reg(np.array([-3, 5, 8, 6]))
reg(np.array([-6, 2, 8, 3, 5]))

reg(np.array([5, 10, 4]))

vals = np.array([1,30,20,99,100,4])
n = np.linalg.norm(vals)

vals_n = vals * (1/n)
print(f"Norm {n}")
print(f"Norm vals {vals_n},  {np.linalg.norm(vals_n)}")


import numpy as np
import matplotlib.pyplot as plt

# function
def f(x):
  return 2*x[0]**2 + 10*x[1]**2

# gradient (normalized)
def fp(x):
  fx1 = 4*x[0]
  fx2 = 20*x[1]
  return np.array([fx1, fx2])

alpha = 0.01 # step size
x = np.array([3,3]) # initial point
xt = [] # empty lists to keep track of points we search
ft = []

# this is a "max-time" approach ...
# could also loop until the gradient is within a tolerance
normgrad = 9999
i = 0
maxiter = 1000

while normgrad > 10**-8 and i < maxiter:
  xt.append(x)
  ft.append(f(x))
  grad = fp(x)
  x = x - alpha*grad
  normgrad = np.sqrt(np.sum(grad**2))
  i += 1

print(i)

# just plotting stuff below here ...
plt.subplot(1,2,1)
xx = np.arange(-5,5,0.005)
X1,X2 = np.meshgrid(xx, xx)
Z = 2*X1**2 + 10*X2**2
plt.contour(X1,X2,Z,50,cmap=plt.cm.Blues_r)
xt = np.array(xt)
plt.plot(xt[:,0], xt[:,1], color='k', linewidth=2)
plt.xlabel('X1')
plt.ylabel('X2')

plt.subplot(1,2,2)
plt.plot(ft, color='k', linewidth=2)
plt.xlabel('Iterations')
plt.ylabel('Objective Value')
plt.show()


#!/usr/bin/env python

import numpy as np

#===================================================

def dejong(vector):
	# takes k-dimensional vector
	# continuous, convex, unimodal
	# global minimum: x_i = 0 for all i in range(k)
	vector = np.array(vector)
	# rescale onto [-5, 5]
	vector = 10 * vector - 5
	result = np.sum(vector**2)
	return result

#===================================================

def hyper_ellipsoid(vector):
	# takes k-dimensional vector
	# continuous, convex, unimodal
	# global minimum: x_i = 0 for all i in range(k)
	# rescale onto [-5, 5]
	vector = np.array(vector)
	vector = 10 * vector - 5
	weights = np.arange(1, len(vector) + 1)
	result = 0
	for index, element in enumerate(vector):
		result += weights[index] * element**2
	return result

#===================================================

def rosenbrock_function(vector):
	# takes k-dimensional vector
	# continuous, convex, unimodal, narrow valley
	# global minimum: x_i = 0 for all i in range(k)
	# rescale onto [-2, 2]
	vector = np.array(vector)
	vector = 4 * vector - 2
	result = 0
	for index, element in enumerate(vector[:-1]):
		result += 100 * (vector[index + 1] - element**2)**2  + (1 - element)**2
	return result

#===================================================

def rastrigin_function(vector):
	# takes k-dimensional vector
	# continuous, multi-modal, regular distribution of minima
	# global minimum: x_i = 0 for all i in range(k)
	result = 10. * len(vector)
	# rescale onto [-5, 5]
	vector = 10 * np.array(vector) - 5
	for index, element in enumerate(vector):
		result += element**2 - 10 * np.cos(2 * np.pi * element)
	return result

#===================================================

def schwefel_function(vector):
	# takes k-dimensional vector
	# continuous, multi-modal
	# global minimum far away from local minima
	# global minimum: 418.9829 * n at x_i = 420.9687
	# rescale onto [-500, 500]
	vector = 1000 * np.array(vector) - 500
	result = 0
	for index, element in enumerate(vector):
		result += - element * np.sin(np.sqrt(np.abs(element)))
	return result

#===================================================

def ackley_path_function(vector):
	# takes k-dimensional vector
	# continuous, multi-model
	# global minimum: x_i = 0
	# rescale onto [-32, 32]
	vector = 64 * np.array(vector) - 32
	a = 20.
	b = 0.2
	c = 2 * np.pi
	n = float(len(vector))
	vector = np.array(vector)
	result = - a * np.exp( - b * np.sqrt(np.sum(vector**2) / n ) ) - np.exp( np.sum(np.cos(c * vector)) / n ) + a + np.exp(1.)
	return result

#===================================================

def linear_funnel(vector):
	# takes k-dimensional vector
	vector = np.array(vector)
	vector = 10 * vector - 5
	bounds = [0.5, 1.5, 2.5, 3.5, 4.5]
	result = 5
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1
	result = np.amin([4, result])
	return result

def narrow_funnel(vector):
	# takes k-dimensional vector
	vector = np.array(vector)
	vector = 100 * vector - 50
	bounds = [1.0, 2.0, 3.0, 4.0, 5.0]
	bounds = np.array(bounds)**2
	result = 5
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1
	result = np.amin([4, result])
	return result

def discrete_ackley(vector):
	# takes k-dimensional vector
	vector = np.array(vector)
	vector = 100 * vector - 50
	bounds = [1.0, 2.0, 3.0, 4.0, 5.0]
	bounds = np.array(bounds)**2
	result = 5
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1
	bounds = [1.25, 2.0, 2.5]
	bounds = np.array(bounds)**2

	domain = np.linspace(-50, 50, 10)
	dx = domain[1] - domain[0]
	imaged = np.array([np.amin(np.abs(element - domain)) for element in vector])
	new_res = 5
	for bound in bounds[::-1]:
		if np.amax(np.abs(imaged)) < bound:
			new_res -= 1
		result = np.amin([result, new_res])
	result = np.amin([4, result])
	return result

def discrete_michalewicz(vector):
	vector = np.array(vector)
	vector = 100 * vector - 40

	bounds = [1.0, 2.0, 3.0, 4.0, 5.0]
	bounds = np.array(bounds)**2
	result = 5
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1
	bounds = [1.25, 2.0, 2.5, 3.0]
	bounds = np.array(bounds)**2
	new_res = 5
	for bound in bounds[::-1]:
		if np.amin(np.abs(vector)) < bound:
			new_res -= 1
	result = np.amin([result, new_res, 4])
	return result

def double_well(vector):
	vector = np.array(vector)
	vector = 100 * vector - 15

	bounds = [1.25, 1.75, 2.0, 2.5, 2.75]
	bounds = np.array(bounds)**2
	result = 5
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1

	vector -= 50
	vector[0] += 10
	bounds = [2.5, 4.0, 5.0, 6.5]
	bounds = np.array(bounds)**2
	new_res = 5
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			new_res -= 1

	result = np.amin([result, 4, new_res])
	return result

def discrete_valleys(vector):
	vector = np.array(vector)
	vector = 100 * vector - 50
	result = 5

	vector = vector - 50
	bounds = [1.25, 2.0, 2.5, 3.0]
	bounds = np.array(bounds)**2
	new_res = 5
	for bound in bounds[::-1]:
		if np.amin(np.abs(vector)) < bound:
			new_res -= 1
	result = np.amin([result, new_res, 4])

	vector = vector + 25
	bounds = [1.25, 2.0, 2.5, 3.0]
	bounds = np.array(bounds)**2
	new_res = 5
	for bound in bounds[::-1]:
		if np.amin(np.abs(vector)) < bound:
			new_res -= 1
	result = np.amin([result, new_res, 4])

	vector = vector + 25
	bounds = [1.25, 2.0, 2.5, 3.0]
	bounds = np.array(bounds)**2
	new_res = 5
	for bound in bounds[::-1]:
		if np.amin(np.abs(vector)) < bound:
			new_res -= 1
	result = np.amin([result, new_res, 4])

	vector = vector + 25
	bounds = [1.25, 2.0, 2.5, 3.0]
	bounds = np.array(bounds)**2
	new_res = 5
	for bound in bounds[::-1]:
		if np.amin(np.abs(vector)) < bound:
			new_res -= 1
	result = np.amin([result, new_res, 4])

	vector = vector + 25
	bounds = [1.25, 2.0, 2.5, 3.0]
	bounds = np.array(bounds)**2
	new_res = 5
	for bound in bounds[::-1]:
		if np.amin(np.abs(vector)) < bound:
			new_res -= 1
	result = np.amin([result, new_res, 4])

	vector = vector - 12.5
	bounds = [1.25, 1.5, 1.75, 2.0, 2.25]
	bounds = np.array(bounds)**2
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1

	vector[1] -= 50.
	bounds = [1.5, 1.75, 2.0]
	bounds = np.array(bounds)**2
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1

	vector[0] -= 50.
	bounds = [1.5, 1.75, 2.0]
	bounds = np.array(bounds)**2
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1

	vector[1] -= 25.
	bounds = [1.5, 1.75, 2.0]
	bounds = np.array(bounds)**2
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1

	vector[1] += 75.
	bounds = [1.5, 1.75, 2.0]
	bounds = np.array(bounds)**2
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1

	vector[1] -= 25.
	vector[0] += 25.
	bounds = [1.5, 1.75, 2.0]
	bounds = np.array(bounds)**2
	for bound in bounds[::-1]:
		if np.amax(np.abs(vector)) < bound:
			result -= 1

	if result < 0:
		result = 0
	return result

#======================================================

if __name__ == '__main__':

	import matplotlib.pyplot as plt
	import seaborn as sns
	import matplotlib.cm as cm
	sns.set_context('paper', font_scale = 2.0, rc = {'lines.linewidth': 3})
	sns.set_style('ticks')

	fig = plt.figure(figsize = (9, 6))
	ax0 = plt.subplot2grid((2, 3), (0, 0))
	ax1 = plt.subplot2grid((2, 3), (0, 1))
	ax2 = plt.subplot2grid((2, 3), (0, 2))
	ax3 = plt.subplot2grid((2, 3), (1, 0))
	ax4 = plt.subplot2grid((2, 3), (1, 1))
	ax5 = plt.subplot2grid((2, 3), (1, 2))
	axs = [ax0, ax1, ax2, ax3, ax4, ax5]

	plt.setp(ax1.get_yticklabels(), visible = False)
	plt.setp(ax2.get_yticklabels(), visible = False)
	plt.setp(ax4.get_yticklabels(), visible = False)
	plt.setp(ax5.get_yticklabels(), visible = False)
	plt.setp(ax0.get_xticklabels(), visible = False)
	plt.setp(ax1.get_xticklabels(), visible = False)
	plt.setp(ax2.get_xticklabels(), visible = False)

	losses = [linear_funnel, narrow_funnel, double_well, discrete_ackley, discrete_michalewicz, discrete_valleys]

	for loss_index, loss in enumerate(losses):
		domain = np.linspace(0, 1, 100)

		X, Y = np.meshgrid(domain, domain)
		Z = np.zeros((len(domain), len(domain)))
		for x_index, x in enumerate(domain):
			for y_index, y in enumerate(domain):
				value = loss([x, y])
				Z[x_index, y_index] = value

		levels = np.linspace(-0.5, 4.5, 6)

		pc = axs[loss_index].contourf(X, Y, Z, cmap = cm.RdYlBu, levels = levels)

	cbaxes = fig.add_axes([1.0, 0.115, 0.03, 0.815])
	cbar = plt.colorbar(pc, cax = cbaxes)
	cbar.set_ticks(range(5))
	cbar.set_ticklabels(range(5))

	plt.tight_layout()
	plt.savefig('discrete_loss_functions.png', bbox_inches = 'tight')
	plt.show()


"""
Name  : fd4.py
Author: Ajay Lotekar
e-mail: ablotekar@gmail.com
Date  : 2021-07-04

DESC  :
"""
import numpy as np

def fd4(z, xd):
    """ Forth order central finite difference

    Function compute first order derivative of given array by using 4th
    order central finite difference scheam

    :param z: Input array
    :param xd: Grid size
    :return: y: output vector
    """
    y = np.zeros(z.shape)
    for i in range(2, max(z.shape) - 3):
        y[i] = (8 * z[i + 1] - 8 * z[i - 1] - z[i + 2] + z[i - 2]) / (12 * xd)
    return y


import matplotlib.pyplot as plt
import numpy as np

SHAPE = (256, 256)
RES = (16, 16)

def generate_perlin_noise_2d(shape, res):
    def f(t):
        return 6 * t ** 5 - 15 * t ** 4 + 10 * t ** 3

    delta = (res[0] / shape[0], res[1] / shape[1])
    d = (shape[0] // res[0], shape[1] // res[1])
    grid = np.mgrid[0:res[0]:delta[0], 0:res[1]:delta[1]].transpose(1, 2, 0) % 1
    # Gradients
    angles = 2 * np.pi * np.random.rand(res[0] + 1, res[1] + 1)
    gradients = np.dstack((np.cos(angles), np.sin(angles)))
    g00 = gradients[0:-1, 0:-1].repeat(d[0], 0).repeat(d[1], 1)
    g10 = gradients[1:, 0:-1].repeat(d[0], 0).repeat(d[1], 1)
    g01 = gradients[0:-1, 1:].repeat(d[0], 0).repeat(d[1], 1)
    g11 = gradients[1:, 1:].repeat(d[0], 0).repeat(d[1], 1)
    # Ramps
    n00 = np.sum(grid * g00, 2)
    n10 = np.sum(np.dstack((grid[:, :, 0] - 1, grid[:, :, 1])) * g10, 2)
    n01 = np.sum(np.dstack((grid[:, :, 0], grid[:, :, 1] - 1)) * g01, 2)
    n11 = np.sum(np.dstack((grid[:, :, 0] - 1, grid[:, :, 1] - 1)) * g11, 2)
    # Interpolation
    t = f(grid)
    n0 = n00 * (1 - t[:, :, 0]) + t[:, :, 0] * n10
    n1 = n01 * (1 - t[:, :, 0]) + t[:, :, 0] * n11
    return np.sqrt(2) * ((1 - t[:, :, 1]) * n0 + t[:, :, 1] * n1)

def generate_fractal_noise_2d(shape, res, octaves=1, persistence=0.5):
    noise = np.zeros(shape)
    frequency = 1
    amplitude = 1
    for _ in range(octaves):
        noise += amplitude * generate_perlin_noise_2d(shape, (frequency*res[0], frequency*res[1]))
        frequency *= 2
        amplitude *= persistence
    return noise

# image = generate_perlin_noise_2d(SHAPE, (4, 4))
# plt.imshow(image, cmap='gray')
# plt.show()
#
# image[image > 0.5] = 1.0
# image[image <= 0.5] = 0.0
# plt.imshow(image, cmap='gray')
# plt.show()

fig, axs = plt.subplots(2, 2)

image = generate_perlin_noise_2d(SHAPE, (1, 1))
axs[0][0].imshow(image, cmap='gray')

image = generate_perlin_noise_2d(SHAPE, (2, 2))
axs[0][1].imshow(image, cmap='gray')

image = generate_perlin_noise_2d(SHAPE, (8, 8))
axs[1][0].imshow(image, cmap='gray')

image = generate_perlin_noise_2d(SHAPE, (16, 16))
axs[1][1].imshow(image, cmap='gray')

plt.show()

fig, axs = plt.subplots(2, 2)

image = generate_fractal_noise_2d((256, 256), (2, 2), octaves=3, persistence=0.25)
axs[0][0].imshow(image, cmap='gray')
print(np.max(image))

image = generate_fractal_noise_2d((256, 256), (4, 4), octaves=3, persistence=0.25)
axs[0][1].imshow(image, cmap='gray')
print(np.max(image))

image = generate_fractal_noise_2d((256, 256), (8, 8), octaves=3, persistence=0.25)
axs[1][0].imshow(image, cmap='gray')
print(np.max(image))

image = generate_fractal_noise_2d((256, 256), (16, 16), octaves=3, persistence=0.25)
axs[1][1].imshow(image, cmap='gray')
print(np.max(image))

plt.show()


'''
scalar_quantization_utils.py

'''

from scipy.integrate import quad
from scipy.stats import uniform, expon, gamma, rayleigh, norm
import numpy as np
from numpy import linalg as LA

def ReconLevel(func, a, b, mu, sigma):
    # Optimum quantizer
    # Calculates and returns a tupil (recon_level, recon_error)
    # func = sample pdf ('uniform', 'expon', 'gamma', 'rayleigh', 'norm')
    # a = quantization region left boundary
    # b = quantization region right boundary
    # mu = pdf mean
    # sigma = pdf standard deviation

    if func == 'uniform':
        y1, _ = quad(lambda x : x * uniform.pdf(x, mu, sigma), a, b);
        y2, _ = quad(uniform.pdf, a, b, args = (mu, sigma));
        recon_level = y1 / y2;

        mse, _ = quad(lambda x : (x - recon_level) ** 2 * uniform.pdf(x, mu, sigma), a, b);
    elif func == 'expon':
        y1, _ = quad(lambda x : x * expon.pdf(x, mu, sigma), a, b);
        y2, _ = quad(expon.pdf, a, b, args = (mu, sigma));
        recon_level = y1 / y2;

        mse, _ = quad(lambda x : (x - recon_level) ** 2 * expon.pdf(x, mu, sigma), a, b);
    elif func == 'gamma':
        y1, _ = quad(lambda x : x * gamma.pdf(x, mu, sigma), a, b);
        y2, _ = quad(gamma.pdf, a, b, args = (mu, sigma));
        recon_level = y1 / y2;

        mse, _ = quad(lambda x : (x - recon_level) ** 2 * gamma.pdf(x, mu, sigma), a, b);
    elif func == 'rayleigh':
        y1, _ = quad(lambda x : x * rayleigh.pdf(x, mu, sigma), a, b);
        y2, _ = quad(rayleigh.pdf, a, b, args = (mu, sigma));
        recon_level = y1 / y2;

        mse, _ = quad(lambda x : (x - recon_level) ** 2 * rayleigh.pdf(x, mu, sigma), a, b);
    elif func == 'norm':
        y1, _ = quad(lambda x : x * norm.pdf(x, mu, sigma), a, b);
        y2, _ = quad(norm.pdf, a, b, args = (mu, sigma));
        recon_level = y1 / y2;

    recon_error, _ = quad(lambda x : (x - recon_level) ** 2 * norm.pdf(x, mu, sigma), a, b);

    return recon_level, recon_error;

def MaxLloydQuantizer(func, limit_left, limit_right, N_levels, mu, sigma, tol = 10e-6):
    # Optimum iterative quantizer
    # Calculates and returns a tupil (quant_intervals, recon_level, recon_error)
    # func = sample pdf ('uniform', 'expon', 'gamma', 'rayleigh', 'norm')
    # limit_left = quantization left support region
    # limit_right = quantization right support region
    # N_levels = number of quantization levels
    # mu = mean
    # sigma = standard deviation
    # tol = tolerance

    step = (limit_right - limit_left) / N_levels;
    quant_intervals = [limit_left + step * i for i in range(N_levels)] + [limit_right];

    recon_levels = [ReconLevel(func, quant_intervals[i], quant_intervals[i + 1], mu, sigma)[0] for i in range(N_levels)];
    recon_error_old = limit_right - limit_left;
    recon_error = sum([ReconLevel(func, quant_intervals[i], quant_intervals[i + 1], mu, sigma)[1] for i in range(N_levels)]);

    while (abs(recon_error - recon_error_old) < tol * recon_error):
        recon_error_old = recon_error;
        quant_intervals = [limit_left] + [(recon_levels[i] + recon_levels[i + 1]) / 2 for i in range(N_levels - 1)] + [limit_right];
        recon_levels = [ReconLevel(func, quant_intervals[i], quant_intervals[i + 1], mu, sigma)[0] for i in range(N_levels)];
        recon_error = sum([ReconLevel(func, quant_intervals[i], quant_intervals[i + 1], mu, sigma)[1] for i in range(N_levels)]);

    return quant_intervals, recon_levels, recon_error

def ApplyQuantization(quant_intervals, recon_levels, signal_input):
    # Returns a tupil (signal_output, quant_error, SQNR)
    # quant_intervals = array of quantization intervals
    # recon_levels = array of quantization levels
    # signal_input = array of input signal

    signal_output = np.zeros(signal_input.shape);

    N_levels = len(recon_levels);
    for i in range(N_levels):
        if i != N_levels - 1:
            mask1 = signal_input >= quant_intervals[i];
            mask2 = signal_input < quant_intervals[i + 1];
            mask = mask1 * mask2;
        else:
            mask1 = signal_input >= quant_intervals[i];
            mask2 = signal_input <= quant_intervals[i + 1];
            mask = mask1 * mask2;

        signal_output[mask] = recon_levels[i];

    quant_error = signal_input - signal_output;
    SQNR = 20 * np.log10(LA.norm(signal_input) / LA.norm(quant_error));

    return signal_output, quant_error, SQNR;


import numpy as np
import time
import pandas as pd

def main():
    # initi conditions---------------------------------------------------------
    A=2
    B=0.01
    t0=0.0
    t_end=5
    #--------------------------------------------------------------------------

    # Analytic solution
    analytic = lambda t_ini, y_ini, t, a, b: 1/(b/a+(1/y_ini-b/a)*np.exp(-a*(t-t_ini)))
    #--------------------------------------------------------------------------
    # ODE
    f = lambda y: A*y-B*y*y
    #--------------------------------------------------------------------------
    # initial valuse
    y0_list = np.array([8, 10, 12])
    #y0_list = np.array([100, 190, 250])
    #--------------------------------------------------------------------------

    timesteps = np.array([0.05, 0.025, 0.0125, 0.00625, 0.003125])

    # maps of 8th order for different time steps (A=2, B=0.01)
    # M(0,:) is map for dt=0.05     (N=100)
    # M(1,:) is map for dt=0.025    (N=200)
    # M(2,:) is map for dt=0.0125   (N=400)
    # M(3,:) is map for dt=0.00625  (N=800)
    # M(4,:) is map for dt=0.003125 (N=1600)
    Maps = np.array([[-4.99601921667887e-24,1.84935964133497e-20,-4.23629183829682e-17,8.39174629410321e-14,-1.60590216195728e-10,3.05590456029527e-7,-0.000581157957616179,1.10517085683809,1.34511594564678e-6],
                    [-4.87731111635685e-26,2.78047377145932e-22,-1.15484975507109e-18,4.5376519355888e-15,-1.77104564948567e-11,6.90875050990476e-8,-0.000269499102660927,1.05127109608626,6.4017323733666e-9],
                    [-4.26178099855912e-28,4.13410513256993e-24,-3.32748495857332e-20,2.631715681418e-16,-2.07925096139956e-12,1.64269661317723e-8,-0.000129779879232919,1.02531512052318,2.76214379044979e-11],
                    [-3.52165980224463e-30,6.23344675198481e-26,-9.96204381136311e-22,1.58422251710012e-17,-2.51895391085821e-13,4.00518933883553e-9,-6.36833449188686e-5,1.01257845154063,2.10469969587637e-13],
                    [-2.82967746195301e-32,9.53610182697698e-28,-3.04610757006006e-23,9.71730546503066e-19,-3.09983113737682e-14,9.88849362981926e-10,-3.15443976843614e-5,1.00626957200376,-6.11683690894885e-14],
                    ])

    #--------------------------------------------------------------------------
    # resulting table
    # result(:, 0) is N
    # result(:, 1) is err_RK4
    # result(:, 2) is err_TM8
    # result(:, 3) is time_RK4
    # result(:, 4) is time_TM8
    # result(:, 5) is time_ratio = time_RK4/time_TM8
    result = np.zeros((len(timesteps), 6))

    for k, dt in enumerate(timesteps):  # for each time stemp dt
        M = Maps[k, :]                 # get TM8 for this dt
        N = int((t_end-t0)/dt)
        t = np.arange(t0, t_end, dt)

        result[k, 0] = N

        for y0 in y0_list: # for each y0
            # analytic solution------------------------------------------------
            y_sol = analytic(t0, y0, t, A, B)
            #------------------------------------------------------------------

            # RK4 integration--------------------------------------------------
            y_rk4 = np.zeros(N)

            y_rk4[0]=y0
            tic = time.time()
            for i in range(N-1):
                y = y_rk4[i]
                k1 = f(y)
                k2 = f(y+dt*k1/2)
                k3 = f(y+dt*k2/2)
                k4 = f(y+dt*k3)
                y_rk4[i+1] = y + dt*(k1+2*k2+2*k3+k4)/6

            elapsed_time = time.time()-tic
            result[k, 3] += elapsed_time # time_RK4
            #------------------------------------------------------------------

            # Mapping----------------------------------------------------------
            y_map = np.zeros(N)
            y_map[0] = y0
            tic = time.time()
            for i in range(N-1):
                y=y_map[i]
                #y_map[i+1] = M[0]*y**8+M[1]*y**7+M[2]*y**6+M[3]*y**5+M[4]*y**4+M[5]*y**3+M[6]*y**2+M[7]*y+M[8]
                #y_map[i+1] = +M[8]
                y2 = y*y
                y3 = y2*y
                y4 = y3*y
                y_map[i+1] = (y4*(M[0]*y4 + M[4]) +
                            y3*(M[1]*y4 + M[5]) +
                            y2*(M[2]*y4 + M[6]) +
                            y *(M[3]*y4 + M[7]) +
                            M[8]
                            )

            elapsed_time = time.time()-tic
            result[k, 4] += elapsed_time # time_RK4
            #------------------------------------------------------------------
            result[k, 1] += np.abs(y_rk4 - y_sol).max() # err_RK4
            result[k, 2] += np.abs(y_map - y_sol).max() # err_TM8

    result[:, 1:] /= len(y0_list) # get average results
    result[:, 5] = result[:, 3]/result[:, 4] # get time_ratio

    result = pd.DataFrame(data=result[:,1:], index=result[:,0], columns=np.array(['err_RK4', 'err_TM8', 'time_RK4', 'time_TM8', 'time_ratio']))
    return result

if __name__ == "__main__":
    print(main())


import numpy as np
import scipy.stats.mstats
import matplotlib.pyplot as plt
from PIL import Image
import os

def create_dir(dir_name):
    if not os.path.exists(dir_name):
        os.makedirs(dir_name)

def corr2_coeff(A, B):
    # Rowwise mean of input arrays & subtract from input arrays themeselves
    A_mA = A - A.mean()
    B_mB = B - B.mean()

    # Sum of squares across rows
    ssA = (A_mA ** 2).mean()
    ssB = (B_mB ** 2).mean()

    # Finally get corr coeff
    coef = (A_mA * B_mB).mean() / np.sqrt(ssA * ssB)
    return coef

def r2coef(gt, pred):
    gt_mean = gt.mean()
    r2coef = 1 - np.sum((gt - pred) ** 2) / np.sum((gt - gt_mean) ** 2)
    return r2coef

def get_rmse(gt, pred):
    return np.sqrt(np.mean((gt - pred) ** 2))

def get_kl(gt, pred, chance=1e-5): # Kullback-Leibler divergence
    kl = np.sum(gt * np.where(gt > chance, np.log(gt), 0) - gt * np.where(pred >= chance, np.log(pred), 0))
    return kl

def get_spearmanr(gt, pred):
    try:
        return scipy.stats.spearmanr(gt, pred)[0]
    except:
        return 0

def label_accuracy(label_trues, label_preds):
    """Returns accuracy score evaluation result."""

    gt = label_trues.astype(np.float64)
    pred = label_preds.astype(np.float64)
    gt_1d, pred_1d = gt.flatten() / gt.sum(), pred.flatten() / pred.sum()
    gt_1d_01, pred_1d_01 = gt.flatten() / 255.0, pred.flatten() / 255.0

    cc = corr2_coeff(gt_1d, pred_1d)
    kl = get_kl(gt_1d, pred_1d)
    kl_01 = get_kl(gt_1d_01, pred_1d_01)
    spearman = get_spearmanr(gt_1d, pred_1d)
    r2 = r2coef(gt_1d, pred_1d)
    rmse = get_rmse(gt_1d_01, pred_1d_01)

    return kl, kl_01, cc, rmse, r2, spearman

def overlay_imp_on_img(img, imp, fname, colormap='jet'):

    cm = plt.get_cmap(colormap) # https://matplotlib.org/examples/color/colormaps_reference.html
    img2 = np.array(img, dtype=np.uint8)
    imp2 = np.array(imp, dtype=np.uint8)
    imp3 = (cm(imp2)[:, :, :3] * 255).astype(np.uint8)
    img3 = Image.fromarray(img2)
    imp3 = Image.fromarray(imp3)
    im_alpha = Image.blend(img3, imp3, 0.5)
    im_alpha.save(fname)


import numpy as np
from algorith.arrays.binary_srch import binary_search

def find_k_ix(a1,i1,j1,a2,i2,j2,k):
    n1=j1-i1+1
    n2=j2-i2+1
    if k==0:
        return min(a1[i1],a2[i2])
    elif k==n1+n2-1:
        return max(a1[j1],a2[j2])
    elif k>n1+n2-1:
        raise Exception("Out of bounds")
    if n1<n2:
        n_s=n1; n_l=n2; i_s=i1; i_l=i2
        a_s=a1; a_l=a2; j_s=j1; j_l=j2
    else:
        n_s=n2; n_l=n1; i_s=i2; i_l=i1
        a_s=a2; a_l=a1; j_s=j2; j_l=j1
    if n_s==1:
        if a_l[k]<a_s[i_s] and a_s[i_s]<a_l[k+1]:
            return a_s[i_s]
        elif a_s[i_s]<a_l[k]:
            return a_l[k+1]
        else:
            return a_l[k]
    if k>=n_s:
        cand = a_l[i_l+k-n_s]
        ix = binary_search(a_s,cand,i_s,j_s)
        if ix==len(a_s)-1:
            return cand
        else:
            return find_k_ix(a_s,ix+1,j_s,a_l,i_l+k-n_s+1,j_l,n_s+i_s-ix-2)
    else:
        if a_s[i_s]<a_l[i_l]:
            return find_k_ix(a_s,i_s+1,i_s+k,a_l,i_l,i_l+k,k-1)
        else:
            return find_k_ix(a_s,i_s,i_s+k,a_l,i_l+1,i_l+k,k-1)

def find_median(a1,a2):
    n = len(a1)+len(a2)
    if n%2==1:
        return find_k_ix(a1,0,len(a1)-1,a2,0,len(a2)-1,n//2)
    else:
        num1=find_k_ix(a1,0,len(a1)-1,a2,0,len(a2)-1,n//2)
        num2=find_k_ix(a1,0,len(a1)-1,a2,0,len(a2)-1,n//2+1)
        return (num1+num2)/2

if __name__=="__main__":
    a11=np.array([ 3,  6,  8, 11, 13, 14])
    a21=np.array([ 0,  1,  2,  4,  5,  7,  9, 10, 12])
    mm1 = find_k_ix(a11,0,len(a11)-1,a21,0,len(a21)-1,5)
    print(mm1)


# Simulate a Cox point process on a disk.
# This Cox point process is formed by first simulating a uniform Poisson
# line process. On each line (or segment) an independent Poisson point
# process is then simulated.
# Author: H. Paul Keeler, 2019.
# Website: hpaulkeeler.com
# Repository: github.com/hpaulkeeler/posts
# For more details, see the post:
# hpaulkeeler.com/simulating-a-cox-point-process-based-on-a-poisson-line-process/

import numpy as np;  # NumPy package for arrays, random number generation, etc
import matplotlib.pyplot as plt  # for plotting
from matplotlib import collections  as mc  # for plotting line segments

plt.close('all');  # close all figures

###START -- Parameters -- START###
# Cox (ie Poisson line-point) process parameters
lambda0 = 4;  # intensity (ie mean density) of the Poisson *line* process
mu = 5;  # intensity (ie mean density) of the Poisson *point* process

# Simulation disk dimensions
xx0 = 0;
yy0 = 0;  # center of disk
r = 1;  # disk radius
massLine = 2 * np.pi * r * lambda0;
###END -- Parameters -- END###

###START -- Simulate a Poisson line process on a disk -- START###
# Simulate Poisson point process
numbLines = np.random.poisson(massLine);  # Poisson number of points

theta = 2 * np.pi * np.random.rand(numbLines);  # choose angular component uniformly
p = r * np.random.rand(numbLines);  # choose radial component uniformly
q = np.sqrt(r ** 2 - p ** 2);  # distance to circle edge (alonge line)

# calculate trig values
sin_theta = np.sin(theta);
cos_theta = np.cos(theta);

# calculate segment endpoints of Poisson line process
xx1 = xx0 + p * cos_theta + q * sin_theta;
yy1 = yy0 + p * sin_theta - q * cos_theta;
xx2 = xx0 + p * cos_theta - q * sin_theta;
yy2 = yy0 + p * sin_theta + q * cos_theta;
###END Simulate a Poisson line process on a disk END###

###START Simulate a Poisson point processes on each line START###
lengthLine = 2 * q;  # length of each segment
massPoint = mu * lengthLine;  # mass on each line
numbLinePoints = np.random.poisson(massPoint);  # Poisson number of points on each line
numbLinePointsTotal = sum(numbLinePoints);
uu = 2 * np.random.rand(numbLinePointsTotal) - 1;  # uniform variables on (-1,1)

# replicate values to simulate points all in one step
xx0_all = np.repeat(xx0, numbLinePointsTotal);
yy0_all = np.repeat(yy0, numbLinePointsTotal);
p_all = np.repeat(p, numbLinePoints);
q_all = np.repeat(q, numbLinePoints);
sin_theta_all = np.repeat(sin_theta, numbLinePoints);
cos_theta_all = np.repeat(cos_theta, numbLinePoints);

# position points on Poisson lines/segments
xxPP_all = xx0_all + p_all * cos_theta_all + q_all * uu * sin_theta_all;
yyPP_all = yy0_all + p_all * sin_theta_all - q_all * uu * cos_theta_all;
### END Simulate a Poisson point processes on each line ###END

### START Plotting ###START
# plot circle
t = np.linspace(0, 2 * np.pi, 200);
xp = r * np.cos(t);
yp = r * np.sin(t);
fig, ax = plt.subplots();
ax.plot(xx0 + xp, yy0 + yp, color='k');
plt.xlabel('x');
plt.ylabel('y');
plt.axis('equal');

# plot segments of Poisson line process
# need to create a list to plot the segments (probably a better way to do this)
segments = [[(xx1[i], yy1[i]), (xx2[i], yy2[i])] for i in range(numbLines)];
lc = mc.LineCollection(segments, colors='b')
ax.add_collection(lc)  # plot segments

# plot Poisson points on line process
plt.plot(xxPP_all, yyPP_all, 'r.');
###END Plotting END###


import math
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.io import loadmat
import scipy.optimize as opt

data = loadmat('data/ex5data1.mat')
x_train = data['X']
y_train = data['y']
x_val = data['Xval']
y_val = data['yval']
x_test = data['Xtest']
y_test = data['ytest']
#print(x_train.shape)
#print(y_train.shape)
#print(x_val.shape)
#print(y_val.shape)
#print(x_test.shape)
#print(y_test.shape)

x_train = np.insert(x_train, 0, np.ones(x_train.shape[0],), axis=1)
x_val = np.insert(x_val, 0, np.ones(x_val.shape[0],), axis=1)
x_test = np.insert(x_test, 0, np.ones(x_test.shape[0],), axis=1)
y_train = y_train[:,0]
y_val = y_val[:,0]
y_test = y_test[:,0]
#print(x_train)

def J(theta, x, y, λ):
	m = x.shape[0]
	tmp = (x @ theta) - y
	j = (tmp @ tmp) / np.float64(2 * m) + (λ / np.float64(2 * m)) * (theta[1:] @ theta[1:])
	return j

def gradient(theta, x, y, λ):
	m = x.shape[0]
	tmp = (x @ theta - y)
	grad = (x.transpose() @ tmp) / np.float64(m)
	tmp = (λ / np.float64(m)) * theta
	tmp[0] = 0
	grad += tmp
	return grad

def plotLine(slope, intercept):
    #Plot a line from slope and intercept
    axes = plt.gca()
    x_vals = np.array(axes.get_xlim())
    y_vals = intercept + slope * x_vals
    plt.plot(x_vals, y_vals, '-')

def learningCurveData(x_train, y_train, x_val, y_val, λ):
	m = x_train.shape[0]
	n = x_train.shape[1]
	theta = np.zeros((n,))
	training_error = np.zeros(m - 1)
	validation_error = np.zeros(m - 1)
	for i in range(1, m):
		x = x_train[:i,:]
		y = y_train[:i]
		fmin = opt.fmin_tnc(func=J, x0=theta, fprime=gradient, args=(x, y, λ))
		training_error[i - 1] = J(fmin[0], x, y, 0)
		validation_error[i - 1] = J(fmin[0], x_val, y_val, 0)
	return training_error, validation_error

def polyFeatures(x, p):
	m = x.shape[0]
	x_poly = np.zeros((m, p + 1))
	for i in range(p + 1):
		x_poly[:,i] = np.power(x, i)
	return x_poly

def featureScaling(data):
	m = data.shape[0]
	mean = data.sum(axis=0) / m
	std = data.std(axis=0)
	mean[0], std[0] = 0.0, 1.0
	normalized_data = (data - mean) / std
	return normalized_data, mean, std

def plotLearntFunc(theta, mean, std):
	delta = 0.25
	xrange = np.arange(-100.0, 100.0, delta)
	yrange = np.arange(-100.0, 100.0, delta)
	X, Y = np.meshgrid(xrange,yrange)
	print(X.shape)
	# F is one side of the equation, G is the other
	tmp = [0.0 for el in range(p + 1)]
	lhs = theta[0]
	for i in range(1, p + 1):
		tmp[i] = (np.power(X, i) - mean[i]) / std[i]
		lhs += theta[i] * tmp[i]
	F = lhs
	G = Y
	plt.contour(X, Y, (F - G), [0], colors=["k"])

def plotLearningCurve(x_train, y_train, x_val, y_val, λ):
	training_error, validation_error = learningCurveData(x_train, y_train, x_val, y_val, λ)
	plt.plot(training_error, 'b')
	plt.plot(validation_error, 'g')

p = 8
theta = np.zeros((p + 1,), dtype=np.float64)
#try different values of lamda to see how the learning curves vary
λ = 0
x_train_poly = polyFeatures(x_train[:,1], p)
x_val_poly = polyFeatures(x_val[:,1], p)
#mean-normalize x_train_poly & x_val_poly
x_train_poly_norm, mean_train, std_train = featureScaling(x_train_poly)
x_val_poly_norm = (x_val_poly - mean_train) / std_train

fmin = opt.fmin_tnc(func=J, x0=theta, fprime=gradient, args=(x_train_poly_norm, y_train, λ))
theta = fmin[0]

plt.plot(x_train[:,1], y_train,'rx')
plotLearntFunc(theta, mean_train, std_train)
#plotLearningCurve(x_train_poly_norm, y_train, x_val_poly_norm, y_val, λ)
#plotLine(fmin[0][1], fmin[0][0])
plt.show()


"""Workout for Statistical tests to estimate population proportions

These functions are created from using basic python libraries

The module contains useful functions for evaluating AB tests
1. one_prop_conf_interval
2. one_prop_hypothesis
3. two_props_diff_conf_interval
4. two_props_hypothesis

"""
import math
import numpy as np
import classical.workout.utils as utils

def one_prop_conf_interval(values: np.ndarray,
                           conf_level: float = 0.95) -> tuple:
    """calculates confidence interval for a proportion

    Args:
        values (np.array): sample values (values are of values 0 and 1)
        conf_level (float): confidence level

    Returns:
        tuple: lower and upper values of confidence interval
    """
    p = np.mean(values)
    n = len(values)
    se = math.sqrt(p*(1-p)/n)
    z_critical = utils.get_z_critical(conf_level)
    lower_ci = p - z_critical * se
    upper_ci = p + z_critical * se
    return (lower_ci, upper_ci)

def one_prop_hypothesis(values: np.ndarray,
                        null_val: float = 0,
                        alternative: str = "two-sided"):
    p = np.mean(values)
    n = len(values)
    se = math.sqrt(null_val*(1-null_val)/n)
    z_statistic = (p - null_val)/se
    p_value = utils.get_norm_pvalue(z_statistic, alternative)
    return (z_statistic, p_value)

def two_props_diff_conf_interval(values1: np.ndarray, values2: np.ndarray,
                                 conf_level: float) -> tuple:
    """Calculates the confidence interval for the diff between two proportions

    Args:
        values1 (np.array): sample 1 binary(0/1) values
        values2 (np.array): sample 2 binary(0/1) values
        conf_level (float): confidence level

    Returns:
        tuple: lower and upper values of confidence interval
    """
    p1 = np.mean(values1)
    p2 = np.mean(values2)
    p_diff = p1-p2
    n1 = len(values1)
    n2 = len(values2)
    SE = math.sqrt(((p1*(1-p1))/n1) + ((p2*(1-p2))/n2))
    z_critical = utils.get_z_critical(conf_level)
    lower = p_diff - z_critical*SE
    upper = p_diff + z_critical*SE
    return (lower, upper)

def two_props_hypothesis(values1: np.ndarray,
                         values2: np.ndarray,
                         alternative: str = "two-sided") -> tuple:
    """Perform two sided z test to comparing two proportions

    Args:
        values1 (np.array): sample 1 binary(0/1) values
        values2 (np.array): sample 2 binaray(0/1) values
        alternative (str, optional): two-sided/larger/smaller.
                                     Defaults to "two-sided".

    Returns:
        tuple: z_statistic and p value of the test
    """
    p1 = np.mean(values1)
    p2 = np.mean(values2)
    n1 = len(values1)
    n2 = len(values2)
    p_pool = (np.sum(values1) + np.sum(values2))/(n1+n2)
    se = math.sqrt((p_pool*(1-p_pool))/n1 + (p_pool*(1-p_pool))/n2)
    z_statistic = (p1-p2)/se
    p_value = utils.get_norm_pvalue(z_statistic, alternative)
    return (z_statistic, p_value)


import numpy as np
from matplotlib import use
use('TkAgg')
import matplotlib.pyplot as plt
from gradientDescentMulti import gradientDescentMulti
from normalEqn import normalEqn
from featureNormalize import featureNormalize
from featureNormalize import normEntry
# ================ Part 1: Feature Normalization ================

print('Loading data ...')

# Load Data
data = np.loadtxt('ex1data2.txt', delimiter=',')
X = data[:, :2]
y = data[:, 2]
m = y.T.size

# Print out some data points
print('First 10 examples from the dataset:')
print(np.column_stack( (X[:10], y[:10]) ))
input("Program paused. Press Enter to continue...")

# Scale features and set them to zero mean
print('Normalizing Features ...')

X, mu, sigma = featureNormalize(X)
print('[mu] [sigma]')
print(mu, sigma)

# Add intercept term to X
X = np.concatenate((np.ones((m, 1)), X), axis=1)

print('Running gradient descent ...')
# Choose some alpha value
alpha = 0.01
num_iters = 9000
# Init Theta and Run Gradient Descent
theta = np.zeros(3)
theta, J_history = gradientDescentMulti(X, y, theta, alpha, num_iters)
# Plot the convergence graph
plt.plot(J_history, '-b')

''' #Plot for some different alphas
plt.gca().set_color_cycle(['red', 'green', 'blue', 'yellow'])
for alpha in np.linspace(0.001, 0.6, 10):
    num_iters = 500
    # Init Theta and Run Gradient Descent
    theta = np.zeros(3)
    theta, J_history = gradientDescentMulti(X, y, theta, alpha, num_iters)
    # Plot the convergence graph
    plt.plot(J_history, '-b')
'''
plt.xlabel('Number of iterations')
plt.ylabel('Cost J')
plt.show()
input("Program paused. Press Enter to continue...")

# Display gradient descent's result
print('Theta computed from gradient descent: ')
print(theta)

# Estimate the price of a 1650 sq-ft, 3 br house
nPrice = normEntry([1, 1650, 3], mu, sigma)
price = np.array(nPrice).dot(theta)

print('Predicted price of a 1650 sq-ft, 3 br house')
print('(using gradient descent): ')
print(price)

input("Program paused. Press Enter to continue...")

# ================ Part 3: Normal Equations ================

# ====================== YOUR CODE HERE ======================
# Instructions: The following code computes the closed form
#               solution for linear regression using the normal
#               equations. You should complete the code in
#               normalEqn.m
#
#               After doing so, you should complete this code
#               to predict the price of a 1650 sq-ft, 3 br house.
#

print('Solving with normal equations...')

# Load Data
data = np.loadtxt('ex1data2.txt', delimiter=',')
X = data[:, :2]
y = data[:, 2]
m = y.T.size

# Add intercept term to X
X = np.concatenate((np.ones((m,1)), X), axis=1)

# Calculate the parameters from the normal equation
theta = normalEqn(X, y)

# Display normal equation's result
print('Theta computed from the normal equations:')
print(' %s \n' % theta)

# Estimate the price of a 1650 sq-ft, 3 br house
price = np.array([1, 1650, 3]).dot(theta)

# ============================================================

print("Predicted price of a 1650 sq-ft, 3 br house ")
print('(using normal equations):\n $%f\n' % price)

input("Program paused. Press Enter to continue...")


from statistics import mean
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import style
import random

style.use('fivethirtyeight')

#xs = [1, 2, 3, 4, 5, 6]
#ys = [5, 4, 6, 4, 5, 7]

# plt.plot(xs, ys)
# plt.scatter(xs, ys)
# plt.show()

#xs = np.array([1, 2, 3, 4, 5, 6], dtype = np.float64)
#ys = np.array([5, 4, 6, 5, 6, 7], dtype = np.float64)

def create_dataset(number_of_datapoint, variance, step=2, correlation=False):
    val = 1
    ys = []
    for i in range(number_of_datapoint):
        y = val + random.randrange(-variance, variance)
        ys.append(y)
        if correlation and correlation == 'pos':
            val+=step
        elif correlation and correlation == 'neg':
            val-=step
    xs = [i for i in range(len(ys))]    # we can give the len(number_of_datapoint)
    return np.array(xs, dtype=np.float64), np.array(ys, dtype=np.float64)

def best_fit_slope_and_intercept(xs, ys):

    x_mean = mean(xs)
    y_mean = mean(ys)

    xy_means = (x_mean*y_mean)
    mean_of_xy = mean(xs*ys)

    x_mean_sq = x_mean*x_mean
    x_sq_mean = mean(xs*xs)

    m = ((xy_means - mean_of_xy)/ (x_mean_sq - x_sq_mean))
    b = y_mean - m*x_mean

    return m, b

def sqaured_error(ys_original, ys_line):
    return sum((ys_line-ys_original)**2)

def coefficient_of_determination(ys_original, ys_line):
    y_mean_line = [mean(ys_original) for y in ys_original]
    squared_error_of_regression_line = sqaured_error(ys_original, ys_line)
    sqaured_error_y_mean_line = sqaured_error(ys_original, y_mean_line)

    return 1 - (squared_error_of_regression_line / sqaured_error_y_mean_line)

xs, ys = create_dataset(40, 40, 2, correlation='neg')

m, b = best_fit_slope_and_intercept(xs, ys)
#print(m, b)

#for x in xs:
#    regression_line.append(m*x)+b)

regression_line = [(m*x)+b for x in xs]

predict_x = 8
predict_y = m*predict_x+b

# Coefficient of determination
r_sqaured = coefficient_of_determination(ys, regression_line)
print(r_sqaured)

plt.scatter(xs, ys)
plt.plot(xs, regression_line)
plt.scatter(predict_x, predict_y, s=100, color='g')
plt.show()

Linear Regression from Scratch/Linear Regression from Scratch.py