# 1. Vectorized Linear algebra
import numpy as np
import time
import functools

x = np.random.randn(1000000)
y = np.random.randn(1000000)

# First versions: simple loop
t1 = time.time()
z1 = sum(x[i] * y[i] for i in range(len(x)))
print("Duration of sum(x[i] * y[i] for i in range(len(x))): {}".format(time.time() - t1))
# 521 ms ± 5.28 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

# Using numpy's dot product
t1 = time.time()
z2 = np.dot(x, y) # elapsed time: 0.0045 seconds
print("Duration of np.dot(x, y): {}".format(time.time() - t1))
# 2.69 ms ± 88.8 µs per loop (mean ± std. dev. of 7 runs, 1 loop each)

print("z1 - z2: {}".format(z1 - z2))
# 2.546585164964199e-11

####
# 2. Simulation
# Finding the mean and standard deviatoion of the number of heads after 10000 coin flips
np.random.seed(42)
samples = np.random.randint(0,2,10000)

print("\n\nMean and STD of binomial dist with numpy")

t1 = time.time()
np.mean(samples)
print("Duration of samples.mean(): {}".format(time.time() - t1))
# 30.5 µs ± 470 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)

t1 = time.time()
np.std(samples)
print("Duration of samples.std(): {}".format(time.time() - t1))
# 94.4 µs ± 442 ns per loop (mean ± std. dev. of 7 runs, 10000 loops each)

####
# 3. Recursion
print("\n\nRecursions")
# Fibonacci loop
def fib(n):
    if n < 2:
        return 1
    else:
        return fib(n-1) + fib(n-2)

t1 = time.time()
fib(30)
print("Duration of fib(30): {}".format(time.time() - t1))
# 493 ms ± 10.3 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)

# Average number of heads
# Solving the above coin flipping problem with recursion
def average_heads(n):
    if n == 1:
        return 0.5
    else:
        return np.mean([average_heads(n-1) + 1./n, average_heads(n-1) - 1./n])

# Takes a lot of time even for 10 flips
t1 = time.time()
average_heads(10)
print("Duration of average_heads(10): {}".format(time.time() - t1))
# 11 ms ± 85.3 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)

@functools.lru_cache()
def average_heads_m(n):
    if n == 1:
        return 0.5
    else:
        print("-"*20)
        print("average_heads_m(n-1):\n{}".format(average_heads_m(n-1)))
        print("average_heads_m(n-1):\n{}".format(average_heads_m(n-1)))
        print("1./n:\n{}".format(1./n))
        print("average_heads_m(n-1) + 1./n, average_heads_m(n-1) - 1./n:\n{}".format(average_heads_m(n-1) + 1./n, average_heads_m(n-1) - 1./n))
        print("np.mean([average_heads_m(n-1) + 1./n, average_heads_m(n-1) - 1./n]):\n{}".format(np.mean([average_heads_m(n-1) + 1./n, average_heads_m(n-1) - 1./n])))
        return np.mean([average_heads_m(n-1) + 1./n, average_heads_m(n-1) - 1./n])

t1 = time.time()
average_heads_m(10)
print("Duration of average_heads(10), memoized: {}".format(time.time() - t1))

# Graph traversal algorithm
print("Graph traversal algorithm")
# How many ways are there to get a given number by drawing coins and summing their values (if the order counts as well)?

coins = [1, 5, 10, 25, 50]
def count(remainder):
    if remainder < 0:
        return 0
        if remainder == 0:
            return 1
        return sum(count(remainder - coin) for coin in coins)

t1 = time.time()
count(50)
print("Duration of count(50):{}".format(time.time() - t1))

####
# 4. Memoization
## Manually
known = {0:1, 1:1}
def fib_man(n):
    if n in known:
        return known[n]
    else:
        known[n] = fib_man(n-2) + fib_man(n-1)
        return known[n]

## With decorator: DOES NOT WORK?
# @memoized Does not work

@functools.lru_cache()
def fib_dec(n):
    if n < 2:
        return 1
    else:
        return fib_dec(n-2) + fib_dec(n-1)

t1 = time.time()
fib_man(30)
print("Duration of manual memoization, fib_man(30): {}".format(time.time() - t1))
# 269 ns ± 1.22 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)

t1 = time.time()
fib_dec(30)
print("Duration of functools memoization, fib_dec(30): {}".format(time.time() - t1))
# 204 ns ± 3.01 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)

####
# 5. Divide and conquer
M = np.matrix([[1,1], [1,0]])

def fib_dc(n):
    if n < 2:
        return 1

    MProd = M.copy()
    for _ in range(n-2):
        MProd *= M

    return MProd[0,0] + MProd[0,1]

t1 = time.time()
fib_dc(30)
print("Duration of, divide an conquer fib_dc(30): {}".format(time.time() - t1))
# 330 µs ± 1.83 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)


#
import numpy as np
import matplotlib.pyplot as plt

def powerResidue(N, seed=None, a=273673163155, c=13, M=2**48):
    """ Calculate a series of random numbers
    """
    import datetime
    if seed == None:
        #print("Seed value set to NONE, defaulting to system time.")
        seed=int(datetime.datetime.now().strftime("%Y%M%d%H%M%S"))
    else:
        pass

    r = seed
    rand = []
    for i in range(N):
        rand.append(((a*r + c) % M)/M)
        r = (a*r + c) % M
    return rand[0:N]

#Generate a seq of random numbers and print them, also plotted
first_seq = powerResidue(50)
print(first_seq)
x_values = np.arange(0,50)
plt.plot(x_values, first_seq)
plt.title("First Sequence Test")
plt.show()
#The list when printed seems somewhat random

#Change value of constants and seed value to produce a sequence that repeats itself
#We can change the values of a and M to see if we can get a repeating pattern
#Initial values
#a = 273673163155
#M = 2**48

second_seq = powerResidue(50,a=27367, M=2**12)
print(second_seq)

plt.plot(x_values, second_seq)
plt.title("Second Sequence Test with Changes in a and M")
plt.show()

#Now change the seed value to see if we can produce a sequence that repeats itself
third_seq = powerResidue(50,seed=9729834523)
print(third_seq)
plt.plot(x_values, third_seq)
plt.title("Third Sequence Test with Changes in Seed")
plt.show()

#Now change all three values to see if we can produce a seq that repeats
fourth_seq = powerResidue(50, seed=987087257133932, a=273, M=2**4)
print(fourth_seq)
plt.plot(x_values, fourth_seq)
plt.title("Fourth Sequence Test Looking for Patterns")
plt.show()

def new_pResidue(N, seed=None, a=273673163155, c=13, M=2**48):
    """ Calculate a series of random numbers
    """
    #import datetime
    #if seed == None:
        #print("Seed value set to NONE, defaulting to system time.")
        #seed=int(datetime.datetime.now().strftime("%Y%M%d%H%M%S"))
    #else:
        #seed = seed + 23
    r = seed
    rand = []
    for i in range(N):
        rand.append(((a*r + c) % M)/M)
        r = (a*r + c) % M
    return rand[0:N]

import datetime
seed = seed=int(datetime.datetime.now().strftime("%Y%M%d%H%M%S"))
for each_one in range(4):
    new_seq = new_pResidue(50,seed)
    seed = seed+23
    plt.plot(new_seq)
    plt.title("Multiple Sequences at same length")

plt.show()
#When looking at the plot with multiple sequences, There seems to be random sequences
    #when you look closer, you can notice there are times in all of the sequences that
    #each tend to drop in values or climb in values. Otherwise the sequences seem to vary
    #in decreasing or increasing successive values and their severity of the jumps as well.

#To try and fix the times where all sequences tend to drop in values at the same time, we can
    #try changing the values of the constants. Seems like increasing a and M worked, or at least
    #made it difficult to see a pattern.
for each_one in range(4):
    new_seq = new_pResidue(50,seed, a=70928509284734092, M=7**48)
    seed = seed+23
    plt.plot(new_seq)
    plt.title('Fixed Multiple Sequences at same length')

plt.show()

#Now use the numpy random function and scatter-plot against this function to see difference
xvalues = np.arange(0,75)
random_values = np.random.random(75)
our_values = powerResidue(75)
plt.scatter(xvalues,random_values,s=2, color = "blue", label="Numpy Function")
plt.scatter(xvalues,our_values,s=2, color = "red", label="Our Function")
plt.title("Numpy Random function versus Our function")
plt.xlim(0,80)
plt.ylim(0.0,1.25)
plt.legend()

#When looking at the distribution of values for the numpy random function and our function
#I believe our function almost seems better at the "randomness" Our function does not seem
#to have as many values clumped together like the numpy function does. There seems to be
#a more uniform distribution to the values for our function. THen again, uniform is not necessarily
#random, and random values can occur in clumps. So perhaps with that point stated, numpy
#function could actually be better at generating random values.


from numpy import array, linalg, dot
from scipy.linalg import lu

# System of Equations
A = array([[3,  -.1, -.2],
           [.1,  7,  -.3],
           [.3, -.2, 10]], float)
B = array([10.3, 33.6, 60.2], float)

# Linalg.Solve Function:

X = linalg.solve(A, B)
print("linalg.solve(A, B) = ", X, end='\n\n')

# Dot and Linalg.inv Function:

X = dot(linalg.inv(A), B)
print("dot(linalg.inv(A), B)", X, end='\n\n')

# Scipy.linalg.lu Function:

P, L, U = lu(A)
print("P, L, U = lu(A)")
print("[P] =\n", P, sep='')
print("[L] =\n", L, sep='')
print("[U] =\n", U, sep='')

5. Systems of Linear Equations/0. Linear System solving functions of SciPy.py


import numpy as np

def pca_train(raw_data, k):
    """
    输入：原始数据矩阵raw_data、降维后的维度k
    输出：样本均值和投影向量
    参考：https://www.cnblogs.com/lzllovesyl/p/5235137.html
    """
    # 计算均值向量
    mean_vecs = np.mean(raw_data, axis=0)
    # 样本中心化
    centralized_data = raw_data - mean_vecs
    # 计算协方差矩阵
    cov_mat = np.cov(centralized_data, rowvar=False)
    # 计算特征值和特征向量
    eig_vals, eig_vecs = np.linalg.eig(np.mat(cov_mat))
    # 特征值排序
    eig_vals_idx = np.argsort(eig_vals)
    topk_idx = eig_vals_idx[:-(k+1):-1]
    # 取对应的特征向量
    topk_eig_vecs= eig_vecs[:,topk_idx]
    return mean_vecs, topk_eig_vecs

def pca_test(new_data, mean_vecs, eig_vecs):
    """
    输入：新样本new_data、均值向量mean_vec、和投影向量eig_vec
    输出：投影后的数据
    """
    # 数据中心化
    centralized_data = new_data - mean_vecs
    # 降维
    return centralized_data.dot(eig_vecs)


import numpy as np

def linear_kernel(x, y, **kwargs):
    return np.dot(x, y)

def polynomial_kernel(x, y, **kwargs):
    degree = kwargs['degree']
    gamma = kwargs['gamma']
    coef0 = kwargs['coef0']
    return (coef0 + gamma * np.dot(x, y)) ** degree

def gaussian_kernel(x, y, **kwargs):
    gamma = kwargs['gamma']
    return np.exp(-gamma * (np.linalg.norm(x-y)**2))


# basic implementation of explicit extrapolation with Euler's and midpoint methods
# a convergence test on the example y' = lam*y is preformed at the end of this file

from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
import math

debugging = False

# solve the first order ODE y' = f(y, t) on the interval [t0, tf] with
# the initial value y(t0) = y0 and time step size h
def euler_method (f, y0, t0, tf, h):
    assert abs(((tf -t0)/h) % 1) < 1.e-15, "Step size %r does not divide \
        the interval [%r, %r] evenly." %(h,t0,tf)

    ts = np.linspace(t0, tf, (tf-t0)/h + 1)
    ys = np.zeros(len(ts), dtype=complex)
    ys[0] = y0

    for i in range(len(ts) - 1):
        ys[i+1] = ys[i] + h*f(ys[i], ts[i])
    return (ts, ys)

# solve the first order ODE y' = f(y, t) on the interval [t0, tf] with
# the intial value y(t0) = y0  and time step size h
def midpoint_method (f, y0, t0, tf, h):
    assert abs(((tf -t0)/h) % 1) < 1.e-15, "Step size %r does not divide \
        the interval [%r, %r] evenly." %(h,t0,tf)
    ts = np.linspace(t0, tf, (tf-t0)/h + 1)
    ys = np.zeros(len(ts), dtype=complex)
    ys[0] = y0
    ys[1] = ys[0] + h*f(ys[0], ts[0]) # use Euler to get second initial value
    for i in range(len(ts) - 2):
        ys[i+2] = ys[i] + 2*h*f(ys[i+1], ts[i+1])
    return (ts, ys)

# preforms a convergence test solving the IVP y' = f(y) and y(t0) = y0 with
# the [method] provided, where the function exact is the solution the IVP.
# The function plots the ||error|| vs time step size, and a reference line
# of the given order to compare. The graph is titled with graph_title
def test_convergence(f, exact, t0, tf, y0, method, order, graph_title):
    hs = np.asarray([2**(-k) for k in range(1, 16)])
    err = np.zeros(len(hs))

    for i in range(len(hs)):
        ts, ys = method(f, y0, t0, tf, hs[i])
        err[i] = abs(ys[-1] - exact(tf))

    plt.hold('true')
    method_err,  = plt.loglog(hs, err, 's-')
    order_line,  = plt.loglog(hs, (hs**order)*(err[5]/hs[5]))
    plt.legend([method_err, order_line], ['Method Error', 'order ' + str(order)], loc=2)
    plt.title(graph_title)
    plt.ylabel('||error||')
    plt.xlabel('time step size')

    plt.show()

# Convergence Tests
if debugging:
    #### test 1 ####
    lam = -2
    f = lambda y,t: lam*y
    y0 = 1
    exact = lambda t: y0*np.exp(lam*t)
    t0 = 1
    tf = 5
    test_convergence(f, exact, t0, tf, exact(t0), euler_method, 1, "TEST 1: euler_method")
    test_convergence(f, exact, t0, tf, exact(t0), midpoint_method, 2, "TEST 1: midpoint_method")

    ### test 2 ####
    f = lambda y,t: 4.*y*float(np.sin(t))**3*np.cos(t)
    y0 = 1
    exact = lambda t: y0*np.exp((np.sin(t))**4)
    t0 = 0
    tf = 5
    test_convergence(f, exact, t0, tf, exact(t0), euler_method, 1, "TEST 2: euler_method")
    test_convergence(f, exact, t0, tf, exact(t0), midpoint_method, 2, "TEST 2: midpoint_method")

    #### test 3 ####
    f = lambda y,t: 4.*t*np.sqrt(y)
    y0 = 1
    exact = lambda t: (1.+t**2)**2
    t0 = 0
    tf = 5
    test_convergence(f, exact, t0, tf, exact(t0), euler_method, 1, "TEST 3: euler_method")
    test_convergence(f, exact, t0, tf, exact(t0), midpoint_method, 2, "TEST 3: midpoint_method")

    #### test 4 ####
    f = lambda y,t:  y/t*np.log(y)
    y0 = np.exp(1.)
    exact = lambda t: np.exp(2.*t)
    t0 = 0.5
    tf = 5
    test_convergence(f, exact, t0, tf, exact(t0), euler_method, 1, "TEST 4: euler_method")
    test_convergence(f, exact, t0, tf, exact(t0), midpoint_method, 2, "TEST 4: midpoint_method")

if __name__ == "__main__":
    import doctest
    doctest.testmod()


import numpy as np
import math
from scipy import stats

# ------------------------ EXPONENTIALS AND LOGARITHMS ----------------------- #
def fexponential(x, a, b, c):
    return a * np.exp(-b * x) + c

def exponential(x, a, b, c, d):
    return a * np.exp(-c * (x - b)) + d

def logarithmic(x, a, b, c):
    return a * np.log(b * x) + c

# --------------------------------- SIGMOIDS --------------------------------- #
def logistic(x, L, x0, k, b):
    """
    L -> shirnks the function on the Y axis.
    x0 -> x shift.
    k  -> slope. the smaller the flatter. Vals > 5 are good
    b -> y shift. Shifts thw whole curve up and donw
    """
    y = L / (1 + np.exp(-k * (x - x0))) + b
    return y

def centered_logistic(x, L, x0, k):
    """
    L -> shirnks the function on the Y axis.
    x0 -> x shift.
    k  -> slope. the smaller the flatter. Vals > 5 are good
    """
    b = (1 - L) / 2
    y = L / (1 + np.exp(-k * (x - x0))) + b
    return y

def hill_function(x, n, L, b):  # ? no work
    return L / (1 + x ** -n) + b

def hyperbolic_tangent(x, L=1, b=0, x0=0):
    return np.tanh(x - x0) / L + b

def arctangent(x, L=2, b=0, x0=0):
    return np.arctan(x - x0) / L + b

def gudermannian(x, L=2, b=0, x0=0):
    return L * np.arctan(np.tanh((x - x0) / L)) + b

def generalised_logistic(x, a, x0):  # ? no work
    """
        a > 0
    """
    if a <= 0:
        raise ValueError("Paramter 'a' should be > 0")
    return (1 + np.exp(-(x - x0))) ** -a

def algebraic_sigmoid(x, L=1, b=0, x0=0):
    return (x - x0 / (math.sqrt(1 + (x - x0) ** 2))) / L + b

def error_function(x, x0=0, scale=1, L=1, b=0):
    norm = stats.norm(x0, scale)
    return norm.cdf(x) * L + b

def centered_error_function(x, x0=0, scale=1, L=1):
    b = (1 - L) / 2
    norm = stats.norm(x0, scale)
    return norm.cdf(x) * L + b

# ------------------------------ OTHER FUNCTIONS ----------------------------- #
def linear_func(x, a, b):
    return x * a + b

def step_function(x, a, b, c):
    # Step function
    """
    a: value at x = b
    f(x) = 0 if x<b, a if x=b and 2*a if  x > a
    """
    return a * (np.sign(x - b) + c)


import numpy as np
from scipy.spatial.distance import cdist

class KMeans:
    def __init__(
            self,
            k: int,
            metric: str = "euclidean",
            tol: float = 1e-6,
            max_iter: int = 100):
        """
        inputs:
            k: int
                the number of centroids to use in cluster fitting
            metric: str
                the name of the distance metric to use
            tol: float
                the minimum error tolerance from previous error during optimization to quit the model fit
            max_iter: int
                the maximum number of iterations before quitting model fit
        """
        self.k = k
        self.metric = metric
        self.tol = tol
        self.max_iter = max_iter

        # model should not run if k<=0
        if self.k<=0:
            raise ValueError("k must be greater than zero")

    def fit(self, mat: np.ndarray):
        """
        fits the kmeans algorithm onto a provided 2D matrix

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features
        """

        # model should not run if k > number of observations
        if self.k > np.shape(mat)[0]:
            raise ValueError("k must be less than the number of observations in the array")

        # create random centers
        self.centers = np.random.uniform(np.min(mat), np.max(mat), size=(self.k,2))

        # set baseline values
        i=1
        mse=1
        mse_diff=1
        self.mat = mat # necessary to access mat in get_centroids

        # iteratively select new centers
        while i<self.max_iter and mse_diff>self.tol:
            # assign points to each cluster based on their distance to the centers
            self.dist_mat = self._calc_distances(mat, self.centers)
            self.labels = self.predict(mat)

            # calculate error - calculate squared distance from each point to its corresponding centroid,
            # using the supplied metric, then take the mean of those values
            old_mse = mse
            mse = self.get_error()
            mse_diff = abs(old_mse-mse)

            # update centers based on cluster membership of datapoints
            self.centers = self.get_centroids()

            # increment
            i+=1

    def predict(self, mat: np.ndarray) -> np.ndarray:
        """
        predicts the cluster labels for a provided 2D matrix

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features

        outputs:
            np.ndarray
                a 1D array with the cluster label for each of the observations in `mat`
        """
        return np.argmin(self.dist_mat, axis=1) # find index of minmum value in each row

    def get_error(self) -> float:
        """
        returns the final squared-mean error of the fit model

        outputs:
            float
                the squared-mean error of the fit model
        """
        return np.mean(np.square(np.choose(self.labels, self.dist_mat.T))) # numpy.choose which constructs an array from an index array - https://stackoverflow.com/questions/17074422/select-one-element-in-each-row-of-a-numpy-array-by-column-indices

    def get_centroids(self) -> np.ndarray:
        """
        returns the centroid locations of the fit model

        outputs:
            np.ndarray
                a `k x m` 2D matrix representing the cluster centroids of the fit model
        """
        return np.array([self.mat[self.labels==j].mean(0) for j in range(self.k)])

    def _calc_distances(self, mat: np.ndarray, centers: np.ndarray) -> np.ndarray:
        """
        calculates the distance from each point to each center
        """
        return cdist(mat, centers, metric=self.metric)


#!/usr/bin/env python3
#
# Script to demonstrate effects of matrix conditioning in floating-point arithmetic.
#
# Daniel R. Reynolds
# SMU Mathematics
# Math 4315

# imports
import numpy

# utility function to create Hilbert matrix of dimension n
def Hilbert(n):
  H = numpy.zeros([n,n])
  for i in range(n):
    for j in range(n):
      H[i,j] = 1/(1+i+j)
  return H

# set matrix sizes for tests
nvals = [6, 8, 10, 12, 14]

# run tests for each matrix size
for n in nvals:

  # create matrix, solution and right-hand side vector
  A = Hilbert(n)
  x = numpy.random.rand(n,1)
  b = A@x

  # ouptut condition number
  print("Hilbert matrix of dimension", n, ": condition number = ",
        format(numpy.linalg.cond(A),'e'))

  # solve the linear system
  x_comp = numpy.linalg.solve(A,b)

  # output relative solution error
  print("  relative solution error = ",
        numpy.linalg.norm(x-x_comp)/numpy.linalg.norm(x))

# end of script


y = dependent variable
x = independent variables

y = b0 + b1.x1 # Simple linear regression
y = b0 + b1.x1 + b2.x2 + .. + bn.xn # Multiple Linear Regression

'''
Assumptions of linear regression
Before we test the assumptions, we’ll need to fit our linear regression models
1. linearity
2. homoskedasticity
3. multivariate normality
4. independence of errors
5. lack of multicollinearity
'''

# Importing the libraries
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
from sklearn import datasets

# Importing the dataset
dataset = pd.read_csv('Data/Ecommerce_Customers.csv')

# Explore the dataset
dataset.info()
dataset.Address

# Visualize the dataset
sns.pairplot(dataset) # hue=None, palette=None, vars=[None], x_vars=[None], y_vars=[None], dropna=True, kind = 'reg' to find linearity, 'scatter' is default
# Here the Length of Membership seems to have a linear relationship with Yearlt amount spent

# One thing to note is that I’m assuming outliers have been removed

# Manipulate the dataset (add new feaures based on data, etc.)
# Extract state from address
def str_split(row):
    return row.Address.split(' ')[-2]
dataset['state'] = dataset.apply(str_split, axis=1) # apply the function to the dataset
dataset.groupby('state').count()[['Address']].index # all are states so the function worked

# defining x and y variables of the linear regression equation
X = dataset.iloc[:, [3,4,5,6]].values
y = dataset.iloc[:, 7].values

# Encoding categorical data
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder
# 4 because we are building the transformer based on 4th column
ct = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [4])], remainder='passthrough')
X = np.array(ct.fit_transform(X))

# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)

# Fitting the model
# Training the Multiple Linear Regression model on the Training set
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(X_train, y_train)

# Returning the R^2 for the model
# .score() automatically scores y_train_pred, and compares those with y_train to calculate the R^2
regressor_r2 = regressor.score(X_train, y_train)
print('R^2:', regressor_r2)
# Instead of using .score() we could also use r2_score()
# But we will first need to manually score y_train_pred values
from sklearn.metrics import r2_score
y_train_pred = regressor.predict(X_train)
r2 = r2_score(y_train, y_train_pred)
print('R^2:', r2)

# Predicting the Test set results
y_test_pred = regressor.predict(X_test)
np.set_printoptions(precision=2)
print(np.concatenate((y_test_pred.reshape(len(y_test_pred),1), y_test.reshape(len(y_test),1)),1))

def calculate_residuals(model, features, label):
    """
    Creates predictions on the features with the model and calculates residuals
    """
    predictions = model.predict(features)
    df_results = pd.DataFrame({'Actual': label, 'Predicted': predictions})
    df_results['Residuals'] = abs(df_results['Actual']) - abs(df_results['Predicted'])
    return df_results

'''
LinearityPermalink
- This assumes that there is a linear relationship between the predictors (e.g. independent variables or features) and the response variable (e.g. dependent variable or label). This also assumes that the predictors are additive.
- Why it can happen: There may not just be a linear relationship among the data. Modeling is about trying to estimate a function that explains a process, and linear regression would not be a fitting estimator (pun intended) if there is no linear relationship.
- What it will affect: The predictions will be extremely inaccurate because our model is underfitting. This is a serious violation that should not be ignored.
- How to detect it: If there is only one predictor, this is pretty easy to test with a scatter plot. Most cases aren’t so simple, so we’ll have to modify this by using a scatter plot to see our predicted values versus the actual values (in other words, view the residuals). Ideally, the points should lie on or around a diagonal line on the scatter plot.
- How to fix it: Either adding polynomial terms to some of the predictors or applying nonlinear transformations . If those do not work, try adding additional variables to help capture the relationship between the predictors and the label.
'''
def linear_assumption(model, features, label):
    """
    Linearity: Assumes that there is a linear relationship between the predictors and
               the response variable. If not, either a quadratic term or another
               algorithm should be used.
    """
    print('Assumption 1: Linear Relationship between the Target and the Feature', '\n')

    print('Checking with a scatter plot of actual vs. predicted.',
           'Predictions should follow the diagonal line.')

    # Calculating residuals for the plot
    df_results = calculate_residuals(model, features, label)

    # Plotting the actual vs predicted values
    sns.lmplot(x='Actual', y='Predicted', data=df_results, fit_reg=False, size=7)

    # Plotting the diagonal line
    line_coords = np.arange(df_results.min().min(), df_results.max().max())
    plt.plot(line_coords, line_coords,  # X and y points
             color='darkorange', linestyle='--')
    plt.title('Actual vs. Predicted')
    plt.show()

linear_assumption(regressor, X_train, y_train)
'''
How to read the output
- The predicted values are equally distributed above and below the diagonal, hence the relationship seems linear
- If there were a higher concentration of predicted values below the diagonal, then we say the predictions are biased towards lower values
- If there were a higher concentration of predicted values above the diagonal, then we say the predictions are biased towards higher values
'''

'''
Normality of the Error TermsPermalink
- More specifically, this assumes that the error terms of the model are normally distributed. Linear regressions other than Ordinary Least Squares (OLS) may also assume normality of the predictors or the label, but that is not the case here.
- Why it can happen: This can actually happen if either the predictors or the label are significantly non-normal. Other potential reasons could include the linearity assumption being violated or outliers affecting our model.
- What it will affect: A violation of this assumption could cause issues with either shrinking or inflating our confidence intervals.
- How to detect it: There are a variety of ways to do so, but we’ll look at both a histogram and the p-value from the Anderson-Darling test for normality.
- How to fix it: It depends on the root cause, but there are a few options. Nonlinear transformations of the variables, excluding specific variables (such as long-tailed variables), or removing outliers may solve this problem.
'''
def normal_errors_assumption(model, features, label, p_value_thresh=0.05):
    """
    Normality: Assumes that the error terms are normally distributed. If they are not,
    nonlinear transformations of variables may solve this.

    This assumption being violated primarily causes issues with the confidence intervals
    """
    from statsmodels.stats.diagnostic import normal_ad
    print('Assumption 2: The error terms are normally distributed', '\n')

    # Calculating residuals for the Anderson-Darling test
    df_results = calculate_residuals(model, features, label)

    print('Using the Anderson-Darling test for normal distribution')

    # Performing the test on the residuals
    p_value = normal_ad(df_results['Residuals'])[1]
    print('p-value from the test - below 0.05 generally means non-normal:', p_value)

    # Reporting the normality of the residuals
    if p_value < p_value_thresh:
        print('Residuals are not normally distributed')
    else:
        print('Residuals are normally distributed')

    # Plotting the residuals distribution
    plt.subplots(figsize=(12, 6))
    plt.title('Distribution of Residuals')
    sns.distplot(df_results['Residuals'])
    plt.show()

    print()
    if p_value > p_value_thresh:
        print('Assumption satisfied')
    else:
        print('Assumption not satisfied')
        print()
        print('Confidence intervals will likely be affected')
        print('Try performing nonlinear transformations on variables')

normal_errors_assumption(regressor, X_train, y_train)
# Output seems normal so assumption satisfied

'''
No Multicollinearity among PredictorsPermalink
- This assumes that the predictors used in the regression are not correlated with each other. This won’t render our model unusable if violated, but it will cause issues with the interpretability of the model.
- Why it can happen: A lot of data is just naturally correlated. For example, if trying to predict a house price with square footage, the number of bedrooms, and the number of bathrooms, we can expect to see correlation between those three variables because bedrooms and bathrooms make up a portion of square footage.
- What it will affect: Multicollinearity causes issues with the interpretation of the coefficients. Specifically, you can interpret a coefficient as “an increase of 1 in this predictor results in a change of (coefficient) in the response variable, holding all other predictors constant.” This becomes problematic when multicollinearity is present because we can’t hold correlated predictors constant. Additionally, it increases the standard error of the coefficients, which results in them potentially showing as statistically insignificant when they might actually be significant.
- How to detect it: There are a few ways, but we will use a heatmap of the correlation as a visual aid and examine the variance inflation factor (VIF).
- How to fix it: This can be fixed by other removing predictors with a high variance inflation factor (VIF) or performing dimensionality reduction.
'''
def multicollinearity_assumption(model, features, label, feature_names=None):
    """
    Multicollinearity: Assumes that predictors are not correlated with each other. If there is
                       correlation among the predictors, then either remove prepdictors with high
                       Variance Inflation Factor (VIF) values or perform dimensionality reduction

                       This assumption being violated causes issues with interpretability of the
                       coefficients and the standard errors of the coefficients.
    """
    from statsmodels.stats.outliers_influence import variance_inflation_factor
    print('Assumption 3: Little to no multicollinearity among predictors')

    # Plotting the heatmap
    plt.figure(figsize = (10,8))
    sns.heatmap(pd.DataFrame(features, columns=feature_names).corr(), annot=True)
    plt.title('Correlation of Variables')
    plt.show()

    print('Variance Inflation Factors (VIF)')
    print('> 10: An indication that multicollinearity may be present')
    print('> 100: Certain multicollinearity among the variables')
    print('-------------------------------------')

    # Gathering the VIF for each variable
    VIF = [variance_inflation_factor(features, i) for i in range(features.shape[1])]
    for idx, vif in enumerate(VIF):
        print('{0}: {1}'.format(feature_names[idx], vif))

    # Gathering and printing total cases of possible or definite multicollinearity
    possible_multicollinearity = sum([1 for vif in VIF if vif > 10])
    definite_multicollinearity = sum([1 for vif in VIF if vif > 100])
    print()
    print('{0} cases of possible multicollinearity'.format(possible_multicollinearity))
    print('{0} cases of definite multicollinearity'.format(definite_multicollinearity))
    print()

    if definite_multicollinearity == 0:
        if possible_multicollinearity == 0:
            print('Assumption satisfied')
        else:
            print('Assumption possibly satisfied')
            print()
            print('Coefficient interpretability may be problematic')
            print('Consider removing variables with a high Variance Inflation Factor (VIF)')

    else:
        print('Assumption not satisfied')
        print()
        print('Coefficient interpretability will be problematic')
        print('Consider removing variables with a high Variance Inflation Factor (VIF)')

# Additional variable being created to test multicollinearity
regressor_feature_names = np.array(['Avg. Session Length', 'Time on App', 'Time on Website', 'Length of Membership'])

multicollinearity_assumption(regressor, X_train, y_train, regressor_feature_names)

'''
No Autocorrelation of the Error TermsPermalink
- This assumes no autocorrelation of the error terms. Autocorrelation being present typically indicates that we are missing some information that should be captured by the model.
- Why it can happen: In a time series scenario, there could be information about the past that we aren’t capturing. In a non-time series scenario, our model could be systematically biased by either under or over predicting in certain conditions. Lastly, this could be a result of a violation of the linearity assumption.
- What it will affect: This will impact our model estimates.
- How to detect it: We will perform a Durbin-Watson test to determine if either positive or negative correlation is present. Alternatively, you could create plots of residual autocorrelations.
- How to fix it: A simple fix of adding lag variables can fix this problem. Alternatively, interaction terms, additional variables, or additional transformations may fix this.
'''
def autocorrelation_assumption(model, features, label):
    """
    Autocorrelation: Assumes that there is no autocorrelation in the residuals. If there is
                     autocorrelation, then there is a pattern that is not explained due to
                     the current value being dependent on the previous value.
                     This may be resolved by adding a lag variable of either the dependent
                     variable or some of the predictors.
    """
    from statsmodels.stats.stattools import durbin_watson
    print('Assumption 4: No Autocorrelation', '\n')

    # Calculating residuals for the Durbin Watson-tests
    df_results = calculate_residuals(model, features, label)

    print('\nPerforming Durbin-Watson Test')
    print('Values of 1.5 < d < 2.5 generally show that there is no autocorrelation in the data')
    print('0 to 2< is positive autocorrelation')
    print('>2 to 4 is negative autocorrelation')
    print('-------------------------------------')
    durbinWatson = durbin_watson(df_results['Residuals'])
    print('Durbin-Watson:', durbinWatson)
    if durbinWatson < 1.5:
        print('Signs of positive autocorrelation', '\n')
        print('Assumption not satisfied')
    elif durbinWatson > 2.5:
        print('Signs of negative autocorrelation', '\n')
        print('Assumption not satisfied')
    else:
        print('Little to no autocorrelation', '\n')
        print('Assumption satisfied')

autocorrelation_assumption(regressor, X_train, y_train)

'''
Homoscedasticity
- This assumes homoscedasticity, which is the same variance within our error terms. Heteroscedasticity, the violation of homoscedasticity, occurs when we don’t have an even variance across the error terms.
- Why it can happen: Our model may be giving too much weight to a subset of the data, particularly where the error variance was the largest.
- What it will affect: Significance tests for coefficients due to the standard errors being biased. Additionally, the confidence intervals will be either too wide or too narrow.
- How to detect it: Plot the residuals and see if the variance appears to be uniform.
- How to fix it: Heteroscedasticity (can you tell I like the scedasticity words?) can be solved either by using weighted least squares regression instead of the standard OLS or transforming either the dependent or highly skewed variables. Performing a log transformation on the dependent variable is not a bad place to start.
'''
def homoscedasticity_assumption(model, features, label):
    """
    Homoscedasticity: Assumes that the errors exhibit constant variance
    """
    print('Assumption 5: Homoscedasticity of Error Terms', '\n')

    print('Residuals should have relative constant variance')

    # Calculating residuals for the plot
    df_results = calculate_residuals(model, features, label)

    # Plotting the residuals
    plt.subplots(figsize=(12, 6))
    ax = plt.subplot(111)  # To remove spines
    plt.scatter(x=df_results.index, y=df_results.Residuals, alpha=0.5)
    plt.plot(np.repeat(0, df_results.index.max()), color='darkorange', linestyle='--')
    ax.spines['right'].set_visible(False)  # Removing the right spine
    ax.spines['top'].set_visible(False)  # Removing the top spine
    plt.title('Residuals')
    plt.show()

# Plotting the residuals of our ideal dataset:
homoscedasticity_assumption(regressor, X_train, y_train)
# residuals have relative constant variance

#############################################################################
#######################  Code for the Master Function #######################
#############################################################################

def linear_regression_assumptions(features, label, feature_names=None):
    """
    Tests a linear regression on the model to see if assumptions are being met
    """
    from sklearn.linear_model import LinearRegression

    # Setting feature names to x1, x2, x3, etc. if they are not defined
    if feature_names is None:
        feature_names = ['X'+str(feature+1) for feature in range(features.shape[1])]

    print('Fitting linear regression')
    # Multi-threading if the dataset is a size where doing so is beneficial
    if features.shape[0] < 100000:
        model = LinearRegression(n_jobs=-1)
    else:
        model = LinearRegression()

    model.fit(features, label)

    # Returning linear regression R^2 and coefficients before performing diagnostics
    r2 = model.score(features, label)
    print()
    print('R^2:', r2, '\n')
    print('Coefficients')
    print('-------------------------------------')
    print('Intercept:', model.intercept_)

    for feature in range(len(model.coef_)):
        print('{0}: {1}'.format(feature_names[feature], model.coef_[feature]))

    print('\nPerforming linear regression assumption testing')

    # Creating predictions and calculating residuals for assumption tests
    predictions = model.predict(features)
    df_results = pd.DataFrame({'Actual': label, 'Predicted': predictions})
    df_results['Residuals'] = abs(df_results['Actual']) - abs(df_results['Predicted'])

    def linear_assumption():
        """
        Linearity: Assumes there is a linear relationship between the predictors and
                   the response variable. If not, either a polynomial term or another
                   algorithm should be used.
        """
        print('\n=======================================================================================')
        print('Assumption 1: Linear Relationship between the Target and the Features')

        print('Checking with a scatter plot of actual vs. predicted. Predictions should follow the diagonal line.')

        # Plotting the actual vs predicted values
        sns.lmplot(x='Actual', y='Predicted', data=df_results, fit_reg=False, size=7)

        # Plotting the diagonal line
        line_coords = np.arange(df_results.min().min(), df_results.max().max())
        plt.plot(line_coords, line_coords,  # X and y points
                 color='darkorange', linestyle='--')
        plt.title('Actual vs. Predicted')
        plt.show()
        print('If non-linearity is apparent, consider adding a polynomial term')

    def normal_errors_assumption(p_value_thresh=0.05):
        """
        Normality: Assumes that the error terms are normally distributed. If they are not,
        nonlinear transformations of variables may solve this.

        This assumption being violated primarily causes issues with the confidence intervals
        """
        from statsmodels.stats.diagnostic import normal_ad
        print('\n=======================================================================================')
        print('Assumption 2: The error terms are normally distributed')
        print()

        print('Using the Anderson-Darling test for normal distribution')

        # Performing the test on the residuals
        p_value = normal_ad(df_results['Residuals'])[1]
        print('p-value from the test - below 0.05 generally means non-normal:', p_value)

        # Reporting the normality of the residuals
        if p_value < p_value_thresh:
            print('Residuals are not normally distributed')
        else:
            print('Residuals are normally distributed')

        # Plotting the residuals distribution
        plt.subplots(figsize=(12, 6))
        plt.title('Distribution of Residuals')
        sns.distplot(df_results['Residuals'])
        plt.show()

        print()
        if p_value > p_value_thresh:
            print('Assumption satisfied')
        else:
            print('Assumption not satisfied')
            print()
            print('Confidence intervals will likely be affected')
            print('Try performing nonlinear transformations on variables')

    def multicollinearity_assumption():
        """
        Multicollinearity: Assumes that predictors are not correlated with each other. If there is
                           correlation among the predictors, then either remove prepdictors with high
                           Variance Inflation Factor (VIF) values or perform dimensionality reduction

                           This assumption being violated causes issues with interpretability of the
                           coefficients and the standard errors of the coefficients.
        """
        from statsmodels.stats.outliers_influence import variance_inflation_factor
        print('\n=======================================================================================')
        print('Assumption 3: Little to no multicollinearity among predictors')

        # Plotting the heatmap
        plt.figure(figsize = (10,8))
        sns.heatmap(pd.DataFrame(features, columns=feature_names).corr(), annot=True)
        plt.title('Correlation of Variables')
        plt.show()

        print('Variance Inflation Factors (VIF)')
        print('> 10: An indication that multicollinearity may be present')
        print('> 100: Certain multicollinearity among the variables')
        print('-------------------------------------')

        # Gathering the VIF for each variable
        VIF = [variance_inflation_factor(features, i) for i in range(features.shape[1])]
        for idx, vif in enumerate(VIF):
            print('{0}: {1}'.format(feature_names[idx], vif))

        # Gathering and printing total cases of possible or definite multicollinearity
        possible_multicollinearity = sum([1 for vif in VIF if vif > 10])
        definite_multicollinearity = sum([1 for vif in VIF if vif > 100])
        print()
        print('{0} cases of possible multicollinearity'.format(possible_multicollinearity))
        print('{0} cases of definite multicollinearity'.format(definite_multicollinearity))
        print()

        if definite_multicollinearity == 0:
            if possible_multicollinearity == 0:
                print('Assumption satisfied')
            else:
                print('Assumption possibly satisfied')
                print()
                print('Coefficient interpretability may be problematic')
                print('Consider removing variables with a high Variance Inflation Factor (VIF)')
        else:
            print('Assumption not satisfied')
            print()
            print('Coefficient interpretability will be problematic')
            print('Consider removing variables with a high Variance Inflation Factor (VIF)')

    def autocorrelation_assumption():
        """
        Autocorrelation: Assumes that there is no autocorrelation in the residuals. If there is
                         autocorrelation, then there is a pattern that is not explained due to
                         the current value being dependent on the previous value.
                         This may be resolved by adding a lag variable of either the dependent
                         variable or some of the predictors.
        """
        from statsmodels.stats.stattools import durbin_watson
        print('\n=======================================================================================')
        print('Assumption 4: No Autocorrelation')
        print('\nPerforming Durbin-Watson Test')
        print('Values of 1.5 < d < 2.5 generally show that there is no autocorrelation in the data')
        print('0 to 2< is positive autocorrelation')
        print('>2 to 4 is negative autocorrelation')
        print('-------------------------------------')
        durbinWatson = durbin_watson(df_results['Residuals'])
        print('Durbin-Watson:', durbinWatson)
        if durbinWatson < 1.5:
            print('Signs of positive autocorrelation', '\n')
            print('Assumption not satisfied', '\n')
            print('Consider adding lag variables')
        elif durbinWatson > 2.5:
            print('Signs of negative autocorrelation', '\n')
            print('Assumption not satisfied', '\n')
            print('Consider adding lag variables')
        else:
            print('Little to no autocorrelation', '\n')
            print('Assumption satisfied')

    def homoscedasticity_assumption():
        """
        Homoscedasticity: Assumes that the errors exhibit constant variance
        """
        print('\n=======================================================================================')
        print('Assumption 5: Homoscedasticity of Error Terms')
        print('Residuals should have relative constant variance')

        # Plotting the residuals
        plt.subplots(figsize=(12, 6))
        ax = plt.subplot(111)  # To remove spines
        plt.scatter(x=df_results.index, y=df_results.Residuals, alpha=0.5)
        plt.plot(np.repeat(0, df_results.index.max()), color='darkorange', linestyle='--')
        ax.spines['right'].set_visible(False)  # Removing the right spine
        ax.spines['top'].set_visible(False)  # Removing the top spine
        plt.title('Residuals')
        plt.show()
        print('If heteroscedasticity is apparent, confidence intervals and predictions will be affected')

    linear_assumption()
    normal_errors_assumption()
    multicollinearity_assumption()
    autocorrelation_assumption()
    homoscedasticity_assumption()

#############################################################################
########################  Code for Model Validation  ########################
#############################################################################
from sklearn.model_selection import RepeatedKFold

#############################################################################
########################  Code for Model Evaluation  ########################
#############################################################################
'''
sklearn package provides various model evaluation metrics. Following are the important ones:
- Max_error
- Mean Absolute Error
- Mean Squared Error
- Median Squared Error
- R Squared
'''

def model_evaluation(original_data,predicted_data):
    # Max_error
    from sklearn.metrics import max_error
    me = max_error(original_data,predicted_data)
    print('1. Max_error: \t\t\t',me)
    # Mean Absolute Error
    from sklearn.metrics import mean_absolute_error
    mae = mean_absolute_error(original_data,predicted_data)
    print('2. Mean absolute error: \t',mae)
    # Mean Squared Error
    from sklearn.metrics import mean_squared_error
    mse = mean_squared_error(original_data,predicted_data)
    print('3. Mean Squared Error: \t\t',mse)
    # Median Squared Error
    from sklearn.metrics import median_absolute_error
    med_ae = median_absolute_error(original_data,predicted_data)
    print('4. Median Squared Error: \t',med_ae)
    # R Squared
    from sklearn.metrics import r2_score
    r2 = r2_score(original_data,predicted_data)
    print('5. R Squared: \t\t\t',r2)

model_evaluation(y_test, y_test_pred)

#############################################################################
#####################  Overfitting: R^2 is too high  ########################
#############################################################################
# One of the most common problems that you’ll encounter when building models is
# multicollinearity. This occurs when two or more predictor variables in a dataset
# are highly correlated.

import seaborn as sns
sns.get_dataset_names()
tips = sns.load_dataset("tips")
tips.plot(x='total_bill', y='tip', kind='scatter')

'''
Methods for correlation analyses:
1. Parametric Correlation : It measures a linear dependence between two variables
(x and y) is known as a parametric correlation test because it depends on the
distribution of the data.
'''
# a. Pearson correlation
from scipy.stats import pearsonr
corr, _ = pearsonr(tips.total_bill, tips.tip)   # Apply the pearsonr()
print('Pearsons correlation: %.5f' % corr)

'''
2. Non-Parametric Correlation: Kendall(tau) and Spearman(rho), which are
rank-based correlation coefficients, are known as non-parametric correlation.
'''
# a. Kendall
from scipy.stats import kendalltau
corr, _ = kendalltau(tips.total_bill, tips.tip)
print('Kendall Rank correlation: %.5f' % corr)
# b. Spearman
from scipy.stats import spearmanr
corr, _ = spearmanr(tips.total_bill, tips.tip)
print('Spearman Rank correlation: %.5f' % corr)

# Calculate the correlation matrix and VIF values for the predictor variables

# correlation matrix (based on Pearson correlation coefficient)
tips.corr()
tips.corr().round(3)
tips.corr().style.background_gradient(cmap='coolwarm') # cmap='RdYlGn', 'bwr', 'PuOr',

# VIF: high VIF values (some texts define a “high” VIF value as 5 while others use 10) indicate multicollinearity
#create dataset
df = pd.DataFrame({'rating': [90, 85, 82, 88, 94, 90, 76, 75, 87, 86],
                   'points': [25, 20, 14, 16, 27, 20, 12, 15, 14, 19],
                   'assists': [5, 7, 7, 8, 5, 7, 6, 9, 9, 5],
                   'rebounds': [11, 8, 10, 6, 6, 9, 6, 10, 10, 7]})
from patsy import dmatrices
from statsmodels.stats.outliers_influence import variance_inflation_factor
#find design matrix for linear regression model using 'rating' as response variable
y, X = dmatrices('rating ~ points+assists+rebounds', data=df, return_type='dataframe')
#calculate VIF for each explanatory variable
vif = pd.DataFrame()
vif['VIF'] = [variance_inflation_factor(X.values, i) for i in range(X.shape[1])]
vif['variable'] = X.columns
#view VIF for each explanatory variable
vif
# How to Interpret VIF Values:
# A value of 1 indicates there is no correlation between a given explanatory variable and any other explanatory variables in the model.
# A value between 1 and 5 indicates moderate correlation between a given explanatory variable and other explanatory variables in the model, but this is often not severe enough to require attention.
# A value greater than 5 indicates potentially severe correlation between a given explanatory variable and other explanatory variables in the model. In this case, the coefficient estimates and p-values in the regression output are likely unreliable.

#############################################################################
###############  Avoid Overfitting using Subset Selection  ##################
#############################################################################

'''
Best subset selection
Given a set of p total predictor variables, there are 2^p models that we could
potentially build. One method that we can use to pick the best model is known
as best subset selection and it works as follows:
1. Let M0 denote the null model, which contains no predictor variables.
2. For k = 1, 2, … p:
    Fit all pCk models that contain exactly k predictors.
    Pick the best among these pCk models and call it Mk. Define “best” as the model
    with the highest R2 or equivalently the lowest RSS.
3. Select a single best model from among M0…Mp using cross-validation prediction
error, Cp, BIC, AIC, or adjusted R2.
Note that for a set of p predictor variables, there are 2p possible models.

Criteria for Choosing the “Best” Model
1. Cp: (RSS+2dσ̂) / n
2. AIC: (RSS+2dσ̂2) / (nσ̂2)
3. BIC: (RSS+log(n)dσ̂2) / n
4. Adjusted R2: 1 – ( (RSS/(n-d-1)) / (TSS / (n-1)) )
where:
    d: The number of predictors
    n: Total observations
    σ̂: Estimate of the variance of the error associate with each response measurement in a regression model
    RSS: Residual sum of squares of the regression model
    TSS: Total sum of squares of the regression model
'''

'''
Stepwise selection - there are two approaches: forward and backward stepwise
Forward stepwise selection works as follows:
1. Let M0 denote the null model, which contains no predictor variables.
2. For k = 0, 2, … p-1:
    Fit all p-k models that augment the predictors in Mk with one additional predictor variable.
    Pick the best among these p-k models and call it Mk+1. Define “best” as the model with the highest R2 or equivalently the lowest RSS.
3. Select a single best model from among M0…Mp using cross-validation prediction error, Cp, BIC, AIC, or adjusted R2.

Backward stepwise selection works as follows:
1. Let Mp denote the full model, which contains all p predictor variables.
2. For k = p, p-1, … 1:
    Fit all k models that contain all but one of the predictors in Mk, for a total of k-1 predictor variables.
    Pick the best among these k models and call it Mk-1. Define “best” as the model with the highest R2 or equivalently the lowest RSS.
3. Select a single best model from among M0…Mp using cross-validation prediction error, Cp, BIC, AIC, or adjusted R2.
'''

#############################################################################
###############  Avoid Overfitting using Regularization  ####################
#############################################################################

# Regularization can be achieved using either of the following two models
# Ridge Regression
# Lasso Regression

# Original model - ordinary least square regression model
# Y = β0 + β1X1 + β2X2 + … + βpXp + ε
# Y: The response variable
# Xj: The jth predictor variable
# βj: The average effect on Y of a one unit increase in Xj, holding all other predictors fixed
# ε: The error term
# The values for β0, β1, B2, … , βp are chosen using the least square method, which minimizes the sum of squared residuals (RSS)
# least squares regression tries to find coefficient estimates that minimize the sum of squared residuals (RSS):
# RSS = Σ(yi – ŷi)^2
# yi: The actual response value for the ith observation
# ŷi: The predicted response value based on the multiple linear regression model
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(X_train, y_train)
print('Without Regularization -> r2 for training set:',regressor.score(X_train, y_train),'and test set:',regressor.score(X_test, y_test))

# Ridge regression - Regularizing the linear model
# The basic idea of ridge regression is to introduce a little bias so that the variance can be substantially reduced, which leads to a lower overall MSE.
# ridge regression, seeks to minimize the following:
# RSS + λΣβj2
# where j ranges from 1 to p and λ ≥ 0
# This second term in the equation is known as a shrinkage penalty.
# When λ = 0, this penalty term has no effect and ridge regression produces the same coefficient estimates as least squares. However, as λ approaches infinity, the shrinkage penalty becomes more influential and the ridge regression coefficient estimates approach zero.
# In general, the predictor variables that are least influential in the model will shrink towards zero the fastest.
# Regularization, significantly reduces the variance of the model, without substantial increase in its bias.
# Using Ridge()
from sklearn.linear_model import Ridge
#alpha =0.5
ridge_reg=Ridge(alpha=0.5,normalize=True)
ridge_reg.fit(X_train,y_train)
print('Ridge Regres alpha=0.5 -> r2 for training set:',ridge_reg.score(X_train, y_train),'and test set:',ridge_reg.score(X_test, y_test))
#alpha =1
ridge_reg=Ridge(alpha=1,normalize=True)
ridge_reg.fit(X_train,y_train)
print('Ridge Regressi alpha=1 -> r2 for training set:',ridge_reg.score(X_train, y_train),'and test set:',ridge_reg.score(X_test, y_test))
#alpha =2
ridge_reg=Ridge(alpha=2,normalize=True)
ridge_reg.fit(X_train,y_train)
print('Ridge Regressi alpha=2 -> r2 for training set:',ridge_reg.score(X_train, y_train),'and test set:',ridge_reg.score(X_test, y_test))

# Using RidgeCV() - https://www.statology.org/ridge-regression-in-python/
from sklearn.linear_model import Ridge
from sklearn.linear_model import RidgeCV
from sklearn.model_selection import RepeatedKFold
#define cross-validation method to evaluate model
cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)
#define model
ridge_reg = RidgeCV(alphas=np.arange(0, 1, 0.01), cv=cv, scoring='neg_mean_absolute_error')
#fit model
ridge_reg.fit(X_train, y_train)
#display lambda that produced the lowest test MSE
print('Ridge Regression -> best alpha (that produced the lowest test MSE):', ridge_reg.alpha_)
# R^2 of the model
print('Ridge Regression -> r2 for training set:',ridge_reg.score(X_train, y_train),'and test set:',ridge_reg.score(X_test, y_test))

# Lasso Regression - use to fit a regression model when multicollinearity is present in the data
# lasso regression seeks to minimize the following:
# RSS + λΣ|βj|
# where j ranges from 1 to p predictor variables and λ ≥ 0.
# This second term in the equation is known as a shrinkage penalty.
# We select a value for λ that produces the lowest possible test MSE (mean squared error).
# Note: “alpha” is used instead of “lambda” in Python
from sklearn.linear_model import LassoCV
from sklearn.model_selection import RepeatedKFold # use the RepeatedKFold() function to perform k-fold cross-validation to find the optimal alpha value to use for the penalty term
#define cross-validation method to evaluate model
cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)
#define model
lasso_reg = LassoCV(alphas=np.arange(0, 1, 0.01), cv=cv, n_jobs=-1)
#fit model
lasso_reg.fit(X_train, y_train)
#display lambda that produced the lowest test MSE
print('Lasso Regression -> best alpha (that produced the lowest test MSE):', lasso_reg.alpha_)
# R^2 of the model
print('Lasso Regression -> r2 for training set:',lasso_reg.score(X_train, y_train),'and test set:',lasso_reg.score(X_test, y_test))

#############################################################################
########  Deal with multicollinearity using dimension reduction  ############
####################  principal components regression  ######################
#############################################################################

# principal components regression, which finds M linear combinations (known as “principal components”) of the original p predictors and then uses least squares to fit a linear regression model using the principal components as predictors.
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import scale
from sklearn import model_selection
from sklearn.model_selection import RepeatedKFold
from sklearn.model_selection import train_test_split
from sklearn.decomposition import PCA
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

# We’ll use hp as the response variable and the following variables as the predictors: mpg, disp, drat, wt, qsec
data_full = pd.read_csv('https://raw.githubusercontent.com/Statology/Python-Guides/main/mtcars.csv')
data = data_full[["mpg", "disp", "drat", "wt", "qsec", "hp"]]
'''
Fit the PCR Model:
1. pca.fit_transform(scale(X)): This tells Python that each of the predictor variables should be scaled to have a mean of 0 and a standard deviation of 1. This ensures that no predictor variable is overly influential in the model if it happens to be measured in different units.
2. cv = RepeatedKFold(): This tells Python to use k-fold cross-validation to evaluate the performance of the model. For this example we choose k = 10 folds, repeated 3 times.
'''
#define predictor and response variables
X = data[["mpg", "disp", "drat", "wt", "qsec"]]
y = data[["hp"]]
#scale predictor variables
pca = PCA()
X_reduced = pca.fit_transform(scale(X))
#define cross validation method
cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)

regr = LinearRegression()
mse = []
# Calculate MSE with only the intercept
score = -1*model_selection.cross_val_score(regr,
           np.ones((len(X_reduced),1)), y, cv=cv,
           scoring='neg_mean_squared_error').mean()
mse.append(score)
# Calculate MSE using cross-validation, adding one component at a time
for i in np.arange(1, 6):
    score = -1*model_selection.cross_val_score(regr,
               X_reduced[:,:i], y, cv=cv, scoring='neg_mean_squared_error').mean()
    mse.append(score)
# Plot cross-validation results
plt.plot(mse) # The plot displays the number of principal components along the x-axis and the test MSE (mean squared error) along the y-axis.
plt.xlabel('Number of Principal Components')
plt.ylabel('MSE')
plt.title('hp')
# Interpret the plot: From the plot we can see that the test MSE decreases by
# adding in two principal components, yet it begins to increase as we add more
# than two principal components. Thus, the optimal model includes just the first
# two principal components.

# We can also use the following code to calculate the percentage of variance in the response variable explained by adding in each principal component to the model:
np.cumsum(np.round(pca.explained_variance_ratio_, decimals=4)*100)
# Interpreting the result:
# By using just the first principal component, we can explain 69.83% of the variation in the response variable.
# By adding in the second principal component, we can explain 89.35% of the variation in the response variable.

#Use the Final Model to Make Predictions
#split the dataset into training (70%) and testing (30%) sets
X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.3,random_state=0)
#scale the training and testing data
X_reduced_train = pca.fit_transform(scale(X_train))
X_reduced_test = pca.transform(scale(X_test))[:,:1]
#train PCR model on training data
regr = LinearRegression()
regr.fit(X_reduced_train[:,:1], y_train)
#calculate RMSE
pred = regr.predict(X_reduced_test)
np.sqrt(mean_squared_error(y_test, pred))
# RMSE turns out to be 40.2096. This is the average deviation between the
# predicted value for hp and the observed value for hp for the observations
# in the testing set.

import numpy,from scipy,from statsmodels


import numpy as np
import matplotlib.pyplot as plt

def plot_loss(loss, reg_loss=np.array([]), title='Loss'):
	if reg_loss.size == 0:
		reg_loss = np.zeros(loss.shape)

	fig, ax1 = plt.subplots()
	plt.title(title)

	color = 'tab:red'
	ax1.set_xlabel('epoch')
	ax1.set_ylabel('loss', color=color)
	ax1.plot(loss - reg_loss, color=color)
	ax1.tick_params(axis='y', labelcolor=color)

	ax2 = ax1.twinx()  # instantiate a second axes that shares the same x-axis

	color = 'tab:blue'
	ax2.set_ylabel('reg loss', color=color)  # we already handled the x-label with ax1
	ax2.plot(reg_loss, color=color)
	ax2.tick_params(axis='y', labelcolor=color)

	fig.tight_layout()  # otherwise the right y-label is slightly clipped

def map_features(x1, x2, deg, add_intercept=True):
	"""
	Return a matrix with x1 and x2 and all combination of x1 and x2 up to the "deg" degree
	e/g: X1, X2, X1^2, X2^2, X1*X2, X1*X2^2, etc..
	:param x1: Array of features
	:param x2: Array of features
	:param deg: The polynomial degree to map the features to
	:param add_intercept: whether to add the intercept column to the result
	:return: Matrix
	"""
	x1 = x1.reshape((x1.shape[0], 1))
	x2 = x2.reshape((x2.shape[0], 1))
	out = np.ones((x1.shape[0], 1))
	for i in range(1, deg + 1):
		for j in range(0, i + 1):
			new_col = (x1 ** (i - j)) * (x2 ** j)
			out = np.insert(out, [out.shape[1]], new_col, axis=1)

	if not add_intercept:
		return out[:, 1:]

	return out

def sigmoid(z):
	return 1 / (1 + np.exp(-1 * z))

def calc_loss(x_mat, y, theta, _lambda=0):
	"""
	Calculate the loss and gradients per given theta array and regularization factor "lambda"
	:param x_mat: shape is (m,n)
	:param y: shape is (m,1)
	:param theta: shape is (n,1) where n is number of features
	:param _lambda: regularization term
	:return: loss, grads
	"""
	# h(i) = sigmoid(theta0*x0 + theta1*x1 + theta2*x2 + ... thetaN*xN)
	# Loss = J = -1/m * sum[ y(i) * log(h(i)) + (1 - y(i)) * log(1 - h(i)) ] + (lambda/2m) * sum[theta(j)^2]
	# Note in the above that theta(0) is not regularized
	m = x_mat.shape[0]
	hi = sigmoid(x_mat.dot(theta))

	# We break the component in the Sum of the loss function into 2 parts: left and right.
	left = y * np.log(hi)
	right = (1 - y) * (np.log(1 - hi))

	# Finally compute J.
	j = -1 * (1/m) * np.sum(left + right)

	reg_loss = 0
	if _lambda > 0:
		reg_loss = ((_lambda / 2 * m) * np.sum(theta[1:] ** 2))  # theta[0] should not regularized

	j += reg_loss

	# Calculate the partial derivative of loss per each theta (j)
	# dJ/dTheta(j) = 1/m * sum[ (h(i) - y(i)) * x(j,i) ]
	# basically we sum the prediction error per sample multiplied by the j's feature, and then divide by m
	# this calculate what is the "contribution" of each feature to the total error
	# grads shape is (n, 1)
	grads = (1 / m) * x_mat.T.dot(hi - y)

	if _lambda > 0:
		grads += ((_lambda / m) * theta)
		grads[0] -= ((_lambda / m) * theta[0])  # theta[0] is not regularized so we undo it only for it

	return j, reg_loss, grads

def learn(x_mat, y, epochs, alpha, _lambda=0):
	lost_arr = np.zeros(epochs)
	reg_loss_arr = np.zeros(epochs)
	(m, n) = x_mat.shape
	theta = np.zeros((n, 1))
	for e in range(epochs):
		(lost, reg_loss, grads) = calc_loss(x_mat, y, theta, _lambda)
		lost_arr[e] = lost
		reg_loss_arr[e] = reg_loss
		theta = theta - (alpha * grads)

	return lost_arr, reg_loss_arr, theta


"""

https://en.wikipedia.org/wiki/Leslie_matrix
http://bandicoot.maths.adelaide.edu.au/Leslie_matrix/leslie.cgi

"""

import matplotlib.pyplot as plt
import numpy as np
from sympy import *

"""

f is the birth rate, range is >= 0
s is the survival rate, range is [0, 1]
n0 is the initial population of age classes

To get how the population grows into different classes
n[l+1] = L*n[l]

"""
def leslie_matrix(f, s):
    w = len(f)
    L = [[0 for i in range(w)] for j in range(w)]
    for i in range(w):
        L[0][i] = f[i]
    for i in range(w-1):
        L[i+1][i] = s[i]
    return L

def print_leslie_property(L):
    M = Matrix(L)
    print("Leslie Matrix Properties")

    pprint(M)
    print("Eigenvalues")
    ev = M.eigenvals()
    for k in ev:
        pprint(k.evalf())

    print("Eigenvectors")
    eg = M.eigenvects()
    for k in eg:
        pprint(k[2])

def plot_leslie_iteration(name, L, n0, iters):
    print("Plotting Leslie Matrix Iteration")
    pprint(Matrix(L))
    pprint(n0)
    print()
    print()

    M = np.array(L)
    n = n0
    w = len(n)
    x = []
    y = [[0 for i in range(iters)] for j in range(w)]
    p = [0] * iters
    for i in range(iters):
        n = np.dot(M, n)
        x.append(i)
        p[i] = sum(n)
        for j in range(w):
            y[j][i] = n[j]

    x = np.array(x)
    p = np.array(p)

    fig, ax = plt.subplots(1, 2)
    ax[0].plot(x, p, label='Total Population')
    ax[0].legend(loc='best')
    for i in range(w):
        ax[1].plot(x, np.array(y[i]), label='Age Classes={}'.format(i))
        ax[1].legend(loc='best')

    plt.savefig(name)

init_printing()

n0 = [0, 100, 100, 100, 0, 0, 0, 0]
f = [0, 0, 0.8, 0.6, 0, 0, 0, 0]
s = [0.95, 0.9, 0.75, 0, 0, 0, 0, 0]
L = leslie_matrix(f, s)
plot_leslie_iteration('leslie-1.png', L, n0, 100)

n0 = [0, 1000, 200, 0, 0, 0, 0, 0]
f = [0, 0, 0.2, 0.3, 0.5, 0.1, 0, 0]
s = [1, 0.95, 0.95, 0.7, 0.4, 0.2, 0, 0]
L = leslie_matrix(f, s)
plot_leslie_iteration('leslie-2.png', L, n0, 100)

n0 = [0, 100, 100, 100, 0, 0, 0, 0]
f = [0, 0, 0.8, 0.5, 0, 0, 0, 0]
s = [0.95, 0.9, 0.739, 0, 0, 0, 0, 0]
L = leslie_matrix(f, s)
plot_leslie_iteration('leslie-3.png', L, n0, 100)


import numpy as np

def euler(n: int):
    """
    Compute an approximation of e, Euler's number, using Taylor series.

    Parameters
    ----------
    n : int
        Number of terms to use in Taylor series approximation.

    Returns
    -------
    e_approx : float
        Approximation of e.
    """
    if n < 0:
        raise ValueError('Must use a non-negative value for n.')
    e_approx = np.sum([1/np.math.factorial(i) for i in range(n+1)])
    return e_approx

def pi(n: int):
    """
    Compute an approximation of pi using Monte Carlo sampling.

    Parameters
    ----------
    n : int
        Number of Monte Carlo samples.

    Returns
    -------
    pi_approx : float
        Approximation of pi.
    """
    pi_approx = 4 * np.mean(np.linalg.norm(np.random.rand(n, 2), axis=-1) < 1)
    return pi_approx


'''
Machine Learning - Polynomial Regression
Polynomial Regression

If your data points clearly will not fit a linear regression (a straight line through all data points), it might be ideal for polynomial regression.

Polynomial regression, like linear regression, uses the relationship between the variables x and y to find the best way to draw a line through the data points.
How Does it Work?

Python has methods for finding a relationship between data-points and to draw a line of polynomial regression. We will show you how to use these methods instead of going through the mathematic formula.

In the example below, we have registered 18 cars as they were passing a certain tollbooth.

We have registered the car's speed, and the time of day (hour) the passing occurred.

The x-axis represents the hours of the day and the y-axis represents the speed:
Example

Start by drawing a scatter plot:
import matplotlib.pyplot as plt

x = [1,2,3,5,6,7,8,9,10,12,13,14,15,16,18,19,21,22]
y = [100,90,80,60,60,55,60,65,70,70,75,76,78,79,90,99,99,100]

plt.scatter(x, y)
plt.show()
Result:

'''
'''
xample Explained

Import the modules you need.

You can learn about the NumPy module in our NumPy Tutorial.

You can learn about the SciPy module in our SciPy Tutorial.

import numpy
import matplotlib.pyplot as plt

Create the arrays that represent the values of the x and y axis:

x = [1,2,3,5,6,7,8,9,10,12,13,14,15,16,18,19,21,22]
y = [100,90,80,60,60,55,60,65,70,70,75,76,78,79,90,99,99,100]

NumPy has a method that lets us make a polynomial model:

mymodel = numpy.poly1d(numpy.polyfit(x, y, 3))

Then specify how the line will display, we start at position 1, and end at position 22:

myline = numpy.linspace(1, 22, 100)

Draw the original scatter plot:

plt.scatter(x, y)

Draw the line of polynomial regression:

plt.plot(myline, mymodel(myline))

Display the diagram:

plt.show()
R-Squared

It is important to know how well the relationship between the values of the x- and y-axis is, if there are no relationship the polynomial regression can not be used to predict anything.

The relationship is measured with a value called the r-squared.

The r-squared value ranges from 0 to 1, where 0 means no relationship, and 1 means 100% related.

Python and the Sklearn module will compute this value for you, all you have to do is feed it with the x and y arrays:
Example

How well does my data fit in a polynomial regression?
import numpy
from sklearn.metrics import r2_score

x = [1,2,3,5,6,7,8,9,10,12,13,14,15,16,18,19,21,22]
y = [100,90,80,60,60,55,60,65,70,70,75,76,78,79,90,99,99,100]

mymodel = numpy.poly1d(numpy.polyfit(x, y, 3))

print(r2_score(y, mymodel(x)))

Note: The result 0.94 shows that there is a very good relationship, and we can use polynomial regression in future predictions.
Predict Future Values

Now we can use the information we have gathered to predict future values.

Example: Let us try to predict the speed of a car that passes the tollbooth at around 17 P.M:

To do so, we need the same mymodel array from the example above:

mymodel = numpy.poly1d(numpy.polyfit(x, y, 3))
Example

Predict the speed of a car passing at 17 P.M:
import numpy
from sklearn.metrics import r2_score

x = [1,2,3,5,6,7,8,9,10,12,13,14,15,16,18,19,21,22]
y = [100,90,80,60,60,55,60,65,70,70,75,76,78,79,90,99,99,100]

mymodel = numpy.poly1d(numpy.polyfit(x, y, 3))

speed = mymodel(17)
print(speed)

The example predicted a speed to be 88.87, which we also could read from the diagram:
Bad Fit?

Let us create an example where polynomial regression would not be the best method to predict future values.
Example

These values for the x- and y-axis should result in a very bad fit for polynomial regression:
import numpy
import matplotlib.pyplot as plt

x = [89,43,36,36,95,10,66,34,38,20,26,29,48,64,6,5,36,66,72,40]
y = [21,46,3,35,67,95,53,72,58,10,26,34,90,33,38,20,56,2,47,15]

mymodel = numpy.poly1d(numpy.polyfit(x, y, 3))

myline = numpy.linspace(2, 95, 100)

plt.scatter(x, y)
plt.plot(myline, mymodel(myline))
plt.show()
Result:

And the r-squared value?
Example

You should get a very low r-squared value.
import numpy
from sklearn.metrics import r2_score

x = [89,43,36,36,95,10,66,34,38,20,26,29,48,64,6,5,36,66,72,40]
y = [21,46,3,35,67,95,53,72,58,10,26,34,90,33,38,20,56,2,47,15]

mymodel = numpy.poly1d(numpy.polyfit(x, y, 3))

print(r2_score(y, mymodel(x)))

The result: 0.00995 indicates a very bad relationship, and tells us that this data set is not suitable for polynomial regression.

COLOR PICKER
colorpicker
LIKE US

Get certified
by completing
a course today!
w3schools CERTIFIED . 2021
CODE GAME
Code Game
'''


from math import lgamma
from numba import jit
import numpy as np
import matplotlib.pyplot as plt

@jit
def h(a, b, c, d):
    """
    This is the equation number 3 of the article

    :param a:
    :param b:
    :param c:
    :param d:
    :return:
    """
    num = lgamma(a + c) + lgamma(b + d) + lgamma(a + b) + lgamma(c + d)
    den = lgamma(a) + lgamma(b) + lgamma(c) + lgamma(d) + lgamma(a + b + c + d)
    return np.exp(num - den)

@jit
def g0(a, b, c):
    """
    This is the first equation of chapter 2

    :param a:
    :param b:
    :param c:
    :return:
    """
    return np.exp(lgamma(a + b) + lgamma(a + c) - (lgamma(a + b + c) + lgamma(a)))

@jit
def hiter(a, b, c, d):
    """
    This function iterate using the recurrence equations (page 3) until it reaches the case of the first equation
    of chapter 2

    :param a:
    :param b:
    :param c:
    :param d:
    :return:
    """
    while d > 1:
        d -= 1
        yield h(a, b, c, d) / d

def g(a, b, c, d):
    """
    This is the equation...

    :param a:
    :param b:
    :param c:
    :param d:
    :return:
    """
    return g0(a, b, c) + sum(hiter(a, b, c, d))

def calc_prob_between(beta1, beta2) -> float:
    """
    This function calculate the probability for beta1 to be greater than beta2.
    In this function the beta functions (scipy.stats.beta) are needed only for extracting the arguments

    The output will be a number. 0.5 means the distributions are the same,
    more than 0.5 means that beta1 is better than beta2, less than 0.5 means the opposite.

    For having an affordable result, that probability should be greater than 0.95.

    Details about the math: https://www.johndcook.com/UTMDABTR-005-05.pdf

    :param beta1: the first beta distribution
    :param beta2: the second beta distribution
    :return: the probability.
    """
    return g(beta1.args[0], beta1.args[1], beta2.args[0], beta2.args[1])

def calc_beta_mode(a: int, b: int) -> float:
    """
    This function calculate the mode (i.e. the peak) of the beta distribution.

    :param a: First shape parameter of the Beta distribution
    :param b: Second shape parameter of the Beta distribution
    :return: The mode
    """
    return (a-1)/(a+b-2)

def plot_beta(betas, names, linf=0, lsup=0.01):
    """
    This function plots the Beta distribution(s)

    :param betas: a list containing the beta distributions
    :param names: a list of the same size of `betas`, with the names associated to the samples as strings
    :param linf: the left limit for the horizontal (`x`) axis.
    :param lsup: the right limit for the horizontal (`x`) axis.
    :return: nothing, just plot the beta(s)
    """""
    x = np.linspace(linf, lsup, 100)
    for f, name in zip(betas, names):
        y = f.pdf(x)
        y_mode = calc_beta_mode(f.args[0], f.args[1])
        y_var = f.var()
        plt.plot(x, y, label=f"{name}, peak @ {y_mode:0.1E}, var = {y_var:0.1E}")
        plt.yticks([])
    plt.legend()
    plt.show()


#IIR Analog and Digital Butterworth LPF filter design
from scipy import signal
import numpy as np
import matplotlib.pyplot as plt
from pylab import clf
#Analog low pass butterworth filter, order =3, fc=50Hz
#b = numerator coefficients, a = denominator coefficients
#w = frequency variable, h= frequency response

b,a = signal.iirfilter(3,50,btype ='low',analog=True, ftype ='butter')
w,h = signal.freqs(b,a,1000)

fig1 = plt.figure(1)
ax = fig1.add_subplot(1,1,1)
ax.semilogx(w,20*np.log10(np.maximum(abs(h),1e-5)))
ax.set_title('Butterworth IIR Analog LPF frequency response')
ax.set_xlabel('Frequency in Hz---->')
ax.set_ylabel('Amplitude in dB----->')
#ax.axis((10,1000,-100,10))
ax.grid(which='both',axis='both')
plt.show()

#Digital low pass butterworth filter, order =3, fc=50Hz
fs =1000
sos = signal.iirfilter(3,50/fs,btype ='low',analog=False, ftype ='butter',output='sos')
w,h = signal.sosfreqz(sos,1000)

fig2 = plt.figure(2)
clf()
ax = fig2.add_subplot(1,1,1)
ax.plot(w/np.pi,20*np.log10(np.maximum(abs(h),1e-5)))
ax.set_title('Butterworth IIR Digital LPF frequency response')
ax.set_xlabel('Normalized Digital Frequency ---->')
ax.set_ylabel('Amplitude in dB----->')
#ax.axis((10,1000,-100,10))
ax.grid(which='both',axis='both')
plt.show()

#Result
##>>> b
##array([125000.])
##>>> a
##array([1.00e+00, 1.00e+02, 5.00e+03, 1.25e+05])
##>>> sos
##array([[ 4.16546139e-04,  8.33092278e-04,  4.16546139e-04,
##         1.00000000e+00, -8.54080685e-01,  0.00000000e+00],
##       [ 1.00000000e+00,  1.00000000e+00,  0.00000000e+00,
##         1.00000000e+00, -1.83207671e+00,  8.54913778e-01]])
##>>>


import logging
import math
import numpy as np

def euclidian_dist(x,y):
    # Just for members of the flat earth society ...
    return math.sqrt((x[0]-y[0]) ** 2 + (x[1]-y[1]) ** 2)

def orthodromic_dist(x, y, radius=1):
    # convert degree to radian
    lon1 = math.radians(x[0])
    lon2 = math.radians(y[0])
    lat1 = math.radians(x[1])
    lat2 = math.radians(y[1])

    delta_lon = lon2 - lon1
    delta_lat = lat2 - lat1
    a = math.sin(delta_lat / 2) ** 2 + math.cos(lat1) * math.cos(lat2) * math.sin(delta_lon / 2) ** 2
    c = 2 * math.asin(math.sqrt(a))
    #delta_omega = 2 * math.sqrt(math.asin(math.sin(delta_lat/2)**2 + math.cos(lat1)*math.cos(lat2)*math.sin(delta_lon/2)**2))
    return radius * c

def compute_distance_matrix(xs, ys, method='euclidian', radius='1'):
    num_points = len(xs)
    # Check feasability
    if not len(xs) == len(ys):
        logging.error("Length of x-coordinates and y-coordinates do not match: " + str(len(xs)) + " vs. " + str(len(ys)))

    d = np.zeros((num_points, num_points))
    for i in range(num_points):
        for j in range(num_points):
            # xs: longitudes
            # ys: latitudes
            p = [xs[i], ys[i]]
            q = [xs[j], ys[j]]
            if i == j:
                d_pq = 0
            elif method == 'euclidian':
                d_pq = euclidian_dist(p, q)
            elif method == 'orthodromic':
                d_pq = orthodromic_dist(p, q, radius=radius)
            d[i,j] = d_pq
            d[j,i] = d_pq
    return d


import numpy as np
import matplotlib.pyplot as plt
import math

import pandas as pd
import statistics

df = pd.read_csv('sbi_data.csv', usecols = ['Date', 'Close'], nrows = 66)

U = []
for i in range(65):
    U.append( math.log(df.Close[i+1]/df.Close[i]) )

sigma_sq = statistics.variance(U)
sigma = math.sqrt(sigma_sq)

mean = statistics.mean(U)

mu = sigma_sq/2 + mean
print("Mu = {}, Sigma_Sq = {}, Sigma = {}".format(mu, sigma_sq, sigma))

lambda_ = [0.01, 0.05, 0.1, 0.2]

# Algorithm I: Using Poisson Distribution
# (Simulating at Fixed Dates)
for i in range(4):
    X = {}
    t = 0
    X[0] = math.log(df.Close[65])

    xAxis = [t]
    yAxis = [df.Close[65]]

    for ctr in range(1000):
        z = np.random.normal(0, 1, 1)
        n = np.random.poisson(lambda_[i])

        M = 0.00
        if n!=0:
            for j in range(n):
                y = np.random.normal(mu, sigma)
                M = M + y

        key =  X[t] + (mu - sigma_sq/2) + sigma*np.random.normal(0, 1, 1) + M
        t = t + 1
        X[t] = key
        yAxis.append(math.exp(key))
        xAxis.append(t)

    plt.plot(xAxis, yAxis)
    plt.title("For lambda = {}".format(lambda_[i]))
    plt.ylabel('S (t)')
    plt.xlabel('Time (t)')
    plt.show()
    print()


import control
import scipy
from src.InvertedPendulum import *

# This script computes the optimal control gain matrix K for LQR control

def lqr(A, B, Q, R):

    """Solve the continuous time lqr controller.

    dx/dt = A x + B u

    cost = integral x.T*Q*x + u.T*R*u
    """
    # first, try to solve the ricatti equation
    X = np.mat(scipy.linalg.solve_continuous_are(A, B, Q, R))

    # compute the LQR gain
    K = np.mat(scipy.linalg.inv(R) * (B.T * X))

    return K

if __name__ == "__main__":

    # Insert model
    model = InvertedPendulum()

    # Get system and input matrices
    A = model.A_cont
    B = model.B_cont

    # Control parameters, Q = state penalty, R = control penalty
    Q = np.array([[500, 0, 0, 0], [0, 250, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1]])
    R = np.identity(1)

    # Check stability of the system by calculating eigenvalues and eigenvectors, mention positive eigenvalues (unstable)
    w, v = np.linalg.eig(A)
    print("Eigenvalues of the system matrix: \n" + str(w))

    # Controllability
    print('\nControllability')
    print('Rank of ctrb(A,b): ', np.linalg.matrix_rank(control.ctrb(A, B)))

    # Pole placement, compute optimal control gain
    K = lqr(A, B, Q, R)
    print('\nK= ', K)

    # Verification of Eigen values of A-BK
    w_, v_ = np.linalg.eig(A - B * K)
    print("\nEigenvalues of A-BK: \n" + str(w_))


# # Polynomial Regression - python, scipy
# We have a couple options we could use to solve [Polynomial Regression](https://en.wikipedia.org/wiki/Polynomial_regression) using [SciPy](https://www.scipy.org/). Below is one appraoch.
import numpy as np
import pandas as pd
from scipy import optimize

# ## Generate Data
# We'll generate sample sample data with known coefficients and randomly generated error. Specifically;
# $$ y = \beta_0 + \sum_i (\beta_i x_i) + \epsilon   \thinspace \thinspace \thinspace \thinspace \thinspace     \forall i \in {1..3}$$
# $$ \beta_0: 4 $$
# $$ \beta_1: 6 $$
# $$ \beta_2: 2 $$
# $$ \beta_3: -1 $$

x1 = np.random.uniform(low=-2.0, high=4.0, size=10000)
x2 = np.power(x1,2)
x3 = np.power(x1,3)
y0  = 4 + 6*x1 + 2*x2 - 1*x3 + np.random.normal(0,4,10000)

# ## Initialize & Run Optimizer
# SciPy optimize can solve problems more general that [linear regression](https://en.wikipedia.org/wiki/Linear_regression). While we are technically solving the same problem when minimizing squared residuals, this form of the analysis is more commonly refer to as [linear least squares](https://en.wikipedia.org/wiki/Linear_least_squares_(mathematics)).
# Our solutions invovles definining the function we want to minimize, residual. The optimizer leastsq function will solve for a solution which minimizes the residual sum of squares.
predict  = lambda b, x1, x2, x3: b[0] + b[1]*x1 + b[2]*x2 + b[3]*x3
residual = lambda b, y,x1,x2,x3: predict(b, x1,x2,x3) - y

# This appraoch requires that we initialize our coefficients, we'll choose to randomly initialize.
b0 = np.random.normal(0,1,4)

b, success = optimize.leastsq(residual, b0[:], args=(y0,x1,x2,x3))

# That's it. **b** is an array of our coefficients and it closely matches our known true values.
dat = pd.DataFrame(data=np.dstack((b, [4,6,2,-1]))[0])
dat.columns = ['scipy_coef', 'true_coef']
dat

# ## Plot Predict Function Over Observations
# Now, let's take a look at a plot of our trained parameters. We'll use bokeh so you can interactively see how slight changes in the coefficient can produce very bad results. Running [create_bokeh_poly_reg.py](../files/py/create_bokeh_poly_reg.py).
execfile('py/create_bokeh_poly_reg.py')


"""
Author: Peratham Wiriyathammabhum

"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
eps = np.finfo(float).eps

"""
Importance sampling is a framework. It enables estimation of an r.v. X from a black box target distribution f
using a known proposal distribution g (importance distribution) representing an r.v. Y.
Well, we cannot sample from f but we can evaluate using f.

That is, we sample Y from g to estimate probabilities of X in f.
We multiply some function h(x) with the probability value for each sample using the importance weight f/g.

See: http://www.acme.byu.edu/wp-content/uploads/2016/12/Vol1B-MonteCarlo2-2017.pdf
	https://machinelearning1.wordpress.com/2017/10/22/importance-sampling-a-tutorial/
"""
def estimate_p_gt_3_for_gaussian(nsamples=2000):
	"""
	 The answer should approach 0.0013499 for sufficiently large samples.
	 See: http://www.acme.byu.edu/wp-content/uploads/2016/12/Vol1B-MonteCarlo2-2017.pdf
	"""
	h = lambda x: x > 3
	f = lambda x: stats.norm().pdf(x)
	g = lambda x: stats.norm(loc=4, scale=1).pdf(x)

	X = np.random.normal(4, scale=1, size=nsamples) # samples from g

	est = np.sum(h(X) * (f(X) / g(X)))
	est /= nsamples
	return est

def main(opts):
	nsamples = opts['nsamples']
	est = estimate_p_gt_3_for_gaussian(nsamples=nsamples)
	print('[Info] estimate P(X > 3)=0.0013499 for normal distribution: {:.7f}'.format(est))
	print('[Info] estimation difffer by: {:.7f}'.format(np.absolute(est - 0.0013499)))
	return

if __name__ == '__main__':
	import argparse

	parser = argparse.ArgumentParser(description='run importance sampling.')

	parser.add_argument('--num_samples', dest='nsamples',
					  help='number of samples',
					  default=2000, type=int)
	args = parser.parse_args()
	opts = vars(args)

	main(opts)


# =========Machine Learning - Exercise 1: Linear Regression========
# x refers to the population size in 10,000s
# y refers to the profit in $10,000s

import numpy as np
import matplotlib.pyplot as plt
from numpy import mat

from computeCost import computeCost
from gradientDescent import gradientDescent

# ==================== Part 1: Basic Function ====================
print('5x5 Identity Matrix: \n');
A = np.eye(5)
print(A)

input('Program paused. Press enter to continue.\n')

# ======================= Part 2: Plotting =======================
print('Plotting Data ...\n')
data = np.loadtxt('ex1data1.txt', delimiter=',')
X = data[:, 0]
y = data[:, 1]
m = len(y) # number of training examples

# plot data
plt.scatter(X,y)
plt.xlabel('Population size in 10,000s')
plt.ylabel('Profit in $10,000s')
plt.show()

input('Program paused. Press enter to continue.\n')

# =================== Part 3: Cost and Gradient descent ===========
B = np.ones([m,1])
X = np.c_[B, X]  # Add a column of ones to x
y = np.reshape(y,(-1,1)) # reshape y to get (97,1)
theta = np.zeros([2,1])

# some gradient descent setting
iterarions = 1500
alpha = 0.01

print('\nTesting the cost function...\n')
# compute and display initial cost
J = computeCost(X, y, theta)
print('With theta = [0 ; 0]\nCost computed = %f\n' % J)
print('Expected cost value (approx) 32.07\n')

# compute and display initial cost

J = computeCost(X, y, np.array([[-1],[2]]) )
print('\nWith theta = [-1 ; 2]\nCost computed = %f \n' % J)
print('Expected cost value (approx) 54.24\n')

input('Program paused. Press enter to continue.\n')

print('\nRunning Gradient Descent ...\n')
# run gradient descent
theta, J_history = gradientDescent(X, y,theta, alpha, iterarions)

# print theta to screen
print('Theta found by gradient descent:\n')
print(theta,'\n')
print('Expected theta values (approx)\n')
print(' -3.6303\n  1.1664\n\n')

input('Program paused. Press enter to continue.\n')

# Plot the linear fit
plt.scatter(X[:,1], y)
plt.plot(X[:,1], np.dot(X, theta), 'r--')
plt.xlabel('Population size in 10,000s')
plt.ylabel('Profit in $10,000s')
plt.show()

# Predict values for population sizes of 35,000 and 70,000
predict1 = np.dot(np.reshape(np.array([1, 3.5]), (1,2)), theta)[0][0]
print('For population = 35,000, we predict a profit of %f\n' % (predict1*10000))
predict2 = np.dot(np.reshape(np.array([1, 7]), (1,2)), theta)[0][0]
print('For population = 70,000, we predict a profit of %f\n' % (predict2*10000))

input('Program paused. Press enter to continue.\n')


import numpy as np
import matplotlib.pyplot as plt

def f(x):                                                       # We define the function first
    return np.sin(x)

class Bisection:
    def __init__(self, epsilon, intervalBegin, intervalEnd, function):
        self.epsilon = epsilon
        self.intervalBegin = intervalBegin                      # This is a variable which determins x1 in (x1, x2) interval
        self.intervalEnd = intervalEnd                          # This is x2
        self.originalBegin = intervalBegin                      # This does not change, used as reference
        self.originalEnd = intervalEnd                          # Same
        self.function = function

    def __str__(self):
        return str(self.result)                                 # To be able to print the actual result instead of the memory location

    def solve(self):
        f1, f2 = self.function(self.intervalBegin), self.function(self.intervalEnd) # Declare these two variables
        if f1 == 0:                                             # If either f1 or f2 are 0, then we have our answer
            self.result = self.intervalBegin
            return                                              # Used to exit the function
        if f2 == 0:
            self.result = self.intervalEnd
            return
        if 0 < f1 * f2:                                         # If there isn't one negative and one positive value, then the bisection method does not work for that interval
            raise ValueError("The root cannot be found within this interval.")

        while (self.intervalEnd-self.intervalBegin) > self.epsilon:     # Keep going until we reach the desired accuracy
            a = 0.5 * (self.intervalBegin + self.intervalEnd)   # We find the center point of those two
            f_of_a = f(a)                                       # Evaluate that point

            if f_of_a == 0:                                     # If that point is 0, there's our answer
                self.result = a
                break
            elif 0 > f2 * f_of_a:                               # This means the sign between midpoint and intervalEnd has changed, thus we keep going this way
                self.intervalBegin, f1 = a, f_of_a
            else:                                               # This means the sign between midpoint and intervalBegin has changed, thus we keep going this way
                self.intervalEnd, f2 = a, f_of_a
        self.result = (self.intervalBegin + self.intervalEnd)/2 # If the interval is small enough, we return the midpoint (the answer)

    def graphit(self):
        plt.title("Graph of the Function f")                    # Give it a title and then labels
        plt.xlabel("x")
        plt.ylabel("y")
        x = np.linspace(self.originalBegin, self.originalEnd, int((self.originalEnd - self.originalBegin)*100)) # Plot for values in interval, then plot for every 0.01
        y = f(x)                                                # Compute the function
        plt.xlim(self.originalBegin, self.originalEnd)          # Set the graph limits
        plt.plot(x,y)                                           # Plot, then show
        plt.show()

b = Bisection(0.00001, 1, 99, f)
b.solve()
print(b)
b.graphit()


# Source: https://github.com/kayoyin/signal-processing/blob/master/dimensionality_reduction.py
import numpy as np

class PCA():
    """
    Perform PCA dimensionality reduction on the input image
    :param img: a numpy array of shape (n_samples, dim_images)
    :param n_components: number of principal components for projection
    :return: image in PCA projection, a numpy array of shape (n_samples, n_components)
    """

    def __init__(self, img, n_comp=2):
        self.img = img
        self.n_comp = n_comp

    def run(self):
        # Compute the covariance matrix
        cov_mat = np.cov(self.img.T)

        # Compute the eigenvectors and eigenvalues
        eig_val_cov, eig_vec_cov = np.linalg.eig(cov_mat)

        # Make a list of (eigenvalue, eigenvector) tuples
        eig_pairs = [
            (np.abs(eig_val_cov[i]), eig_vec_cov[:, i]) for i in range(len(eig_val_cov))
        ]

        # Select n_components eigenvectors with largest eigenvalues, obtain subspace transform matrix
        eig_pairs.sort(key=lambda x: x[0], reverse=True)
        eig_pairs = np.array(eig_pairs)
        matrix_w = np.hstack(
            [eig_pairs[i, 1].reshape(self.img.shape[1], 1) for i in range(self.n_comp)]
        )

        # Return samples in new subspace
        return np.dot(self.img, matrix_w), matrix_w

    def inverse_pca(self, pca_img, components):
        """
        Obtain the reconstruction after PCA dimensionality reduction
        :param img: Reduced image, a numpy array of shape (n_samples, n_components)
        :param components: a numpy array of size (original_dimension, n_components)
        :return:
        """
        reconstruct = np.dot(pca_img, components.T).astype(int)
        return reconstruct.reshape(-1, 28, 28)

    def mean_square_error(self, A, B):
        """
        Compute the mean square error between two images
        :param A: a numpy array of shape (self.img_height, img_width)
        :param B: a numpy array of shape (self.img_height, img_width)
        :return: a scalar
        """
        return ((A - B) ** 2).mean()