import numpy as np

from gd import *
from sgd import *
from costs import *

def least_squares_GD(y, tx, initial_w,
                     max_it, gamma, verbose=False):
    """Linear Regression with Gradient Descent

    Uses Mean Squared Error as the loss function.
    """
    losses, ws = gradient_descent(
        y=y,
        tx=tx,
        initial_w=initial_w,
        max_iters=max_it,
        gamma=gamma,
        verbose=verbose
    )

    return ws[-1], losses[-1]

def least_squares_SGD(y, tx, initial_w,
                      max_iters, gamma, verbose=False):
    """Linear regression with Stochastic Gradient Descent (SGD)

    Current implementation uses Mean Squared Error as the loss.
    """
    # Use batch_size = 1 as per the project instructions.
    losses, ws = stochastic_gradient_descent(
        y=y,
        tx=tx,
        initial_w=initial_w,
        max_iters=max_iters,
        gamma=gamma,
        batch_size=1,
        verbose=verbose
    )

    return ws[-1], mse(y, tx, ws[-1])

def least_squares(y, tx):
    """Linear regression fit using normal equations."""
    a = tx.T @ tx
    b = tx.T @ y
    w = np.linalg.solve(a, b)
    loss = mse(y, tx, w)
    return w, loss

def ridge_regression(y, tx, lambda_):
    """ Ridge regression fit using normal equations """
    a = (tx.T @ tx) + lambda_*2*tx.shape[0] * np.eye(tx.shape[1])
    b = tx.T @ y
    w = np.linalg.solve(a, b)
    return w, mse(y, tx, w)

def logistic_regression(y, tx, initial_w, max_iters,
                        gamma, batch_size=None, verbose=False):
    """ Logistic regression with gradient descent or stochastic gradient descent"""

    if batch_size:
        losses, ws = stochastic_gradient_descent_logistic(
            y=y,
            tx=tx,
            initial_w=initial_w,
            batch_size=batch_size,
            max_iters=max_iters,
            gamma=gamma,
            verbose=verbose
        )
    else:
        losses, ws = gradient_descent_logistic(
            y=y,
            tx=tx,
            initial_w=initial_w,
            max_iters=max_iters,
            gamma=gamma,
            verbose=verbose
        )

    return ws[-1], logistic_error(y, tx, ws[-1])

def reg_logistic_regression(y, tx, lambda_, reg, initial_w,
                            max_iters, gamma, verbose=False,
                            early_stopping=True, tol = 0.0001,
                            patience = 5):
    """ Regularized logistic regression with gradient descent"""

    losses, ws = reg_gradient_descent_logistic(
        y=y,
        tx=tx,
        initial_w=initial_w,
        max_iters=max_iters,
        gamma=gamma,
        lambda_=lambda_,
        reg=reg,
        verbose=verbose,
        early_stopping=early_stopping,
        tol=tol,
        patience=patience
    )

    return ws[-1], losses[-1]


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.cm import get_cmap

plt.rcParams['figure.figsize'] = [8, 8]
plt.rcParams.update({'font.size': 18})

# Define Domain
dx = 0.001
L = np.pi
x = L * np.arange(-1+dx, 1+dx, dx)
n = len(x)
nquart = int(np.floor(n/4))

# Definition of hat function (in this case im defining a hat funtion)
f = np.zeros_like(x)
f[nquart:2*nquart] = (4/n)*np.arange(1, nquart+1) # positive slope
f[2*nquart:3*nquart] = np.ones(nquart) - (4/n)*np.arange(0, nquart) # negative slope

fig, ax = plt.subplots()
ax.plot(x, f, '-', color='k', LineWidth=2)

# Computation of the fourier series (this is an approximation for the limit has been set to 20 iterations)
name = "Accent"
cmap = get_cmap('tab10')
colors = cmap.colors
ax.set_prop_cycle(color=colors)

A0 = np.sum(f * np.ones_like(x)) * dx
fFS = A0/2

A = np.zeros(20)
B = np.zeros(20)
for k in range(20):
    A[k] = np.sum(f * np.cos(np.pi * (k+1) * x / L)) * dx # inner product a(k)
    B[k] = np.sum(f * np.sin(np.pi * (k+1) * x / L)) * dx # inner product b(k)
    fFS = fFS + A[k] * np.cos((k+1)*np.pi*x/L) + B[k]*np.sin((k+1)*np.pi*x/L)
    ax.plot(x, fFS, '-')

# Amplitudes and reconstruction error rate

fFS = (A0/2) * np.ones_like(f)
kmax = 100
A = np.zeros(kmax)
B = np.zeros(kmax)
ERR = np.zeros(kmax)

A[0] = A0/2
ERR[0] = np.linalg.norm(f-fFS)/np.linalg.norm(f)

for k in range(1, kmax):
    A[k] = np.sum(f * np.cos(np.pi * k * x / L)) * dx
    B[k] = np.sum(f * np.sin(np.pi * k * x / L)) * dx
    fFS += A[k] * np.cos(np.pi * k * x / L) + B[k] * np.sin(k * np.pi * x / L)
    ERR[k] = np.linalg.norm(f-fFS)/np.linalg.norm(f)

thresh = np.median(ERR) * np.sqrt(kmax) * (4/np.sqrt(3))
r = np.max(np.where(ERR > thresh))

fig, axs = plt.subplots(2, 1)
axs[0].semilogy(np.arange(kmax), A, color='k', LineWidth=2)
axs[0].semilogy(r, A[r], 'o', color='b', MarkerSize=10)
plt.sca(axs[0])
plt.title('Fourier Coefficients')

axs[1].semilogy(np.arange(kmax), ERR, color='k', LineWidth=2)
axs[1].semilogy(r, ERR[r], 'o', color='b', MarkerSize=10)
plt.sca(axs[1])
plt.title('Error')
plt.show()


from sklearn import linear_model
import numpy as np
import math
import matplotlib.pyplot as plt
from pylab import scatter, show, legend, xlabel, ylabel

# https://www.youtube.com/watch?v=-BQCB6Uch1g
# from __future__ import division

def logistic_func(theta, x):
  return float(1) / (1 + math.e**(-x.dot(theta)))

def log_gradient(theta, x, y):
  first_calc = logistic_func(theta, x) - np.squeeze(y)
  final_calc = first_calc.T.dot(x)
  return final_calc

def cost_func(theta, x, y):
  log_func_v = logistic_func(theta,x)
  y = np.squeeze(y)
  step1 = y * np.log(log_func_v)
  step2 = (1-y) * np.log(1 - log_func_v)
  final = -step1 - step2
  return np.mean(final)

def grad_desc(theta_values, X, y, lr=.001, limit=10):
  #normalize
  X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
  #setup cost iter
  cost_iter = []
  cost = cost_func(theta_values, X, y)
  cost_iter.append([0, cost])
  i = 0
  while(i < limit):
    # old_cost = cost
    theta_values = theta_values - (lr * log_gradient(theta_values, X, y))
    cost = cost_func(theta_values, X, y)
    cost_iter.append([i, cost])
    i+=1
  return theta_values, np.array(cost_iter)

def pred_values(theta, X, hard=True):
  #normalize
  X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
  pred_prob = logistic_func(theta, X)
  pred_value = np.where(pred_prob >= .5, 1, 0)
  if hard:
    return pred_value
  return pred_prob

def logistic_regression(Y, X, limit=100):
  shape = X.shape[1]
  # print("Dimension = (",shape,")")
  # y_flip = np.logical_not(Y)
  betas = np.zeros(shape)
  fitted_values, cost_iter = grad_desc(betas, X, Y,limit=limit)
  # print("Fit values =",fitted_values)
  predicted_y = pred_values(fitted_values, X)
  # print("Y =",Y)
  # print("Predict Y =",predicted_y)

  print("Y =",np.sum(Y))
  print("Predict Y =", np.sum(predicted_y))
  print("Equal =", np.sum(Y == predicted_y))

  # print(cost_iter)
  plt.plot(cost_iter[:,0], cost_iter[:,1],'-', linewidth=3)
  plt.ylabel("Cost")
  plt.xlabel("Iteration")
  plt.show()
  plt.savefig('cost_iter.png')

  from scipy.optimize import fmin_l_bfgs_b
  #normalize data
  norm_X = (X - np.mean(X, axis=0)) / np.std(X, axis=0)
  myargs = (norm_X, Y)
  betas = np.zeros(norm_X.shape[1])
  lbfgs_fitted = fmin_l_bfgs_b(cost_func, x0=betas, args=myargs, fprime=log_gradient)

  lbfgs_predicted = pred_values(lbfgs_fitted[0], norm_X, hard=True)

  print("\n2a part")
  # print("l_bfgs_b =",lbfgs_fitted[0])
  print("lbfgs_predicted == Y =", sum(lbfgs_predicted == Y))

  logreg = linear_model.LogisticRegression()
  logreg.fit(norm_X, Y)
  print("From 'sklearn' --> logreg.predict == Y =", sum(Y == logreg.predict(norm_X,)))
  print(logreg.score(norm_X, Y))

  fitted_values, cost_iter = grad_desc(betas, norm_X, Y)
  predicted_y = pred_values(fitted_values, norm_X)
  print("From 'my function' --> Predict Y =",sum(predicted_y == Y))

  plt.plot(cost_iter[:,0], cost_iter[:,1],'-', linewidth=3)
  plt.ylabel("Cost")
  plt.xlabel("Iteration")
  plt.show()
  plt.savefig("cost_iter_second.png")

def test():
  import seaborn as sns
  #matplotlib inline
  sns.set(style='ticks', palette='Set2')

  data = datasets.load_iris()
  X = data.data[:100, :2]
  y = data.target[:100]
  X_full = data.data[:100, :]

  setosa = plt.scatter(X[:50,0], X[:50,1], c='b')
  versicolor = plt.scatter(X[50:,0], X[50:,1], c='r')
  plt.xlabel("Sepal Length")
  plt.ylabel("Sepal Width")
  plt.legend((setosa, versicolor), ("Setosa", "Versicolor"))
  sns.despine()

  shape = X.shape[1]
  print("Dimension = (",shape,")")
  y_flip = np.logical_not(y) #flip Setosa to be 1 and Versicolor to zero to be consistent
  betas = np.zeros(shape)
  fitted_values, cost_iter = grad_desc(betas, X, y)
  print("Fit values =",fitted_values)

  predicted_y = pred_values(fitted_values, X)

  print("Predict Y =",predicted_y)

  # print(np.sum(y_flip == predicted_y))

  plt.plot(cost_iter[:,0], cost_iter[:,1])
  plt.ylabel("Cost")
  plt.xlabel("Iteration")
  sns.despine()

  plt.show()
  # plt.savefig('foo.png')

  from scipy.optimize import fmin_l_bfgs_b
  #normalize data
  norm_X = (X_full - np.mean(X_full, axis=0)) / np.std(X_full, axis=0)
  myargs = (norm_X, y_flip)
  betas = np.zeros(norm_X.shape[1])
  lbfgs_fitted = fmin_l_bfgs_b(cost_func, x0=betas, args=myargs, fprime=log_gradient)
  print("l_bfgs_b =",lbfgs_fitted[0])

  lbfgs_predicted = pred_values(lbfgs_fitted[0], norm_X, hard=True)
  print(sum(lbfgs_predicted == y_flip))

  logreg = linear_model.LogisticRegression()
  logreg.fit(norm_X, y_flip)
  print(sum(y_flip == logreg.predict(norm_X)))

  fitted_values, cost_iter = grad_desc(betas, norm_X, y_flip)
  predicted_y = pred_values(fitted_values, norm_X)
  print(sum(predicted_y == y_flip))


import numpy as np

def cross_entropy_sigmoid(y, y_hat):
    m = y.shape[1]
    #cost = np.sum(y*np.log(y_hat) + (1-y)*np.log(1 - y_hat)) / (-1*m)
    cost = (1./m) * (-np.dot(y,np.log(y_hat).T) - np.dot(1-y, np.log(1-y_hat).T))

    # So that we have a real number at the end instead of a singleton; e.g. [[3]] => 3
    cost = np.squeeze(cost)
    assert(cost.shape == ())

    return cost

def cross_entropy_softmax(y, y_hat):
    # y is a vector of dimension [1 x num_of_inputs]
    # IT IS NOT ONE HOT VECTOR !!!

    num_inputs = y_hat.shape[1]

    # get the probabilities indexed by classes, y_hat
    probs = y_hat[y.squeeze(), range(num_inputs)]

    log_probs = np.log(probs)

    cost = np.sum(log_probs)/(-1*num_inputs)

    # So that we have a real number at the end instead of a singleton; e.g. [[3]] => 3
    cost = cost.squeeze()
    assert(cost.shape == ())

    return cost

def cross_entropy_sigmoid_derivative(y, y_hat):
    m = y.shape[1]
    return (-(np.divide(y, y_hat) - np.divide(1 - y, 1 - y_hat)))

def cross_entropy_softmax_derivative(y, y_hat):
    # y is a vector of dimension [1 x num_of_inputs]
    # IT IS NOT ONE HOT VECTOR !!!

    num_inputs = y_hat.shape[1]

    d = np.zeros(y_hat.shape)

    d[y, range(num_inputs)] = 1/y_hat[y, range(num_inputs)]

    return d/num_inputs

def mean_squared(y, y_hat):
    return  np.sum((y - y_hat)**2 ).squeeze() / (y_hat.shape[1]*2)

def d_mean_squared(y, y_hat):
    return (y_hat - y)

cost_functions = {"cross_entropy_sigmoid" : (cross_entropy_sigmoid, cross_entropy_sigmoid_derivative),
                 "cross_entropy_softmax" : (cross_entropy_softmax, cross_entropy_softmax_derivative),
                  "mean_squared" : (mean_squared, d_mean_squared)
                 }


"""
  Name     : c7_07_optimization_01.py
  Book     : Hands-on Data Science with Anaconda )
  Publisher: Packt Publishing Ltd.
  Author   : Yuxing Yan and James Yan
  Date     : 3/1/2018
  email    : yany@canisius.edu
             paulyxy@hotmail.com
"""

import numpy as np
from scipy.optimize import minimize
def rosen(x):
    """The Rosenbrock function"""
    return sum(100.0*(x[1:]-x[:-1]**2.0)**2.0 + (1-x[:-1])**2.0)
#
x0 = np.array([1.3, 0.7, 0.8, 1.9, 1.2])
solution= minimize(rosen, x0, method='nelder-mead',options={'xtol': 1e-8, 'disp': True})

print(solution.x)


import numpy as np

def solver(eps, Nx, method='centered'):
    """
    Solver for the two point boundary value problem u'=eps*u'',
    u(0)=0, u(1)=1.
    """
    x = np.linspace(0, 1, Nx+1)       # Mesh points in space
    # Make sure dx and dt are compatible with x and t
    dx = x[1] - x[0]
    u   = np.zeros(Nx+1)

    # Representation of sparse matrix and right-hand side
    diagonal = np.zeros(Nx+1)
    lower    = np.zeros(Nx)
    upper    = np.zeros(Nx)
    b        = np.zeros(Nx+1)

    # Precompute sparse matrix (scipy format)
    if method == 'centered':
        diagonal[:] = 2*eps/dx**2
        lower[:] = -1/dx - eps/dx**2
        upper[:] =  1/dx - eps/dx**2
    elif method == 'upwind':
        diagonal[:] = 1/dx + 2*eps/dx**2
        lower[:] =  1/dx - eps/dx**2
        upper[:] = - eps/dx**2

    # Insert boundary conditions
    upper[0] = 0
    lower[-1] = 0
    diagonal[0] = diagonal[-1] = 1
    b[-1] = 1.0

    # Set up sparse matrix and solve
    diags = [0, -1, 1]
    import scipy.sparse
    import scipy.sparse.linalg
    A = scipy.sparse.diags(
        diagonals=[diagonal, lower, upper],
        offsets=[0, -1, 1], shape=(Nx+1, Nx+1),
        format='csr')
    u[:] = scipy.sparse.linalg.spsolve(A, b)
    return u, x

def u_exact(x, eps):
    return (np.exp(x/eps)-1)/(np.exp(1.0/eps)-1)

def demo(eps = 0.01, method='centered'):
    import matplotlib.pyplot as plt
    x_fine = np.linspace(0, 1, 2001)
    for Nx in (20, 40):
        u, x = solver(eps, Nx, method=method)
        plt.figure()
        plt.plot(x, u, 'o-')
        plt.hold('on')
        plt.plot(x_fine, u_exact(x_fine, eps), 'k--')
        plt.legend(['$N_x=%d$' % Nx, 'exact'], loc='upper left')
        plt.title(method + ' difference scheme, ' + r'$\epsilon=%g$' % eps)
        plt.xlabel('x');  plt.ylabel('u')
        stem = 'tmp1_%s_%d_%s' % (method, Nx, str(eps).replace('.','_'))
        plt.savefig(stem + '.png'); plt.savefig(stem + '.pdf')
    plt.show()

if __name__ == '__main__':
    demo(eps=0.1, method='upwind')
    demo(eps=0.01, method='upwind')
    #demo(eps=0.1, method='centered')
    #demo(eps=0.01, mehtod='centered')


# lagrange interpolation example

import math
import numpy
import pylab

# globals to control some behavior
func_type = "tanh"   # can be sine or tanh
points = "fixed"  # can be variable or fixed

npts = 15

def fun_exact(x):
    """ the exact function that we sample to get the points to
    interpolate through """

    if func_type == "sine":
        return numpy.sin(x)
    elif func_type == "tanh":
        return 0.5*(1.0+numpy.tanh((x-1.0)/0.1))

def get_interp_points(N, xmin, xmax):
    """ get the x points that we interpolate at """
    if points == "fixed":
        x = numpy.linspace(xmin, xmax, N)

    elif points == "variable":
        # the Chebyshev nodes
        x = 0.5*(xmin + xmax) + \
            0.5*(xmax - xmin)*numpy.cos(2.0*numpy.arange(N)*math.pi/(2*N))

    return x

def lagrange_poly(x, xp, fp):
    """ given points (xp, fp), fit a lagrange polynomial and return
        the value at point x """

    f = 0.0

    # sum over points
    m = 0
    while (m < len(xp)):

        # create the Lagrange basis polynomial for point m
        l = None

        n = 0
        while (n < len(xp)):
            if n == m:
                n += 1
                continue

            if l == None:
                l = (x - xp[n])/(xp[m] - xp[n])
            else:
                l *= (x - xp[n])/(xp[m] - xp[n])

            n += 1

        f += fp[m]*l

        m += 1

    return f

if func_type == "sine":
    xmin = 0.0
    xmax = 2.0*math.pi
elif func_type == "tanh":
    xmin = 0.0
    xmax = 2.0

# xp, fp are the points that we build the interpolant from
xp = get_interp_points(npts, xmin, xmax)
fp = fun_exact(xp)

# xx are the finely grided data that we will interpolate at to get
# the interpolated function values ff
xx = numpy.linspace(xmin, xmax, 200)
ff = numpy.zeros(len(xx))

n = 0
while (n < len(xx)):
    ff[n] = lagrange_poly(xx[n], xp, fp)
    n += 1

# exact function values at the interpolated points
fexact = fun_exact(xx)

# error
e = fexact-ff

pylab.subplot(211)

pylab.scatter(xp, fp, marker="x", color="r", s=30)
pylab.plot(xx, ff, color="k")

pylab.plot(xx, fexact, color="0.5")

pylab.xlim(xmin, xmax)

pylab.subplot(212)

pylab.plot(xx, e)

pylab.xlim(xmin, xmax)

pylab.savefig("lagrange.png")


#!/usr/bin/env python
# coding: utf-8

# In[1]:

# @gyleodhis=====gyleodhis@outlook.com=====
import numpy as np

# np.__version__ Checks numpy vesion
# Generate an array of 10 random integers less than 100
rand = np.random.RandomState(42)
x = rand.randint(100, size=10)
x

# Remember that unlike Python lists, NumPy is constrained to arrays that all contain
# the same type. If types do not match, NumPy will upcast if possible as below:

# In[2]:

L=np.array([3.14, 4, 2, 3])
L

# In[3]:

##If we want to explicitly set the data type of the resulting array, we can use the dtype keyword:
L=np.array([1, 2, 3, 4], dtype='float32')
L

# Accesiing four different elemets in a array

# In[4]:

k=[x[2], x[5], x[9], x[0]]
k
#We can also do it this way
z=[2,5,9,1]
x[z]

# In[5]:

#Multidimentional Array.
L=np.array([range(i, i + 3) for i in [2, 4, 6]])
L

# In[6]:

# Create a length-10 integer array filled with zeros
L=np.zeros(10, dtype=int)
L

# In[7]:

# Create a 3x5 floating-point array filled with 1s
L=np.ones((3, 5), dtype=float)
L

# In[8]:

# Create a 3x5 array filled with 3.14
L=np.full((3, 5), 3.14)
L

# In[9]:

# Create an array filled with a linear sequence
# Starting at 0, ending at 20, stepping by 2
# (this is similar to the built-in range() function)
L=np.arange(0, 20, 2)
L

# In[10]:

# Create an array of five values evenly spaced between 0 and 1
np.linspace(0, 1, 5)

# In[11]:

# Create a 3x3 array of uniformly distributed
# random values between 0 and 1
L=np.random.random((3, 3))
L

# In[12]:

# Create a 3x3 array of normally distributed random values
# with mean 0 and standard deviation 1
np.random.normal(0, 1, (3, 3))

# In[13]:

# Create a 3x3 array of random integers in the interval [0, 10)
np.random.randint(0, 10, (3, 3))

# In[14]:

# Create a 3x3 identity matrix
np.eye(3)

# In[15]:

# Create an uninitialized array of three integers
# The values will be whatever happens to already exist at that
# memory location
np.empty(3)

# In[16]:

x1 = np.random.randint(10, size=6) # One-dimensional array
x2 = np.random.randint(10, size=(3, 4)) # Two-dimensional array
x3 = np.random.randint(10, size=(3, 4, 5)) # Three-dimensional array
#Each array has attributes ndim (the number of dimensions), shape (the size of each dimension), and size (the total size of the array):
print("x3 ndim: ", x3.ndim)
print("x3 shape:", x3.shape)
print("x3 size: ", x3.size)
print("x3 type: ", x3.dtype)
print("x3 type: ", x3.itemsize)

# In[17]:

# Array Indexing: Accessing Single Elements
print(x1[4])
# To index from the end of the array, you can use negative indices:
print(x1[-1])

# In[18]:

# In a multidimensional array, you access items using a comma-separated tuple of indices:
x2[2, 0]

# In[19]:

# You can also modify values using any of the above index notation:
x2[0, 0] = 12
x2[0,0]

# ## Keep in mind that, unlike Python lists, NumPy arrays have a fixed type. This means,for example, that if you attempt to insert a floating-point value to an integer array, the value will be silently truncated. Don’t be caught unaware by this behavior!
# x1[0] = 3.14159 # this will be truncated to 3!

# Fancy indexing also works in multiple dimensions. Consider the following array:

# In[20]:

x = np.arange(12).reshape((3,4))
x

# Like with standard indexing, the first index refers to the row, and the second to the
# column:

# In[21]:

row = np.array([0, 1, 2])
col = np.array([2, 1, 3])
x[row, col]

# Selecting Random Points.

# In[22]:

mean = [0,0]
cov = [[1,2], [2,5]]
x = rand.multivariate_normal(mean, cov, 100)
x.shape

# we can visualize these points as a scatter plot

# In[23]:

get_ipython().run_line_magic('matplotlib', 'inline')
import matplotlib.pyplot as plt
#import seaborn; seaborn.set() #for plot styling
plt.scatter(x[:,0], x[:,1]);

# Let’s use fancy indexing to select 20 random points. We’ll do this by first choosing 20
# random indices with no repeats, and use these indices to select a portion of the origi‐
# nal array:

# In[24]:

indices = np.random.choice(x.shape[0], 20, replace=False)
indices

# In[25]:

selection = x[indices] # Fancy indexing here
selection.shape

# Now to see which points were selected, let’s over-plot large circles at the locations of
# the selected points

# In[26]:

plt.scatter(x[:, 0], x[:, 1], alpha=0.3)
plt.scatter(selection[:, 0], selection[:, 1],
            facecolor='none', s=200);

# This sort of strategy is often used to quickly partition datasets, as is often needed in
# train/test splitting for validation of statistical models

# Modifying Values with Fancy Indexing. Just as fancy indexing can be used to access parts of an array, it can also be used to
# modify parts of an array

# In[27]:

x = np.arange(10)
i = np.array([2, 1, 8, 4])
x[i] = 99
x

# In[28]:

i

# We can use any assignment-type operator for this. For example:

# In[29]:

x[i] +=20
x

# imagine we have 1,000 values and would like to quickly find where they fall within an array of bins. We could compute it using ufunc.at like this:

# In[30]:

np.random.seed(42)
x=np.random.randn(100)
#compute a histogram by hand
bins = np.linspace(-5,5,20)
counts= np.zeros_like(bins)
# find the appropriate bin for each x
i=np.searchsorted(bins,x)
# add 1 to each of these bins
np.add.at(counts, i, 1)
#The counts now reflect the number of points within each bin—in other words, a histogram
#plt.plot(bins, counts, linestyle='steps');
plt.hist(x, bins, histtype='step');

# Fast sorting pf arrays with np.sort and np.argsort. To return a sorted version of the array without modifying the input, you can use
# np.sort:

# In[31]:

x = np.array([2, 1, 4, 3, 5])
np.sort(x) #x.sort() produces the same result.

#  argsort,returns the indices of the sorted elements:

# In[32]:

x = np.array([2, 1, 4, 3, 5])
x.argsort()

# Sorting along rows or columns

# In[33]:

rand = np.random.RandomState(42)
x = rand.randint(0, 10, (4, 6))
x

# In[34]:

# sort each column of X. Replace axix=0 with 1 to sort rows
np.sort(x, axis=0)

# Sometimes we’re not interested in sorting the entire array, but simply want to find the
# K smallest values in the array.

# In[35]:

x = np.array([7, 2, 3, 1, 6, 5, 4])
np.partition(x,4)

# Similarly to sorting, we can partition along an arbitrary axis of a multidimensional
# array:

# In[36]:

np.partition(x, 2, axis=0)

# The result is an array where the first two slots in each row contain the smallest values
# from that row, with the remaining values filling the remaining slots.

# Example: k-Nearest Neighbors. We will start by creating random set of 10 points on a 2 dimentional plane.

# In[37]:

x= rand.rand(10, 2)
x

# In[38]:

plt.scatter(x[:,0], x[:,1], s=100)

# Now we will compute the distance between each pair of points

# In[39]:

dist_sq = np.sum((x[:,np.newaxis,:] - x[np.newaxis,:,:]) ** 2, axis=-1)
dist_sq

# In[40]:

plt.scatter(x[:, 0], x[:, 1], s=100)
# draw lines from each point to its two nearest neighbors
K = 2
nearest_partition = np.argpartition(dist_sq, K + 1, axis=1)
for i in range (x.shape[0]):
    for j in nearest_partition[i, :k+1]:
        # plot a line from X[i] to X[j]
        # use some zip magic to make it happen:
        plt.plot(*zip(X[j], X[i]), color='black')

# The above cell is supposed to draw connections between the dots. Some how it has refused to do this.

# Structured Arrays.Imagine that we have several categories of data on a number of people (say, name,
# age, and weight). We can create a structured array using a compound data type specification:
#

# In[41]:

data = np.zeros(4, dtype={'names':('name', 'age', 'weight'),
                          'formats':('U10', 'i4', 'f8')}) #formats':((np.str_, 10), int, np.float32) does the same.
print(data.dtype)

# Here 'U10' translates to “Unicode string of maximum length 10,” 'i4' translates to “4-byte (i.e., 32 bit) integer,” and 'f8' translates to “8-byte (i.e., 64 bit) float.”

# In[42]:

name = ['Alice', 'Bob', 'Cathy', 'Doug']
age = [25, 45, 37, 19]
weight = [55.0, 85.5, 68.0, 61.5]
data['name'] = name
data['age'] = age
data['weight'] = weight
print(data)

# In[43]:

data[1] #remember array indexes are not quoted.

# Using Boolean masking, this even allows you to do some more sophisticated opera‐
# tions such as filtering on age:

# In[44]:

# Get names where age is under 30
data[data['age'] < 30]['name']

# ## Array Slicing: Accessing Subarrays

# In[45]:

x = np.arange(20)
print(x[:5]) # first five elements
print(x[4:7]) # middle subarray
print(x[::2]) # Every other element in additions of 2
print(x[3::2]) # Every other element begining with 3 in additions of 2

# ## Multidimensional subarrays

# In[46]:

print(x2[:2, :3]) # two rows, three columns
print(x2[:3, ::2]) # all rows, every column in additions of two

# ## Accessing array rows and columns.

# In[47]:

print(x2[:, 0]) # first column of x2
print(x2[1,:]) # print second row of x2.

# In the case of row access, the empty slice can be omitted for a more compact syntax:
# print(x2[0]) # equivalent to x2[0, :]

# ## Creating copies of arrays

# In[48]:

x2_copy = x2[:2, :2].copy() #the copy command is used.
x2_copy

# ## Reshaping of Arrays
# The most flexible way of doing this is with the reshape() method. For example, if you want to put the numbers 1 through 9 in a 3×3 grid, you can do the following:
# grid = np.arange(1, 10).reshape((3, 3))
#
# [Note that for this to work, the size of the initial array must match the size of the
# reshaped array.]
# Another common reshaping pattern is the conversion of a one-dimensional array into a two-dimensional row or column matrix.

# ## ARRAY CONCATINATION AND SPLITTING

# In[49]:

# Look at the following example
x = np.array([1, 2, 3])
y = np.array([3, 2, 1])
np.concatenate([x, y])

# ### When working with arrays of mixed dimensions, it can be clearer to use the np.vstack(vertical stack; the no of columns has to be the same) and np.hstack (horizontal stack; the no of rows has to be the same) functions. Similarly, np.dstack will stack arrays along the third axis:

# In[50]:

x = np.array([[1, 2, 3,11,14,56],
             [4,5,6,33,54,12,]])
grid = np.array([[9, 8, 7],
                 [6, 5, 4]])
y = np.hstack([x, grid])
y

# ## Splitting of arrays
# This is implemented by the functions np.split, np.hsplit, and np.vsplit

# In[51]:

x = [1, 2, 3, 99, 99, 3, 2, 1]
x1, x2, x3 = np.split(x, [3, 5])
print(x1, x2, x3)

# ### Notice that N split points lead to N + 1 subarrays.
# The related functions np.hsplit and np.vsplit are similar:
# Similarly, np.dsplit will split arrays along the third axis.

# In[52]:

grid = np.arange(16).reshape((4, 4))
upper, lower = np.vsplit(grid, [2]) # Splits the array into two. (hsplit splits horizontally)
print(upper)
print(lower)

# ## Absolute value
# Just as NumPy understands Python’s built-in arithmetic operators, it also understands Python’s built-in absolute value function:

# In[59]:

x = np.array([-2, -1, 0, 1, 2])
abs(x)

# In[60]:

# This ufunc can also handle complex data, in which the absolute value returns the magnitude:
x = np.array([3 - 4j, 4 - 3j, 2 + 0j, 0 + 1j])
abs(x)

# ## Trigonometric functions
# NumPy provides a large number of useful ufuncs, and some of the most useful for the
# data scientist are the trigonometric functions. We’ll start by defining an array of
# angles:

# In[62]:

theta = np.linspace(0, np.pi, 3)
# Now we can compute some trigonometric functions on these values:
print("theta = ", theta)
print("sin(theta) = ", np.sin(theta))
print("cos(theta) = ", np.cos(theta))
print("tan(theta) = ", np.tan(theta))

# In[64]:

# Another common type of operation available in a NumPy ufunc are the exponentials:
x = [1, 2, 3]
print("x =", x)
print("e^x =", np.exp(x))
print("2^x =", np.exp2(x))
print("3^x =", np.power(3, x))
print("ln(x) =", np.log(x))
print("log2(x) =", np.log2(x))
print("log10(x) =", np.log10(x))

# ### The Reduce Operation
# A reduce repeatedly applies a given operation to the elements of an array until only a single result remains.
# For example, calling reduce on the add ufunc returns the sum of all elements in thet array:

# In[65]:

x = np.arange(1, 6)
np.add.reduce(x)

# In[67]:

#np.multiply.reduce(x) # returns a product of all array elements:
np.add.accumulate(x) # returns x together with the final value of addition

# ## Summing the Values in an Array
#

# In[68]:

L = np.random.random(100)
sum(L) #np.sum(L) returns the same result but it is however faster.

# ## Minimum and Maximum

# In[72]:

print(min(L))
print(max(L))
#np.min(K) and np.max(K) generate same results though much more faster

# ## Aggregation
# For example, we can find the minimum value within each column in a two dimentional array by specifying axis=0:

# In[74]:

M = np.random.random((3, 4))
print(M.min(axis=0)) #returns minimum value per column(.max returns maximum value)
print(M.min(axis=0)) #returns minimum value per row.(.max returns maximum value)

# ### Other Usefull aggregation functions
# 		print("25th percentile: ", np.percentile(heights, 25)) #heights is an array.
# 		print("Median: ", np.median(heights))
# 		print("75th percentile: ", np.percentile(heights, 75))

# ## ARRAY COMPUTATIONS
# ### Broadcasting: A set of rules for applying binary ufuncs on arrays of different sizes.Example:

# In[78]:

a = np.array([0, 1, 2])
M = np.ones((3, 3))
M + a

# Note that while we’ve been focusing on the + operator here, these broadcasting rules apply to any binary ufunc.


"""
Computes equilibrium price and quantities.
"""
from numpy import exp

## Parameters
a = 1
b = 0.1
epsilon = 1

def supply(price):
   return exp(b*price) - 1

def demand(price):
   return a*(price**(-epsilon))

mxiter = 30
toler = 1.0e-6

plow = 0.1
phigh = 10.0

niter = mxiter

for i in range(mxiter):

    pcur = (plow + phigh)/2
    yd = demand(pcur)
    ys = supply(pcur)
    excesssupply = ys - yd

    if excesssupply > 0:
        phigh = pcur
    else:
        plow = pcur

    diff = abs(phigh - plow)

    if diff <= toler:
        niter = i
        break

pclear = (plow + phigh)/2
yd = demand(pcur)
ys = supply(pcur)
excesssupply = ys - yd

print(niter, pclear, yd, ys, excesssupply)


import numpy as np
import math

class LogisticRegression:

	""" Fits a logistic regression model to binary classification data

	Args:
		use_bias: bool
			A flag to enable or disable a bias term in the regression
	"""

	def __init__(self, use_bias: bool=True):
		self.use_bias = use_bias
		self.weights = None

	def augment_matrix(self, X: np.ndarray) -> np.ndarray:
		""" Augment a matrix with a dummy feature to include a bias term in the regression

		Parameters
		----------
		X: np.ndarray
			numpy array corresponding to the feature matrix. Each row is a data point
			each column is a feature/variable
		"""

		# create a matrix full of 1's with one more column than X
		augmented_X = np.full((X.shape[0], X.shape[1] + 1), 1.0)
		# copy X over the first N column of the new matrix, preserving the last 1's column
		augmented_X[:, :-1] = X

		return augmented_X

	def fit(self, X: np.ndarray, y: np.ndarray):
		"""Will fit a logistic regression model on an input matrix X and
		a target vector y

		Parameters
		----------
		X: np.ndarray
			numpy array corresponding to the feature matrix. Each row is a data point
			each column is a feature/variable
		y: np.ndarray
			numpy array corresponding to the 1D target vector
		"""

		if self.use_bias:
			X = self.augment_matrix(X)

		# create a vector of 0's of the same size as the target
		# will be used to convert True/False classification into
		# a numerical target for OLS
		y_inverted = np.zeros(y.shape[0])

		# True/False values are converted to 1 and 0
		# Next, the 1/0 values are converted to 0.9 and 0.1 (see label smoothing for more info)
		# Then, 0.9 and 0.1 are passed through the inverse sigmoid function to obtain a numerical
		# target for OLS linear regression
		true_value = math.log(0.9 / (1 - 0.9))
		false_value = math.log(0.1 / (1 - 0.1))

		# set true labels to true value
		y_inverted[y == True] = true_value
		# set false labels to false value
		y_inverted[y == False] = false_value

		# compute matrices for OLS regression
		gram_matrix = np.matmul(X.transpose(), X)
		moment_matrix = np.matmul(X.transpose(), y_inverted)

		# compute weights of linear model
		self.weights = np.matmul(np.linalg.inv(gram_matrix), moment_matrix)

		return self

	def predict(self, X: np.ndarray) -> np.ndarray:
		""" Classify a matrix using the trained Logistic model

		Parameters
		----------
		X: np.ndarray
			numpy array corresponding to the feature matrix. Each row is a data point
			each column is a feature/variable

		Returns
		----------
		predictions: np.ndarray
			a numpy array corresponding to the prediction vector
		"""

		if self.use_bias:
			X = self.augment_matrix(X)

		raw_predictions = np.matmul(X, self.weights)

		# use logistic equation to get true/false score
		return 1.0 / (1.0 + np.exp(-raw_predictions))


import numpy as np

def estimate_coef(x, y):
    # number of observations/points
    n = np.size(x)

    # mean of x and y vector
    m_x, m_y = np.mean(x), np.mean(y)

    # calculating cross-deviation and deviation about x
    SS_xy = np.sum(y*x) - n*m_y*m_x
    SS_xx = np.sum(x*x) - n*m_x*m_x

    # calculating regression coefficients
    b_1 = SS_xy / SS_xx
    b_0 = m_y - b_1*m_x

    return(b_0, b_1)

if __name__ == "__main__":
    # observations
    x = np.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
    y = np.array([1, 3, 2, 5, 7, 8, 8, 9, 10, 12])

    # estimating coefficients
    b = estimate_coef(x, y)
    print("Estimated coefficients in Python:\n b_0 = {} \n b_1 = {}".format(b[0], b[1]))


import numpy as np
from numpy.fft import fft

# Steve and Rosie's Radix-2 Decimation In Time (DIT) algorithm
def fft_radix2(arr):
    # assume len(arr) is a power of 2
    n = arr.shape[0]
    if n == 1:
        return arr[0]
    else:
        even = fft_radix2(arr[::2]) # this is the FFT of even indices
        odd = fft_radix2(arr[1::2]) * np.exp(-2*np.pi*1.j / n * np.arange(n/2)) # FFT of odd indices
        return np.concatenate([even + odd, even - odd])

# Radix-2 DIT with padding
def fft_radix2_pad(arr):
    log2len = np.log2(len(arr)) # first check if len pow 2
    zeros = np.zeros(int(2**np.ceil(log2len)) - len(arr))
    padded_arr = np.concatenate((arr,zeros)) # pad with zeros
    return fft_radix2(padded_arr)

# TESTS
if __name__ == "__main__":
    x1 = np.array([0,1,2,3,4,5,6,7])
    x2 = np.array([1,2,3,4,5])
    x2_pad = np.array([1,2,3,4,5,0,0,0])

    assert (np.round(fft(x1),5) == np.round(fft_radix2(x1),5)).all()
    assert (np.round(fft(x1),5) == np.round(fft_radix2_pad(x1),5)).all()
    assert (np.round(fft(x2_pad),5) == np.round(fft_radix2(x2_pad),5)).all()
    assert (np.round(fft(x2_pad),5) == np.round(fft_radix2_pad(x2),5)).all()
    print("All tests passed")


import numpy as np

# https://docs.scipy.org/doc/numpy/reference/generated/numpy.eye.html

e = np.eye(4)
print(type(e))
print(e)
print(e.dtype)
# <class 'numpy.ndarray'>
# [[ 1.  0.  0.  0.]
#  [ 0.  1.  0.  0.]
#  [ 0.  0.  1.  0.]
#  [ 0.  0.  0.  1.]]
# float64

e = np.eye(4, M=3, k=1, dtype=np.int8)
print(e)
print(e.dtype)
# [[0 1 0]
#  [0 0 1]
#  [0 0 0]
#  [0 0 0]]
# int8

# https://docs.scipy.org/doc/numpy/reference/generated/numpy.identity.html

i = np.identity(4)
print(i)
print(i.dtype)
# [[ 1.  0.  0.  0.]
#  [ 0.  1.  0.  0.]
#  [ 0.  0.  1.  0.]
#  [ 0.  0.  0.  1.]]
# float64

i = np.identity(4, dtype=np.uint8)
print(i)
print(i.dtype)
# [[1 0 0 0]
#  [0 1 0 0]
#  [0 0 1 0]
#  [0 0 0 1]]
# uint8

a = [3, 0, 8, 1, 9]
a_one_hot = np.identity(10)[a]
print(a)
print(a_one_hot)
# [3, 0, 8, 1, 9]
# [[ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.]
#  [ 1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]
#  [ 0.  0.  0.  0.  0.  0.  0.  0.  1.  0.]
#  [ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]
#  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  1.]]

a = [2, 2, 0, 1, 0]
a_one_hot = np.identity(3)[a]
print(a)
print(a_one_hot)
# [2, 2, 0, 1, 0]
# [[ 0.  0.  1.]
#  [ 0.  0.  1.]
#  [ 1.  0.  0.]
#  [ 0.  1.  0.]
#  [ 1.  0.  0.]]


""""

    This program analysis the condition number of Hilbert Matrices.
    Marina von Steinkirch, spring/2013 (based on Mike Zingale's codes)

"""

import numpy as npy
from scipy import linalg
from gaussElimination import gaussElim
from createHilbert import createHilbert
from calculatePrecision import calculatePrecision

def main():

    aCond = []
    aOlimit = []
    epsilon = calculatePrecision()

    for N in range(2,16):
        H, x = createHilbert(N)

        # calculate the error
        b = npy.dot(H,x)
        xtilde = gaussElim(H, b)
        error_calculated = abs(npy.linalg.norm(x - xtilde))

        # verifies when it reaches O(1)
        if (error_calculated >= 1.0):
            aOlimit.append([N, x, xtilde])

        # calculates the conditional number use the numpy API to get the conditional number
        cond_npy = npy.linalg.cond(H, p="fro")
        cond_formal = npy.linalg.norm(npy.dot(abs(H), abs(linalg.inv(H))))
        cond_comp = (npy.linalg.norm(x -xtilde)/npy.linalg.norm(x))/epsilon
        m = npy.log10(cond_npy)
        aCond.append([error_calculated,cond_formal, cond_comp, cond_npy, m])

    for N in range(2,16):
        print "\n*** Hilbert Matrix N x N =", N,"x",N, ": ***"
        print "Calculated error =",aCond[N-2][0]
        print "Condition number:\n    Formal definition (cond(A)=|A||A^-1|) =", aCond[N-2][1]
        print "    Calculated with the machine precision =",  aCond[N-2][2]
        print "    From the Numpy API =", aCond[N-2][3]
        print "    Number of digit of accuracy lost in solving Hx=b, giving by log(cond(H)) is ", aCond[N-2][4]

    print "\n\n\nThe error for the N x N Hilbert matrix becomes O(1) when N =", aOlimit[0][0]
    print "as we can see when comparing \nx =", aOlimit[0][1], " to \nxtilde =", aOlimit[0][2]
    print "\n\nDone!"

if __name__ == "__main__":
    main()


from numpy import random
import numpy as np
import matplotlib.pyplot as plt
import math

### Defining theta
theta = math.pi/4

### Generates count number of random values in the range [0, 1]
def getU(count):
	u = []
	for i in range(count):
		key = random.rand()
		u.append(key)

	return u

def getX(u):
	x = []
	for t in u:
		res = -theta*(math.log(1-t))
		x.append(res)

	return x

def getSampleMeanVariance(x):
	sum = 0.00
	for i in x:
		sum += i

	avg = sum/(len(x))

	cnt = 0.00
	for i in x:
		cnt += (i-avg)**2

	variance = cnt/(len(x)-1)
	return avg, variance

def plotCDF(data):
	data_size = len(data)

	data_set = sorted(set(data))
	bins = np.append(data_set, data_set[-1]+1)

	counts, bin_edges = np.histogram(data, bins=bins, density=False)

	counts = counts.astype(float)/data_size

	cdf = np.cumsum(counts)

	plt.plot(bin_edges[0:-1], cdf, linestyle='--', marker='o', color='b')
	plt.ylim((0, 1))
	plt.ylabel("CDF")
	plt.grid(True)

	plt.show()

# Plots y = 1 - e^(-x/theta)
def plotActualDistributionFunction():
	a = -1
	b = 1/theta
	c = 1
	x = np.linspace(0, 10, 256, endpoint = True)
	y = (a * np.exp(-b*x)) + c

	plt.plot(x, y, '-r', label=r'$y =  1 - e^{-x/theta}$')

	axes = plt.gca()
	axes.set_xlim([x.min(), x.max()])
	axes.set_ylim([y.min(), y.max()])

	plt.xlabel('x')
	plt.ylabel('y')
	plt.title('Actual Distribution')
	plt.legend(loc='upper left')

	plt.show()

def execute(cnt):
	print("For input size of : " + str(cnt))
	u = getU(cnt)
	u.sort()
	# print(u)
	x = getX(u)
	# print(x)

	sMean, sVariance = getSampleMeanVariance(x)
	# Actual Mean is theta
	print("Sample Mean: " + str(sMean) + " "  + "Actual Mean: " + str(theta))
	print("Abs. Difference : " + str(abs(sMean-theta)))
	# Actual Variance is theta^2
	print("Sample Variance: " + str(sVariance) + " " + "Actual Variance: " + str(theta**2))
	print("Abs. Difference : " + str(abs(sVariance-theta**2)))
	print()

	plotCDF(x)

def main():
	plotActualDistributionFunction()

	execute(10)
	execute(100)
	execute(1000)
	execute(10000)
	execute(100000)
	# execute(1000000)

if __name__ == '__main__':
	main()


# # Applying Bayes' theorem to iris classification
#
# Can **Bayes' theorem** help us to solve a **classification problem**, namely predicting the species of an iris?

# ## Preparing the data
#
# We'll read the iris data into a DataFrame, and **round up** all of the measurements to the next integer:

import pandas as pd
import numpy as np

# read the iris data into a DataFrame
url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data'
col_names = ['sepal_length', 'sepal_width', 'petal_length', 'petal_width', 'species']
iris = pd.read_csv(url, header=None, names=col_names)
iris.head()

# apply the ceiling function to the numeric columns
iris.loc[:, 'sepal_length':'petal_width'] = iris.loc[:, 'sepal_length':'petal_width'].apply(np.ceil)
iris.head()

# ## Deciding how to make a prediction
#
# Let's say that I have an **out-of-sample iris** with the following measurements: **7, 3, 5, 2**. How might I predict the species?

# show all observations with features: 7, 3, 5, 2
iris[(iris.sepal_length==7) & (iris.sepal_width==3) & (iris.petal_length==5) & (iris.petal_width==2)]

# count the species for these observations
iris[(iris.sepal_length==7) & (iris.sepal_width==3) & (iris.petal_length==5) & (iris.petal_width==2)].species.value_counts()

# count the species for all observations
iris.species.value_counts()

# Let's frame this as a **conditional probability problem**: What is the probability of some particular species, given the measurements 7, 3, 5, and 2?
#
# $$P(species \ | \ 7352)$$
#
# We could calculate the conditional probability for **each of the three species**, and then predict the species with the **highest probability**:
#
# $$P(setosa \ | \ 7352)$$
# $$P(versicolor \ | \ 7352)$$
# $$P(virginica \ | \ 7352)$$

# ## Calculating the probability of each species
#
# **Bayes' theorem** gives us a way to calculate these conditional probabilities.
#
# Let's start with **versicolor**:
#
# $$P(versicolor \ | \ 7352) = \frac {P(7352 \ | \ versicolor) \times P(versicolor)} {P(7352)}$$
#
# We can calculate each of the terms on the right side of the equation:
#
# $$P(7352 \ | \ versicolor) = \frac {13} {50} = 0.26$$
#
# $$P(versicolor) = \frac {50} {150} = 0.33$$
#
# $$P(7352) = \frac {17} {150} = 0.11$$
#
# Therefore, Bayes' theorem says the **probability of versicolor given these measurements** is:
#
# $$P(versicolor \ | \ 7352) = \frac {0.26 \times 0.33} {0.11} = 0.76$$
#
# Let's repeat this process for **virginica** and **setosa**:
#
# $$P(virginica \ | \ 7352) = \frac {0.08 \times 0.33} {0.11} = 0.24$$
#
# $$P(setosa \ | \ 7352) = \frac {0 \times 0.33} {0.11} = 0$$
#
# We predict that the iris is a versicolor, since that species had the **highest conditional probability**.

# ## Summary
#
# 1. We framed a **classification problem** as three conditional probability problems.
# 2. We used **Bayes' theorem** to calculate those conditional probabilities.
# 3. We made a **prediction** by choosing the species with the highest conditional probability.

# ## Bonus: The intuition behind Bayes' theorem
#
# Let's make some hypothetical adjustments to the data, to demonstrate how Bayes' theorem makes intuitive sense:
#
# Pretend that **more of the existing versicolors had measurements of 7352:**
#
# - $P(7352 \ | \ versicolor)$ would increase, thus increasing the numerator.
# - It would make sense that given an iris with measurements of 7352, the probability of it being a versicolor would also increase.
#
# Pretend that **most of the existing irises were versicolor:**
#
# - $P(versicolor)$ would increase, thus increasing the numerator.
# - It would make sense that the probability of any iris being a versicolor (regardless of measurements) would also increase.
#
# Pretend that **17 of the setosas had measurements of 7352:**
#
# - $P(7352)$ would double, thus doubling the denominator.
# - It would make sense that given an iris with measurements of 7352, the probability of it being a versicolor would be cut in half.


import numpy as np

def minmax(X, low, high, minX=None, maxX=None, dtype=np.float):
    X = np.asarray(X)
    if minX is None:
        minX = np.min(X)
    if maxX is None:
        maxX = np.max(X)
    # normalize to [0...1].
    X -= float(minX)
    X /= float((maxX - minX))
    # scale to [low...high].
    X = X * (high - low)
    X = X + low
    return np.asarray(X, dtype=dtype)

def zscore(X, mean=None, std=None):
    """
    Mean Normalization + Feature Scaling

    :param X: ndarray
    :param mean: mean
    :param std: std dev
    :return: normalized ndarry
    """
    X = np.asarray(X)
    if mean is None:
        mean = X.mean()
    if std is None:
        std = X.std()
    X = (X - mean) / std
    return X

def gaussian(X, mu, sig):
    return (1/(sig*np.sqrt(2*np.pi)))*\
           np.exp(-(X-mu)**2/(2*sig**2))

def inverse_dissim(X):
    """

    :param X: int or np.array
    :return:
    """
    X = np.asarray(X)
    X = zscore(X)
    X = minmax(X, 0, 10)
    return 1./(1+X)

def vector_normalize(x):
    return x / np.linalg.norm(x)

def gaussian_kernel(X, mu=None, sig=None):
    """
    gaussian kernel.
    convert distance to similarity by setting mu=0
    :param X:
    :param mu:
    :param sig:
    :return:
    """
    X = np.asarray(X)
    if mu is None:
        mu = X.mean()
    if sig is None:
        sig = X.std()

    return np.exp(-np.power(X-mu, 2)/(2*sig**2))


from matplotlib import pyplot as plt
import pandas as pd
import numpy as np
import math
import scipy.stats as stats

def con_level(x_bar, sigma, n, sig_level, show=True):
    """
Input: x_bar (x_mean), sigma (sample sigma), sample size, sig_level (alpha), show=True
Return the confidence level at alpha. Return a dictionary: {"lcl": lcl, "ucl": ucl, "x_bar": x_bar, "t_value": t_value, "sig_x_bar": sig_x_bar}

+ `show`: default is `True`. Set to `False` to disable rendering.
    """
    a = sig_level
    df_v = n - 1
    con_coef = 1 - a
    t_value = stats.t.ppf(1 - a / 2, df=df_v)
    sig_x_bar = sigma / math.sqrt(n)
    lcl = x_bar - t_value * sig_x_bar
    ucl = x_bar + t_value * sig_x_bar
    result = f"""{con_coef * 100:.1f}% Confidence Interval: [{lcl:.4f}, {ucl:.4f}]
Mean: {x_bar:.4f}
Std. Dev. = {sigma:.4f}
Sample Size: {n}
t (Critical value): {t_value:.4f}
    """
    if show:
        print(result)
    return {"lcl": lcl, "ucl": ucl, "x_bar": x_bar, "t_value": t_value, "sig_x_bar": sig_x_bar}

def rejection_region_method(x_mean, mu, std, n, alpha, option='left', precision=4, show=True, ignore=False):
    """
    Input: x_mean, mu, std, n, alpha, option='left', precision=4, show=True, ignore=False
    Output:
        if opt == 't':
            return x_l, x_u
        else:
            return x_c
    """
    opt = option.lower()[0]
    df_v = n - 1
    if opt == 't':
        option = 'Two-Tail Test'
        t_value = stats.t.ppf(1 - alpha / 2, df=df_v)
        x_u = mu + t_value * std / math.sqrt(n)
        x_l = mu - t_value * std / math.sqrt(n)
        flag = x_mean < x_l or x_mean > x_u
        if not ignore:
            result = f'''======= The Rejection Region Method =======
Significant Level (alpha) = {alpha:.{precision}f}
t (Critical value) = {t_value:.{precision}f}

Using {option}:
x̄ =  {x_mean:.{precision}f}
x_l (Lower bound for the critical value) = {x_l:.{precision}f}
x_u (Upper bound for the critical value) = {x_u:.{precision}f}
Reject H_0 → {flag}
            '''
        else:
            result = f'''======= The Rejection Region Method =======
Significant Level (alpha) = {alpha:.{precision}f}
t (Critical value) = {t_value:.{precision}f}

Using {option}:
x_l (Lower bound for the critical value) = {x_l:.{precision}f}
x_u (Upper bound for the critical value) = {x_u:.{precision}f}
            '''

    else:
        if opt == 'l':
            # left tail
            option = 'One-Tail Test (left tail)'
            t_value = stats.t.ppf(alpha, df=df_v)  # negative
            x_c = mu + t_value * std / math.sqrt(n)
            flag = x_mean < x_c
        elif opt == 'r':
            option = 'One-Tail Test (right tail)'
            t_value = stats.t.ppf(1 - alpha, df=df_v)
            x_c = mu + t_value * std / math.sqrt(n)
            flag = x_mean > x_c
        if not ignore:
            result = f'''======= The Rejection Region Method =======
Significant Level (alpha) = {alpha:.{precision}f}
t (Critical value) = {t_value:.{precision}f}

Using {option}:
x̄ =  {x_mean:.{precision}f}
x_c (Critical value) = {x_c:.{precision}f}
Reject H_0 → {flag}
            '''
        else:
            result = f'''======= The Rejection Region Method =======
Significant Level (alpha) = {alpha:.{precision}f}
t (Critical value) = {t_value:.{precision}f}

Using {option}:
x_c (Critical value) = {x_c:.{precision}f}
            '''

    if show:
        print(result)

    if opt == 't':
        return x_l, x_u
    else:
        return x_c

def testing_statistic_method(x_mean, mu, std, n, alpha, option='left', precision=4, ignore=False):
    """
    Input: x_mean, mu, std (sample std), n, alpha, option='left', precision=4, ignore=False
    Output:
        if opt == 't':
            return t, t_l, t_u
        else:
            return t, t_value
    """
    df_v = n - 1
    opt = option.lower()[0]
    t = (x_mean - mu)/(std / math.sqrt(n))
    if opt == 't':
        option = 'Two-Tail Test'
        t_value = stats.t.ppf(1 - alpha / 2, df=df_v)
        t_u = t_value
        t_l = -t_value
        flag = t < t_l or t > t_u

        if not ignore:
            result = f'''======= Testing Statistic Method =======
Significant Level (alpha) = {alpha:.{precision}f}
t (Critical value) = {t_value:.{precision}f}

Using {option}:
t (Observed value) =  {t:.{precision}f}
t_l (Lower bound for the critical value) = {t_l:.{precision}f}
t_u (Upper bound for the critical value) = {t_u:.{precision}f}
Reject H_0 → {flag}
            '''
        else:
            result = f'''======= Testing Statistic Method =======
Significant Level (alpha) = {alpha:.{precision}f}
t (Critical value) = {t_value:.{precision}f}

Using {option}:
t_l (Lower bound for the critical value) = {t_l:.{precision}f}
t_u (Upper bound for the critical value) = {t_u:.{precision}f}
            '''

    else:
        if opt == 'l':
            # left tail
            option = 'One-Tail Test (left tail)'
            t_value = stats.t.ppf(alpha, df=df_v)  # negative
            flag = t < t_value
        elif opt == 'r':
            option = 'One-Tail Test (right tail)'
            t_value = stats.t.ppf(1 - alpha, df=df_v)
            flag = t > t_value

        if not ignore:
            result = f'''======= Testing Statistic Method =======
Significant Level (alpha) = {alpha:.{precision}f}
t (Critical value) = {t_value:.{precision}f}

Using {option}:
t (Observed value) =  {t:.{precision}f}
t (Critical value) = {t_value:.{precision}f}
Reject H_0 → {flag}
            '''

        else:
            result = f'''======= Testing Statistic Method =======
Significant Level (alpha) = {alpha:.{precision}f}
t (Critical value) = {t_value:.{precision}f}

Using {option}:
t (Critical value) = {t_value:.{precision}f}
            '''

    print(result)
    if opt == 't':
        return t, t_l, t_u
    else:
        return t, t_value

def inter_p_value(p_value):
    # interpretation
    if p_value >= 0 and p_value < 0.01:
        inter_p = 'Overwhelming Evidence'
    elif p_value >= 0.01 and p_value < 0.05:
        inter_p = 'Strong Evidence'
    elif p_value >= 0.05 and p_value < 0.1:
        inter_p = 'Weak Evidence'
    elif p_value >= .1:
        inter_p = 'No Evidence'
    return inter_p

def p_value_method(x_mean, h0_mean, std, samp_num, siglevel, option='left', precision=4):
    """
    Input: x_mean, h0_mean, std (standard deviation of sample), samp_num (sample size), siglevel (alpha), option='left', precision=4):
    Output: zcv, p_value
    """
    df_v = samp_num - 1
    t_value = (x_mean - h0_mean) / (std/(samp_num ** 0.5))
    alpha = siglevel
    opt = option.lower()[0]
    if opt == 't':
        # two-tail test
        option = 'Two-Tail Test'
        p_value = (1 - stats.t.cdf(t_value, df=df_v)) * 2
        if (p_value > 1):
            p_value = (stats.t.cdf(t_value, df=df_v)) * 2
        tcv = stats.t.ppf(1 - siglevel/2, df=df_v)
        flag = p_value < alpha
        sub_result = f'''Using {option}:
Difference = {x_mean - h0_mean}
t (Critical value) = {-tcv:.{precision}f}, {tcv:.{precision}f}
t (Observed value) = {t_value:.{precision}f}
p-value = {p_value:.{precision}f} ({inter_p_value(p_value)})
Reject H_0 → {flag}
        '''
    else:
        if opt == 'l':
            option = 'One-Tail Test (left tail)'
            p_value = stats.t.cdf(t_value, df=df_v)
            tcv = stats.t.ppf(siglevel, df=df_v)
        elif opt == 'r':
            option = 'One-Tail Test (right tail)'
            p_value = stats.t.sf(t_value, df=df_v)
            tcv = stats.t.ppf(1 - siglevel, df=df_v)
        flag = p_value < alpha
        sub_result = f'''Using {option}:
Difference = {x_mean - h0_mean}
t (Critical value) = {tcv:.{precision}f}
t (Observed value) = {t_value:.{precision}f}
p-value = {p_value:.{precision}f} ({inter_p_value(p_value)})
Reject H_0 → {flag}
        '''

    result = f"""======= p-value Method =======
Mean = {x_mean:.{precision}f}
Number of Observation = {samp_num:.{precision}f}
Hypothesized Mean (H0 Mean) = {h0_mean:.{precision}f}
Sample Standard Deviation = {std:.{precision}f}
Significant Level (alpha) = {siglevel:.{precision}f}

""" + sub_result

    print(result)

    return tcv, p_value

def type2_plot(h0_mean, psigma, nsizes, alpha, ranges, option='right', figsize=(12, 6), pf=False, label=True, show=True):
    """
    Caution: 外面要自己 plt.show()

    Input: h0_mean, psigma, nsizes (list or one value), alpha, ranges, option='right', figsize=(12, 6), pf=False, label=True, show=True
    → set show to false to only get the values for powers
    Output: (if pf=True: means, betas, xticks, yticks)
    """

    try:
        _ = iter(nsizes)
    except TypeError as te:
        nsizes = [nsizes]

    opt = option.lower()[0]
    # options

    means = np.arange(ranges[0], ranges[1], 0.1)
    betas = np.zeros(means.shape[0])
    powers = betas.copy()
    if show:
        fig, ax = plt.subplots(figsize=figsize)

    for nsize in nsizes:
        df_v = nsize - 1
        if opt == 'r':
            tcv = stats.t.ppf(1 - alpha, df=df_v)
        elif opt == 'l':
            tcv = stats.t.ppf(alpha, df=df_v)
        elif opt == 't':
            tcv = stats.t.ppf(1 - alpha / 2, df=df_v)
        means = np.arange(ranges[0], ranges[1], 0.1)
        betas = np.zeros(means.shape[0])
        powers = betas.copy()
        i = 0
        if opt == 'r':
            x_c = h0_mean + tcv * psigma / (nsize ** 0.5)
            for h1_mean in means:
                t_type2 = (x_c - h1_mean) / (psigma / (nsize ** 0.5))
                type2_p = stats.t.cdf(t_type2, df=df_v)
                betas[i] = type2_p
                powers[i] = 1 - type2_p
                i += 1
        elif opt == 'l':
            x_c = h0_mean + tcv * psigma / (nsize ** 0.5)
            for h1_mean in means:
                t_type2 = (x_c - h1_mean) / (psigma / (nsize ** 0.5))
                type2_p = 1 - stats.t.cdf(t_type2, df=df_v)
                betas[i] = type2_p
                powers[i] = 1 - type2_p
                i += 1
        elif opt == 't':
            x_u = h0_mean + tcv * psigma / math.sqrt(nsize)
            x_l = h0_mean - tcv * psigma / math.sqrt(nsize)
            # x_l, x_u = rejection_region_method(_, h0_mean, psigma, nsize, alpha, option=opt, precision=4, show=False, ignore=True)
            for h1_mean in means:
                t_type2_l = (x_l - h1_mean) / (psigma / (nsize ** 0.5))
                t_type2_u = (x_u - h1_mean) / (psigma / (nsize ** 0.5))
                type2_p_l = stats.t.cdf(t_type2_l, df=df_v)
                type2_p_u = stats.t.cdf(t_type2_u, df=df_v)
                type2_p = type2_p_u - type2_p_l
                betas[i] = type2_p
                powers[i] = 1 - type2_p
                i += 1

        if show:
            if pf:
                plt.plot(means, betas, label=f'OC ({nsize})')
                plt.plot(means, powers, label=f'PF ({nsize})')
            else:
                plt.plot(means, betas, label=f'n = {nsize}')

    if len(ranges) == 3:
        xticks = np.arange(ranges[0], ranges[1] + 1, ranges[2])
    else:  # default
        xticks = np.arange(ranges[0], ranges[1] + 1, 1)
    yticks = np.arange(0, 1.1, .1)

    if show:
        plt.xlabel("H1 Mean")
        plt.xticks(xticks, rotation=45, fontsize=8)
        plt.yticks(yticks, fontsize=8)
        plt.ylabel("Probability of a Type II Error")
        plt.margins(x=.01, tight=False)
        if label:
            plt.legend()

    if pf:
        return means, betas, xticks, yticks

def power_test(x_mean, h0_mean, std, n, alpha, h1_mean, option='left', precision=4, show=True, ignore=True):
    """
    Input: x_mean (not necessary if ignore=True), h0_mean, std, n, alpha, h1_mean, option='left', precision=4, show=True, ignore=True
    Output: type2_p (beta), ptest (power of a test)
    """
    opt = option.lower()[0]
    df_v = (n - 1)
    if opt == 't':
        option = 'Two-Tail Test'
        x_l, x_u = rejection_region_method(
            x_mean, h0_mean, std, n, alpha, option=opt, precision=precision, show=show, ignore=ignore)
        t_value = stats.t.ppf(1 - alpha / 2, df=df_v)
        t_l = -t_value
        t_u = t_value
        t_type2_l = (x_l - h1_mean) / (std / (n ** 0.5))
        t_type2_u = (x_u - h1_mean) / (std / (n ** 0.5))
        type2_p_l = stats.t.cdf(t_type2_l, df=df_v)
        type2_p_u = stats.t.cdf(t_type2_u, df=df_v)
        type2_p = type2_p_u - type2_p_l
        ptest = 1 - type2_p
        result = f'''======= Evaluating Type II Errors ({option}) =======
μ = {h1_mean}
t (lower bound) = {t_type2_l:.{precision}f}
t (upper bound) = {t_type2_u:.{precision}f}

t_l (Lower bound for the critical value) = {t_l:.{precision}f}
t_u (Upper bound for the critical value) = {t_u:.{precision}f}

x_l (Lower bound for x critical value) = {x_l:.{precision}f}
x_u (Upper bound for x critical value) = {x_u:.{precision}f}

P(Type II Error) = {type2_p:.{precision}f}
Power of a Test = {ptest:.{precision}f}
        '''
    else:
        x_c = rejection_region_method(
            x_mean, h0_mean, std, n, alpha, option=opt, precision=precision, show=show, ignore=ignore)
#         if x_c > h1_mean:
#             opt = 'l'
#         else:
#             opt = 'r'

        if opt == 'l':
            option = 'One-Tail Test (left tail)'

            t_c = stats.t.ppf(alpha, df=df_v)
            t_type2 = (x_c - h1_mean) / (std / (n ** 0.5))
            type2_p = 1 - stats.t.cdf(t_type2, df=df_v)
            ptest = 1 - type2_p

        elif opt == 'r':
            option = 'One-Tail Test (right tail)'
            t_c = stats.t.ppf(1 - alpha, df=df_v)
            t_type2 = (x_c - h1_mean) / (std / (n ** 0.5))
            type2_p = stats.t.cdf(t_type2, df=df_v)
            ptest = 1 - type2_p

        result = f'''======= Evaluating Type II Errors ({option}) =======
μ = {h1_mean}
t = {t_type2:.{precision}f}

t critical value = {t_c:.{precision}f}
x critical value = {x_c:.{precision}f}

P(Type II Error) = {type2_p:.{precision}f}
Power of a Test = {ptest:.{precision}f}
'''

    if show:
        print(result)

    return type2_p, ptest

def power_plot(h0_mean, psigma, nsizes, alpha, ranges, option='r', figsize=(12, 6), show=True):
    means, betas, xticks, yticks = type2_plot(
        h0_mean, psigma, nsizes, alpha, ranges, option=option, figsize=figsize, pf=True, label=True, show=show)
    if show:
        plt.clf()
        plt.plot(means, 1 - betas)
        plt.xticks(xticks, rotation=45, fontsize=8)
        plt.yticks(yticks, fontsize=8)
        plt.title('Power Function Curve')
        plt.margins(x=.01, tight=False)


from scipy import stats
import statistics
import timeit

# samples from: https://docs.python.org/3/library/statistics.html

#==================================================================================
# Arrange
data = [1, 2, 3, 4, 4]
print ("data          = ", data)
start = timeit.default_timer()
# Act
mean_arithmetic = statistics.mean(data)
end = timeit.default_timer()
# Assert
print ("mean_arithmetic = ", mean_arithmetic)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(mean_arithmetic == 2.8)
#==================================================================================

#==================================================================================
# Arrange
data = [-1.0, 2.5, 3.25, 5.75]
print ("data          = ", data)
start = timeit.default_timer()
# Act
mean_arithmetic = statistics.mean(data)
end = timeit.default_timer()
# Assert
print ("mean_arithmetic = ", mean_arithmetic)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(mean_arithmetic == 2.625)
#==================================================================================

#==================================================================================
# Arrange
data = [2.5, 3, 10]
print ("data          = ", data)
start = timeit.default_timer()
# Act
mean_harmonic = stats.hmean(data)
end = timeit.default_timer()
# Assert
print ("mean_harmonic = ", mean_harmonic)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(mean_harmonic == 3.5999999999999996)
#==================================================================================

#==================================================================================
# Arrange
data = [1, 3, 5]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median = statistics.median(data)
end = timeit.default_timer()
# Assert
print ("median = ", median)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median == 3)
#==================================================================================

#==================================================================================
# Arrange
data = [1, 3, 5, 7]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median = statistics.median(data)
end = timeit.default_timer()
# Assert
print ("median = ", median)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median == 4)
#==================================================================================

#==================================================================================
# Arrange
data = [1, 3, 5]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median_low = statistics.median_low(data)
end = timeit.default_timer()
# Assert
print ("median_low = ", median_low)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median_low == 3)
#==================================================================================

#==================================================================================
# Arrange
data = [1, 3, 5, 7]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median_low = statistics.median_low(data)
end = timeit.default_timer()
# Assert
print ("median_low = ", median_low)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median_low == 3)
#==================================================================================

#==================================================================================
# Arrange
data = [1, 3, 5]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median_high = statistics.median_high(data)
end = timeit.default_timer()
# Assert
print ("median_high = ", median_high)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median_high == 3)
#==================================================================================

#==================================================================================
# Arrange
data = [1, 3, 5, 7]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median_high = statistics.median_high(data)
end = timeit.default_timer()
# Assert
print ("median_high     = ", median_high)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median_high == 5)
#==================================================================================

#==================================================================================
# Arrange
data = [52, 52, 53, 54]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median_grouped = statistics.median_grouped(data)
end = timeit.default_timer()
# Assert
print ("median_grouped  = ", median_grouped)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median_grouped == 52.5)
#==================================================================================

#==================================================================================
# Arrange
data = [1, 2, 2, 3, 4, 4, 4, 4, 4, 5]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median_grouped = statistics.median_grouped(data)
end = timeit.default_timer()
# Assert
print ("median_grouped  = ", median_grouped)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median_grouped == 3.7)
#==================================================================================

#
# https://help.gnome.org/users/gnumeric/stable/gnumeric.html#gnumeric-function-SSMEDIAN
# https://mail.gnome.org/archives/gnumeric-list/2011-April/msg00018.html
#==================================================================================
# Arrange
data = [1, 3, 3, 5, 7]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median_grouped = statistics.median_grouped(data, interval=1)
end = timeit.default_timer()
# Assert
print ("median_grouped  = ", median_grouped)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median_grouped == 3.25)
#==================================================================================

#==================================================================================
# Arrange
data = [1, 3, 3, 5, 7]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median_grouped = statistics.median_grouped(data, interval=2)
end = timeit.default_timer()
# Assert
print ("median_grouped  = ", median_grouped)
print ("      time [us] = ", (end - start) * 100000)
assert(median_grouped == 3.5)
#==================================================================================

#==================================================================================
# Arrange
data = [7, 8, 8]
print ("data          = ", data)
start = timeit.default_timer()
# Act
median_grouped = statistics.median_grouped(data, interval=1)
end = timeit.default_timer()
# Assert
print ("median_grouped  = ", median_grouped)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(median_grouped == 7.75)
#==================================================================================

#==================================================================================
# Arrange
data = [1, 1, 2, 3, 3, 3, 3, 4]
print ("data          = ", data)
start = timeit.default_timer()
# Act
mode = statistics.mode(data)
end = timeit.default_timer()
# Assert
print ("mode            = ", mode)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(mode == 3)
#==================================================================================

#==================================================================================
# Arrange
data = ["red", "blue", "blue", "red", "green", "red", "red"]
print ("data          = ", data)
start = timeit.default_timer()
# Act
mode = statistics.mode(data)
end = timeit.default_timer()
# Assert
print ("mode            = ", mode)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(mode == "red")
#==================================================================================

#==================================================================================
# Arrange
data = [1.5, 2.5, 2.5, 2.75, 3.25, 4.75]
print ("data          = ", data)
start = timeit.default_timer()
# Act
pstdev = statistics.pstdev(data)
end = timeit.default_timer()
# Assert
print ("pstdev          = ", pstdev)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(abs(pstdev - 0.98689) < 0.00001)
#==================================================================================

#==================================================================================
# Arrange
mu = statistics.mean(data)
start = timeit.default_timer()
# Act
pstdev = statistics.pstdev(data, mu)
end = timeit.default_timer()
# Assert
print ("pstdev          = ", pstdev)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(abs(pstdev - 0.98689) < 0.00001)
#==================================================================================

#==================================================================================
# Arrange
data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25]
print ("data          = ", data)
start = timeit.default_timer()
# Act
pvariance = statistics.pvariance(data)
end = timeit.default_timer()
# Assert
print ("pvariance       = ", pvariance)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(abs(pvariance - 1.25) < 0.001)
#==================================================================================

#==================================================================================
# Arrange
mu = statistics.mean(data)
start = timeit.default_timer()
# Act
pvariance = statistics.pvariance(data, mu)
end = timeit.default_timer()
# Assert
print ("pvariance       = ", pvariance)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(abs(pvariance - 1.25) < 0.001)
#==================================================================================

#==================================================================================
# Arrange
data = [1.5, 2.5, 2.5, 2.75, 3.25, 4.75]
print ("data          = ", data)
start = timeit.default_timer()
# Act
stdev = statistics.stdev(data)
end = timeit.default_timer()
# Assert
print ("stdev           = ", stdev)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(abs(stdev - 1.08108) < 0.00001)
#==================================================================================

#==================================================================================
# Arrange
mu = statistics.mean(data)
start = timeit.default_timer()
# Act
stdev = statistics.stdev(data, mu)
end = timeit.default_timer()
# Assert
print ("stdev           = ", stdev)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(abs(stdev - 1.08108) < 0.00001)
#==================================================================================

#==================================================================================
# Arrange
data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5]
print ("data          = ", data)
start = timeit.default_timer()
# Act
variance = statistics.variance(data)
end = timeit.default_timer()
# Assert
print ("variance        = ", variance)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(abs(variance - 1.37202) < 0.00001)
#==================================================================================

#==================================================================================
# Arrange
mu = statistics.mean(data)
start = timeit.default_timer()
# Act
variance = statistics.variance(data, mu)
end = timeit.default_timer()
# Assert
print ("variance        = ", variance)
print (u"      time [\u00B5s] = ", (end - start) * 100000)
assert(abs(variance - 1.37202) < 0.00001)
#==================================================================================


# =================================================
# minso.jeong@daum.net
# 29. 푸리에 변환 응용
# Reference : samsjang@naver.com
# =================================================
import numpy as np
import cv2 as cv
import matplotlib.pyplot as plt

# A. 푸리에 변환으로 이미지 작업하기
def fourier_edge():
	img = cv.imread('../Images/34.Ji.jpg', cv.IMREAD_GRAYSCALE)

	f = np.fft.fft2(img)
	fshift = np.fft.fftshift(f)

	rows, cols = img.shape
	crow, ccol = int(rows/2), int(cols/2)

	fshift[crow-30:crow+30, ccol-30:ccol+30] = 0
	f_ishift = np.fft.ifftshift(fshift)
	img_back = np.fft.ifft2(f_ishift)
	img_back = np.abs(img_back)

	plt.subplot(1, 3, 1), plt.imshow(img, cmap='gray')
	plt.title('Original Image'), plt.xticks([]), plt.yticks([])

	plt.subplot(1, 3, 2), plt.imshow(img_back, cmap='gray')
	plt.title('After HPF'), plt.xticks([]), plt.yticks([])

	plt.subplot(1, 3, 3), plt.imshow(img_back)
	plt.title('Result in JET'), plt.xticks([]), plt.yticks([])

	plt.show()

#fourier_edge()

# B. 푸리에 변환을 이용한 이미지 필터 효과
# Averagint Filter
# Gaussian Filter
# Scharr Filter
# sobel_x, sobel_y Filter
# Laplacian Filter
def check_kernel():
	# Simple Averagint Filter without Scaling Parameter
	mean_filter = np.ones((3,3))
	# Creating a Gaussian FIlter
	x = cv.getGaussianKernel(3,3)
	gaussian = x*x.T
	# Filters for Edge Detection
	# Laplacian
	laplacian = np.array([[0, 1, 0],
						  [1,-4, 1],
						  [0, 1, 0]])
	# Scharr in x-direction
	scharr = np.array([[-3, 0, 3],
					   [-10,0,10],
					   [-3, 0, 3]])
	# Sobel in x-direction
	sobel_x = np.array([[-1, 0, 1],
					    [-2, 0, 2],
					    [-1, 0, 1]])
	# Sobel in y-direction
	sobel_y = np.array([[-1,-2,-1],
					    [0, 0, 0],
					    [1, 2, 1]])

	filters = [mean_filter, gaussian, laplacian, sobel_x, sobel_y, scharr]
	filter_name = ['mean_filter', 'gaussian', 'laplacian', 'sobel_x', 'sobel_y', 'scharr_x']

	fft_filters = [np.fft.fft2(x) for x in filters]
	fft_shift = [np.fft.fftshift(y) for y in fft_filters]
	mag_spectrum = [np.log(np.abs(z)+1) for z in fft_shift]

	for i in range(6):
		plt.subplot(2, 3, i+1), plt.imshow(mag_spectrum[i], cmap='gray')
		plt.title(filter_name[i]), plt.xticks([]), plt.yticks([])
	plt.show()

check_kernel()


import numpy

# -------------------------------------------------------------------------

def simple_quadratic_function(x):
    """
    this function accepts a 2D vector as input.
    Its outputs are:
       value: h(x1, x2) = x1^2 + 3*x1*x2
       grad: A 2x1 vector that gives the partial derivatives
             of h with respect to x1 and x2
    Note that when we pass simple_quadratic_function(x) to
    compute_gradient_numerical_estimate, we're assuming
    that compute_gradient_numerical_estimate will use only
    the first returned value of this function.
    :param x:
    :return:
    """
    value = x[0] ** 2 + 3 * x[0] * x[1]

    grad = numpy.zeros(shape=2, dtype=numpy.float32)
    grad[0] = 2 * x[0] + 3 * x[1]
    grad[1] = 3 * x[0]

    return value, grad

# -------------------------------------------------------------------------

def compute_gradient_numerical_estimate(J, theta, epsilon=0.0001):
    """
    :param J: a loss (cost) function that computes the real-valued loss given parameters and data
    :param theta: array of parameters
    :param epsilon: amount to vary each parameter in order to estimate
                    the gradient by numerical difference
    :return: array of numerical gradient estimate
    """

    gradient = numpy.zeros(theta.shape)

    # Symmetric Difference Quotient
    # https://en.wikipedia.org/wiki/Numerical_differentiation
    # slope ~= [ f(x+h) - f(x-h) ]  /  [ 2h ]

    # Iterates through all parameters in theta to estimate their respective
    #   gradients, using the symmetric difference quotient.
    for num, elem in enumerate(theta):

        theta[num] = elem + epsilon
        cost_plus  = J(theta)[0]

        theta[num] = elem - epsilon
        cost_minus = J(theta)[0]

        theta[num] = elem
        gradient[num] = (cost_plus - cost_minus) / (2 * epsilon)

    return gradient

# -------------------------------------------------------------------------

def test_compute_gradient_numerical_estimate():
    """
    Test of compute_gradient_numerical_estimate.
    This provides a test for your numerical gradient implementation
    in compute_gradient_numerical_estimate
    It analytically evaluates the gradient of a very simple function
    called simple_quadratic_function and compares the result with
    your numerical estimate. Your numerical gradient implementation
    is incorrect if your numerical solution deviates too much from
    the analytical solution.
    :return:
    """
    print("test_compute_gradient_numerical_estimate(): Start Test")
    print("    Testing that your implementation of ")
    print("        compute_gradient_numerical_estimate()")
    print("        is correct")
    x = numpy.array([4, 10], dtype=numpy.float64)
    (value, grad) = simple_quadratic_function(x)

    print("    Computing the numerical and actual gradient for 'simple_quadratic_function'")
    num_grad = compute_gradient_numerical_estimate(simple_quadratic_function, x)
    print("    The following two 2d arrays should be very similar:")
    print("        ", num_grad, grad)
    print("    (Left: numerical gradient estimate; Right: analytical gradient)")

    diff = numpy.linalg.norm(num_grad - grad)
    print("    Norm of the difference between numerical and analytical num_grad:")
    print("        ", diff)
    print("    (should be < 1.0e-09 ; I get about 1.7e-10)")
    print("test_compute_gradient_numerical_estimate(): DONE\n")

# -------------------------------------------------------------------------


import math

import numpy as np

def shannon_entropy(string):
    """
    Calculates the Shannon entropy for the given string.
    :param string: any string, such as '000101001', '12402', or 'aBcd1234ef5g'
    :return: a real number representing the Shannon entropy
    """
    symbols = dict.fromkeys(list(string))
    symbol_probabilities = [float(string.count(symbol)) / len(string) for symbol in symbols]
    H = -sum([p_symbol * math.log(p_symbol, 2.0) for p_symbol in symbol_probabilities])
    return H + 0  # add 0 as a workaround so we don't end up with -0.0

def average_cell_entropy(cellular_automaton):
    """
    Calculates the average cell entropy in the given cellular automaton, where entropy is the Shannon entropy.
    In the case of a 1D cellular automaton, the state of a cell over time is represented as a string, and its entropy
     is calculated. The same is done for all cells in this cellular automaton, and the average entropy is returned.
    :param cellular_automaton: the cellular automaton to perform this operation on
    :return: a real number representing the average cell Shannon entropy
    """
    num_cols = cellular_automaton.shape[1]
    entropies = []
    for i in range(0, num_cols):
        cell_states_over_time = ''.join([str(x) for x in cellular_automaton[:, i]])
        entropy = shannon_entropy(cell_states_over_time)
        entropies.append(entropy)
    return np.mean(entropies)

def joint_shannon_entropy(stringX, stringY):
    """
    Calculates the joint Shannon entropy between the given strings, which must be of the same length.
    :param stringX: any string, such as '000101001', '12402', or 'aBcd1234ef5g'
    :param stringY: any string, such as '000101001', '12402', or 'aBcd1234ef5g'
    :return: a real number representing the joint Shannon entropy between the given strings
    """
    X = np.array(list(stringX))
    Y = np.array(list(stringY))
    joint_symbol_probabilities = []
    for x in set(X):
        for y in set(Y):
            joint_symbol_probabilities.append(np.mean(np.logical_and(X == x, Y == y)))
    return np.sum(-p * np.log2(p) for p in joint_symbol_probabilities if p != 0)

def mutual_information(stringX, stringY):
    """
    Calculates the mutual information between the given strings, which must be of the same length.
    :param stringX: any string, such as '000101001', '12402', or 'aBcd1234ef5g'
    :param stringY: any string, such as '000101001', '12402', or 'aBcd1234ef5g'
    :return: a real number representing the mutual information between the given strings
    """
    return shannon_entropy(stringX) + shannon_entropy(stringY) - joint_shannon_entropy(stringX, stringY)

def average_mutual_information(cellular_automaton, temporal_distance=1):
    """
    Calculates the average mutual information between a cell and itself at the next n time steps, given by the
    specified temporal distance. A temporal distance of 1 means the next time step.
    For example, consider the following string, '00101010110', which represents the state of a cell over 11 time steps.
     The strings which will be used for the computation of the mutual information between a cell and itself at the
     next time step are: '0010101011' and '0101010110', since we pair each time-step value with its next value:
     " 00101010110"
     "00101010110 "
    :param cellular_automaton: the cellular automaton to perform this operation on
    :param temporal_distance: the size of temporal separation, where the value must be greater than 0 and
                              less than the number of time steps.
    :return: a real number representing the average mutual information between a cell and itself at the next time step
    """
    num_cols = cellular_automaton.shape[1]
    if not (0 < temporal_distance < num_cols):
        raise Exception("the temporal distance must be greater than 0 and less than the number of time steps")
    mutual_informations = []
    for i in range(0, num_cols):
        cell_states_over_time = ''.join([str(x) for x in cellular_automaton[:, i]])
        mi = mutual_information(cell_states_over_time[:-temporal_distance], cell_states_over_time[temporal_distance:])
        mutual_informations.append(mi)
    return np.mean(mutual_informations)


'''
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Taking user input ~~~~~~~~~~~~~~~~~~~~~~~~~~~
while True:
    try:
        N = int(input("Enter the Number of Points: "))
        if N > 1: break
        else: print("Number of Points must be greater than 1")
    except ValueError:
        print("Enter a Natural Number greater than 1")
X, Y = [], []
for i in range(N):
    while True:
        try:
            x = float(input("Enter the X coordinate of the point "+str(i+1)+": "))
        except ValueError:
                print("Enter a Real Number")
                continue
        else: break
    X.append(x)
    while True:
        try:
            y = float(input("Enter the Y coordinate of the point "+str(i+1)+": "))
        except ValueError:
                print("Enter a Real Number")
                continue
        else: break
    Y.append(y)
while True:
    try:
        n = int(input("Enter the Degree of polynomial: "))
        if n > 0: break
        else: print("Degree of polynomial must be greater than 1")
    except ValueError:
        print("Enter a Natural Number greater than 0")
print()
'''
from numpy import array, zeros, linalg

# X = [0, 1, 2, 3, 4, 5]
# Y = [2.1, 7.7, 13.6, 27.2, 40.9, 61.1]

X = [0, 1, 2, 3, 4, 5]      # numpy.arange(6)
Y = [2, 8, 14, 28, 39, 62]

X = array(X, float)
Y = array(Y, float)

N = len(X)  # Number of data points
n = 2       # Degree of polynomial
# n = 3

# [A]{a} = {B}
A = zeros((n+1, n+1))
B = zeros(n+1)
a = zeros(n+1)

#     [   N     ∑ xi      ∑ xi^2    ∑ xi^n   ]      [∑      yi]
# A = [ ∑ xi    ∑ xi^2    ∑ xi^3    ∑ xi^n+1 ], B = [∑ xi   yi]
#     [ ∑ xi^n  ∑ xi^n+1  ∑ xi^n+2  ∑ xi^2n  ]      [∑ xi^n yi]

# A[row, col] = ∑ xi^(row+col)             B[row] = ∑ xi^row yi
# where row, col = 0 to n, except [0,0]

A[0,0] = N
for row in range(n+1):

    for col in range(n+1):
        if row == 0 and col == 0: continue

        A[row,col] = sum(X**(row+col))

    B[row] = sum(X**row * Y)

a = linalg.solve(A, B)

print("The polynomial equation :")
print('y = %f' % a[0], end=' ')
for i in range(1, n+1):
    print('%+f x^%d' % (a[i], i), end=' ')
print()

'''
For 2nd order polynomial:
y = a0 + a1 x + a2 x^2 + e
e = y - a0 - a1 x - a2 x^2

Criteria for a best fit:
∑ (ei) = ∑ (yi - a0 - a1 xi - a2 xi^2)

Sr = ∑ (ei)^2 = ∑ (yi - a0 - a1 xi - a2 xi^2)^2

∂Sr/∂a0 = -2      ∑ (yi - a0 - a1 xi - a2 xi^2)
∂Sr/∂a1 = -2 xi   ∑ (yi - a0 - a1 xi - a2 xi^2)
∂Sr/∂a2 = -2 xi^2 ∑ (yi - a0 - a1 xi - a2 xi^2)

0 = ∑      yi - ∑ a0      - ∑ a1 xi   - ∑ a2 xi^2
0 = ∑ xi   yi - ∑ a0 xi   - ∑ a1 xi^2 - ∑ a2 xi^3
0 = ∑ xi^2 yi - ∑ a0 xi^2 - ∑ a1 xi^3 - ∑ a2 xi^4

(   n  ) a0 + (∑ xi  ) a1 + (∑ xi^2) a2 = ∑      yi    # ∑ a0 = n a0
(∑ xi  ) a0 + (∑ xi^2) a1 + (∑ xi^3) a2 = ∑ xi   yi
(∑ xi^2) a0 + (∑ xi^3) a1 + (∑ xi^4) a2 = ∑ xi^2 yi
Any method of solving systems of linear equations will give a0, a1, a2

For nth order polynomial:
y = a0 + a1 x + a2 x^2 + ... + an x^n

(  N   ) a0 + (∑ xi    ) a1 + (∑ xi^2  ) a2 + (∑ xi^n  ) an = ∑      yi
(∑ xi  ) a0 + (∑ xi^2  ) a1 + (∑ xi^3  ) a2 + (∑ xi^n+1) an = ∑ xi   yi
(∑ xi^2) a0 + (∑ xi^3  ) a1 + (∑ xi^4  ) a2 + (∑ xi^n+2) an = ∑ xi^2 yi
(∑ xi^n) a0 + (∑ xi^n+1) a1 + (∑ xi^n+2) a2 + (∑ xi^2n ) an = ∑ xi^n yi

# Standard Error, Sy/x = √( Sr / (n-(m+1)) ) = √(3.74657/(6-(2+1))) = 1.12
# where n = number of points, m = degree of polynomial

'''

2. Interpolation and Curve Fitting/6. Polynomial (Least Squares) Regression or Fit.py


import numpy as np
import matplotlib.pyplot as plt

# Ejercicio 1

def FD(f, x, h):
    return (f(x+h)-f(x))/h

def CD(f, x, h):
    return (f(x+h/2)-f(x-h/2))/h

def ED(f, x, h):
    return (4*CD(f, x, h/2)-CD(f, x, h))/3.0

def get_error(f, deriva_f_analitica, deriva_f_numerica, x):
    h_range = np.logspace(-15,-1, 200)
    analitica = deriva_f_analitica(x)
    error_range = np.abs((deriva_f_numerica(f, x, h_range) - analitica)/analitica)
    return h_range, error_range

plt.figure(figsize=(15,10))

i = 1
ff = [np.exp, np.sin]
gg = [np.exp, np.cos]
names = ["exp(x)", "sin(x)"]
xx = [0.1, 1.0, 100.0]

for f, g, n, in zip(ff, gg, names):
    for x in xx:
        plt.subplot(2,3,i)

        h, e = get_error(f, g, FD, x); plt.plot(h, e, label="FD")
        h, e = get_error(f, g, CD, x); plt.plot(h, e, label="CD")
        h, e = get_error(f, g, ED, x); plt.plot(h, e, label="ED")

        plt.title("f(x)={}, x={:.1f}".format(n,x))
        plt.xlim([1E-15,1E-1])
        plt.ylim([1E-15,1E-1])
        plt.loglog()
        plt.legend()
        plt.grid()
        plt.xlabel("x")
        plt.ylabel("|error primera derivada|")
        i += 1

plt.savefig("primera_derivada.png", bbox_inches='tight')

# Ejercicio 2

def CD2(f, x, h):
    return (CD(f, x+h/2, h) - CD(f, x-h/2, h))/h

def CD2_bis(f, x, h):
    return (f(x+h) + f(x-h) - 2*f(x))/(h**2)

def minus_cos(x):
    return -np.cos(x)

plt.figure(figsize=(15,10))

i = 1
xx = [0.1, 1.0, 100.0]
for x in xx:
    plt.subplot(2,3,i)
    h, e = get_error(np.cos, minus_cos, CD2, x); plt.plot(h,e, label="CD2")
    h, e = get_error(np.cos, minus_cos, CD2_bis, x); plt.plot(h,e, label="CD2_bis")
    plt.xlim([1E-15,1E-1])
    plt.ylim([1E-15,1E-1])
    plt.loglog()
    plt.legend()
    plt.grid()
    plt.xlabel("x")
    plt.ylabel("|error segunda derivada|")
    plt.title("f(x)=cos(x), x={:.1f}".format(x))
    i += 1

plt.savefig("segunda_derivada.png", bbox_inches='tight')


import math
import matplotlib.pyplot as plt
import numpy as np

data = [[0,20.1],[60,23],[120,26.9],[180,30],[240,33.1],[300,36.9]]

datax = []
datay = []

class LinearRegression():

    def __init__(self):
        pass

    def sigmax(self):
        a1 = 0
        for a in range(len(data)):
            a1 = a1+(data[a][0])
        return a1

    def xbar(self):
        return self.sigmax() / len(data)

    def sigmaxsq(self):
        b1 = 0
        for b in range(len(data)):
            b1 = b1+(data[b][0])**2
        return b1

    def sigmay(self):
        d1 = 0
        for d in range(len(data)):
            d1 += data[d][1]
        return d1

    def ybar(self):
        return self.sigmay() / len(data)

    def sigmaysq(self):
        e1 = 0
        for e in range(len(data)):
            e1 += (data[e][1])**2
        return e1

    def sigmaxy(self):
        g1 = 0
        for g in range(len(data)):
            g1 += (data[g][0])*(data[g][1])
        return g1

    def SXX(self):
        return ((self.sigmaxsq()) - (self.sigmax()**2) / len(data))

    def SYY(self):
        return ((self.sigmaysq()) - ((self.sigmay())**2 / len(data)))

    def SXY(self):
        return ((self.sigmaxy()) - ((self.sigmax()*self.sigmay()) / len(data)))

    def b(self):
        return self.SXY() / self.SXX()

    def a(self):
        return (self.b()*-self.xbar()) + (self.ybar())

    def linregres(self):
        return ('y =  '+str(self.b())+'x'+' + '+str(self.a()))

    def PMCC(self):
        pmcc = ((self.SXY()) / math.sqrt(self.SXX()*self.SYY()))
        return ('PMCC = '+str(pmcc))

    def scatter(self):
        for i in range(len(data)):
            datax.append(data[i][0])
        for p in range(len(data)):
            datay.append(data[p][1])

if __name__ == '__main__':
    lr = LinearRegression()
    lr.scatter()
    print(lr.linregres())
    print(lr.PMCC())
    m = lr.b()
    c = lr.a()

    x = np.linspace(0,300,10)
    y = m*x + c
    plt.plot(x, y, 'r')
    plt.scatter(datax, datay)
    plt.show()

    inp = input('predict values?')
    if inp == 'y':
        x = float(input('value for x?'))
        y = (m*x) + c
        print(y)
    else:
        pass


from IMLearn.learners import UnivariateGaussian, MultivariateGaussian
import numpy as np
import plotly.graph_objects as go
import plotly.express as px
import plotly.io as pio
pio.templates.default = "simple_white"

def test_univariate_gaussian():
    # Question 1 - Draw samples and print fitted model
    mu = 10
    sigma = 1
    sample_size = 1000
    samples = np.random.normal(mu, sigma, sample_size)
    model = UnivariateGaussian()
    model.fit(samples)
    print(f'({model.mu_}, {model.var_})')

    # Question 2 - Empirically showing sample mean is consistent
    def estimated_expected_value(samples):
        model = UnivariateGaussian()
        model.fit(samples)
        return model.mu_

    sample_counts = np.arange(10, sample_size+1, 10)
    expected_values = np.array([estimated_expected_value(samples[:i]) for i in sample_counts])
    distance = abs(expected_values - mu)

    x = sample_counts
    y = distance
    fig = px.line(x=x, y=y,
                  title='Estimation Distance',
                  labels={'x':'sample size', 'y':'estimation error'})
    fig.show()

    # Question 3 - Plotting Empirical PDF of fitted model
    x = samples
    y = model.pdf(samples)
    fig = px.scatter(x=x, y=y, title='Sample PDFs', labels={'x':'sample', 'y':'pdf'})
    fig.show()

def test_multivariate_gaussian():
    # Question 4 - Draw samples and print fitted model
    mu = np.array([0, 0, 4, 0])
    sigma = np.array([[1.0, 0.2, 0, 0.5],
                      [0.2, 2.0, 0,   0],
                      [0,   0,   1,   0],
                      [0.5, 0,   0,   1]])
    sample_size = 1000
    samples = np.random.multivariate_normal(mu, sigma, sample_size)

    model = MultivariateGaussian()
    model.fit(samples)
    print(model.mu_)
    print(model.cov_)

    # Question 5 - Likelihood evaluation
    f1 = np.linspace(-10, 10, 200)
    f3 = np.linspace(-10, 10, 200)
    # very slow try to vectorize
    data = np.zeros((200, 200))
    for i, v1 in enumerate(f1):
        for j, v3 in enumerate(f3):
            data[i, j] = MultivariateGaussian.log_likelihood(np.array([v1, 0, v3, 0]), sigma, samples)

    fig = px.imshow(data, labels=dict(x="f3", y="f1", color="Log-likelihood"),
                    x=f1, y=f3)
    fig.show()

    # Question 6 - Maximum likelihood
    max_index = np.unravel_index(data.argmax(), data.shape)
    max_param = (f1[max_index[0]], f3[max_index[1]])
    print(f'({max_param[0]:.3}, {max_param[1]:.3})')

if __name__ == '__main__':
    np.random.seed(0)
    test_univariate_gaussian()
    test_multivariate_gaussian()


import numpy as np

array_1d = np.array([1.2, 2.4, 3.5, 4.7, 6.1, 7.2, 8.3, 9.5])
print(array_1d)

array_2d = np.array([[6, 5], [11, 7], [4, 8]])
print(array_2d)

# populate arrays with sequences of number
array_seq = np.arange(5, 12)
print(array_seq)

# populate with random numbers
array_random_int_50_to_100 = np.random.randint(low=50, high=101, size=(6))
print(array_random_int_50_to_100)

array_random_float_0_to_1 = np.random.random((6))
print(array_random_float_0_to_1)

# math op
# NumPy uses a trick called broadcasting to virtually expand the smaller operand to dimensions compatible for linear algebra
random_floats_2_to_3 = np.random.random((6)) + 2
print('random_floats_2_to_3', random_floats_2_to_3)

# Create a Linear Dataset
# Your goal is to create a simple dataset consisting of a single feature and a label as follows:
# Assign a sequence of integers from 6 to 20 (inclusive) to a NumPy array named feature.
# Assign 15 values to a NumPy array named label such that:
#    label = (3)(feature) + 4
# For example, the first value for label should be:
#   label = (3)(6) + 4 = 22

features = np.arange(6, 21)
print("features", features)
labels = features * 3 + 4
print("labels", labels)

# Add Some Noise to the Dataset
# To make your dataset a little more realistic, insert a little random noise into each element
# of the label array you already created. To be more precise, modify each value assigned to label
# by adding a different random floating-point value between -2 and +2.
# Don't rely on broadcasting. Instead, create a noise array having the same dimension as label.
noise = np.random.random(size=labels.size) * 4 - 2
print('noise', noise)
noised_label = noise + labels
print('noised_label', noised_label)


import numpy as np

def bisection(f,a,b,TOL,NMAX):
    # THIS FUNCTION PRINTS THE a,b,c VALUES FOR EACH ITERATION:

    #Approximates root using Bisection Method
    #In an interval[a,b] with a tolerance TOL
    #|f(c)| < TOL, where m is the midpoint

    #INPUT : f , a , b , TOL , NMAX
    # f : function / polynomial
    # a : interval start
    # b : interval end
    # TOL : tolerance value
    # NMAX : maximum number of iterations

    #Print Table Header:
    print('--------------------------------------------------------------------------')
    print('iter \t\t a \t\t b \t\t c \t\t f(c)        ')
    print('--------------------------------------------------------------------------')
    #satisfy the conditions needed to apply the Bisection Method:
    if np.sign(f(a)) == np.sign(f(b)):
        print("No root found in the given interval!")
        exit()

    for i in range(NMAX):
        #Compute Midpoint
        c = (a+b)/2
        #print line for the table:
        N.append(1+i)
        f_x.append(f(c))
        print(str(1+i)+'\t% 15.12f\t% 15.12f\t% 15.12f\t% 15.12f\t' %(a, b, c, f(c)))
        #Check stopping condition:
        if np.abs(f(c)) < TOL:
            print('------------------------------------------------------------------------')
            print('Root Found: '+str(c))
            break
        #Implement Recursion:
        elif np.sign(f(a)) == np.sign(f(c)):
            #Improvement on a
            a = c
        elif np.sign(f(b)) == np.sign(f(c)):
            #Improvement on b
            b = c

    if i == NMAX -1:
        print("MAX NUMBER OF ITERATIONS REACHED!")
        print('Approximaiton to the Root after max iterations is : '+str(c))
        exit()

def newton(f,fp,x0,TOL,NMAX):
    #INPUT : f , fp , x0 , TOL , NMAX
    # f : function / polynomial
    # fp : derivative of f
    # x0 : initial guess
    # TOL : tolerance value
    # NMAX : maximum number of iterations

    #Approximates root using Newton's Method
    #Recursive Program

    print('--------------------------------------------------------------------------')
    print('iter \t\t xi \t\t   correction \t\t   rdiff        ')
    print('--------------------------------------------------------------------------')
    # initiate values for  iteration loop:
    rdiff = 1
    xi = x0
    counter = 0
    while rdiff > TOL and counter < NMAX:
        # get the number of necessary iterations at that particular x0
        # compute relative difference
        rdiff = np.abs(f(xi)/fp(xi)/xi)
        # next xi:
        x1 = xi - f(xi)/fp(xi)
        N.append(counter+1)
        f_x.append(f(x1))
        # print iteration data:
        print('%i \t %15.12f \t %15.12f \t %15.12f' % (counter+1, x1, np.abs(f(xi)/fp(xi)),  rdiff))
        # prepare for the next iteration:
        xi = x1
        counter += 1
        if counter == NMAX:
            print("MAX NUMBER OF ITERATIONS REACHED!")
            print('Approximaiton to the Root after max iterations is : ' , x1)
            exit()

    print('------------------------------------------------------------------------')
    print('Root Found: ', x1)


# Undergraduate Student: Arturo Burgos
# Professor: João Rodrigo Andrade
# Federal University of Uberlândia - UFU, Fluid Mechanics Laboratory - MFLab, Block 5P, Uberlândia, MG, Brazil

# Third exercise: Fibonacci sequence - by a common loop

import numpy as np
import time

n = int(input("Enter the n indices: "))
Fbn=0 # Value of the first box
A = 0 # Values of the second
B = 1 # Value of the third box

##############################
# Variable iterative fibonacci  --> Better performance, why?
##############################

def varloop_fibonacci(k,F1,F2,S):

    for i in range(0,k):
        F1=S
        S = F1+F2
        F2 = F1

    return S

t_initial1 = time.time()
Fbn = varloop_fibonacci(n,A,B,Fbn)

print("Elapsed time is: %s seconds" % (time.time()-t_initial1))
print("The correspond indices number is: ",Fbn)

print("\n")

###########################
# Array iterative fibonacci  --> Worse performance, why?
###########################

Fbn=0 # Value of the first box
A = np.array([0, 1]) # Values of the second an third boxes

def arrayloop_fibonacci(k,F,S):

    for i in range(0,k):
        F[0]=S
        S = sum(F)
        F[1] = F[0]
    return S

t_initial2 = time.time()
Fbn = arrayloop_fibonacci(n,A,Fbn)

print("Elapsed time is: %s seconds" % (time.time()-t_initial2))
print("The correspond indices number is: ",Fbn)


## 2. Calculating expected values ##

males_over50k = .67 * .241 * 32561
males_under50k = .67 * .759 * 32561
females_over50k = .33 * .241 * 32561
females_under50k = .33 * .759 * 32561

## 3. Calculating chi-squared ##

observed = [6662, 1179, 15128, 9592]
expected = [5257.6, 2589.6, 16558.2, 8155.6]
values = []

for i, obs in enumerate(observed):
    exp = expected[i]
    value = (obs - exp) ** 2 / exp
    values.append(value)

chisq_gender_income = sum(values)

## 4. Finding statistical significance ##

import numpy as np
from scipy.stats import chisquare

observed = np.array([6662, 1179, 15128, 9592])
expected = np.array([5257.6, 2589.6, 16558.2, 8155.6])

chisq_value, pvalue_gender_income = chisquare(observed, expected)

## 5. Cross tables ##

import pandas
table = pandas.crosstab(income["sex"], [income["race"]])
print(table)

## 6. Finding expected values ##

import pandas
from scipy.stats import chi2_contingency

table = pandas.crosstab(income["sex"], [income["race"]])
chisq_value, pvalue_gender_race, df, expected = chi2_contingency(table)

Data Analyst in Python/Step 5 - Probability and Statistics/5. Hypothesis Testing Fundamentals/3. Multi category chi-squared tests.py