# Imports
import numpy as np

# Each row is a training example, each column is a feature  [X1, X2, X3]
X=np.array(([0,0,1],[0,1,1],[1,0,1],[1,1,1]),dtype=float)
y=np.array(([0,1],[1,0],[1,0],[0,1]))

# y=np.array(([0,1],[1,0],[1,0],[0,1]), dtype=float)

# Define useful functions

# Activation function
def sigmoid(t):
    return 1 / (1 + np.exp(-t))

# Derivative of sigmoid
def sigmoid_derivative(p):
    return p * (1 - p)

# Class definition
class NeuralNetwork:
    def __init__(self, x, y):
        self.input = x
        self.weights1 = np.random.rand(self.input.shape[1], 6)  # considering we have 4 nodes in the hidden layer
        self.weights2 = np.random.rand(6, 2)
        self.y = y
        self.output = np.zeros(y.shape)

    def feedforward(self):
        self.layer1 = sigmoid(np.dot(self.input, self.weights1))
        self.layer2 = sigmoid(np.dot(self.layer1, self.weights2))
        return self.layer2

    def backprop(self):
        d_weights2 = np.dot(self.layer1.T, 2 * (self.y - self.output) * sigmoid_derivative(self.output))
        d_weights1 = np.dot(self.input.T, np.dot(2 * (self.y - self.output) *
                                                 sigmoid_derivative(self.output), self.weights2.T) *
                            sigmoid_derivative(self.layer1))

        self.weights1 += d_weights1
        self.weights2 += d_weights2

    def train(self, X, y):
        self.output = self.feedforward()
        self.backprop()

    def predict(self,x):
        """

        :param x:
        :return prediction :
        """
        x=np.array(x,ndmin=2)
        r = x @ self.weights1
        r=sigmoid(r)
        p=sigmoid(r@self.weights2)
        return p

NN = NeuralNetwork(X, y)
for i in range(1500):  # trains the NN 1,000 times
    if i % 300 == 0:
        print("for iteration # " + str(i) + "\n")
        print("Input : \n" + str(X))
        print("Actual Output: \n" + str(y))
        print("Predicted Output: \n" + str(NN.feedforward()))
        print("Loss: \n" + str(np.mean(np.square(y - NN.feedforward()))))  # mean sum squared loss
        print("\n")

    NN.train(X, y)
print(NN.weights1)
print(NN.weights2)

w1=NN.weights1
w2=NN.weights2

print(w1)
print(w2)

x1=[0,0,0]
x2=[1,1,1]

print(NN.predict(x1))
print(NN.predict(x2))


import numpy as np

def activation_fn(x):
    f = 1/(1+np.exp(-x)) #-sigmoid fn
    return f

def weighted_biased_act_fn(x,w=1,b=0):
    f= 1/(1+np.exp(-(x*w+b)))
    return f

def looped_nn_calc(n_layers, x, w, b):
    for l in range(n_layers-1):
        #Setup the input array which the weights will be multiplied by for each layer
        #If it's the first layer, the input array will be the x input vector
        #If it's not the first layer, the input to the next layer will be the
        #output of the previous layer
        if l == 0:
            node_in = x
        else:
            node_in = h
        #Setup the output array for the nodes in layer l + 1
        h = np.zeros((w[l].shape[0],))
        #loop through the rows of the weight array
        for i in range(w[l].shape[0]):
            #setup the sum inside the activation function
            f_sum = 0
            #loop through the columns of the weight array
            for j in range(w[l].shape[1]):
                f_sum += w[l][i][j] * node_in[j]
            #add the bias
            f_sum += b[l][i]
            #finally use the activation function to calculate the
            #i-th output i.e. h1, h2, h3
            h[i] = activation_fn(f_sum)
    return h

def matrix_feed_forward_calc(n_layers, x, w, b): #- vectorized
    for l in range(n_layers-1):
        if l == 0:
            node_in = x
        else:
            node_in = h
        z = w[l].dot(node_in) + b[l]
        h = activation_fn(z)
    return h


### The MILP problems can be solved using pulp packages that powered by B&B algorithm
### provides the ability to get the global optimum for not only LP but also MILP
import pandas as pd
from numpy.linalg import norm
import numpy as np
import pulp as plp
#%%
def dist(df):
    dist_mat = np.zeros((df.shape[0], df.shape[0]))
    for i in range(df.shape[0]):
        for j in range(df.shape[0]):
            dist_mat[i,j] = norm(df.iloc[i,:].values - df.iloc[j,:].values)**2
    return dist_mat
def k_median(df, K):
    dist_mat = dist(df)
    n = df.shape[0]
    k = K
    set_I = range(n)
    set_J = range(n)
    opt_model = plp.LpProblem(name="MILP_k-median_Model")
    # if x is Binary
    x_vars  =  {"x_{0}_{1}".format(i,j): plp.LpVariable(cat = plp.LpBinary, name="x_{0}_{1}".format(i,j)) for i in set_I for j in set_J}
    y_vars  =  {"y_{0}".format(j): plp.LpVariable(cat = plp.LpBinary, name="y_{0}".format(j)) for j in set_J}
    vars_   = {**x_vars, **y_vars}
    # == constraints
    con1 = {"con1_{0}".format(i) :
            opt_model.addConstraint(plp.LpConstraint(
                e = plp.lpSum(vars_["x_{0}_{1}".format(i,j)]
                              for j in set_J),
                sense = plp.LpConstraintEQ,
                rhs = 1,
                name="con1_{0}".format(i)))
            for i in set_I}

    con2 = {"con2" : opt_model.addConstraint(plp.LpConstraint(e = plp.lpSum(vars_["y_{0}".format(j)] for j in set_J), sense = plp.LpConstraintEQ , rhs = k, name="con2"))}

    con3 = {"con3_{0}_{1}".format(i,j) : opt_model.addConstraint(plp.LpConstraint(e = plp.lpSum([vars_["x_{0}_{1}".format(i,j)], -vars_["y_{0}".format(j)]]), sense = plp.LpConstraintLE, rhs = 0,name="con3_{0}_{1}".format(i,j))) for i in set_I for j in set_J}
    con = {**con1, **con2, **con3}
    objective = plp.lpSum(vars_["x_{0}_{1}".format(i,j)] * dist_mat[i,j]
                          for i in set_I
                          for j in set_J)
    # for minimization
    opt_model.sense = plp.LpMinimize
    opt_model.setObjective(objective)

    # solving with local cplex
    opt_model.solve()

    opt_model.numConstraints()

    opt_df = pd.DataFrame.from_dict(x_vars, orient="index",
                                    columns = ["variable_object"])

    opt_df["solution_value"] = opt_df["variable_object"].apply(lambda item: item.varValue)

    opt_df['i'] = opt_df['variable_object'].apply(lambda x: int(x.name.split("_")[1]))
    opt_df['j'] = opt_df['variable_object'].apply(lambda x: int(x.name.split("_")[2]))

    return (pd.pivot_table(data = opt_df, values='solution_value', index ='i', columns='j').values*dist_mat).sum()

#%%
def plot_res():
    p1 =       40.41
    p2 =      -816.4
    p3 =        6109
    p4 =  -2.054e+04
    p5 =    2.95e+04
    p6 =  -1.069e+04

    def f(t):
        return p1*t**5 + p2*t**4 + p3*t**3 + p4*t**2 + p5*t**1 + p6

    import matplotlib.pyplot as plt
    x = np.arange(2,7,.1)
    y = [f(t) for t in x]
    #print(x)
    #print(y)
    plt.figure(figsize=(10,6))
    plt.ylim(1200,3250)
    plt.plot(x,y)
    plt.scatter(x[10],y[10], marker="d", lw = 5, color = 'blue')
    plt.scatter(x[12],y[12], marker="*", lw = 5, color = 'r')
    plt.scatter(x[43],y[43], marker="o", lw = 5, color = 'green')
    plt.grid(color='r', linestyle='-', linewidth=.1, axis='x')
    plt.text(x=2.1, y=1556, s='k-median', color = 'blue')
    plt.text(x=3.1, y=1356, s='Global minimum', color = 'r')
    plt.text(x=6.2, y=2276, s='k-means', color = 'green')
    plt.ylabel('Cost')
    plt.xticks([1,2,3,4,5,6,7],["","","","","","","",""])
    #print(np.array(y).argmin())


import numpy as np

def snell(theta_inc, n1, n2):
    """
    This is an awesome docstring.
    Compute the refraction angle using Snell's law
    See https://en.wikipedia.org/Wiki/Snell%27s_law

    Parameters
    ----------
    theta_inc : float
        Incident angle in radians

    n1, n2 : float
        The refractive index of medium of origin and destination medium

    Returns
    -------
    theta : float
        Refraction angle

    Examples
    --------
    A ray enters an air-water boundary at pi/4 radians (45 degrees), compute exit angle

    >>> snell(np.pi/4, 1..00, 1.33)
    >>> 0.560558

    """

    return np.arcsin(n1 / n2 * np.sin(theta_inc))


import numpy as np

def sigmoid(z):
#SIGMOID Compute sigmoid functoon
#   J = SIGMOID(z) computes the sigmoid of z.

    g = 1/(1 + np.exp(-z))
    return g


# coding=utf-8
"""
Daniel Calderon, CC3501, 2019-1
Transformation matrices for computer graphics
v2.0
"""

import numpy as np

pi = 3.1415

def identity():
    return np.identity(4, dtype=np.float32)

def uniformScale(s):
    return np.array([
        [s,0,0,0],
        [0,s,0,0],
        [0,0,s,0],
        [0,0,0,1]], dtype = np.float32)

def scale(sx, sy, sz):
    return np.array([
        [sx,0,0,0],
        [0,sy,0,0],
        [0,0,sz,0],
        [0,0,0,1]], dtype = np.float32)

def rotationX(theta):
    sin_theta = np.sin(theta)
    cos_theta = np.cos(theta)

    return np.array([
        [1,0,0,0],
        [0,cos_theta,-sin_theta,0],
        [0,sin_theta,cos_theta,0],
        [0,0,0,1]], dtype = np.float32)

def rotationY(theta):
    sin_theta = np.sin(theta)
    cos_theta = np.cos(theta)

    return np.array([
        [cos_theta,0,sin_theta,0],
        [0,1,0,0],
        [-sin_theta,0,cos_theta,0],
        [0,0,0,1]], dtype = np.float32)

def rotationZ(theta):
    sin_theta = np.sin(theta)
    cos_theta = np.cos(theta)

    return np.array([
        [cos_theta,-sin_theta,0,0],
        [sin_theta,cos_theta,0,0],
        [0,0,1,0],
        [0,0,0,1]], dtype = np.float32)

def rotationA(theta, axis):
    s = np.sin(theta)
    c = np.cos(theta)

    assert axis.shape == (3,)

    x = axis[0]
    y = axis[1]
    z = axis[2]

    return np.array([
        # First row
        [c + (1 - c) * x * x,
        (1 - c) * x * y - s * z,
        (1 - c) * x * z + s * y,
        0],
        # Second row
        [(1 - c) * x * y + s * z,
        c + (1 - c) * y * y,
        (1 - c) * y * z - s * x,
        0],
        # Third row
        [(1 - c) * x * z - s * y,
        (1 - c) * y * z + s * x,
        c + (1 - c) * z * z,
        0],
        # Fourth row
        [0,0,0,1]], dtype = np.float32)

def translate(tx, ty, tz):
    return np.array([
        [1,0,0,tx],
        [0,1,0,ty],
        [0,0,1,tz],
        [0,0,0,1]], dtype = np.float32)

def shearing(xy, yx, xz, zx, yz, zy):
    return np.array([
        [ 1, xy, xz, 0],
        [yx,  1, yz, 0],
        [zx, zy,  1, 0],
        [ 0,  0,  0, 1]], dtype = np.float32)

def matmul(mats):
    out = mats[0]
    for i in range(1, len(mats)):
        out = np.matmul(out, mats[i])

    return out

def composes(A,B):
    return np.matmul(A,B)

def frustum(left, right, bottom, top, near, far):
    r_l = right - left
    t_b = top - bottom
    f_n = far - near
    return np.array([
        [ 2 * near / r_l,
        0,
        (right + left) / r_l,
        0],
        [ 0,
        2 * near / t_b,
        (top + bottom) / t_b,
        0],
        [ 0,
        0,
        -(far + near) / f_n,
        -2 * near * far / f_n],
        [ 0,
        0,
        -1,
        0]], dtype = np.float32)

def perspective(fovy, aspect, near, far):
    halfHeight = np.tan(np.pi * fovy / 360) * near
    halfWidth = halfHeight * aspect
    return frustum(-halfWidth, halfWidth, -halfHeight, halfHeight, near, far)

def ortho(left, right, bottom, top, near, far):
    r_l = right - left
    t_b = top - bottom
    f_n = far - near
    return np.array([
        [ 2 / r_l,
        0,
        0,
        -(right + left) / r_l],
        [ 0,
        2 / t_b,
        0,
        -(top + bottom) / t_b],
        [ 0,
        0,
        -2 / f_n,
        -(far + near) / f_n],
        [ 0,
        0,
        0,
        1]], dtype = np.float32)

def lookAt(eye, at, up):

    forward = (at - eye)
    forward = forward / np.linalg.norm(forward)

    side = np.cross(forward, up)
    side = side / np.linalg.norm(side)

    newUp = np.cross(side, forward)
    newUp = newUp / np.linalg.norm(newUp)

    return np.array([
            [side[0],       side[1],    side[2], -np.dot(side, eye)],
            [newUp[0],     newUp[1],   newUp[2], -np.dot(newUp, eye)],
            [-forward[0], -forward[1], -forward[2], np.dot(forward, eye)],
            [0,0,0,1]
        ], dtype = np.float32)


# -*- coding: utf-8 -*-
"""
Spyder Editor

This is a temporary script file.
"""
import numpy as np
import math
import matplotlib.pyplot as plt
import random

def kmeans(sample,K,maxiter):
    N = sample.shape[0]
    D = sample.shape[1]
    C = np.zeros((K,D))
    L = np.zeros((N,1))
    L1 = np.zeros((N,1))
    dist = np.zeros((N,K))
    idx = random.sample(range(N),K)
    C = sample[idx,:]
    iter = 0
    while(iter<maxiter):
        for i in range(K):
            dist[:,i] = np.sum((sample-np.tile(C[i,:],(N,1)))**2,1)
        L1 = np.argmin(dist,1)
        if(iter>0 and np.array_equal(L,L1)):
            break
        L = L1
        for i in range(K):
            idx = np.nonzero(L==i)[0]
            if(len(idx)>0):
                C[i,:] = np.mean(sample[idx,:],0)
        iter += 1
    wicd = np.sum(np.sqrt(np.sum((sample-C[L,:])**2,1)))
    return C,L,wicd

G1 = np.random.normal(0,1,(5000,2))
G1 = G1+4
G2 = np.random.normal(0,1,(3000,2))
G2[:,1] = G2[:,1]*3 - 3
G = np.append(G1,G2,axis=0)
G3 = np.random.normal(0,1,(2000,2))
G3[:,1] = G3[:,1]*4
c45 = math.cos(-45/180*math.pi)
s45 = math.sin(-45/180*math.pi)
R = np.array([[c45,-s45],[s45,c45]])
G3 = G3.dot(R)
G3[:,0] = G3[:,0]-4
G3[:,1] = G3[:,1]+6
G = np.append(G,G3,axis=0)
plt.plot(G[:,0],G[:,1],'.')
C,L,wicd = kmeans(G,3,1000)
G1 = G[L==0,:]
G2 = G[L==1,:]
G3 = G[L==2,:]
plt.plot(G1[:,0],G1[:,1],'r.',G2[:,0],G2[:,1],'g.',G3[:,0],G3[:,1],'b.',C[:,0],C[:,1],'kx')
print(wicd)

G = (G-np.tile(G.mean(0),(G.shape[0],1)))/np.tile(G.std(0),(G.shape[0],1))
for i in range(G.shape[1]):
    meanv = np.mean(G[:,i])
    stdv = np.std(G[:,i])
    G[:,i] = (G[:,i]-meanv)/stdv


"""Generate the exercice results on the Gumbell distribution
"""
import numpy as np
from scipy.interpolate import UnivariateSpline
import pylab as pl

def gumbell_dist(arr):
    return -np.log(-np.log(arr))

years_nb = 21
wspeeds = np.load('data/sprog-windspeeds.npy')
max_speeds = np.array([arr.max() for arr in np.array_split(wspeeds, years_nb)])
sorted_max_speeds = np.sort(max_speeds)

cprob = (np.arange(years_nb, dtype=np.float32) + 1)/(years_nb + 1)
gprob = gumbell_dist(cprob)
speed_spline = UnivariateSpline(gprob, sorted_max_speeds, k=1)
nprob = gumbell_dist(np.linspace(1e-3, 1-1e-3, 1e2))
fitted_max_speeds = speed_spline(nprob)

fifty_prob = gumbell_dist(49./50.)
fifty_wind = speed_spline(fifty_prob)

pl.figure()
pl.bar(np.arange(years_nb) + 1, max_speeds)
pl.axis('tight')
pl.xlabel('Year')
pl.ylabel('Annual wind speed maxima [$m/s$]')
pl.savefig('intro/summary-exercices/sprog-annual-maxima.png')

pl.figure()
pl.plot(sorted_max_speeds, gprob, 'o')
pl.plot(fitted_max_speeds, nprob, 'g--')
pl.plot([fifty_wind], [fifty_prob], 'o', ms=8., mfc='y', mec='y')
pl.plot([fifty_wind, fifty_wind], [pl.axis()[2], fifty_prob], 'k--')
pl.text(35, -1, r'$V_{50} = %.2f \, m/s$' % fifty_wind)
pl.xlabel('Annual wind speed maxima [$m/s$]')
pl.ylabel('Gumbell cumulative probability')
pl.savefig('intro/summary-exercices/gumbell-wind-speed-prediction.png')


"""Linear Regression code
Find slope and intercept
"""

import numpy as np
import matplotlib.pyplot as plt
from matplotlib import style
import random

style.use('fivethirtyeight')

def create_dataset(how_many_points, variance, step=2, correlation='pos'):
    val = 1
    ys = []
    for i in range(how_many_points):
        yt = val + random.randrange(-variance, variance)
        ys.append(yt)
        if correlation and correlation == 'pos':
            val += step
        elif correlation and correlation == 'neg':
            val -= step
    xs = [i for i in range(len(ys))]
    return np.array(xs, dtype=np.float64), np.array(ys, dtype=np.float64)

def best_fit_slope_and_intercept(xs, ys):
    """
    Calculates slope and intercept from input points
    :param xs: x points
    :param ys: y points
    :return: slope m, intercept b
    """
    get_m = (((np.mean(xs) * np.mean(ys)) - np.mean(xs * ys)) /
             ((np.mean(xs) * np.mean(xs)) - np.mean(xs * xs)))
    get_b = np.mean(ys) - get_m * np.mean(xs)
    return get_m, get_b

def squared_error(y_orig, y_line):
    """
    Total sum of squared errors
    :param y_orig: original y points
    :param y_line: fitted y points
    :return: int square error
    """
    return sum((y_line - y_orig) ** 2)

def coefficient_of_determination(y_orig, y_line):
    """
    :param y_orig: original y points
    :param y_line: fitted y points
    :return: int coefficient of determination
    """
    y_mean_line = np.array([np.mean(y_orig) for _ in y_orig])
    square_error_regression = squared_error(y_orig, y_line)
    square_error_y_mean = squared_error(y_orig, y_mean_line)
    return 1 - (square_error_regression / square_error_y_mean)

# Points
# x = np.array([1, 2, 3, 4, 5, 6])
# y = np.array([5, 4, 6, 5, 6, 7])

x, y = create_dataset(40, 10, 2, 'pos')

# Fit and predict
m, b = best_fit_slope_and_intercept(x, y)

regression_line = np.array([(m * x1) + b for x1 in x])

predict_x = 45
predict_y = m * predict_x + b

r_squared = coefficient_of_determination(y, regression_line)
print(r_squared)

# Plot
plt.scatter(x, y)
plt.scatter(predict_x, predict_y, color='g')
plt.plot(x, regression_line)
plt.show()


from sympy import *

x, y = symbols('x y')
all_symbols = [x]

psi=0.1*(x)*(x)*(x) + 0.5*(x)*(x)
target_value = 0
model_function = y - psi
obs_eq = Matrix([target_value - model_function]).vec()
obs_eq_jacobian = obs_eq.jacobian(all_symbols)

print(psi)
print(obs_eq)
print(obs_eq_jacobian)
print(latex(psi))
print(latex(obs_eq_jacobian))

with open("example_func_x_jacobian.h",'w') as f_cpp:
    f_cpp.write("inline void example_func_x(double &psi, double x)\n")
    f_cpp.write("{")
    f_cpp.write("psi = %s;\n"%(ccode(psi)))
    f_cpp.write("}")
    f_cpp.write("\n")
    f_cpp.write("inline void observation_equation_example_func_x(double &delta, double x, double y)\n")
    f_cpp.write("{")
    f_cpp.write("delta = %s;\n"%(ccode(obs_eq[0,0])))
    f_cpp.write("}")
    f_cpp.write("\n")
    f_cpp.write("inline void observation_equation_example_func_x_jacobian(Eigen::Matrix<double, 1, 1> &j, double x)\n")
    f_cpp.write("{")
    for i in range (1):
        for j in range (1):
            f_cpp.write("j.coeffRef(%d,%d) = %s;\n"%(i,j, ccode(obs_eq_jacobian[i,j])))
    f_cpp.write("}")


import numpy as np

from hand_crafted_models.activations import sigmoid
from hand_crafted_models.loss_functions import log_loss
from hand_crafted_models.optimization import gradient_descent, GradientStep, WeightsAndBias
from hand_crafted_models.utils import forward_pass

def _step(
        x: np.ndarray,
        y: np.ndarray,
        weights: np.ndarray,
        bias: np.ndarray,
        one: np.ndarray
) -> GradientStep:
    """
    Log-loss: Calculate gradients for a given step.

    :param x: Input data [Batch, Features]
    :param y: Label data [Batch, 1]
    :param weights: Feature parameters [1, Features]
    :param bias: Bias parameter [1, 1]
    :param one: (Ignore) Vector of ones
    :return: step loss, weight gradients, bias gradient
    """
    # Make prediction
    y_hat = sigmoid(logits=forward_pass(x=x, weights=weights, bias=bias))
    # Derive total loss value for current parameter values (i.e., Binary Cross-Entropy loss cost fn)
    loss = log_loss(y_hat=y_hat, y=y)
    # Perform back-propagation to parameters
    d_y = (y - y_hat)  # [B, 1]
    # Weights derivative w.r.t. Log-Loss loss fn
    d_w = d_y.T @ x  # [1, B] @ [B, N] -> [1, N]
    # Bias derivative w.r.t. Log-Loss loss fn
    d_b = d_y.T @ one  # [B, 1] -> [1]
    return loss, d_w, d_b

def get_beta_sgd(
        x: np.ndarray,
        y: np.ndarray,
        lr: float = 0.001,
        tol: float = 1e-6,
        max_grad: float = 10.0,
        max_loops: int = 10000
) -> WeightsAndBias:
    """
    Fit parameters using gradient descent.

    :param x: Input data [Batch, Features]
    :param y: Label data [Batch, 1]
    :param lr: Learning rate (i.e., optimizer step size)
    :param tol: Tolerance for early-stopping
    :param max_grad: (Optional) Max size of gradient
    :param max_loops: Maximum number of steps to take
    :return: weight gradients, bias gradient
    """
    return gradient_descent(
        x=x,
        y=y,
        fn=_step,
        lr=lr,
        tol=tol,
        max_grad=max_grad,
        max_loops=max_loops
    )


from RK4 import RK4
import numpy as np

# Removal rate (beta) 1/3 day-1
# Infected people that cease to take part in the transmission per day
beta = 1. / 3.  # day^{-1}

# R0=3 ,
# leads to alpha = 1 day-1
# One S<->I contact lead to a new infection per day
def alpha(time, beta):
    R0 = 3.0
    return R0 * beta

def testSystem(time, vector_S_I_R):
    """
    dI/dt = alpha * I * S - beta*I
    dR/dt = beta * I
    dS/dt = -dI/dt-dR/dt
    """
    dI = alpha(time, beta) * vector_S_I_R[1] * \
        vector_S_I_R[0] - beta * vector_S_I_R[1]
    dR = beta * vector_S_I_R[1]
    dS = -1 * dI - 1 * dR
    return np.array([dS, dI, dR])

if __name__ == "__main__":
    # NSteps
    time = 80
    Step = 0.1
    NSteps = int(time / Step)

    initI = 1e-05  # Initial percentage of Infected
    initS = 1. - initI  # Initial percentage of Susceptible
    initR = 0  # Initial percentage of Recovered
    myrk4 = RK4(testSystem, Step, 0, np.array([initS, initI, initR]))

    t = np.arange(0.0, time, Step)
    S = np.zeros(t.shape)
    I = np.zeros(t.shape)
    R = np.zeros(t.shape)
    for i in range(0, NSteps):
        S[i] = myrk4.currentValues()[0]
        I[i] = myrk4.currentValues()[1]
        R[i] = myrk4.currentValues()[2]
        myrk4.integrateStep()

    # plotting
    import matplotlib.pyplot as plt
    yticks = np.arange(0, 1.1, 1e-01)
    fig, ax = plt.subplots()
    ax.plot(t, S, label='S')
    ax.plot(t, I, label='I')
    ax.plot(t, R, label='R')
    ax.set_yticks(yticks)
    from matplotlib.ticker import FormatStrFormatter
    ax.set(xlabel='time (days)', ylabel='Fraction', title='SIR')
    ax.yaxis.set_major_formatter(FormatStrFormatter('%1.2f'))
    # ax.set_yscale('log')
    legend = ax.legend(loc='center right')
    fig.savefig("SIR.png")

    fig, ax = plt.subplots()
    ax.plot(t, I, label='I')
    legend = ax.legend(loc='center right')
    fig.savefig("I.png")


import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
#%matplotlib inline

# Define uma função que calcula o erro na aproximação theta
def computeCost(X, y, theta):
    inner = np.power(((X * theta.T) - y), 2)
    return np.sum(inner) / (2 * len(X))

# Define a Função de Gradiente Descent
def gradientDescent(X, y, theta, alpha, iters):
    temp = np.matrix(np.zeros(theta.shape))
    #print(temp)
    parameters = int(theta.ravel().shape[1])
    #print(parameters)
    cost = np.zeros(iters)
    #print(cost)

    for i in range(iters):
        error = (X * theta.T) - y

        for j in range(parameters):
            term = np.multiply(error, X[:,j])
            temp[0,j] = theta[0,j] - ((alpha / len(X)) * np.sum(term))

        theta = temp
        cost[i] = computeCost(X, y, theta)

    return theta, cost

path = os.getcwd() + '\data\ex1data1.txt'
data = pd.read_csv(path, header=None, names=['Population', 'Profit'])

#print(data.head())
#print(data.describe())

#data.plot(kind='scatter', x='Population', y='Profit', figsize=(12,8))

# Acrescenta uma coluna de 1s na frente dos dados
data.insert(0, 'Ones', 1)
#print(data.head())

# Define X (dados de treinamento) e y (objetivo)
cols = data.shape[1]
X = data.iloc[:,0:cols-1]
y = data.iloc[:,cols-1:cols]
#print(cols)
#print(X)
#print(y)

# Converte os frames de dados para matrizes numpy
X = np.matrix(X.values)
y = np.matrix(y.values)
theta = np.matrix(np.array([0,0]))
#print(X)
#print(y)
#print(theta)

#print(computeCost(X, y, theta))

# Inicializa as variáveis de learning rate e número de iterações
alpha = 0.01
iters = 1000

# Executa o gradiente descent para adequar ao modelo de parâmetros
g, cost = gradientDescent(X, y, theta, alpha, iters)
print(g) # Novo theta calculado após as iterações
# print(cost)

# Plotando os resultados
x = np.linspace(data.Population.min(), data.Population.max(), 100)
f = g[0, 0] + (g[0, 1] * x)

fig, ax = plt.subplots(figsize=(12,8))
ax.plot(x, f, 'r', label='Prediction')
ax.scatter(data.Population, data.Profit, label='Traning Data')
ax.legend(loc=2)
ax.set_xlabel('Population')
ax.set_ylabel('Profit')
ax.set_title('Predicted Profit vs. Population Size')

fig1, ax1 = plt.subplots(figsize=(12,8))
ax1.plot(np.arange(iters), cost, 'r')
ax1.set_xlabel('Iterations')
ax1.set_ylabel('Cost')
ax1.set_title('Error vs. Training Epoch')

plt.show()


from collections import Counter
import math
from Stats.CramersV import *
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from scipy.stats import entropy

# note for Theils u the given is the x or columns. so entropy of (y|x)
def conditional_entropy(x, y):
    """
    Calculates the conditional entropy of x given y: S(x|y)
    Wikipedia: https://en.wikipedia.org/wiki/Conditional_entropy
    **Returns:** float
    Parameters
    ----------
    x : list / NumPy ndarray / Pandas Series
        A sequence of measurements
    y : list / NumPy ndarray / Pandas Series
        A sequence of measurements
    """
    y_counter = Counter(y)
    xy_counter = Counter(list(zip(x,y)))
    total_occurrences = sum(y_counter.values())
    entropy = 0.0
    for xy in xy_counter.keys():
        p_xy = xy_counter[xy] / total_occurrences
        p_y = y_counter[xy[1]] / total_occurrences
        entropy += p_xy * math.log(p_y/p_xy)
#         entropy += p_xy * math.log(p_xy) # ali changed
    return entropy

def TheilsU(x, y):
    """
    Calculates Theil's U statistic (Uncertainty coefficient) for categorical-categorical association.
    This is the uncertainty of x given y: value is on the range of [0,1] - where 0 means y provides no information about
    x, and 1 means y provides full information about x.
    This is an asymmetric coefficient: U(x,y) != U(y,x)
    Wikipedia: https://en.wikipedia.org/wiki/Uncertainty_coefficient
    **Returns:** float in the range of [0,1]
    Parameters
    ----------
    x : list / NumPy ndarray / Pandas Series
        A sequence of categorical measurements
    y : list / NumPy ndarray / Pandas Series
        A sequence of categorical measurements
    """
    s_xy = conditional_entropy(x,y)
    x_counter = Counter(x)
    total_occurrences = sum(x_counter.values())
    p_x = list(map(lambda n: n/total_occurrences, x_counter.values()))
    s_x = entropy(p_x)
    if s_x == 0:
        return 1
    else:
        return (s_x - s_xy) / s_x

# https://github.com/shakedzy/dython/blob/master/build/lib/dython/nominal.py

def associations(dataset, nominal_columns=None, mark_columns=False, Theils_U=False, plot=True,
                          return_results = False,
                          savefigure=False, path_name=None,
                          title_auto=False,
                          cmap='YlOrRd',
                          **kwargs):
    """
    Calculate the correlation/strength-of-association of features in data-set with both categorical (eda_tools) and
    continuous features using:
     * Pearson's R for continuous-continuous cases
     * Correlation Ratio for categorical-continuous cases
     * Cramer's V or Theil's U for categorical-categorical cases
    **Returns:** a DataFrame of the correlation/strength-of-association between all features
    **Example:** see `associations_example` under `dython.examples`
    Parameters
    ----------
    dataset : NumPy ndarray / Pandas DataFrame
        The data-set for which the features' correlation is computed
    nominal_columns : string / list / NumPy ndarray
        Names of columns of the data-set which hold categorical values. Can also be the string 'all' to state that all
        columns are categorical, or None (default) to state none are categorical
    mark_columns : Boolean, default = False
        if True, output's columns' names will have a suffix of '(nom)' or '(con)' based on there type (eda_tools or
        continuous), as provided by nominal_columns
    theil_u : Boolean, default = False
        In the case of categorical-categorical feaures, use Theil's U instead of Cramer's V
    plot : Boolean, default = True
        If True, plot a heat-map of the correlation matrix
    return_results : Boolean, default = False
        If True, the function will return a Pandas DataFrame of the computed associations
    kwargs : any key-value pairs
        Arguments to be passed to used function and methods

    use as such:
    associations(X, nominal_columns='all', mark_columns=False, TheilsU=True, plot=True,
                          return_results = False)
    """

    import matplotlib.pyplot as plt
    csfont = {'fontname':'Times New Roman'}

    #dataset = convert(dataset, 'dataframe')
    columns = dataset.columns
    if nominal_columns is None:
        nominal_columns = list()
    elif nominal_columns == 'all':
        nominal_columns = columns
    corr = pd.DataFrame(index=columns, columns=columns)
    for i in range(0,len(columns)):
        for j in range(i,len(columns)):
            if i == j:
                corr[columns[i]][columns[j]] = 1.0
            else:
                if columns[i] in nominal_columns:
                    if columns[j] in nominal_columns:
                        if Theils_U:
                            title = 'Theil\'s U \n%d doctors, %d categorical specialties and religious affiliations'%(dataset.shape[0], dataset.shape[1])
                            corr[columns[j]][columns[i]] = TheilsU(dataset[columns[i]],dataset[columns[j]])
                            corr[columns[i]][columns[j]] = TheilsU(dataset[columns[j]],dataset[columns[i]])
                        else:
                            confusion_matrix = pd.crosstab(dataset[columns[i]],dataset[columns[j]])
                            cell = cramers_corrected_stat(confusion_matrix)
                            corr[columns[i]][columns[j]] = cell
                            corr[columns[j]][columns[i]] = cell
                    else:
                        cell = correlation_ratio(dataset[columns[i]], dataset[columns[j]])
                        corr[columns[i]][columns[j]] = cell
                        corr[columns[j]][columns[i]] = cell
                else:
                    if columns[j] in nominal_columns:
                        cell = correlation_ratio(dataset[columns[j]], dataset[columns[i]])
                        corr[columns[i]][columns[j]] = cell
                        corr[columns[j]][columns[i]] = cell
                    else:
                        cell, _ = pearsonr(dataset[columns[i]], dataset[columns[j]])
                        corr[columns[i]][columns[j]] = cell
                        corr[columns[j]][columns[i]] = cell
    corr.fillna(value=np.nan, inplace=True)
    if mark_columns:
        marked_columns = ['{} (nom)'.format(col) if col in nominal_columns else '{} (con)'.format(col) for col in columns]
        corr.columns = marked_columns
        corr.index = marked_columns
    if plot:
        plt.figure(figsize=kwargs.get('figsize',None))
        sns.heatmap(corr, annot=kwargs.get('annot',False), fmt=kwargs.get('fmt','.2f'), cmap=cmap)

        if title_auto:
            # title = 'Categorical Correlation: Theil\'s U for top X%s of Diagnoses\n\%d Patients, %d Diagnoses'%("%", dataset.shape[0], dataset.shape[1])
            # title = 'Categorical Correlation: Theil\'s U Semiology, HS, EZ \n%d TEST patients, %d variables'%(dataset.shape[0], dataset.shape[1])
            title = 'Categorical Correlation: Semiology, HS, EZ \n%d TEST patients, %d variables'%(dataset.shape[0], dataset.shape[1])

        else:
            # title = 'Theil\'s U for Features, Models and Temporal-EZ'
            if not title:
                title = ''
        plt.title(title, fontsize=14, fontweight='bold')
        plt.xticks(np.arange(len(dataset.columns)), dataset.columns, rotation='vertical', fontsize=12, fontweight='ultralight')
        plt.yticks(np.arange(len(dataset.columns)), dataset.columns, fontsize=12)

        if savefigure:
            # plt.savefig('L:\\Parashkev_top_2%_TheilsU.eps',
            #             format='eps', bbox_inches='tight', dpi=1200)

            plt.savefig(path_name,
                        format='jpg', bbox_inches='tight', dpi=1200)

        plt.show()
    if return_results:
        return corr

def Theils_significant_above_threshold(corr, threshold=0.5):
    """
    corr from associations above.

    """
    # for the 126 ground truth unmerged cases pruned:

    for semio1 in list(corr.columns):   # semio 1 is the given
        for semio2 in list(corr.columns):
            if (semio1 != semio2) &(corr[semio1][semio2] >threshold):
                print ("("+semio2,"|", semio1+")",corr[semio1][semio2])


"""This script implements the Logistic Regression Algorithm from scratch."""

# importing required libraries
import numpy as np

# Constructing the Logistic regression model

class LogisticRegression:
    """Logistic Regression Class"""

    def __init__(self, learning_rate=0.05, iterations=1000):
        """
        Logistic Regression

        Args:
            learning_rate (float, optional): Defaults to 0.05.
            iterations (int, optional): Defaults to 1000.

        Functions:
            fit : (X, y) -> Train the X
            predict : (X) -> Predict the new y from the new X

        """
        self.learning_rate = learning_rate
        self.iterations = iterations
        self.weights = None
        self.bias = None

    def fit(self, X, y):
        """
        This function trains a logistic regression model.

        Parameters
        ----------
            X : array-like, shape = [n_samples, n_features]
                Training vectors, with n_samples  and n_features.
            y : array-like, shape = [n_samples, n_target_values]
                Target values, with n_samples and n_target_values.

        Returns
        -------
            self : object
        """
        n_samples, n_features = X.shape

        # init parameters
        self.weights, self.bias = np.zeros(n_features), 0

        # Gradient Descent
        for i in range(self.iterations):
            # Linear forward propagation
            linear_model = np.dot(X, self.weights) + self.bias

            # Sigmoid function
            y_pred = self.sigmoid(linear_model)

            # Compute gradients
            grad_weights = (1 / n_samples) * np.dot(X.T, (y_pred - y))
            grad_bias = (1 / n_samples) * np.sum(y_pred - y)

            # Update parameters
            self.weights -= self.learning_rate * grad_weights
            self.bias -= self.learning_rate * grad_bias

    def predict(self, X):
        """ Predict the new X from the previously trained model

        Parameters
        ----------
            X : array-like, shape = [n_samples, n_features]
                Testing vectors, with n_samples and n_features.

        Returns
        -------
            y_pred : array-like, shape = [n_samples, n_target_values]
                Predicted values using the trained linear model.
        """
        linear_model = np.dot(X, self.weights) + self.bias
        y_pred = self.sigmoid(linear_model)
        pred_class = np.where(y_pred >= 0.5, 1, 0)
        return np.array(pred_class, dtype=int)

    def sigmoid(self, z):
        """ Compute the sigmoid function of the input z

        Parameters
        ----------
            z : array-like, shape = [n_samples, n_features]
                Input values.

        Returns
        -------
            sigmoid : array-like, shape = [n_samples, n_features]
                Sigmoid values of the input z.
        """
        return 1 / (1 + np.exp(-z))

    def accuracy_score(self, X, y):
        """ Compute the Accuracy Score of the model

        Parameters
        ----------
            X : array-like, shape = [n_samples, n_features]
                Training/Testing vectors, with n_samples and n_features.
            y : array-like, shape = [n_samples, n_target_values]
                Training/Testing target values, with n_samples and n_features.

        Returns
        -------
            Accuracy_score : float
                Accuracy Score of the trained model on the provided X.
        """
        y_pred = self.predict(X)
        y_pred = np.round(y_pred)
        return np.mean(y == y_pred)


"""
Author: Peratham Wiriyathammabhum

"""
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
eps = np.finfo(float).eps

def logistic_regression(Xin, yin, opts, thres=10**-5, max_epochs=200):
	"""
	Perform logistic regression on an input row matrix X and a target vector y.
	See: http://www.ciml.info/dl/v0_9/ciml-v0_9-ch06.pdf
	"""
	X = Xin.copy() # safety
	y = yin.copy()
	y = np.expand_dims(y,axis=1)

	ni, nd = X.shape
	X = np.append(X, np.ones((ni, 1)), axis=1) # append the bias/const term

	theta = weight_init(nd, init_mode='random')

	lr = opts['lr']
	bsize = opts['bsize']
	lamb_const = opts['lamb_const']

	n_epoch = 0
	while not terminating_cond(n_epoch, max_epochs):
		ind = 0
		loss = 0
		while ind < ni:
			end = ind + bsize if ind + bsize <= ni else ni
			bX = X[ind:end]
			by = y[ind:end]
			gz = sigmoid(np.matmul(bX, theta))
			pred = gz
			loss += cross_entropy_loss(pred, by) + (lamb_const / 2.0) * np.linalg.norm(theta, ord=2)
			grad = (np.matmul(bX.T, (pred - by))).sum(axis=1)
			grad = np.expand_dims(grad, 1)
			theta -= lr * grad + lamb_const * theta # sgd
			ind += bsize
		n_epoch += 1
		if n_epoch % 10 == 0:
			acc = compute_accuracy(X, y, theta)
			print('[Info] epoch: {} loss: {:.4f} acc: {:2.2f}%'.format(n_epoch, float(loss), float(acc)))
	return theta

def compute_accuracy(X, yin, theta):
	y = yin.copy()
	gz = sigmoid(np.matmul(X,theta))
	acc = (gz == y).mean()
	return acc*100.

def weight_init(nd, init_mode='random'):
	if init_mode == 'random':
		theta = np.random.randn(nd + 1, 1) # plus the bias/const term
	elif init_mode=='xavier':
		"""
		Glorot, Xavier, and Yoshua Bengio.
		"Understanding the difficulty of training deep feedforward neural networks."
		Proceedings of the thirteenth international conference on artificial
		intelligence and statistics. 2010.
		"""
		pass
	else:
		pass
	return theta

def cross_entropy_loss(pred, yin):
	"""
	https://en.wikipedia.org/wiki/Cross_entropy#Cross-entropy_loss_function_and_logistic_regression
	"""
	# n = y.shape[0]
	# loss = -(y*np.log(pred+eps) + (1-y)*np.log(1-pred+eps))/n
	# loss = loss.sum(axis=0)
	# return loss
	"""
	for numerical stability use log sum:
		??max_val = (-input).clamp(min=0)
		??loss = input - input * target + max_val + ((-max_val).exp() + (-input - max_val).exp()).log()
		https://discuss.pytorch.org/t/numerical-stability-of-bcewithlogitsloss/8246
		http://tagkopouloslab.ucdavis.edu/?p=2197
		https://www.xarg.org/2016/06/the-log-sum-exp-trick-in-machine-learning/
	"""
	y = yin.copy()
	n = y.shape[0]
	max_value = np.clip(pred, eps, 1-eps)
	loss = -(y * np.log(max_value) + (1-y) * np.log(1-max_value))/n
	loss = loss.sum(axis=0)
	return loss

def _pos_sigmoid(x):
	z = np.exp(-x)
	return 1 / (1 + z)

def _neg_sigmoid(x):
	z = np.exp(x)
	return z / (1 + z)

def sigmoid(x):
	#return 1.0 / (1.0 + np.exp(-x))
	"""
	Numerically stable sigmoid function.
		see: https://timvieira.github.io/blog/post/2014/02/11/exp-normalize-trick/
	"""
	sx = x.copy()
	pos_ind = sx >= 0
	neg_ind = ~pos_ind

	sx[pos_ind] = _pos_sigmoid(sx[pos_ind])
	sx[neg_ind] = _neg_sigmoid(sx[neg_ind])
	return sx

def terminating_cond(nite, max_iters):
	if nite >= max_iters:
		print('[Info] terminate at iter: {}'.format(nite))
		return True
	# elif np.linalg.norm(mu - nmu) < thres:
	#   print('[Info] terminate at iter: {}'.format(nite))
	#   return True
	else:
		return False

def main(opts):
	max_epochs = opts['max_epochs']
	from sklearn import datasets
	breast_cancer = datasets.load_breast_cancer()
	X, y = breast_cancer.data, breast_cancer.target
	theta = logistic_regression(X, y, opts, thres=10**-5, max_epochs=max_epochs)

	# plot
	# input("Press Enter to continue...")
	"""

	"""
	return

if __name__ == '__main__':
	import argparse

	parser = argparse.ArgumentParser(description='run logistic regression with l2 regularization.')
	parser.add_argument('--lr', dest='lr',
					  help='learning rate',
					  default=1e-3, type=float)
	parser.add_argument('--lambda', dest='lamb_const',
					  help='lambda for l2',
					  default=1e-3, type=float)
	parser.add_argument('--bsize', dest='bsize',
					  help='batch size',
					  default=16, type=int)
	parser.add_argument('--max_epochs', dest='max_epochs',
					  help='number of iterations to train',
					  default=200, type=int)
	args = parser.parse_args()
	opts = vars(args)

	main(opts)


import numpy as np

def sigmoid(x):
    return 1 / (1 + np.exp(-x))

def sigmoid_grad(x):
    return (1.0 - sigmoid(x)) * sigmoid(x)

def softmax(x):
    x = x - np.max(x, axis=-1, keepdims=True)   # オーバーフロー対策
    return np.exp(x) / np.sum(np.exp(x), axis=-1, keepdims=True)

def cross_entropy_error(y, t):
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)

    # 教師データがone-hot-vectorの場合、正解ラベルのインデックスに変換
    if t.size == y.size:
        t = t.argmax(axis=1)

    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size

def numerical_gradient(f, x):
    h = 1e-4 # 0.0001
    grad = np.zeros_like(x)

    it = np.nditer(x, flags=['multi_index'], op_flags=['readwrite'])
    while not it.finished:
        idx = it.multi_index
        tmp_val = x[idx]
        x[idx] = tmp_val + h
        fxh1 = f(x) # f(x+h)

        x[idx] = tmp_val - h
        fxh2 = f(x) # f(x-h)
        grad[idx] = (fxh1 - fxh2) / (2*h)

        x[idx] = tmp_val # 値を元に戻す
        it.iternext()

    return grad


'''
Created on Jun 1, 2011

@author: Peter Harrington
'''
from numpy import *

def loadDataSet(fileName, delim='\t'):
    fr = open(fileName)
    stringArr = [line.strip().split(delim) for line in fr.readlines()]
    datArr = [map(float,line) for line in stringArr]
    return mat(datArr)

def pca(dataMat, topNfeat=9999999):
    meanVals = mean(dataMat, axis=0)
    meanRemoved = dataMat - meanVals #remove mean
    covMat = cov(meanRemoved, rowvar=0)
    eigVals,eigVects = linalg.eig(mat(covMat))
    eigValInd = argsort(eigVals)            #sort, sort goes smallest to largest
    eigValInd = eigValInd[:-(topNfeat+1):-1]  #cut off unwanted dimensions
    redEigVects = eigVects[:,eigValInd]       #reorganize eig vects largest to smallest
    lowDDataMat = meanRemoved * redEigVects#transform data into new dimensions
    reconMat = (lowDDataMat * redEigVects.T) + meanVals
    return lowDDataMat, reconMat

def replaceNanWithMean():
    datMat = loadDataSet('secom.data', ' ')
    numFeat = shape(datMat)[1]
    for i in range(numFeat):
        meanVal = mean(datMat[nonzero(~isnan(datMat[:,i].A))[0],i]) #values that are not NaN (a number)
        datMat[nonzero(isnan(datMat[:,i].A))[0],i] = meanVal  #set NaN values to mean
    return datMat


"""
Linear Quadratic Regulator
"""

import jax.numpy as np
import tigercontrol
from tigercontrol.controllers import Controller

class LQRFiniteHorizon(Controller):
    """
    Description: Computes optimal set of actions for finite horizon using the Linear Quadratic Regulator
    algorithm.
    """

    def __init__(self, A, B, C, T):
        """
        Description: Initialize the dynamics of the method

        Args:
            A (float/numpy.ndarray): past value contribution coefficients
            B (float/numpy.ndarray): control value contribution coefficients
            C (float/numpy.ndarray): quadratic cost coefficients
            T (postive int): number of timesteps
            x (float/numpy.ndarray): initial state
        """

        F = np.hstack((A, B))

        n, m = B.shape # State & Action Dimensions

        self.F, C, self.T = self._extend(F, T), self._extend(C, T), T
        self.K = self._extend(np.zeros((self.m, n)), T)

        ## Initialize V and Q Functions ##
        V = np.zeros((self.F[0].shape[0], self.F[0].shape[0]))
        Q = np.zeros((C[0].shape[0], C[0].shape[1]))

        ## Backward Recursion ##
        for t in range(self.T - 1, -1, -1):
            Q = C[t] + self.F[t].T @ V @ self.F[t]
            self.K[t] = -np.linalg.inv(Q[n :, n :]) @ Q[n :, : n]
            V = Q[: n, : n] + Q[: n, n :] @ self.K[t] + self.K[t].T @ Q[n :, : n] + self.K[t].T @ Q[n :, n :] @ self.K[t]

    def _to_ndarray(self, x):
        """
        Description: If x is a scalar, transform it to a (1, 1) numpy.ndarray;
        otherwise, leave it unchanged.
        Args:
            x (float/numpy.ndarray)
        Returns:
            A numpy.ndarray representation of x
        """
        x = np.asarray(x)
        if(np.ndim(x) == 0):
            x = x[None, None]
        return x

    def _extend(self, x, T):
        """
        Description: If x is not in the correct form, convert it; otherwise, leave it unchanged.
        Args:
            x (float/numpy.ndarray)
            T (postive int): number of timesteps
        Returns:
            A numpy.ndarray representation of x
        """
        #x = self._to_ndarray(x)
        return [x for i in range(T)]

    def plan(self, x, T):
        """
        Description: Updates internal parameters and then returns the estimated optimal set of actions
        Args:
            None
        Returns:
            Estimated optimal set of actions
        """

        assert T == self.T, "ERROR: Can only plan for the initial timeline provided."

        u = self._extend(np.zeros((self.m, 1)), T)

        ## Forward Recursion ##
        for t in range(T):
            u[t] = self.K[t] @ x
            x = self.F[t] @ np.hstack((x, u[t]))

        return u

    def __str__(self):
        return "<Finite Horizion LQR Method>"


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import scipy.optimize as opt    # more on this later

data = pd.read_csv('ex2data1.txt', header = None)
X = data.iloc[:,:-1]
y = data.iloc[:,2]
#print(data.head())

mask = y == 1
adm = plt.scatter(X[mask][0].values, X[mask][1].values)
not_adm = plt.scatter(X[~mask][0].values, X[~mask][1].values)
plt.xlabel('Exam 1 score')
plt.ylabel('Exam 2 score')
plt.legend((adm, not_adm), ('Admitted', 'Not admitted'))
#plt.show()

#Sigmoid Function
def sigmoid(x):
  return 1/(1+np.exp(-x))

#Cost Function
#Let’s implement the cost function for the Logistic Regression.
def costFunction(theta, X, y):
    J = (-1/m) * np.sum(np.multiply(y, np.log(sigmoid(X @ theta)))
        + np.multiply((1-y), np.log(1 - sigmoid(X @ theta))))
    return J
#Note that we have used the sigmoid function in the costFunction above.
#There are multiple ways to code cost function. Whats more important is the underlying mathematical ideas and our ability to translate them into code.

def gradient(theta, X, y):
    return ((1/m) * X.T @ (sigmoid(X @ theta) - y))
(m, n) = X.shape
X = np.hstack((np.ones((m,1)), X))
y = y[:, np.newaxis]
theta = np.zeros((n+1,1)) # intializing theta with all zeros
J = costFunction(theta, X, y)
print(J)

temp = opt.fmin_tnc(func = costFunction,
                    x0 = theta.flatten(),fprime = gradient,
                    args = (X, y.flatten()))
theta_optimized = temp[0]
print(theta_optimized)
#temp = opt.fmin_tnc
J = costFunction(theta_optimized[:,np.newaxis], X, y)
print(J)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

#Linear regression using Scipy
from numpy.random import seed
from numpy.random import randn as rnd

#Set seed value
seed(32)

x = rnd(1000)
y = (3*x + 4) + rnd(1000)
# We add rnd(1000) to make small errors in the values, otherwise
# We would obtain a line

#Let us generate a plot to see the distribution
import matplotlib.pyplot as plt
plt.scatter(x, y)
plt.show()

# To see how much correlated values are
from scipy.stats import pearsonr
corr_coef, p_value = pearsonr(x, y)

print(f"p_value: {p_value}, corr_coef: {corr_coef}")

# We see that corr coef is about 94% and p_value is below 0.05

# Now we will try and predict the linear regression line

from scipy.stats import linregress

slope, intercept, r_value, p_value, std_err = linregress(x,y)

print(f"Equation predicted is : y = {slope}*x + {intercept}")
print(f"Actual Equation is: y = 3*x + 4")

# Now we print the regression line
y_preds = slope*x + intercept
plt.scatter(x, y)
plt.plot(x, y_preds, c = 'orange')
plt.show()

# Let us get the r-squared score to check accuracy of regression
print('accuracy is {}'.format(r_value**2))


#!/bin/python3

import math
import os
import random
import re
import sys
import numpy as np

'''
Given a square matrix, calculate the absolute difference between the sums of its diagonals.

Link: https://www.hackerrank.com/challenges/diagonal-difference/problem
'''

def diagonalDifference(arr):
    # Write your code here
    #print(np.matrix(arr))
    dim = len(arr[0])

    # first diagonal, from 0,0 to n,n
    d1 = 0
    for i in range(dim):
        d1 += arr[i][i]
    print(d1)

    # opposit diagonal
    d2 = 0
    for i in range(dim):
        d2 += arr[i][dim - 1 - i]
    print(d2)

    return abs(d1 - d2)

if __name__ == '__main__':

    # sample case
    n = 3
    arr = [[11, 2, 4], [4, 5, 6], [10, 8, -12]]

    result = diagonalDifference(arr)
    print(result)


import numpy as np
import matplotlib
import datetime
import matplotlib.dates
from scipy.misc import derivative
from sympy import poly
from floodsystem.flood import stations_highest_rel_level
from floodsystem.stationdata import update_water_levels, build_station_list

#Task 2F Uday Singh
def polyfit(dates, levels, p):
	d0 = matplotlib.dates.date2num(dates[1])
	x = matplotlib.dates.date2num(dates)-d0
	y = np.asarray(levels)
	#This section of code finds the coeffiecient for the different terms in the polynomial expression
	p_coeff = np.polyfit(x, y, p)
	#This section of the code converts the coefficents into the polynomial expression
	poly = np.poly1d(p_coeff)
	return poly, d0

#Task 2G Uday Singh
#This section of code tries to find the rate of increase of water, by finding the derivative of the polynomial
def rate_of_increase(dates, levels, p):
    poly, d0 = polyfit(dates, levels, p)
    derivative = np.polyder(poly)
    return derivative[0]
#Task 2G Uday Singh
#This section of code groups the stations by their water level- 'high' level, 'mid' level and 'low' level
def water_level():
    stations = build_station_list()
    update_water_levels(stations)
    high = []
    mid = []
    low = []
    for station in stations:
        if station.relative_water_level() == None:
            pass
        elif station.relative_water_level() <1:
            low.append(station)
        elif station.relative_water_level() > 2:
            high.append(station)
        else:
            mid.append(station)
    return low, mid, high


import numpy as np

# Linear Activations
def Linear(x):
    return x

def LinearPrime(x):
    return np.ones(x.shape)

# Sigmoid Activations
def Sigmoid(x):
    return 1 / (1 + np.exp(-x))

def SigmoidPrime(x):
    return np.exp(-x) / ((1 + np.exp(-x)) ** 2)

# Tanh Activations
def Tanh(x):
    return np.tanh(x)

def TanhPrime(x):
    return 1 / (np.cosh(x)**2)

# Softmax Activations
def Softmax(x):
    exp = np.exp(x - np.max(x))
    return exp / np.sum(exp, axis=1, keepdims=True)

def SoftmaxPrime(x):
    return x / x.shape[0]

# Global Activations Dictionary
allActivations = globals()

# Get Activation Function
def Activations(name):
    name = "".join(name.title().split(" "))
    return allActivations[name], allActivations[name + "Prime"]


import scipy.spatial.distance as dist
import scipy.linalg as linalg
import numpy as np

def rbf_kernel_pca(data, gamma, n_components):
    """
    rbf kernel pca implementation

    params -
    numpy ndarray data: shape = [n_samples, n_features]
    float gamma: tuning param of rbf kernel
    int n_components: num components to return

    returns -
    numpy ndarray projected data, list eigvals: shape = [n_samples, k_features]
    """

    # calc pairwise squared euclidean distances in MxN dataset
    sq_dists = dist.pdist(data, 'sqeuclidean')
    # convert pairwise distances into square matrix
    mat_sq_dists = dist.squareform(sq_dists)
    # compute symmetric kernel matrix
    k_mat = np.exp(-gamma * mat_sq_dists)
    # center kernel matrix
    flat = k_mat.shape[0]
    one_flat = np.ones((flat, flat)) / flat
    k_mat = (k_mat - one_flat.dot(k_mat) -
             k_mat.dot(one_flat) + one_flat.dot(k_mat).dot(one_flat))
    # obtain eigpairs from centered kernel matrix
    # scipy.eigh returns them sorted
    eigvals, eigvecs = linalg.eigh(k_mat)
    # collect top k eigvecs (projected samples, eigvals)
    # these are informally alphas and lambdas
    return (np.column_stack((eigvecs[:, -index]
                             for index in range(1, n_components + 1))),
            [eigvals[-index] for index in range(1, n_components + 1)])

def project_data(data_proj, data, gamma, alphas, lambdas):
    """project a data point"""
    pair_dist = np.array([(np.sum(data_proj - row)**2) for row in data])
    return np.exp(-gamma * pair_dist).dot(alphas / lambdas)


# Собственные значения и векторы
import numpy as np

EPS = 10.0 ** -4
Variant = 5

def initial_input():
    c_matrix = np.array([
        [0.2, 0.0, 0.2, 0.0, 0.0],
        [0.0, 0.2, 0.0, 0.2, 0.0],
        [0.2, 0.0, 0.2, 0.0, 0.2],
        [0.0, 0.2, 0.0, 0.2, 0.0],
        [0.0, 0.0, 0.2, 0.0, 0.2]
    ])
    d_matrix = np.array([
        [2.33, 0.81, 0.67, 0.92, -0.53],
        [0.81, 2.33, 0.81, 0.67, 0.92],
        [0.67, 0.81, 2.33, 0.81, 0.92],
        [0.92, 0.67, 0.81, 2.33, -0.53],
        [-0.53, 0.92, 0.92, -0.53, 2.33]
    ])
    a = Variant * c_matrix + d_matrix
    return a

A = initial_input()
n = len(A)

# 3

ansV = np.eye(n)
while True:
    max_element = (0, 1)
    for i in range(n):
        for j in range(i + 1, n):
            if abs(A[i][j]) > abs(A[max_element]):
                max_element = (i, j)
    (i, j) = max_element
    if A[i][i] == A[j][j]:
        p = np.pi / 4
    else:
        p = 2 * A[i][j] / (A[i][i] - A[j][j])
    c = np.cos(1 / 2 * np.arctan(p))
    s = np.sin(1 / 2 * np.arctan(p))
    V = np.eye(n)
    V[i][i] = c
    V[i][j] = -s
    V[j][i] = s
    V[j][j] = c
    A = V.T @ A @ V  # По практикуму рекомендуется расписать кучей формул
    ansV = ansV @ V
    if (((A ** 2).sum() - (np.diag(A) ** 2).sum()) / n) < EPS:
        ansW = np.diag(A)
        break

np.set_printoptions(suppress=True, precision=4, floatmode="fixed")

A = initial_input()
print("A = ")
print(A)

print("-----")
print("Ответ:")
(W, V) = (ansW, ansV)
print("Вектор собственных значений =")
print(W)
print("Столбцы - собственные векторы =")
print(V)

print("-----")
print("Проверка:")
(W, V) = np.linalg.eig(A)
print(W)
print(V)


"""
Introduction to Artificial Intelligence, Exercise 1:
Comparison of the gradient descent method and the Newton method.
Author: Robert Kaczmarski 293377
"""
import matplotlib.pyplot as plt
import numpy as np
import time

from methods import find_minimum, gradient_search_direction, himmelblau, newton_search_direction

def measure_performence(starting_points, beta, max_iters, search_direction):
    time_start = time.monotonic_ns()
    minimum, visited_points = find_minimum(starting_points, beta, max_iters, search_direction)
    time_stop = time.monotonic_ns()
    total_time = time_stop - time_start
    return minimum, visited_points, total_time

def plot_contour3d(x, y, F, contours):
    fig = plt.figure(figsize=(9, 9))
    ax = plt.axes(projection='3d')
    ax.contour3D(x, y, F, contours)
    ax.set_xlabel('X')
    ax.set_ylabel('Y')
    ax.set_title('Himmelblau Function')
    ax.view_init(50, 50)

def plot_path(x_points, y_points, label_name, color: str):
    X = np.linspace(-5, 5, 100)
    Y = np.linspace(-5, 5, 100)
    x, y = np.meshgrid(X, Y)
    F = himmelblau([x, y])
    contours = 50

    plt.contour(x, y, F, contours)
    plt.plot(x_points, y_points, color+"o", label = label_name, linewidth=1, markersize=2, linestyle='dashed')
    # Plot the starting point
    plt.plot(x_points[0], y_points[0], "yp", markersize=3)
    plt.xlabel("X")
    plt.ylabel("Y")
    plt.grid()
    plt.legend(bbox_to_anchor=(0, 1.02, 1,0), loc=3, ncol=1, borderaxespad=0)

def display_results(method_name, minimum, visited_points, total_time):
    nano = 1e-9
    print(f'Results for {method_name}:')
    print(f'Minimum found at {minimum}, number of points = {len(visited_points)}')
    print(f'Performence time: {total_time*nano:.15f} sec' )

def split_list(list_of_lists):
    return map(list, zip(*list_of_lists))

def main() -> int:
    starting_points = [-4.0, 0.0]
    beta = 0.001
    max_iters = 100000

    minimum_grad, visited_points_grad, time_grad = measure_performence(starting_points, beta, max_iters, gradient_search_direction)
    display_results("Gradient", minimum_grad, visited_points_grad, time_grad)
    minimum_newt, visited_points_newt, time_newt = measure_performence(starting_points, beta, max_iters, newton_search_direction)
    display_results("Newton", minimum_newt, visited_points_newt, time_newt)

    # visited_points_grad = [[x0,y0], [x1,y1], ...]
    # Splitting to two lists
    x_grad, y_grad = split_list(visited_points_grad)
    x_newt, y_newt = split_list(visited_points_newt)

    plot_path(x_grad, y_grad, 'Metoda spadku gradientu', "b")
    plot_path(x_newt, y_newt, 'Metoda Newtona', "r")
    plt.show()
    # plot_contour3d(x, y, F, 80)
    # plt.plot([3.0, -2.805118, -3.779310, 3.584428], [2.0, 3.131312, -3.283186, -1.848126], 'ro')
    # plt.show()

    return 0

if __name__ == "__main__":
    raise(SystemExit(main()))


import numpy as np
import pandas as pd
import random

"""
Генерация сигналов-синусоид с переменными периодами и амплитудами.
"""

def generate(periods, amplitudes, frequencies, ground=0, sample_rate=1e-2):
    """
    Генерирует синусоиду с заданным количеством периодов, случайно изменяющейся
    амплитудой и случайно изменяющейся частотой.

    Параметры:
        periods - количество периодов синусоиды. Тип - int.
        amplitudes - список из двух значений амплитуд - минимальной и
    максимальной. Амплитуды будут генерироваться случайно, между двумя этими
    значениями. Тип - list.
        frequencies - список из двух значений частот - минимальной и
    максимальной. Частоты будут генерироваться случайно, между двумя этими
    значениями. Тип - list.
        ground - точка по оси y, относительно которой строится синусоида.
    Значение по умолчанию: 0. Тип - float.
        sample_rate - частота дискретизации сигнала синусоиды. Значение по
    умолчанию: 1e-2. Тип - float.

    Возвращает синусоиду. Тип - list.
    """
    # Инициализация списка
    sgn = []
    # Каждую итерацию цикла создается один период синусоиды
    for i in range(periods):
        # Генерируется амплитуда и частота
        amplitude = random.uniform(amplitudes[0], amplitudes[1])
        frequency = random.uniform(frequencies[0], frequencies[1])
        period = 1 / frequency
        # Создается период синусоиды и добавляется в общий список
        for t in np.arange(0, 2 * np.pi * period, sample_rate):
            sgn.append(amplitude * np.sin(frequency * t) + ground + amplitudes[1])
    return sgn

def reduce(arrays):
    """
    Укорачивает списки до длины меньшего списка.

    Параметры:
        arrays - список, состоящий из списков. Например, список из синусоид,
    сгеренированных с помощью функции generate. Тип - list.

    Возвращает список, состоящий из списков. Тип - list.
    """
    # Вычисляется минимальная длина среди всех списков
    min = len(arrays[0])
    for array in arrays:
        if len(array) < min:
            min = len(array)
    # Все списки укорачиваются до длины самого короткого
    arrays = [array[0:min] for array in arrays]
    return arrays

def createDF(arrays, keys):
    """
    Создает таблицу из списков.

    Параметры:
        arrays - список, состоящий из списков. Например, список из синусоид,
    сгеренированных с помощью функции generate. Тип - list.
        keys - список из названий для столбцов. Тип - list.

    Возвращает таблицу. Тип - pandas.DataFrame.
    """
    df = pd.DataFrame()
    for i in range(0, len(arrays)):
        df[keys[i]] = arrays[i]
    return df

def generateDF(keys, periods=20, amplitudes=[1, 1.5], frequencies=[1/6, 1/2],
               ground=0, sample_rate=1e-2):
    """
    Генерирует таблицу со столбцами из синусоид с заданным количеством периодов,
    случайно изменяющейся амплитудой и случайно изменяющейся частотой.

    Параметры:
        keys - список из названий для столбцов. Тип - list.
        periods - количество периодов синусоиды. Значение по умолчанию: 20.
    Тип - int.
        amplitudes - список из двух значений амплитуд - минимальной и
    максимальной. Амплитуды будут генерироваться случайно, между двумя этими
    значениями. Значение по умолчанию: [1, 1.5]. Тип - list.
        frequencies - список из двух значений частот - минимальной и
    максимальной. Частоты будут генерироваться случайно, между двумя этими
    значениями. Значение по умолчанию: [1/6, 1/2]. Тип - list.
        ground - точка по оси y, относительно которой строится синусоида.
    Значение по умолчанию: 0. Тип - float.
        sample_rate - частота дискретизации сигнала синусоиды. Значение по
    умолчанию: 1e-2. Тип - float.

    Возвращает таблицу. Тип - pandas.DataFrame.
    """
    # Создание списка из синусоид
    xs = []
    for key in keys:
        xs.append(generate(periods, amplitudes, frequencies, ground, sample_rate))
    # Укорачивание списка из синусоид до длины меньшей синусоиды
    xs = reduce(xs)
    return createDF(xs, keys)


import csv
import numpy as np
import math

def readData():
    rawData = csv.reader(open("kleibers_law_data.csv", 'r'))

    xList = []
    yList = []
    i = 0
    for raw in rawData:
        if i == 0:
            xList = raw
            i += 1
        else:
            yList = raw

    for i in range(len(xList)):
        xList[i] = math.log(float(xList[i]))
        yList[i] = math.log(float(yList[i]))

    return xList, yList

if  __name__  ==  "__main__":
    xList, yList = readData()
    xBar = np.mean(xList)
    yBar = np.mean(yList)

    a = 0
    b = 0
    for i in range(len(xList)):
        a += (xList[i] - xBar) * (yList[i] - yBar)
        b += (xList[i] - xBar) ** 2

    w1 = a / b
    w2 = yBar - w1 * xBar
    print(w1, w2)


""" Some validation functions. """

from __future__ import division

import numpy as np
from scipy.stats import norm

def smse(y_true, y_pred):
    """
    Standardised mean squared error.

    Parameters
    ----------
    y_true: ndarray
        vector of true targets
    y_pred: ndarray
        vector of predicted targets

    Returns
    -------
    float:
        SMSE of predictions vs truth (scalar)

    Example
    -------
    >>> y_true = np.random.randn(100)
    >>> smse(y_true, y_true)
    0.0
    >>> smse(y_true, np.random.randn(100)) >= 1.0
    True
    """

    N = y_true.shape[0]
    return ((y_true - y_pred)**2).sum() / (N * y_true.var())

def mll(y_true, y_pred, y_var):
    """
    Mean log loss under a Gaussian distribution.

    Parameters
    ----------
    y_true: ndarray
        vector of true targets
    y_pred: ndarray
        vector of predicted targets
    y_var: float or ndarray
        predicted variances

    Returns
    -------
    float:
        The mean negative log loss (negative log likelihood)

    Example
    -------
    >>> y_true = np.random.randn(100)
    >>> mean_prob = - norm.logpdf(1e-2, loc=0)  # -ve log prob close to mean
    >>> mll(y_true, y_true, 1) <= mean_prob  # Should be good predictor
    True
    >>> mll(y_true, np.random.randn(100), 1) >= mean_prob  # naive predictor
    True
    """

    return - norm.logpdf(y_true, loc=y_pred, scale=np.sqrt(y_var)).mean()

def msll(y_true, y_pred, y_var, y_train):
    """
    Mean standardised log loss under a Gaussian distribution.

    Parameters
    ----------
    y_true: ndarray
        vector of true targets
    y_pred: ndarray
        vector of predicted targets
    y_var: float or ndarray
        predicted variances
    y_train: ndarray
        vector of *training* targets by which to standardise

    Returns
    -------
    float:
        The negative mean standardised log loss (negative log likelihood)

    Example
    -------
    >>> y_true = np.random.randn(100)
    >>> msll(y_true, y_true, 1, y_true) < 0  # Should be good predictor
    True
    >>> msll(y_true, np.random.randn(100), 1, y_true) >= 0  # naive predictor
    True
    """

    var = y_train.var()
    mu = y_train.mean()

    ll_naive = norm.logpdf(y_true, loc=mu, scale=np.sqrt(var))
    ll_mod = norm.logpdf(y_true, loc=y_pred, scale=np.sqrt(y_var))

    return - (ll_mod - ll_naive).mean()

def lins_ccc(y_true, y_pred):
    """
    Lin's Concordance Correlation Coefficient.

    See https://en.wikipedia.org/wiki/Concordance_correlation_coefficient

    Parameters
    ----------
    y_true: ndarray
        vector of true targets
    y_pred: ndarray
        vector of predicted targets

    Returns
    -------
    float:
        1.0 for a perfect match between :code:`y_true` and :code:`y_pred`, less
        otherwise

    Example
    -------
    >>> y_true = np.random.randn(100)
    >>> lins_ccc(y_true, y_true) > 0.99  # Should be good predictor
    True
    >>> lins_ccc(y_true, np.zeros_like(y_true)) < 0.01  # Bad predictor
    True
    """

    t = y_true.mean()
    p = y_pred.mean()
    St = y_true.var()
    Sp = y_pred.var()
    Spt = np.mean((y_true - t) * (y_pred - p))

    return 2 * Spt / (St + Sp + (t - p)**2)


#!/usr/bin/env python3
import numpy as np
import scipy.optimize as spo

#theta for the question
given_theta = 0.2
#array of target spot rates
target_spot_rates= np.array([0.0249,0.0250,0.0233,0.0245,0.0247,0.0250,0.0254,0.0257,0.0262,0.0265])
#array of rzero guesses
rzero = np.array([0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01,0.01])
years = 10
#r00 = 0.05
t=10

def fillratetree(r0,theta):
    ratetree=[]
    for i,element in enumerate(r0):
        treecolumn=[]
        for j in range(0,i+1):
            apnd = element*np.exp(2*j*theta)
            treecolumn.append(apnd)
        ratetree.append(treecolumn)
    return ratetree

#Cash Flow tree
def cf_tree(t):
    cftree=[]
    for i in range(0, t+1):
        treecolumn=[]
        for j in range(0,i+1):
            if i < t:
                treecolumn.append(0)
            if i == t:
                treecolumn.append(100)
        cftree.append(treecolumn)
    return cftree

#Present value tree
def comppvtree(ratetree,cvtree):
    pvtree=[]
    for element in cvtree:
        parttree=[]
        for element2 in element:
            parttree.append(0)
        pvtree.append(parttree)
    icol=len(pvtree)-1
    for j in range(0,icol+1):
        pvtree[icol][j]=cvtree[icol][j]
    for i in range (1,len(pvtree)):
        icol=len(pvtree)-1-i
        for j in range(0,icol+1):
            pvtree[icol][j]=cvtree[icol][j]+(0.5*pvtree[icol+1][j+1]+0.5*pvtree[icol+1][j])/(1+ratetree[icol][j])
    return pvtree

#Spot Rates
def spotrates(r0):
    spot_array = []
    for i in range(1,t+1):
        pv = comppvtree(fillratetree(r0,given_theta),cf_tree(i))[0][0]
        spot_rate = (100/pv)**(1/i)-1
        spot_array.append(spot_rate)
    return spot_array

def to_min(r0,theta,target):
    return np.sum(np.square((spotrates(r0)) - target))*1000000000

result = spo.minimize(to_min,[rzero],(given_theta,target_spot_rates,),method='SLSQP',options={'disp':True})
answer = result.x
print()
print("The list of r0n rates that minimise the difference between spot rates:")
print(answer)
print()
print("Target Spot Rates from WSJ:")
print(target_spot_rates)
print("Spotrates from model:")
print(spotrates(answer))


# Exercise_1
# Import the fcluster and linkage functions
from scipy.cluster.hierarchy import fcluster, linkage

# Use the linkage() function
distance_matrix = linkage(comic_con[['x_scaled', 'y_scaled']], method = 'ward', metric = 'euclidean')

# Assign cluster labels
comic_con['cluster_labels'] = fcluster(distance_matrix, 2, criterion='maxclust')

# Plot clusters
sns.scatterplot(x='x_scaled', y='y_scaled',
                hue='cluster_labels', data = comic_con)
plt.show()

--------------------------------------------------
# Exercise_2
# Import the fcluster and linkage functions
from scipy.cluster.hierarchy import fcluster, linkage

# Use the linkage() function
distance_matrix = linkage(comic_con[['x_scaled', 'y_scaled']], method = 'single', metric = 'euclidean')

# Assign cluster labels
comic_con['cluster_labels'] = fcluster(distance_matrix, 2, criterion = 'maxclust')

# Plot clusters
sns.scatterplot(x='x_scaled', y='y_scaled',
                hue='cluster_labels', data = comic_con)
plt.show()

--------------------------------------------------
# Exercise_3
# Import the fcluster and linkage functions
from scipy.cluster.hierarchy import fcluster, linkage

# Use the linkage() function
distance_matrix = linkage(comic_con[['x_scaled', 'y_scaled']], method ='complete', metric= 'euclidean')

# Assign cluster labels
comic_con['cluster_labels'] = fcluster(distance_matrix, 2, criterion='maxclust')

# Plot clusters
sns.scatterplot(x='x_scaled', y='y_scaled',
                hue='cluster_labels', data = comic_con)
plt.show()

--------------------------------------------------
# Exercise_4
# Import the pyplot class
from matplotlib import pyplot as plt

# Define a colors dictionary for clusters
colors = {1:'red', 2:'blue'}

# Plot a scatter plot
comic_con.plot.scatter(x = 'x_scaled',
                	   y = 'y_scaled',
                	   c = comic_con['cluster_labels'].apply(lambda x: colors[x]))
plt.show()

--------------------------------------------------
# Exercise_5
# Import the seaborn module
import seaborn as sns

# Plot a scatter plot using seaborn
sns.scatterplot(x='x_scaled',
                y='y_scaled',
                hue = 'cluster_labels',
                data = comic_con)
plt.show()

--------------------------------------------------
# Exercise_6
# Import the dendrogram function
from scipy.cluster.hierarchy import dendrogram

# Create a dendrogram
dn = dendrogram(distance_matrix)

# Display the dendogram
plt.show()

--------------------------------------------------
# Exercise_7
#1
# Fit the data into a hierarchical clustering algorithm
distance_matrix = linkage(fifa[['scaled_sliding_tackle', 'scaled_aggression']], method= 'ward')
#2
# Fit the data into a hierarchical clustering algorithm
distance_matrix = linkage(fifa[['scaled_sliding_tackle', 'scaled_aggression']], 'ward')

# Assign cluster labels to each row of data
fifa['cluster_labels'] = fcluster(distance_matrix, 3, criterion='maxclust')
#3
# Fit the data into a hierarchical clustering algorithm
distance_matrix = linkage(fifa[['scaled_sliding_tackle', 'scaled_aggression']], 'ward')

# Assign cluster labels to each row of data
fifa['cluster_labels'] = fcluster(distance_matrix, 3, criterion='maxclust')

# Display cluster centers of each cluster
print(fifa[['scaled_sliding_tackle', 'scaled_aggression', 'cluster_labels']].groupby('cluster_labels').mean())

#4
# Fit the data into a hierarchical clustering algorithm
distance_matrix = linkage(fifa[['scaled_sliding_tackle', 'scaled_aggression']], 'ward')

# Assign cluster labels to each row of data
fifa['cluster_labels'] = fcluster(distance_matrix, 3, criterion='maxclust')

# Display cluster centers of each cluster
print(fifa[['scaled_sliding_tackle', 'scaled_aggression', 'cluster_labels']].groupby('cluster_labels').mean())

# Create a scatter plot through seaborn
sns.scatterplot(x='scaled_sliding_tackle', y='scaled_aggression', hue='cluster_labels', data=fifa)
plt.show()

--------------------------------------------------

Machine Learning Scientist with Python Track/6. Cluster Analysis in Python/ch2_exercises.py


import os
""" First change the following directory link to where all input files do exist """
os.chdir("D:\\Book writing\\Codes\\Chapter 5")

import numpy as np
import pandas as pd

# KNN Curse of Dimensionality
import random,math

def random_point_gen(dimension):
    return [random.random() for _ in range(dimension)]

def distance(v,w):
    vec_sub = [v_i-w_i for v_i,w_i in zip(v,w)]
    sum_of_sqrs = sum(v_i*v_i for v_i in vec_sub)
    return math.sqrt(sum_of_sqrs)

def random_distances_comparison(dimension,number_pairs):
    return [distance(random_point_gen(dimension),random_point_gen(dimension))
            for _ in range(number_pairs)]

def mean(x):
    return sum(x) / len(x)

dimensions = range(1, 201, 5)

avg_distances = []
min_distances = []

dummyarray = np.empty((20,4))
dist_vals = pd.DataFrame(dummyarray)
dist_vals.columns = ["Dimension","Min_Distance","Avg_Distance","Min/Avg_Distance"]

random.seed(34)
i = 0
for dims in dimensions:
    distances = random_distances_comparison(dims, 1000)
    avg_distances.append(mean(distances))
    min_distances.append(min(distances))

    dist_vals.loc[i,"Dimension"] = dims
    dist_vals.loc[i,"Min_Distance"] = min(distances)
    dist_vals.loc[i,"Avg_Distance"] = mean(distances)
    dist_vals.loc[i,"Min/Avg_Distance"] = min(distances)/mean(distances)

    print(dims, min(distances), mean(distances), min(distances)*1.0 / mean(distances))
    i = i+1

# Ploting Average distances for Various Dimensions
import matplotlib.pyplot as plt
plt.figure()
#plt.title('Avg. Distance Change with Number of Dimensions for 1K Obs')
plt.xlabel('Dimensions')
plt.ylabel('Avg. Distance')
plt.plot(dist_vals["Dimension"],dist_vals["Avg_Distance"])
plt.legend(loc='best')
plt.show()

# 1-Dimension Plot
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

one_d_data = np.random.rand(60,1)
one_d_data_df = pd.DataFrame(one_d_data)
one_d_data_df.columns = ["1D_Data"]
one_d_data_df["height"] = 1

plt.figure()
plt.scatter(one_d_data_df['1D_Data'],one_d_data_df["height"])
plt.yticks([])
plt.xlabel("1-D points")
plt.show()

# 2- Dimensions Plot
two_d_data = np.random.rand(60,2)
two_d_data_df = pd.DataFrame(two_d_data)
two_d_data_df.columns = ["x_axis","y_axis"]

plt.figure()
plt.scatter(two_d_data_df['x_axis'],two_d_data_df["y_axis"])
plt.xlabel("x_axis");plt.ylabel("y_axis")
plt.show()

# 3- Dimensions Plot
three_d_data = np.random.rand(60,3)
three_d_data_df = pd.DataFrame(three_d_data)
three_d_data_df.columns = ["x_axis","y_axis","z_axis"]

from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(three_d_data_df['x_axis'],three_d_data_df["y_axis"],three_d_data_df["z_axis"])
ax.set_xlabel('x_axis')
ax.set_ylabel('y_axis')
ax.set_zlabel('z_axis')
plt.show()

# KNN CLassifier - Breast Cancer
import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score,classification_report

breast_cancer = pd.read_csv("Breast_Cancer_Wisconsin.csv")

print (breast_cancer.head())

breast_cancer['Bare_Nuclei'] = breast_cancer['Bare_Nuclei'].replace('?', np.NAN)
breast_cancer['Bare_Nuclei'] = breast_cancer['Bare_Nuclei'].fillna(breast_cancer['Bare_Nuclei'].value_counts().index[0])

breast_cancer['Cancer_Ind'] = 0
breast_cancer.loc[breast_cancer['Class']==4,'Cancer_Ind'] = 1

x_vars = breast_cancer.drop(['ID_Number','Class','Cancer_Ind'],axis=1)
y_var = breast_cancer['Cancer_Ind']

from sklearn.preprocessing import StandardScaler
x_vars_stdscle = StandardScaler().fit_transform(x_vars.values)
from sklearn.model_selection import train_test_split
x_vars_stdscle_df = pd.DataFrame(x_vars_stdscle, index=x_vars.index, columns=x_vars.columns)
x_train,x_test,y_train,y_test = train_test_split(x_vars_stdscle_df,y_var,train_size = 0.7,random_state=42)

from sklearn.neighbors import KNeighborsClassifier
knn_fit = KNeighborsClassifier(n_neighbors=3,p=2,metric='minkowski')
knn_fit.fit(x_train,y_train)

print ("\nK-Nearest Neighbors - Train Confusion Matrix\n\n",pd.crosstab(y_train,knn_fit.predict(x_train),rownames = ["Actuall"],colnames = ["Predicted"]) )
print ("\nK-Nearest Neighbors - Train accuracy:",round(accuracy_score(y_train,knn_fit.predict(x_train)),3))
print ("\nK-Nearest Neighbors - Train Classification Report\n",classification_report(y_train,knn_fit.predict(x_train)))

print ("\n\nK-Nearest Neighbors - Test Confusion Matrix\n\n",pd.crosstab(y_test,knn_fit.predict(x_test),rownames = ["Actuall"],colnames = ["Predicted"]))
print ("\nK-Nearest Neighbors - Test accuracy:",round(accuracy_score(y_test,knn_fit.predict(x_test)),3))
print ("\nK-Nearest Neighbors - Test Classification Report\n",classification_report(y_test,knn_fit.predict(x_test)))

# Tuning of K- value for Train & Test data
dummyarray = np.empty((5,3))
k_valchart = pd.DataFrame(dummyarray)
k_valchart.columns = ["K_value","Train_acc","Test_acc"]

k_vals = [1,2,3,4,5]
for i in range(len(k_vals)):
    knn_fit = KNeighborsClassifier(n_neighbors=k_vals[i],p=2,metric='minkowski')
    knn_fit.fit(x_train,y_train)

    print ("\nK-value",k_vals[i])

    tr_accscore = round(accuracy_score(y_train,knn_fit.predict(x_train)),3)
    print ("\nK-Nearest Neighbors - Train Confusion Matrix\n\n",pd.crosstab(y_train,knn_fit.predict(x_train),rownames = ["Actuall"],colnames = ["Predicted"]) )
    print ("\nK-Nearest Neighbors - Train accuracy:",tr_accscore)
    print ("\nK-Nearest Neighbors - Train Classification Report\n",classification_report(y_train,knn_fit.predict(x_train)))

    ts_accscore = round(accuracy_score(y_test,knn_fit.predict(x_test)),3)
    print ("\n\nK-Nearest Neighbors - Test Confusion Matrix\n\n",pd.crosstab(y_test,knn_fit.predict(x_test),rownames = ["Actuall"],colnames = ["Predicted"]))
    print ("\nK-Nearest Neighbors - Test accuracy:",ts_accscore)
    print ("\nK-Nearest Neighbors - Test Classification Report\n",classification_report(y_test,knn_fit.predict(x_test)))

    k_valchart.loc[i, 'K_value'] = k_vals[i]
    k_valchart.loc[i, 'Train_acc'] = tr_accscore
    k_valchart.loc[i, 'Test_acc'] = ts_accscore

# Ploting accuracies over varied K-values
import matplotlib.pyplot as plt
plt.figure()
#plt.title('KNN Train & Test Accuracy change with K-value')

plt.xlabel('K-value')
plt.ylabel('Accuracy')
plt.plot(k_valchart["K_value"],k_valchart["Train_acc"])
plt.plot(k_valchart["K_value"],k_valchart["Test_acc"])

plt.axis([0.9,5, 0.92, 1.005])
plt.xticks([1,2,3,4,5])

for a,b in zip(k_valchart["K_value"],k_valchart["Train_acc"]):
    plt.text(a, b, str(b),fontsize=10)

for a,b in zip(k_valchart["K_value"],k_valchart["Test_acc"]):
    plt.text(a, b, str(b),fontsize=10)

plt.legend(loc='upper right')

plt.show()

# Naive Bayes using NLP

# USe following code if it wont work in first place with UTF-8 code error

# import sys
# reload(sys)
# sys.setdefaultencoding('utf-8')

import csv

smsdata = open('SMSSpamCollection.txt','r')
csv_reader = csv.reader(smsdata,delimiter='\t')

smsdata_data = []
smsdata_labels = []

for line in csv_reader:
    smsdata_labels.append(line[0])
    smsdata_data.append(line[1])

smsdata.close()

# Printing top 5 lines
for i in range(5):
    print (smsdata_data[i],smsdata_labels[i])

# Printing Spam & Ham count
from collections import Counter
c = Counter( smsdata_labels )
print(c)

import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
import string
import pandas as pd
from nltk import pos_tag
from nltk.stem import PorterStemmer

def preprocessing(text):
    text2 = " ".join("".join([" " if ch in string.punctuation else ch for ch in text]).split())

    tokens = [word for sent in nltk.sent_tokenize(text2) for word in
              nltk.word_tokenize(sent)]

    tokens = [word.lower() for word in tokens]

    stopwds = stopwords.words('english')
    tokens = [token for token in tokens if token not in stopwds]

    tokens = [word for word in tokens if len(word)>=3]

    stemmer = PorterStemmer()
    tokens = [stemmer.stem(word) for word in tokens]

    tagged_corpus = pos_tag(tokens)

    Noun_tags = ['NN','NNP','NNPS','NNS']
    Verb_tags = ['VB','VBD','VBG','VBN','VBP','VBZ']

    lemmatizer = WordNetLemmatizer()

    def prat_lemmatize(token,tag):
        if tag in Noun_tags:
            return lemmatizer.lemmatize(token,'n')
        elif tag in Verb_tags:
            return lemmatizer.lemmatize(token,'v')
        else:
            return lemmatizer.lemmatize(token,'n')

    pre_proc_text =  " ".join([prat_lemmatize(token,tag) for token,tag in tagged_corpus])

    return pre_proc_text

smsdata_data_2 = []

for i in smsdata_data:
    smsdata_data_2.append(preprocessing(i))

import numpy as np

trainset_size = int(round(len(smsdata_data_2)*0.70))

print ('The training set size for this classifier is ' + str(trainset_size) + '\n')

x_train = np.array([''.join(rec) for rec in smsdata_data_2[0:trainset_size]])
y_train = np.array([rec for rec in smsdata_labels[0:trainset_size]])
x_test = np.array([''.join(rec) for rec in smsdata_data_2[trainset_size+1:len(smsdata_data_2)]])
y_test = np.array([rec for rec in smsdata_labels[trainset_size+1:len(smsdata_labels)]])

# building TFIDF vectorizer
from sklearn.feature_extraction.text import TfidfVectorizer

vectorizer = TfidfVectorizer(min_df=2, ngram_range=(1, 2),  stop_words='english',
                             max_features= 4000,strip_accents='unicode',  norm='l2')

x_train_2 = vectorizer.fit_transform(x_train).todense()
x_test_2 = vectorizer.transform(x_test).todense()

from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB().fit(x_train_2, y_train)

ytrain_nb_predicted = clf.predict(x_train_2)
ytest_nb_predicted = clf.predict(x_test_2)

from sklearn.metrics import classification_report,accuracy_score

print ("\nNaive Bayes - Train Confusion Matrix\n\n",pd.crosstab(y_train,ytrain_nb_predicted,rownames = ["Actuall"],colnames = ["Predicted"]))
print ("\nNaive Bayes- Train accuracy",round(accuracy_score(y_train,ytrain_nb_predicted),3))
print ("\nNaive Bayes  - Train Classification Report\n",classification_report(y_train,ytrain_nb_predicted))

print ("\nNaive Bayes - Test Confusion Matrix\n\n",pd.crosstab(y_test,ytest_nb_predicted,rownames = ["Actuall"],colnames = ["Predicted"]))
print ("\nNaive Bayes- Test accuracy",round(accuracy_score(y_test,ytest_nb_predicted),3))
print ("\nNaive Bayes  - Test Classification Report\n",classification_report(y_test,ytest_nb_predicted))

# printing top features
feature_names = vectorizer.get_feature_names()
coefs = clf.coef_
intercept = clf.intercept_
coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))

print ("\n\nTop 10 features - both first & last\n")
n=10
top_n_coefs = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])
for (coef_1, fn_1), (coef_2, fn_2) in top_n_coefs:
    print('\t%.4f\t%-15s\t\t%.4f\t%-15s' % (coef_1, fn_1, coef_2, fn_2))

Module 1/Chapter05/Chapter 05_KNN n Naive Bayes.py


# -*- coding: utf-8 -*-
# !/usr/bin/env python3

import numpy as np
from matplotlib import pyplot as plt
from scipy import stats as stats

m = [10, 100, 500, 1000, 5000, 10000, 50000]
n = [1, 2, 4, 8, 16, 32]
c = [0, 0, 1, 0, 0]

# Question 1(A)
print('X ~ Exp(1)')
print('E(X) = 1')
x = m
y = list(sum(round(np.random.exponential(scale=1), 2) for j in range(i)) / i for i in x)
fig, ax = plt.subplots()
l1 = ax.plot(x, y)
l2 = ax.axhline(y=1, dashes=[6, 2], color='green')
plt.xlabel('Sample Size')
plt.ylabel('Mean')
plt.title('Exponential Distribution')
plt.show()

# Question 1(B)
print('X ~ Unif(1, 2)')
print('E(X) = 1.5')
x = m
y = list(sum(round(np.random.uniform(1, 2), 2) for j in range(i)) / i for i in x)
fig, ax = plt.subplots()
l1 = ax.plot(x, y)
l2 = ax.axhline(y=1.5, dashes=[6, 2], color='green')
plt.xlabel('Sample Size')
plt.ylabel('Mean')
plt.title('Uniform Distribution')
plt.show()

# Question 1(C)
print('X ~ Bern(0.2)')
print('E(X) = 0.2')
x = m
y = list(sum(np.random.choice(c) for j in range(i)) / i for i in x)
fig, ax = plt.subplots()
l1 = ax.plot(x, y)
l2 = ax.axhline(y=0.2, dashes=[6, 2], color='green')
plt.xlabel('Sample Size')
plt.ylabel('Mean')
plt.title('Bernoulli Distribution')
plt.show()

print('WLLN, verified.')

# Question 2(A)
print('X ~ Exp(1)')
x = n
for i in x:
    dat = sorted(list(sum(np.random.exponential(scale=1) for k in range(i)) / i for j in range(1000)))
    fit = stats.norm.pdf(dat, 1, (1 / i) ** (1 / 2))
    print('For N = {}, E(X) = {:.6f}'.format(i, sum(dat) / len(dat)))
    plt.plot(dat, fit, '-o', label='Normal Distribution')
    plt.hist(dat, bins=20, density=True, label='Approximated Normal Distribution')
    plt.title('Exponential Distribution for N = {}'.format(i))
    plt.legend()
    plt.show()

# Question 2(B)
print('X ~ Unif(1, 2)')
x = n
for i in x:
    dat = sorted(list(sum(np.random.uniform(1, 2) for k in range(i)) / i for j in range(1000)))
    fit = stats.norm.pdf(dat, 1.5, (1 / (i * 12)) ** (1 / 2))
    print('For N = {}, E(X) = {:.6f}'.format(i, sum(dat) / len(dat)))
    plt.plot(dat, fit, '-o', label='Normal Distribution')
    plt.hist(dat, bins=20, density=True, label='Approximated Normal Distribution')
    plt.title('Uniform Distribution for N = {}'.format(i))
    plt.legend()
    plt.show()

# Question 2(C)
print('X ~ Bern(0.2)')
x = n
for i in x:
    dat = sorted(list(sum(np.random.choice(c) for k in range(i)) / i for j in range(1000)))
    fit = stats.norm.pdf(dat, 0.2, (0.16 / i) ** (1 / 2))
    print('For N = {}, E(X) = {:.6f}'.format(i, sum(dat) / len(dat)))
    plt.plot(dat, fit, '-o', label='Normal Distribution')
    plt.hist(dat, bins=20, density=True, label='Approximated Normal Distribution')
    plt.title('Bernoulli Distribution for N = {}'.format(i))
    plt.legend()
    plt.show()

print('CLT, verified.')


import math
import numpy as np
def fp(x,h):
	temp=(2*x+h)/2
	return (2*math.cos(temp)*math.sin(h/2))/h
def f1(x,h):
	return (math.sin(x+h)-math.sin(x))/h
print (" f prime of sin for x= 1.2")
space=np.logspace(-20,0,20)
exact_value = math.cos(1.2)
print (exact_value)
for h  in space:
	print ('f w/o cancellation:',fp(1.2,h), " \t error : ", fp(1.2,h)-exact_value, '\n f with cancellation', f1(1.2,h), '\t error: ', f1(1.2,h)-exact_value)

Nummerical Methods for CSE/PS1/cancellation.py


import numpy as np
from scipy.spatial.distance import cdist

class KMeans:
    def __init__(
            self,
            k: int,
            metric: str = "euclidean",
            tol: float = 1e-6,
            max_iter: int = 100):
        """
        inputs:
            k: int
                the number of centroids to use in cluster fitting
            metric: str
                the name of the distance metric to use
            tol: float
                the minimum error tolerance from previous error during optimization to quit the model fit
            max_iter: int
                the maximum number of iterations before quitting model fit
        """

    def fit(self, mat: np.ndarray):
        """
        fits the kmeans algorithm onto a provided 2D matrix

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features
        """

    def predict(self, mat: np.ndarray) -> np.ndarray:
        """
        predicts the cluster labels for a provided 2D matrix

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features

        outputs:
            np.ndarray
                a 1D array with the cluster label for each of the observations in `mat`
        """

    def get_error(self) -> float:
        """
        returns the final squared-mean error of the fit model

        outputs:
            float
                the squared-mean error of the fit model
        """

    def get_centroids(self) -> np.ndarray:
        """
        returns the centroid locations of the fit model

        outputs:
            np.ndarray
                a `k x m` 2D matrix representing the cluster centroids of the fit model
        """


import numpy as np
import matplotlib.pyplot as plt
import cvxpy as cp
import matplotlib

"""
x = cp.Variable(1)
obj = cp.quad_form(x, [[1]]) + 1
constraints = [
    # (x-2)*(x-4) <= 0,
    cp.quad_form(x, [[1]]) - 6*x + 8 <= 0,
]
prob = cp.Problem(cp.Minimize(obj), constraints=constraints)
prob.solve()
"""
def f0(x):
    return x**2 + 1
def f1(x):
    return (x-2)*(x-4)
def L(x,l):
    return f0(x) + l*f1(x)
"""
x^2 + 1 + l*(x^2 -6x + 8)
(l+1)x^2 -6*x*l + 1 + 8*l
min_x = 6*l / 2*(l+1) = 3l/(l+1)
l_min = -9l^2/(l+1) + 8l + 1
"""
def Lmin(l):
    xMin = 3*l/(l+1)
    return xMin , L(xMin, l)

x = np.linspace(-2,5,100)
obj = f0(x)
fig, ax = plt.subplots(figsize = (15,10))
ax.plot(x,obj, label = 'objective')
for i, l in enumerate([0.1,0.5,1,2,4,8]):
    ax.plot(x, L(x,l), label = f'lambda {l}', c = f"C{i+1}")
    xMin,lMin = Lmin(l)
    ax.scatter(xMin, lMin, s=80, marker = '*', c = f"C{i+1}")

ax.vlines([2,4], 0, max(obj), linestyles='dashed', label = 'feasible')
ax.set_ylim(0,max(obj))
ax.legend()
fig.show()

print(Lmin(2))

_ = input()


# THIS SCRIPT IS SUPPOSED TO RUN IN A JUPYTER NOTEBOOK (WE USED VS CODE)

# %%
import pandas as pd
import numpy as np
from sklearn.preprocessing import PowerTransformer
from sklearn.covariance import MinCovDet
from scipy.stats import chi2

import seaborn as sb
import matplotlib.pyplot as plt

# %%
def add_is_outlier_IQR(data, col_name):
    col_values = data[col_name]

    Q1=col_values.quantile(0.25)
    Q3=col_values.quantile(0.75)
    IQR=Q3-Q1

    outliers_col_name = f'is_{col_name.replace(" ", "_")}_outlier'
    data[outliers_col_name] = ((col_values < (Q1 - 1.5 * IQR)) | (col_values > (Q3 + 1.5 * IQR)))

    return data

def boxPlot(data, varx, vary, title, xlab, ylab, hue = None):
    hplot = sb.boxplot(varx, vary, hue=hue, data=data)
    plt.title(title, fontsize=18)
    plt.xlabel(xlab, fontsize=16)
    plt.ylabel(ylab, fontsize=16)

    return hplot

def yeo_johnson_transf(data):
    pt = PowerTransformer(method='yeo-johnson', standardize=True)
    pt.fit(data)

    lambdas = pt.lambdas_
    df_yeojohnson = pd.DataFrame( pt.transform(data), columns=data.columns.values )

    return df_yeojohnson, lambdas

# %%
# Load red wine data
df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv',
                 sep=';')

# Get numeric column names but the quality
numeric_col_names = df.drop('quality', axis=1).columns.values

df

# %%
sb.set(style="whitegrid")

# Let's plot sulphates boxplot in order to see if
# there are univariate outliers
boxPlot(df, varx='sulphates', vary=None,
        title='Sulphates distribution',
        xlab='sulphates', ylab=None)
# In case you're not using a Jupyter notebook run also the following:
# plt.show()

# %%
# As you see there are outliers, let's add a boolean
# column to the dataframeindicating which row
# has a sulphate outlier
add_is_outlier_IQR(df, 'sulphates')

# Let's plot the boxplot removing the initial outliers
df_no_outliers = df.loc[~df['is_sulphates_outlier']]

boxPlot(df_no_outliers, varx='sulphates', vary=None,
        title='Sulphates distribution without outliers',
        xlab='sulphates', ylab=None)
# In case you're not using a Jupyter notebook run also the following:
# plt.show()

# %%
# Let's now plot boxplots for each quality vote,
# removing the initial outliers
boxPlot(df_no_outliers, varx='quality', vary='sulphates',
        title='Sulphates distribution without outliers by Quality',
        xlab='quality', ylab='sulphates')
# In case you're not using a Jupyter notebook run also the following:
# plt.show()

# %%
# MULTIVARIATE ANALYSIS
#-----------------------
# Let's now plot an histogram for all the variables
# using the dataset without outliers
df_no_outliers.drop('quality', axis=1).hist(figsize=(10,10))
plt.tight_layout()
plt.show()

# %%
# Let's apply Yeo-Johnson transformations
# in order to remove skewness
df_transf, lambda_arr = yeo_johnson_transf(df_no_outliers[numeric_col_names])

# Let's plot an histogram for all the transformed variables
# in order to check if skewness is decreased
df_transf.hist(figsize=(10,10))
plt.tight_layout()
plt.show()

# %%
# # WARNING: The following plots take some minutes to get plotted.
# #
# # If you want to check also the density plots of each variable and
# # the scatter plots between all of them two by two, grouped by quality,
# # you can use the pairplot. This one is using the original dataframe...
# sb.pairplot(df, hue='quality', diag_kind = 'kde',
#             plot_kws = {'alpha': 0.6, 's': 80, 'edgecolor': 'k'})

# # %%
# # ... and this one is generated using the transformed dataframe.
# df_transf_qual = df_transf.copy()
# df_transf_qual['quality'] = df['quality']

# sb.pairplot(df_transf_qual, hue='quality', diag_kind = 'kde',
#             plot_kws = {'alpha': 0.6, 's': 80, 'edgecolor': 'k'})
# # In case you're not using a Jupyter notebook run also the following:
# # plt.show()
# %%
# Let's compute the squared Mahalanobis distances using
# the Minimum Covariance Determinant to calculate a
# robust covariance matrix
robust_cov = MinCovDet(support_fraction=0.7).fit(df_transf)
center = robust_cov.location_

D = robust_cov.mahalanobis(df_transf - center)
D

# %%
# The squared Mahalanobis distance (D) follows a Chi-Square distribution
# (https://markusthill.github.io/mahalanbis-chi-squared/#the-squared-mahalanobis-distance-follows-a-chi-square-distribution-more-formal-derivation)
#
# Given a cutoff value associated with the statistical significance
# with which we want to determine outliers, we obtain the corresponding
# threshold value above which to consider an observation an outlier
cutoff = 0.98
degrees_of_freedom = df_transf.shape[1]  # given by the number of variables (columns)
cut = chi2.ppf(cutoff, degrees_of_freedom) # threshold value

# Squared Mahalanobis distance values of outliers
D[D > cut]

# %%
# Calculate the probability that the distance D[5]
# is an outlier
chi2.cdf(D[5], degrees_of_freedom)

# %%
# Calulate if the observation is an outlier given the cutoff
is_outlier_arr = (D > cut)

# Calculate the probability that an observation is an outlier not by chance
outliers_stat_proba = np.zeros(len(is_outlier_arr))

for i in range(len(is_outlier_arr)):
    outliers_stat_proba[i] = chi2.cdf(D[i], degrees_of_freedom)

# How many outliers with statistical significance greater than the cutoff
len(outliers_stat_proba[outliers_stat_proba > cutoff])

# %%
# Adding outliers info to the dataframe according to
# the squared Mahalanobis distance
df['is_mahalanobis_outlier'] = is_outlier_arr
df['mahalanobis_outlier_proba'] = outliers_stat_proba

df[df['is_mahalanobis_outlier']]

# %%


import numpy as np

from ytgeotools.typing import all_numbers

def sphere2cart(
    phi: all_numbers, theta: all_numbers, radius: all_numbers
) -> all_numbers:
    """
    seis_model.sphere2cart(phi,theta,radius)

    transformation from yt spherical coordinates to cartesian

    Parameters
    ----------
    phi : ndarray or scalar float/ing
        angle from north in radians (0 = north pole)
    theta : ndarray or scalar float/ing
        longitudinal angle in radians
    radius : ndarray or scalar float/ing
        radius in any units

    all arrays must be the same size (or 2 of 3 can be scalars)

    Returns
    -------
    (x,y,z) : tuple of cartesian x,y,z in same units as radius
    """
    xy = radius * np.sin(phi)
    x = xy * np.cos(theta)
    y = xy * np.sin(theta)
    z = radius * np.cos(phi)
    return (x, y, z)

def get_xy_quad(x: all_numbers, y: all_numbers) -> np.ndarray:
    """

    identify the map quadrant of x-y values:

    1 : north-east, 2: north-west, 3: south-west, 4: south-east

    Parameters
    ----------
    x : array_like
        x values
    y : array_like
        y values

    Returns
    -------
    ndarray
        array of identified quadrant, same shape as x, y

    """
    x = np.asarray(x)
    y = np.asarray(y)

    quad = np.ones(x.shape)
    quad[(x < 0) & (y >= 0)] = 2
    quad[(x <= 0) & (y < 0)] = 3
    quad[(x >= 0) & (y < 0)] = 4
    return quad

def cart2sphere(
    x: all_numbers, y: all_numbers, z: all_numbers, geo: bool = True, deg: bool = True
) -> all_numbers:
    """
    seis_model.cart2sphere(x,y,z,geo=True)

    transformation from cartesian to spherical coordinates

    Parameters
    ----------
    x, y, z  : np.ndarray
        cartesian coordinate arrays
    geo  : bool
        if True (default) then latitude is 0 at equator, otherwise 0 at
        the north pole.
    deg : bool
        if True (default) return angles in degrees

    all arrays must be the same size (or 2 of 3 can be scalars)

    Returns
    -------
    (R,lat,lon) : tuple
        radius, lat, lon

    """

    x = np.asarray(x)
    y = np.asarray(y)
    z = np.asarray(z)
    xy = x ** 2 + y ** 2
    R = np.asarray(np.sqrt(xy + z ** 2))
    phi = np.asarray(np.arccos(z / R))
    theta = np.asarray(np.arctan2(y, x))
    theta[theta < 0] = theta[theta < 0] + np.pi * 2

    if deg or geo:
        phi = phi * 180.0 / np.pi
        theta = theta * 180.0 / np.pi

    if geo:
        phi = 90 - phi  # equator is at 0, +90 is N pole

    return (R, phi, theta)

def geosphere2cart(
    lat: all_numbers, lon: all_numbers, radius: all_numbers
) -> all_numbers:
    """
    transformation from latitude, longitude to to cartesian

    Parameters
    ----------
    lat : ndarray or scalar float/int
        latitude, -180 to 180 or 0 to 360
    lon : ndarray or scalar float/int
        longitude, -90 to 90
    radius : ndarray or scalar float/int
        radius in any units

    all arrays must be the same size (or 2 of 3 can be scalars)

    Returns
    -------
    (x,y,z) : tuple of cartesian x,y,z in same units as radius
    """

    phi = (90.0 - lat) * np.pi / 180.0  # lat is now deg from North

    if isinstance(lon, np.ndarray):
        lon[lon < 0.0] = lon[lon < 0.0] + 360.0
    elif lon < 0.0:
        lon = lon + 360.0
    theta = lon * np.pi / 180.0

    return sphere2cart(phi, theta, radius)