# -*- coding: utf-8 -*-
#Example of numerical integration with Gauss-Legendre quadrature
#Translated to Python by Kyrre Ness Sjøbæk

import sys
import numpy
from computationalLib import pylib

#Read input
if len(sys.argv) == 1:
    print "Number of integration points:"
    n = int(sys.stdin.readline())
    print "Integration limits (a, b)"
    (a,b) = sys.stdin.readline().split(",")
    a = int(a)
    b = int(b)
elif len(sys.argv) == 4:
    n = int(sys.argv[1])
    a = int(sys.argv[2])
    b = int(sys.argv[3])
else:
    print "Usage: python", sys.argv[0], "N a b"
    sys.exit(0)

#Definitions
m = pylib(inputcheck=False,cpp=False)

def integrand(x):
    return 4./(1. + x*x)

#Integrate with Gaus-legendre!
(x,w) = m.gausLegendre(a,b,n)
int_gaus = numpy.sum(w*integrand(x))

#Final output
print "integration of f(x) 4/(1+x**2) from",a,"to",b,"with",n,"meshpoints:"
print "Gaus-legendre:", int_gaus
print "Trapezoidal:  ", m.trapezoidal(a,b,n, integrand)
print "Simpson:      ", m.simpson(a,b,n,integrand)


import numpy as np

def sigmoid(z):
    return 1 / (1 + np.exp(-z))


"""Collection of norm functions i..e, ways to measure the norm of a input- (or explanation) vector."""
import numpy as np

def fro_norm(a: np.array) -> float:
    """Calculate Froberius norm for an array."""
    assert a.ndim == 1, "Check that 'fro_norm' receives a 1D array."
    return np.linalg.norm(a)

def l2_norm(a: np.array) -> float:
    """Calculate L2-norm for an array."""
    assert a.ndim == 1, "Check that 'l2_norm' receives a 1D array."
    return np.linalg.norm(a)

def linf_norm(a: np.array) -> float:
    """Calculate L inf-norm for an array."""
    assert a.ndim == 1, "Check that 'linf_norm' receives a 1D array."
    return np.linalg.norm(a, ord=np.inf)


import pdb
import numpy as np

def produto_interno_canonico(u, v):
    resultado = 0
    for i in range(len(u)):
        resultado += u[i] * v[i]
    return resultado

def produto_interno(u, v, funcional_linear = produto_interno_canonico):
    return funcional_linear(u, v)

def norma(v, funcional_linear = produto_interno_canonico):
    return funcional_linear(v, v) ** (0.5)

def angulo(u, v, funcional_linear = produto_interno_canonico):
    return np.arccos(np.around(funcional_linear(u, v) / (norma(u, funcional_linear = funcional_linear) * norma(v, funcional_linear = funcional_linear)), decimals = 6))

def projecao_escalar(u, v, funcional_linear = produto_interno_canonico):
    return np.around(produto_interno(u, v, funcional_linear = funcional_linear)\
         / produto_interno(v, v, funcional_linear = funcional_linear), decimals = 6)

def projecao_vetorial(u, v, funcional_linear = produto_interno_canonico):
    return projecao_escalar(u, v, funcional_linear = funcional_linear) * v

def v_ortogonal_u(u, v, funcional_linear = produto_interno_canonico):
    return v - projecao_escalar(v, u, funcional_linear = funcional_linear) * u

def normalizar(u, funcional_linear = produto_interno_canonico):
    return u / norma(u, funcional_linear = funcional_linear)

def gram_schmidt(i, vetores, funcional_linear = produto_interno_canonico):
    w = [vetores[0]]
    for j in range(1, i + 1):
        v = vetores[j]
        w += [v - sum([projecao_vetorial(v, w[k], funcional_linear = funcional_linear) for k in range(j)])]
    return w[i]

def ortogonalizar_vetores(vetores, funcional_linear = produto_interno_canonico):
    ortogonalizados = []
    for i in range(len(vetores)):
        ortogonalizados += [gram_schmidt(i, vetores, funcional_linear = funcional_linear)]
    return ortogonalizados

def normalizar_vetores(vetores, funcional_linear = produto_interno_canonico):
    normalizados = []
    for v in vetores:
        normalizados += [normalizar(v, funcional_linear = funcional_linear)]
    return normalizados

def ortonormalizar_vetores(vetores, funcional_linear = produto_interno_canonico):
    return normalizar_vetores(ortogonalizar_vetores(vetores, funcional_linear = funcional_linear), funcional_linear = funcional_linear)


"""
Author: Drishika Nadella
Date: 20th April 2021
Time: 16:09PM
Email: drishikanadella@gmail.com
"""

import numpy as np
import matplotlib.pyplot as plt

# Creating a list with the zeroth Catalan number
C = [1]
# Creating the counter
i = 0

# Calculating and printing Catalan numbers upto 1 billion
while C[-1] <= 10**9:
    print(C[-1])
    C.append((4*i + 2)/(i+2)*C[i])
    i+=1

# Plotting the Catalan numbers
plt.plot(range(len(C)), C, marker='o')
plt.title("Visualizing the Catalan numbers upto 1 billion")
plt.show()


import numpy as np
import matplotlib.pyplot as plt
from perceptron import Perceptron

# generate data
x1, x2 = np.meshgrid([0, 1], [0, 1])
x1, x2 = np.hstack(x1), np.hstack(x2)
data = np.vstack([x1, x2])

######################################################################################

# generate labels and train perceptron
y = x1 & x2  # AND
p = Perceptron(2)
p.train(data, y)
w, b = p.weight, p.bias

# plot data
plt.plot(x1[y == 0], x2[y == 0], 'ro', x1[y == 1], x2[y == 1], 'b+')
plt.xlabel('x1'), plt.ylabel('x2'), plt.grid(True)

# plot separation line
left, right = plt.xlim()
plt.plot([left, right], [(-w[0]*left-b)/w[1], (-w[0]*right-b)/w[1]], 'k-')
plt.title('AND logical function')

######################################################################################

# generate labels and train perceptron
y = x1 | x2  # OR
p = Perceptron(2)
p.train(data, y)
w, b = p.weight, p.bias

# plot data
plt.subplots()
plt.plot(x1[y == 0], x2[y == 0], 'ro', x1[y == 1], x2[y == 1], 'b+')
plt.xlabel('x1'), plt.ylabel('x2'), plt.grid(True)

# plot separation line
left, right = plt.xlim()
plt.plot([left, right], [(-w[0]*left-b)/w[1], (-w[0]*right-b)/w[1]], 'k-')
plt.title('OR logical function')
plt.show()


# based on https://sebastianraschka.com/Articles/2014_kernel_pca.html

from __future__ import annotations
from typing import Union
import numpy as np
from scipy.spatial.distance import pdist, squareform
from scipy.linalg import eigh

class KernelPCA:
    """KernelPCA
    Parameters:
    -----------
    n_components: int = 2
        Number of components to keep.
    gamma: float = None
        Kernel coefficient
    """
    def __init__(self, n_components: int = 2, gamma: float = None):
        self.n_components = n_components
        self.gamma = gamma
        self.alphas = None
        self.lambdas = None
        self.X = None

    def fit(self, X: Union[list, np.ndarray]) -> KernelPCA:
        if self.gamma == None:
            self.gamma = 1 / X.shape[1]

        sq_dists = pdist(X, 'sqeuclidean')

        mat_sq_dists = squareform(sq_dists)

        K = np.exp(-self.gamma * mat_sq_dists)

        N = K.shape[0]
        one_n = np.ones((N,N)) / N
        K_norm = K - one_n.dot(K) - K.dot(one_n) + one_n.dot(K).dot(one_n)

        eigenvalues, eigenvectors = eigh(K_norm)

        alphas = np.column_stack((eigenvectors[:,-i] for i in range(1, self.n_components+1)))
        lambdas = [eigenvalues[-i] for i in range(1, self.n_components+1)]

        self.alphas = alphas
        self.lambdas = lambdas
        self.X = X

        return self

    def fit_transform(self, X: Union[list, np.ndarray]) -> np.ndarray:
        self.fit(X)
        return self.alphas * np.sqrt(self.lambdas)

    def transform(self, X: Union[list, np.ndarray]) -> np.ndarray:
        # TODO: Rewrite as this is very inefficient
        def transform_row(X_r):
            pair_dist = np.array([np.sum((X_r-row)**2) for row in self.X])
            k = np.exp(-self.gamma * pair_dist)
            return k.dot(self.alphas / self.lambdas)

        return np.array(list(map(transform_row, X)))

if __name__ == '__main__':
    from sklearn.datasets import make_circles
    import matplotlib.pyplot as plt

    X, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2)

    plt.figure(figsize=(8,6))

    pca = KernelPCA(n_components=3)
    pca.fit(X)
    X = pca.transform(X)

    print(X)


import numpy as np
from scipy.integrate import odeint
import matplotlib.pyplot as plt

# lets define the van der pol oscillator!
# d^2x/tx^2 - mu(1 - x^2)*dx/dt + x = 0
# d^2x/tx^2 = mu(1 - x^2)*dx/dt - x
# a = dv/dt = d^2x/dt^2 = mu*(1 - x^2)*dx/dt - x = mu*(1 - x^2)v - x
# dx/dt = v
# dv/dt = mu*(1 - x^2)v - x

def get_vdp_accel(x, v):
    return mu*(1 - x**2)*v - x

def vdp_rhs(state, t, mu):
    x, v = state[0], state[1]
    a = get_vdp_accel(x, v)
    return np.array([v, a])

# for mu = 0 we get the harmonic oscillator for k = 1
# d^2x/tx^2 - 0(1 - x^2)*dx/dt + x = 0
# d^2x/tx^2 + x = 0
# -d^2x/tx^2 = x

# let's simulate the system
x0 = 2.0
v0 = 0.0
init_state = np.array([x0, v0])

T = 100.0
steps = 1000
ts = np.linspace(0, T, steps)

mu = 5.0

hist = odeint(vdp_rhs, init_state, ts, args=(mu,))
xs, vs = hist[:, 0], hist[:, 1]

# display the evolution of the system over time
plt.subplot(2,1,1)
plt.title("Van der Pol oscillator over time for mu={}".format(mu))
plt.plot(ts, xs)
plt.ylabel("x")
plt.subplot(2,1,2)
plt.xlabel("t")
plt.ylabel("dx/dt")
plt.plot(ts, vs)

# make a direction field [dx/dt, d^2x/dt^2]
res = 20
x = np.linspace(min(xs), max(xs), res)
v = np.linspace(min(vs), max(vs), res)
xx, vv = np.meshgrid(x, v)
aa = get_vdp_accel(xx, vv)
plt.title("Direction field of the van der pol oscillator with mu={}".format(mu))
plt.quiver(xx, vv, vv, aa)
plt.xlabel("x")
plt.ylabel("dx/dt")
plt.plot(xs, vs, label="from [{}, {}]".format(x0, v0))
plt.legend()
plt.grid()

# how does adding an epsilon value effect the system?
mu = 15.0
hist = odeint(vdp_rhs, init_state, ts, args=(mu,))
xs, vs = hist[:, 0], hist[:, 1]
eps = 1e-3
init_state_eps = np.array([x0 + eps, v0 + eps])
res_eps = odeint(vdp_rhs, init_state_eps, ts, args=(mu,))
xs_eps, vs_eps = res_eps[:, 0], res_eps[:, 1]
x_err = xs - xs_eps
v_err = vs - vs_eps
plt.subplot(2,1,1)
plt.title("Error evolution in the van der pol oscillator")
plt.plot(ts, x_err)
plt.ylabel("x - x'")
plt.subplot(2,1,2)
plt.plot(ts, v_err)
plt.ylabel("v - v'")
plt.xlabel("t")
# ok, we have some kind of oscillation going on
plt.plot(ts, xs)
plt.plot(ts, xs_eps) # almost equal
plt.show()


import numpy as np
import numpy.random as npr
import matplotlib.pyplot as plt
# 比较模拟结果的分布特性.
import scipy.stats as scs
# 字体
import matplotlib
matplotlib.rcParams['font.sans-serif'] = ['SimHei']

def print_statistics(a1, a2):
    """
    Prints selected statistics.

    Parameters
    ==========
    al, a2 : ndarray objects
    results object from simulation
    """
    # 主体
    sta1 = scs.describe(a1)
    sta2 = scs.describe(a2)
    print("%14s %14s %14s"%("statistic", "data set 1", "data set 2"))
    print("-" * 45)
    print("%14s %14.3f %14.3f"%("size", sta1[0], sta2[0]))
    print("%14s %14.3f %14.3f"%("min", sta1[1][0], sta2[1][0]))
    print("%14s %14.3f %14.3f"%("max", sta1[1][1], sta2[1][1]))
    print("%14s %14.3f %14.3f"%("mean", sta1[2], sta2[2]))
    print("%14s %14.3f %14.3f"%("std", np.sqrt(sta1[3]), np.sqrt(sta2[3])))
    print("%14s %14.3f %14.3f"%("skew", sta1[4], sta2[4]))
    print("%14s %14.3f %14.3f"%("kurtosis", sta1[5], sta2[5]))

S0    = 100    # initial value
r     = 0.05   # constant short rate
sigma = 0.25   # constant volatility
T     = 2.0    # in years
I     = 10000  # number of random draws

ST1   = S0 * np.exp(
            (r-sigma**2/2)*T + sigma*np.sqrt(T)*npr.standard_normal(I)
        )
ST2   = S0 * npr.lognormal(mean  = (r-sigma**2/2)*T,
                           sigma = sigma*np.sqrt(T),
                           size  = I)
print_statistics(ST1, ST2)

"""
plt.hist(ST1, bins=50)
plt.xlabel("index level")
plt.ylabel("frequency")
plt.grid(True)
plt.show()
"""

# 几何布朗运动
S0    = 100    # initial value
r     = 0.05   # constant short rate
sigma = 0.25   # constant volatility

M  = 50
T  = 2.0
I  = 10000
dt = T / M
S  = np.zeros((M+1, I))
S[0] = S0
for t in range(1, M+1):
    S[t] = S[t-1] * npr.lognormal(mean  = (r-sigma**2/2)*dt,
                                 sigma = sigma*np.sqrt(dt),
                                 size  = I)

plt.plot(S[:, :10], lw=1.5)
plt.title("模拟几何布朗运动路径")
plt.xlabel("time")
plt.ylabel("index level")
plt.grid(True)
plt.show()

# 平方根扩散(欧拉格式)
x0    = 0.05
kappa = 3.0
theta = 0.02
sigma = 0.1

M  = 50
T  = 2.0
I  = 10000
dt = T / M
x  = np.zeros((M+1, I))
x[0] = x0
clamp = lambda x: np.maximum(x, 0)
for t in range(1, M+1):
    x[t] = x[t-1] + kappa*(theta-clamp(x[t-1]))*dt \
         + sigma*np.sqrt(clamp(x[t-1]))*np.sqrt(dt)*npr.standard_normal(I)
x = clamp(x)

plt.plot(x[:, :10], lw=1.5)
plt.title("模拟平方根扩散路径(欧拉格式)")
plt.xlabel("time")
plt.ylabel("index level")
plt.grid(True)
plt.show()

# 平方根扩散(精确格式)
x0    = 0.05
kappa = 3.0
theta = 0.02
sigma = 0.1

M  = 50
T  = 2.0
I  = 10000
dt = T / M
y  = np.zeros((M+1, I))
y[0] = x0
for t in range(1, M+1):
    df = 4*theta*kappa / sigma**2
    c  = (sigma**2 * (1-np.exp(-kappa*dt))) / (4*kappa)
    nc = np.exp(-kappa*dt) * y[t-1] / c
    y[t] = c * npr.noncentral_chisquare(df, nc, size=I)

plt.plot(y[:, :10], lw=1.5)
plt.title("模拟平方根扩散路径(精确格式)")
plt.xlabel("time")
plt.ylabel("index level")
plt.grid(True)
plt.show()

# 比较欧拉格式与精确格式
print_statistics(x[-1], y[-1])

# 随机波动率
r     = 0.05
v0    = 0.1
kappa = 3.0
theta = 0.25
sigma = 0.1
rho   = 0.6

M  = 50
T  = 2.0
I  = 10000
dt = T / M
v  = np.zeros((M+1, I))
v[0] = v0

ran_num = npr.standard_normal((2, M+1, I))
corr_mat = np.array([[1.0,rho], [rho,1.0]])
cho_mat  = np.linalg.cholesky(corr_mat)
clamp    = lambda x: np.maximum(x, 0)
for t in range(1, M+1):
    ran = np.dot(cho_mat, ran_num[:,t,:])
    v[t] = v[t-1] + kappa*(theta-clamp(v[t-1]))*dt \
         + sigma*np.sqrt(clamp(v[t-1]))*np.sqrt(dt)*ran[1]
v = clamp(v)

w0 = 100
w  = np.zeros((M+1, I))
w[0] = w0
for t in range(1, M+1):
    ran = np.dot(cho_mat, ran_num[:,t,:])
    w[t] = w[t-1] * np.exp((r-v[t])*dt +
                           np.sqrt(v[t])*np.sqrt(dt)*ran[0])

fig,(ax1,ax2) = plt.subplots(2, 1, sharex=True, figsize=(7,6))
ax1.plot(w[:,:10], lw=1.5)
ax1.set_title("模拟随机波动率模型路径")
ax1.set_ylabel("index level")
ax1.grid(True)
ax2.plot(v[:,:10], lw=1.5)
ax2.set_xlabel("time")
ax2.set_ylabel("volatility")
ax2.grid(True)

fig.show()


import numpy as np
import matplotlib.pyplot as plt
import os
figdir = os.path.join(os.environ["PYPROBML"], "figures")
def save_fig(fname): plt.savefig(os.path.join(figdir, fname))

from mpl_toolkits.mplot3d import axes3d
from scipy.stats import multivariate_normal

Gs = ["Full", "Diag", "Spherical"]

# Gaussian Parameters:
# Mean:
mu = [0, 0]
# Covariances:
Covs = {'Full': [[2, 1.8], [1.8, 2]],
        'Diag': [[1, 0], [0, 3]],
        'Spherical': [[1, 0], [0, 1]]}

#Multivariate gaussian PDF
def Gpdf(x, y, G):
    return multivariate_normal(mean=mu, cov=Covs[G]).pdf([x, y])

Gpdf = np.vectorize(Gpdf, excluded=['G'])

points = np.linspace(-5, 5, 100)
X, Y = np.meshgrid(points, points)

def make_contour_plot(G):
    Z = Gpdf(X, Y, G)
    fig, ax = plt.subplots()
    ax.contour(X, Y, Z)
    plt.axis('equal')
    plt.title(G)
    plt.draw()
    save_fig('gaussPlot2dDemoContour{}.pdf'.format(G))
    plt.show()

def make_surface_plot(G):
    Z = Gpdf(X, Y, G)
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    ax.plot_surface(X, Y, Z, rstride=2, cstride=2, color='white', edgecolor="black")
    #ax.axis('equal')
    #ax.title(G)f
    plt.draw()
    save_fig('gaussPlot2dDemoSurf{}.pdf'.format(G))
    plt.show()

for g in Gs:
    make_contour_plot(g)
    make_surface_plot(g)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Date    : 2018-12-17 20:22:27
# @Author  : Zhi Liu (zhiliu.mind@gmail.com)
# @Link    : http://blog.csdn.net/enjoyyl
# @Version : $1.0$

import random
import numpy as np

def circle1(x0, y0, r, nps):
    r"""Generates circle (x, y)

    :math:`(x-x_0)^2 + (y-y_0)^2 = r^2`

    Arguments:
       x0 {Float} -- x coordinate of circle center
       y0 {Float} -- y coordinate of circle center
       r {Float} -- radius of circle
       nps {Integer} -- number of points
    """

    xmin = x0 - r
    xmax = x0 + r

    x = np.linspace(xmin, xmax, nps)
    y1 = y0 - np.sqrt(r * r - (x - x0) ** 2)
    y2 = y0 + np.sqrt(r * r - (x - x0) ** 2)

    x = np.concatenate((x, x), axis=0)
    y = np.concatenate((y1, y2), axis=0)
    return x, y

def circle2(x0, y0, r):
    r"""Generates circle (x, y)

    :math:`(x-x_0)^2 + (y-y_0)^2 = r^2`

    Arguments:
       x0 {Float} -- x coordinate of circle center
       y0 {Float} -- y coordinate of circle center
       r {Float} -- radius of circle
       nps {Integer} -- number of points
    """

    angles_circle = [i * np.pi / 180 for i in range(0, 360)]
    x = x0 + r * np.cos(angles_circle)
    y = y0 + r * np.sin(angles_circle)

    return x, y

def ellipse(a, b):
    """[summary]

    [description]

    Arguments:
       a {[type]} -- [description]
       b {[type]} -- [description]
    """
    pass

def ellipse_surface(a, b, x0, y0, nps, mod):
    """generates ellipse surface

    :math:`\frac{(x-x_0)^2}{a^2} + \frac{{y-y_0}^2}{b^2} \ngt 1`

    Arguments:
       a {float} -- axis of ellipse
       b {float} -- axis of ellipse
       x0 {float} -- x center of ellipse
       y0 {float} -- y center of ellipse
       nps {Integer} -- number of points
       mod {float} -- mode: 'rand' or 'order'
    """

    if mod is None:
        mod = 'order'

    aa = a ** 2
    bb = b ** 2

    xmin = x0 - a
    xmax = x0 + a
    ymin = y0 - b
    ymax = y0 + b

    x = []
    y = []

    if mod is 'order':
        xx = np.linspace(xmin, xmax, nps)
        yy = np.linspace(ymin, ymax, nps)
        for xi in xx:
            for yj in yy:
                if (xi - x0) ** 2 / aa + (yj - y0) ** 2 / bb <= 1:
                    x.append(xi)
                    y.append(yj)

    cnt = 0
    if mod is 'rand':
        for i in range(0, 10 * nps):
            print(i)
            xi = (xmax - xmin) * np.random.rand(1) + xmin
            yi = (ymax - ymin) * np.random.rand(1) + ymin
            if (xi - x0) ** 2 / aa + (yi - y0) ** 2 / bb <= 1:
                x.append(xi)
                y.append(yi)
            if cnt < nps:
                cnt = cnt + 1
            else:
                break
    return x, y

if __name__ == '__main__':
    import matplotlib.pyplot as plt

    x, y = circle1(0, 0, 1, 1000)
    plt.figure()
    plt.plot(x, y, '.')
    plt.title('circle')
    plt.show()

    x, y = circle2(0, 0, 1)
    plt.figure()
    plt.plot(x, y, '.r')
    plt.title('circle')
    plt.show()

    x, y = ellipse_surface(8, 2, 1, 1, 1000, 'rand')
    plt.figure()
    plt.scatter(x, y, c='g', marker='+')
    plt.axis('equal')
    plt.title('ellipse surface')
    plt.show()


import typing

import matplotlib.pyplot as plt
import numpy as np
from numpy.polynomial.polynomial import Polynomial

def pi_n(alpha: float, n: int):
    """
    Вычисляет значение предельной вероятности системы при n заявках.
    Значение n <= числа обслуживающих каналов в системе.
    :param alpha: Коэффициент загрузки системы.
    :param n: Количество заявок в системе.
    :return: Значение предельной вероятности системы при n заявках.
    """
    return alpha ** n / (np.math.factorial(n) * np.sum(
        [alpha ** k / np.math.factorial(k) for k in range(n + 1)]) + alpha ** (n + 1) / (
                                 n - alpha))

def calc_ey(mu: float, lam: float, n: int):
    """
    Вычилсяет математическое ожидание заявки от поступления ее в систему
    до начала обслуживания одним из каналов.
    :param mu: Интенсивность обслуживания заявок одним каналом узла.
    :param lam: Интенсивность входящего в узел потока из одиночных пассажиров.
    :param n: Количество каналов обслуживания в узле.
    :return: Значение математического ожидания времени ожидания заявки в очереди.
    """
    alpha = lam / mu
    return n * pi_n(alpha, n) / (mu * (n - alpha) ** 2)

def fit_ey(mu: float, lam: float, n_min: int, n_max: int, p_pow=4, plot=False):
    """
    Аппроксимирует математическое ожидание времени ожидания заявки в очереди
    полиномом заданной степени.
    :param mu: Интенсивность обслуживания заявок одним каналом узла.
    :param lam: Интенсивность входящего в узел потока из одиночных пассажиров.
    :param n_min: Минимально возможное число каналов обслуживания в узле.
    :param n_max: Максимально возможное число каналов обслуживания в узле.
    :param p_pow: Степень полинома для аппроксимации.
    :param plot: Визуализировать приближение полиномом.
    :return: Массив из коэффициентов полинома.
    """
    n = np.arange(n_min, n_max + 1)
    ey = np.array([calc_ey(mu, lam, n_) for n_ in n])
    if len(n) < p_pow:
        p_pow = len(n) - 1
    if p_pow:
        res = Polynomial.fit(n, np.log(ey), p_pow, full=True)
        coef = res[0].convert().coef[::-1]
        rss = res[1][0][0]
    else:
        coef = np.log(ey)
        rss = 0.

    if plot:
        x = np.linspace(n_min - 1, n_max + 1, 100)
        f = np.poly1d(coef)

        plt.figure(figsize=(13, 8))
        plt.scatter(n, np.log(ey), s=55)
        plt.plot(x, f(x), label=f'Аппроксимирующая кривая (полином), RSS = {np.round(rss, 2)}')
        plt.title(
            'Логарифм мат. ожидания времени ожидания начала обслуживания в зависимости от числа обслуживающих каналов в узле')
        plt.xlabel('Количество обслуживающих каналов в узле')
        plt.ylabel('Логарифм мат. ожидания времени ожидания начала обслуживания')
        plt.legend()
        plt.grid()
        plt.show()

        rss = np.sum(np.square(ey - np.array([np.exp(f(n_)) for n_ in n])))
        plt.figure(figsize=(13, 8))
        plt.scatter(n, ey, s=55)
        plt.plot(x, np.exp(f(x)), label=f'Аппроксимирующая кривая, RSS = {np.round(rss, 2)}')
        plt.title(
            'Мат. ожидание времени ожидания начала обслуживания в зависимости от числа обслуживающих каналов в узле')
        plt.xlabel('Количество обслуживающих каналов в узле')
        plt.ylabel('Мат. ожидание времени ожидания начала обслуживания')
        plt.legend()
        plt.grid()
        plt.show()

    return coef

def calc_time(x: list, p: list):
    """
    Рассчет суммарного среднего времени ожидания.
    :param x: Количество каналов обслуживания в каждом узле.
    :param p: Массив из коэффициентов полинома,
    описывающего логарифм среднего времени ожидания в каждом узле.
    :return: Суммарное среднее время ожидания.
    """
    res = 0
    for x_, p_ in zip(x, p):
        f = np.poly1d(p_)
        res += np.exp(f(x_))
    return res

def constraint_cost(x: np.ndarray, p: list):
    """
    Ограничение для затрат на обслуживающий персонал.
    Должно быть >= 0.
    :param x: Массив. Количество каналов обслуживания в каждом узле.
    :param p[0]: Максимальное значение затрат на каналы обслуживания.
    :param p[1]: Массив. Стоимость каждого канала обслуживания.
    :return: Значение ограничения.
    """
    return np.array([p[0] - sum(x * p[1])])

def calc_cost(x: np.ndarray, p: np.ndarray):
    """
    Рассчет затрат на обслуживающий персонал.
    :param x: Количество каналов обслуживания в каждом узле.
    :param p: Массив. Стоимость каждого канала обслуживания.
    :return: Значение затрат.
    """
    return sum(x * p)

def constraint_time(x: list, p: list):
    """
    Ограничение для суммарного среднего времени ожидания. Должно быть >= 0.
    :param x: Количество каналов обслуживания в каждом узле.
    :param p[0]: Максимальное суммарное время ожидания пассажира в очереди.
    :param p[1]: Массив из коэффициентов полинома,
    описывающего логарифм среднего времени ожидания в каждом узле.
    :return: Значение ограничения.
    """
    res = p[0]
    for x_, p_ in zip(x, p[1]):
        f = np.poly1d(p_)
        res -= np.exp(f(x_))
    return res

class ConstraintTime:
    def __init__(self, t_max: float, coefs: np.ndarray):
        """
        Ограничение для суммарного среднего времени ожидания. Должно быть >= 0.
        :param t_max: Максимальное суммарное время ожидания пассажира в очереди.
        :param coefs: Массив из коэффициентов полинома,
        описывающего логарифм среднего времени ожидания в каждом узле.
        """
        self.t_max = t_max
        self.coefs = coefs

    def __call__(self, x: typing.Union[list, np.ndarray]):
        """
        :param x: Количество каналов обслуживания в каждом узле.
        описывающего логарифм среднего времени ожидания в каждом узле.
        :return: Значение ограничения.
        """
        res = self.t_max
        for x_, p_ in zip(x, self.coefs):
            f = np.poly1d(p_)
            res -= np.exp(f(x_))
        return res

def f_ey(n: list, mu: list, lam: float, p: list):
    """
    Вычилсяет суммарное математическое ожидание заявки
    в сети массового обслуживания.
    :param n: Массив. Количество каналов обслуживания в узлах сети.
    :param mu: Массив. Интенсивности обслуживания заявок узлами сети.
    :param lam: Интенсивность входящего в первый узел потока из одиночных пассажиров.
    :param p: Массив из вероятностей перехода в следующие после первого узлы сети.
    :return: Значение суммарного математического ожидания времени ожидания заявки в очередях сети.
    """
    l = np.insert(lam * np.cumprod(p), 0, lam)
    return sum([calc_ey(mu[i], l[i], n[i]) for i in range(len(n))])

def f_conv(n: list, mu: list, lam: float, p: list, n_min: list, n_max: list, s: list, w: list):
    """
    Рассчет взвешенной суммы суммарного времени ожидания и затрат на обслуживающий персонал.
    :param n: Массив. Количество каналов обслуживания в узлах сети.
    :param mu: Массив. Интенсивности обслуживания заявок узлами сети.
    :param lam: Интенсивность входящего в первый узел потока из одиночных пассажиров.
    :param p: Массив из вероятностей перехода в следующие после первого узлы сети.
    :param n_min: Массив. Минимальное число каналов в каждом узле.
    :param n_max: Массив. Максимальное число каналов в каждом узле.
    :param s: Массив. Стоимость каждого канала обслуживания.
    :param w: Массив. Коэффициенты свертки каждого критерия.
    :return: Значение взвешенной суммы критериев.
    """
    a1 = f_ey(n_max, mu, lam, p)
    a1_max = f_ey(n_min, mu, lam, p)
    a2 = calc_cost(n_min, s)
    a2_max = calc_cost(n_max, s)
    return w[0] * (f_ey(n, mu, lam, p) - a1) / (a1_max - a1) + w[1] * (
            calc_cost(n, s) - a2) / (a2_max - a2)

def f_ideal(n: list, mu: list, lam: float, p: list, n_min: list, n_max: list, s: list, w: list, m: str):
    """
    Рассчет расстояния до идеальной точки.
    :param n: Массив. Количество каналов обслуживания в узлах сети.
    :param mu: Массив. Интенсивности обслуживания заявок узлами сети.
    :param lam: Интенсивность входящего в первый узел потока из одиночных пассажиров.
    :param p: Массив из вероятностей перехода в следующие после первого узлы сети.
    :param n_min: Массив. Минимальное число каналов в каждом узле.
    :param n_max: Массив. Максимальное число каналов в каждом узле.
    :param s: Массив. Стоимость каждого канала обслуживания.
    :param w: Массив. Веса каждого критерия.
    :param m: Строка ('2-norm' или 'inf'). Задает норму для вычисления отклонения
    критерия от идеальной точки.
    :return: Расстояние до идеальной точки.
    """
    metrics = ['2-norm', 'inf']
    assert m in metrics
    a1 = f_ey(n_max, mu, lam, p)
    a1_max = f_ey(n_min, mu, lam, p)
    a2 = calc_cost(n_min, s)
    a2_max = calc_cost(n_max, s)
    if m == metrics[0]:
        return w[0] * ((f_ey(n, mu, lam, p) - a1) / (a1_max - a1)) ** 2 + w[1] * (
                (calc_cost(n, s) - a2) / (a2_max - a2)) ** 2
    else:
        return max(w[0] * abs((f_ey(n, mu, lam, p) - a1) / (a1_max - a1)), w[1] *
                   abs((calc_cost(n, s) - a2) / (a2_max - a2)))

def calc_conv(x: list, p: list):
    """
    Рассчет взвешенной суммы времени ожидания и затрат на обслуживающий персонал.
    :param x: Количество каналов обслуживания в каждом узле.
    :param p[0]: Массив. Коэффициенты свертки каждого критерия.
    :param p[1]: Массив из коэффициентов полинома,
    описывающего логарифм среднего времени ожидания в каждом узле.
    :param p[2]: Массив. Стоимость каждого канала обслуживания.
    :param p[3]: Массив. Минимальное число каналов в каждом узле.
    :param p[4]: Массив. Максимальное число каналов в каждом узле.
    :return: Значение взвешенной суммы критериев.
    """
    a1 = calc_time(p[4], p[1])
    a1_max = calc_time(p[3], p[1])
    a2 = calc_cost(p[3], p[2])
    a2_max = calc_cost(p[4], p[2])
    return p[0][0] * (calc_time(x, p[1]) - a1) / (a1_max - a1) + p[0][1] * (
            calc_cost(x, p[2]) - a2) / (a2_max - a2)

def calc_ideal(x: list, p: list):
    """
    Рассчет расстояния до идеальной точки.
    :param x: Количество каналов обслуживания в каждом узле.
    :param p[0]: Массив. Веса каждого критерия.
    :param p[1]: Массив из коэффициентов полинома,
    описывающего логарифм среднего времени ожидания в каждом узле.
    :param p[2]: Массив. Стоимость каждого канала обслуживания.
    :param p[3]: Массив. Минимальное число каналов в каждом узле.
    :param p[4]: Массив. Максимальное число каналов в каждом узле.
    :param p[5]: Строка ('2-norm' или 'inf'). Задает норму для вычисления отклонения
    критерия от идеальной точки.
    :return: Расстояние до идеальной точки.
    """
    metrics = ['2-norm', 'inf']
    assert p[5] in metrics
    a1 = calc_time(p[4], p[1])
    a1_max = calc_time(p[3], p[1])
    a2 = calc_cost(p[3], p[2])
    a2_max = calc_cost(p[4], p[2])
    if p[5] == metrics[0]:
        return p[0][0] * ((calc_time(x, p[1]) - a1) / (a1_max - a1)) ** 2 + p[0][1] * (
                (calc_cost(x, p[2]) - a2) / (a2_max - a2)) ** 2
    else:
        return max(p[0][0] * abs((calc_time(x, p[1]) - a1) / (a1_max - a1)), p[0][1] *
                   abs((calc_cost(x, p[2]) - a2) / (a2_max - a2)))


#!/usr/bin/env python
# coding: utf-8
# # Portfolio Optimization
# “Modern Portfolio Theory (MPT), a hypothesis put forth by Harry Markowitz in his paper “Portfolio Selection,”
# (published in 1952 by the Journal of Finance) is an investment theory based on the idea that risk-averse investors can construct portfolios
# to optimize or maximize expected return based on a given level of market risk, emphasizing that risk is an inherent part of higher reward.
# It is one of the most important and influential economic theories dealing with finance and investment.
# ## Monte Carlo Simulation for Optimization Search
# We could randomly try to find the optimal portfolio balance using Monte Carlo simulation
# In[21]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy.optimize import minimize

# Download and get Daily Returns
aapl = pd.read_csv('Data/09-Python-Finance-Fundamentals_AAPL_CLOSE.csv', index_col='Date', parse_dates=True)
cisco = pd.read_csv('Data/09-Python-Finance-Fundamentals_CISCO_CLOSE.csv', index_col='Date', parse_dates=True)
ibm = pd.read_csv('Data/09-Python-Finance-Fundamentals_IBM_CLOSE.csv', index_col='Date', parse_dates=True)
amzn = pd.read_csv('Data/09-Python-Finance-Fundamentals_AMZN_CLOSE.csv', index_col='Date', parse_dates=True)
stocks = pd.concat([aapl, cisco, ibm, amzn], axis=1)
stocks.columns = ['aapl', 'cisco', 'ibm', 'amzn']
stocks.head()
# In[25]:
mean_daily_ret = stocks.pct_change(1).mean()
mean_daily_ret
# In[26]:
stocks.pct_change(1).corr()
# # Simulating Thousands of Possible Allocations
stocks.head()
# In[28]:
stock_normed = stocks / stocks.iloc[0]
stock_normed.plot()
# In[29]:
stock_daily_ret = stocks.pct_change(1)
stock_daily_ret.head()
# ## Log Returns vs Arithmetic Returns
# We will now switch over to using log returns instead of arithmetic returns, for many of our use cases they are almost the same,
# but most technical analyses require detrending/normalizing the time series and using log returns is a nice way to do that.
# Log returns are convenient to work with in many of the algorithms we will encounter.
# For a full analysis of why we use log returns, check [this great article](https://quantivity.wordpress.com/2011/02/21/why-log-returns/).
log_ret = np.log(stocks / stocks.shift(1))
log_ret.head()
# In[31]:
log_ret.hist_df(bins=100, figsize=(12, 6));
plt.tight_layout()
# In[32]:
log_ret.describe().transpose()
# In[33]:
log_ret.mean() * 252
# Compute pairwise covariance of columns
log_ret.cov()
# In[35]:
log_ret.cov() * 252  # multiply by days
# ## Single Run for Some Random Allocation
# Set seed (optional)
np.random.seed(101)
# Stock Columns
print('Stocks')
print(stocks.columns)
print('\n')
# Create Random Weights
print('Creating Random Weights')
weights = np.array(np.random.random(4))
print(weights)
print('\n')
# Rebalance Weights
print('Rebalance to sum to 1.0')
weights = weights / np.sum(weights)
print(weights)
print('\n')
# Expected Return
print('Expected Portfolio Return')
exp_ret = np.sum(log_ret.mean() * weights) * 252
print(exp_ret)
print('\n')
# Expected Variance
print('Expected Volatility')
exp_vol = np.sqrt(np.dot(weights.T, np.dot(log_ret.cov() * 252, weights)))
print(exp_vol)
print('\n')
# Sharpe Ratio
SR = exp_ret / exp_vol
print('Sharpe Ratio')
print(SR)
# Great! Now we can just run this many times over!
num_ports = 15000
all_weights = np.zeros((num_ports, len(stocks.columns)))
ret_arr = np.zeros(num_ports)
vol_arr = np.zeros(num_ports)
sharpe_arr = np.zeros(num_ports)
for ind in range(num_ports):
    # Create Random Weights
    weights = np.array(np.random.random(4))
    # Rebalance Weights
    weights = weights / np.sum(weights)
    # Save Weights
    all_weights[ind, :] = weights
    # Expected Return
    ret_arr[ind] = np.sum((log_ret.mean() * weights) * 252)
    # Expected Variance
    vol_arr[ind] = np.sqrt(np.dot(weights.T, np.dot(log_ret.cov() * 252, weights)))
    # Sharpe Ratio
    sharpe_arr[ind] = ret_arr[ind] / vol_arr[ind]
sharpe_arr.max()
# In[39]:
sharpe_arr.argmax()
# In[40]:
all_weights[1419, :]
# In[41]:
max_sr_ret = ret_arr[1419]
max_sr_vol = vol_arr[1419]
# ## Plotting the data
plt.figure(figsize=(12, 8))
plt.scatter(vol_arr, ret_arr, c=sharpe_arr, cmap='plasma')
plt.colorbar(label='Sharpe Ratio')
plt.xlabel('Volatility')
plt.ylabel('Return')
# Add red dot for max SR
plt.scatter(max_sr_vol, max_sr_ret, c='red', s=50, edgecolors='black')

# # Mathematical Optimization
# There are much better ways to find good allocation weights than just guess and check! We can use optimization functions to find the ideal weights mathematically!
# ### Functionalize Return and SR operations
def get_ret_vol_sr(weights):
    """
    Takes in weights, returns array or return,volatility, sharpe ratio
    """
    weights = np.array(weights)
    ret = np.sum(log_ret.mean() * weights) * 252
    vol = np.sqrt(np.dot(weights.T, np.dot(log_ret.cov() * 252, weights)))
    sr = ret / vol
    return np.array([ret, vol, sr])

# To fully understand all the parameters, check out:
# https://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.minimize.html
help(minimize)

# Optimization works as a minimization function, since we actually want to maximize the Sharpe Ratio,
# we will need to turn it negative so we can minimize the negative sharpe (same as maximizing the postive sharpe)
def neg_sharpe(weights):
    return get_ret_vol_sr(weights)[2] * -1

# Contraints
def check_sum(weights):
    '''
    Returns 0 if sum of weights is 1.0
    '''
    return np.sum(weights) - 1

# By convention of minimize function it should be a function that returns zero for conditions
cons = ({'type': 'eq', 'fun': check_sum})
# 0-1 bounds for each weight
bounds = ((0, 1), (0, 1), (0, 1), (0, 1))
# Initial Guess (equal distribution)
init_guess = [0.25, 0.25, 0.25, 0.25]
# Sequential Least SQuares Programming (SLSQP).
opt_results = minimize(neg_sharpe, init_guess, method='SLSQP', bounds=bounds, constraints=cons)
opt_results
# In[53]:
opt_results.x
# In[54]:
get_ret_vol_sr(opt_results.x)
# # All Optimal Portfolios (Efficient Frontier)
# The efficient frontier is the set of optimal portfolios that offers the highest expected return for a defined level of risk or
# the lowest risk for a given level of expected return. Portfolios that lie below the efficient frontier are sub-optimal,
# because they do not provide enough return for the level of risk. Portfolios that cluster to the right of the efficient frontier are also sub-optimal,
# because they have a higher level of risk for the defined rate of return.
# Efficient Frontier http://www.investopedia.com/terms/e/efficientfrontier
# Our returns go from 0 to somewhere along 0.3
# Create a linspace number of points to calculate x on
frontier_y = np.linspace(0, 0.3, 100)  # Change 100 to a lower number for slower computers!

# In[56]:
def minimize_volatility(weights):
    return get_ret_vol_sr(weights)[1]

# In[57]:
frontier_volatility = []
for possible_return in frontier_y:
    # function for return
    cons = ({'type': 'eq', 'fun': check_sum},
            {'type': 'eq', 'fun': lambda w: get_ret_vol_sr(w)[0] - possible_return})
    result = minimize(minimize_volatility, init_guess, method='SLSQP', bounds=bounds, constraints=cons)
    frontier_volatility.append(result['fun'])
plt.figure(figsize=(12, 8))
plt.scatter(vol_arr, ret_arr, c=sharpe_arr, cmap='plasma')
plt.colorbar(label='Sharpe Ratio')
plt.xlabel('Volatility')
plt.ylabel('Return')
# Add frontier line
plt.plot(frontier_volatility, frontier_y, 'g--', linewidth=3)
# # Great Job!


from numpy import linalg as _LA
import numpy as _np

def matrix_system(q, N, type='even', period='one'):
    ''' Creates tridianogal matrix system of order NxN, associated with
    each of the four classes of simply-periodic functions.

    Input:
        q: parameter, real or purely imaginary.
        N: Size of the matrix, and thus the order of the highest harmonic in
            the trigonometric series that defines each Mathieu function.
        type: str, `even` or `odd`.
        period: str, `one` or 'two'. If `one`, function is assumed to be
            pi-periodic. If `two` function is taken to be `2pi`-periodic.
    Outout:
        A: ndarray, the square matrix associated with each of the four types
            of simply-periodic Mathieu-functions.
    '''
    if type is 'even':
        if period is 'one':  # ce_{2n}
            d = [(2. * r) ** 2 for r in range(N)]
            e = q * _np.ones(N - 1)
            A = _np.diag(d) + _np.diag(e, k=-1) + _np.diag(e, k=1)
            A[0, 1] = _np.sqrt(2) * A[0, 1]
            A[1, 0] = A[0, 1]
        elif period is 'two':  # se_{2n+2}
            pass
    elif type is 'odd':
        if period is 'one':  # se_{2n+1}
            pass
        elif period is 'two':  # ce_{2n+1}
            d = [((2. * r) + 1)**2 for r in range(N)]
            e = q * _np.ones(N - 1)
            A = _np.diag(d) + _np.diag(e, k=-1) + _np.diag(e, k=1)
    return A

def eig_pairs(A, type='even', period='one'):
    ''' Calculates the characteristic value (eigenvalue) and the Fourier
    coefficients associated with the Mathieu function. Both the eigenvalues
    and Fourier coefficients are given in ascending order.
    '''
    w, V = _LA.eig(A)  # calculates the eigenvalue
    if [type, period] == ['even', 'one']:
        V[0, :] = V[0, :] / _np.sqrt(2)  # remove factor on all first entries
    #  Sort the eigenvalues and in accordance, re-arrange eigenvectors
    ord_w, V = order_check(w, V)
    return ord_w, V

def order_check(a, v):
    """ Check the ordering of the eigenvalue array, from smaller to larger. If
    true, return a unchanged. Ordering also matters if a is complex. If a is
    complex, ordering again is first set according to real(a). If two
    eigenvalues are complex conjugates, then ordering is in accordance to the
    sign of complex(a). Negative sign is first.
    """
    if a.imag.any() == 0:
        ordered_a = a
        nv = v
    else:
        Ind = _np.argsort(_np.round(a, 5))  # sorting through 5 decimals
        ordered_a = a[Ind]
        nv = 0 * _np.copy(v)
        for k in range(len(Ind)):
            nv[:, k] = v[:, Ind[k]]
    return ordered_a, nv


import numpy as np
import cmath
import math
import matplotlib.pyplot as plt
from scipy.linalg import hessenberg

# A function that gets the Termination condition for the Eigen value iterations
def term_eigs(H,Q,R,n):
	flag = True
	for i in range(n):
		temp = np.linalg.norm(np.matmul(H,Q[:,i]) - (R[i,i] * Q[:,i]))
		# If norm of any corresponding index is greater than the threshold
		if temp >= 10**-6:
			flag = False
			break
	return flag

## A function that gets the n smallest Eigen values of A
def eigs(A,n):
	# Performing matrix inversion
	A_orig = A.copy()
	A = np.linalg.inv(A)

	# Getting the Hessenberg matrix (Tridiagonal in case of Symmetric matrix)
	# Also gets out the reflector Q for the transformation
	H, Q = hessenberg(A, calc_q=True)

	# Defining variables for the iteration
	m = A.shape[0]
	Q_eig, R_eig = np.eye(m,m), np.eye(m,m)
	k = 0

	while (k<50) and (not (term_eigs(H, Q_eig, R_eig,n))):
		Q_eig, R_eig = np.linalg.qr(H, mode = 'reduced')
		H = np.matmul(R_eig,Q_eig)
		k +=1

	# Getting the eigen values
	lamda = np.zeros((n,1))
	for i in range(n):
		lamda[i] = 1.0/R_eig[i,i]

	# Reducing the eigen vectors to the original basis
	Qhat = np.matmul(Q.T, np.matmul(Q_eig,Q))

	return lamda,(Qhat[:,:(n)]/np.linalg.norm(Qhat[:,:(n)]))

# A function that checks the working of Eigen QR iterations
def check_eigen_correctness():
	A = np.array([[1,2,4,2,5],[3,5,6,2,5],[6,8,5,4,3],[7,6,5,3,2],[7,5,4,3,2]])
	n = 4
	print(eigs(A,n))

# A function that would compute the 2 norm of a complex number
def complex_norm(temp):
	return math.sqrt(((temp.real)**2) + ((temp.imag)**2))

# A function that computes the value of the function p
def compute_p(a,x0):
	n = a.shape[0]
	temp = complex(0,0)
	z = complex(x0[0],x0[1])

	for i in range(n):
		temp += a[i] * (z ** i)

	return temp

# A function that computes the value of derivative the function p
def compute_derivative_p(a,x0):
	n = a.shape[0]
	temp = complex(0,0)
	z = complex(x0[0],x0[1])

	for i in range(1, n):
		temp += i * a[i] * (z ** (i-1))

	return temp

## A function that computes the Jacobian of the function at a given point
def compute_J_p(a,x0):
	temp = compute_derivative_p(a,x0)
	alpha, beta = temp.real, temp.imag

	J = np.zeros((2,2))
	J[0,0] = alpha
	J[0,1] = -1.0 * beta
	J[1,0] = beta
	J[1,1] = alpha

	return J

# The function that solves Netwon's method
def newton(a,x0):
	k = 0
	temp = compute_p(a,x0)
	xstar = x0
	while k<50 and complex_norm(temp)>=10**-6:
		f_vec = np.array([temp.real, temp.imag])
		xstar = xstar - np.matmul((np.linalg.inv(compute_J_p(a,xstar))),f_vec)
		temp = compute_p(a,xstar)
		k += 1
	return xstar

# The function that would make the colored mesh as required
def create_mesh():
	a = np.array([-1.,-1.,0.,1.])
	x1_list = np.arange(-1.0, 2.05, 0.05)
	x2_list = np.arange(-1.0, 1.05, 0.05)

	# Hard-coding the solutions
	sol_1 = np.array([-0.66235, 0.56227])
	sol_2 = np.array([-0.66235, -0.56227])
	sol_3 = np.array([1.32471, 0.0])

	# Starting the plot
	plt.xlim([-1, 2])
	plt.ylim([-1, 1])

	for x1 in x1_list:
		for x2 in x2_list:
			sol = newton(a,np.array([x1,x2]))
			if np.linalg.norm(sol - sol_1) < 10**-3:
				plt.scatter(x1,x2,s=10,color = 'r')
			elif np.linalg.norm(sol - sol_2) < 10**-3:
				plt.scatter(x1,x2,s=10,color = 'b')
			else:
				plt.scatter(x1,x2,s=10,color = 'g')

	# The configurations required for the plot
	plt.xlabel('x1')
	plt.ylabel('x2')
	plt.title('Netwon\'s method convergence for various starting points')
	plt.savefig('q6_scatter.png')

# The function that checks the correctness of the Newton's algorithm
def check_newton_correctness():
	a = np.array([3.0, -4.0, 1.0])
	x0 = np.array([1.0, 10.0])
	xstar = newton(a,x0)
	print('The solution for the given polynomial is: ', xstar)

# Here the main function is called
if __name__ == '__main__':
	create_mesh()


def matrix_modulus(N, p=2, sigma=None):
    """
    Computes the modulus of a family given its N matrix.

    Parameters
    ----------
    N : numpy array OR scipy sparse matrix
        The N matrix for the family.

    p : float or np.inf
        The modulus energy exponent.

    sigma : numpy array
        The weights sigma.  If sigma = None, all weights
        are treated as 1.

    Returns
    -------
    mod : float
        An approximation of the modulus.

    rho : numpy array
        An optimal rho^* for modulus.

    lam : numpy array
        An optimal dual lambda^* for modulus.

    Notes
    -----
    This function is just a template solver.  For specialized
    problems, it is probably best to implement a particular
    solver based on the energy exponent and the desired
    tolerances in the approximation.
    """

    import cvxpy as cvx
    import numpy as np

    from warnings import warn

    # problem dimension
    m = N.shape[1]

    # set sigma to default if necessary
    if sigma is None:
        sigma = np.ones(m)

    # convert inputs to cvxpy constants
    N = cvx.Constant(N)
    sigma = cvx.Constant(sigma)

    # primal variables
    rho = cvx.Variable(m)

    # objective
    if p is not np.inf:
        obj = cvx.Minimize(sigma.T@rho**p)
    else:
        obj = cvx.Minimize(cvx.max(cvx.multiply(sigma, rho)))

    # constraints
    cons = [rho >= 0, N@rho >= 1]

    # set up the problem
    prob = cvx.Problem(obj, cons)

    # attempt to solve
    prob.solve()
    if prob.status != 'optimal':
        warn('cvxpy solve returned status {}'.format(prob.status))

    return prob.value, np.array(rho.value).flatten(), np.array(cons[1].dual_value).flatten()

def modulus(m, solve_subproblem, find_shortest, p = 2, sigma = None, tol = 1e-3, max_iter = 1000,
            output_every = None):
    """
    Implements the basic algorithm for modulus.

    Parameters
    ----------
    m : int
        The dimension of the modulus problem (number of edges).

    solve_subproblem : callable
        See below.

    find_shortest : callable
        See below.

    p : float or np.inf
        The modulus energy exponent.

    sigma : numpy array
        The weights sigma.  If sigma = None, all weights
        are treated as 1.

    tol : float
        The tolerance.  The modulus algorithm stops when the approximate
        density is within tol of being admissible.

    max_iter : int
        Maximum number of iterations to perform before terminating with an error.

    output_every : int
        Frequency of output to stderr.  If this is set to None, output is supressed.

    Returns
    -------
    mod : float
        Approximation to modulus.

    cons : list
        List of constraints added during the iteration.  The format of the elements of
        this list is determined by the output of the find_violated_constraint function.

    rho : numpy array
        Approximation to an optimal density.

    lam : numpy array
        Approximation to the dual variables for the constraints listed in cons.

    Notes
    -----
    The function solve_subproblem should have the following signature

        mod, rho, lam = solve_subproblem(N, p, sigma)

    See the function matrix_modulus for an example.

    The function find_shortest should have the following signature

        cons, n = find_shortest(rho, tol)

    This function should find a "most violated constraint" using the specified values
    for rho.  Upon return, cons may contain any representation desired for describing
    the constraint.  (This is purely informational for the user and it is acceptable
    for cons to be set to None.)  n should contain the numpy row vector representing the
    violated constraint.  This is the row that will be added to N on the next iteration.

    The argument tol may be ignored.  However, it is acceptable for the function to return
    the tuple (None, None) if every constraint is satisfied to within a tolerance of tol.
    """

    import numpy as np
    import scipy.sparse as sp
    from time import perf_counter

    # timers
    search_time = 0.
    update_time = 0.
    mod_start = perf_counter()

    # initialize variables
    rho = np.zeros(m)
    N = sp.csr_matrix((0,m))
    lam = np.array([])
    mod = 0.
    upper = np.inf
    cons = []

    # default sigma
    if sigma is None:
        sigma = np.ones(m)

    # initialize output table
    if output_every:
        print('| {:>6s} | {:>9s} | {:>9s} | {:>9s} | {:>6s} | {:>9s} |' \
            .format( 'it', 'l bnd', 'u bnd', 'rel gap', '# cons', 'time (s)' ))
        print('+--------+-----------+-----------+-----------+--------+-----------+')

    # loop to at most max_iter
    for iter_count in range(max_iter):

        # find a constraint to add
        start = perf_counter()
        c, n = find_shortest(rho, tol)
        search_time += perf_counter()-start

        # compute the length of the shortest object
        if n is None:
            length = 1.
        else:
            length = n.dot(rho)

        # update the upper bound
        if length > 0:
            if p is np.Inf:
                upper = np.abs(sigma*rho/length)
            else:
                upper = np.sum(sigma*(rho/length)**p)

        # check if we can stop
        if length > 1-tol:
            if output_every:
                print('| {:6d} | {:9.3e} | {:9.3e} | {:9.3e} | {:6d} | {:9.3e} |' \
                    .format( iter_count+1, mod, upper, (upper-mod)/mod, N.shape[0], perf_counter() - mod_start ))

                print()
                print('program running time = {} sec'.format( perf_counter() - mod_start ))
                print('constraint search    = {} sec'.format( search_time ))
                print('solution update      = {} sec'.format( update_time ))

            return mod, cons, rho, lam

        # if not, we need to add a constraint
        N = sp.vstack([N,n], format='csr')
        cons.append(c)

        # re-optimize
        start = perf_counter()
        mod, rho, lam =solve_subproblem(N, p, sigma)
        update_time += perf_counter()-start

        # print some feedback if desired
        if output_every and (iter_count+1) % output_every == 0:
            if mod == 0:
                rel_gap = np.inf
            else:
                rel_gap = (upper-mod)/mod
            print('| {:6d} | {:.3e} | {:.3e} | {:.3e} | {:6d} | {:.3e} |' \
                .format( iter_count+1, mod, upper, rel_gap, N.shape[0], perf_counter() - mod_start ))

    # if we got here, we failed to converge
    raise RuntimeError('Modulus algorithm failed to converge in {} iterations.'.format(max_iter))


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model
from sklearn.metrics import r2_score, mean_squared_error

# Read the table
df = pd.read_csv("fiji_consum_exp.csv")

# Separate 60 percent for train dataset and rest as test dataset
msk = np.random.rand(len(df)) < 0.6
train = df[msk]
test = df[~msk]

# Import Linear Regression model
regr = linear_model.LinearRegression()

# Train dataset
# Convert List -> Array
train_x = np.asanyarray(train[["Year"]])
train_y = np.asanyarray(train[["Value"]])

# Learning
regr.fit(train_x, train_y)
print(f"Coefficient: {float(regr.coef_)}")
print(f"Intercept: {float(regr.intercept_)}")

# Test dataset
test_x = np.asanyarray(test[["Year"]])
test_y = np.asanyarray(test[["Value"]])
test_y_ = regr.predict(test_x)

# Accuracy of this model
mse = mean_squared_error(test_y_, test_y)
r2 = r2_score(test_y_, test_y)
print("Mean Squared Error: %.2f" % mse)
print("R2 Score: %.2f" % r2)

# Graph
plt.scatter(train.Year, train.Value, label="train", color="indigo")
plt.plot(train_x, regr.coef_[0][0] * train_x + regr.intercept_[0], label="prediction", color="red")
plt.scatter(test.Year, test.Value, label="test", color="blue")
plt.xlabel("Year")
plt.ylabel("Government Consumption Expense (10^6 USD)")
plt.title("Fiji Government Consumption Expense 1950 - 2005")
plt.legend()

# Plot
plt.show()


from scipy.spatial.distance import pdist
import numpy as np

def get_module(x):
    """
        x为需要求长度的向量
        欧拉距离/欧式距离(Euclidean Distance)
    """
    return np.sqrt(np.sum(np.square(x)))

def get_angle(x, y):
    """
        一维数据，dot点乘，对数学而言即为求取其数量积
    """
    return np.arccos(np.dot(x, y)/(get_module(x)*get_module(y)))

def get_mudule_len_by_dot():
    print('----- 通过数量积求模长 ----- ')
    # --------------------------------
    # 纯numpy(数学)方式
    OA = np.array([3, 0])
    OB = np.array([0, 4])
    AB = OB - OA  # 指向被减数
    # 长度
    d = get_module(AB)
    print('mlen = ', d)

    # -------------------------------
    # numpy + scipy方式
    xy = np.vstack([OA, OB])
    print(xy)
    dist = pdist(xy)
    print('mlen = ', dist[0])

def get_angle_by_cross():
    """
        已知M(1,1,1),A(2,2,1),B(2,1,2)
        求角度AMB
    """
    print('----- 通过向量积求角度 ----- ')
    M = np.array([1, 1, 1])
    A = np.array([2, 2, 1])
    B = np.array([2, 1, 2])
    MA = A - M
    MB = B - M
    angle = get_angle(MA, MB)
    print(angle)
    if angle - np.pi/3 < 0.0001:
        print('数量积结果正确')

def get_triangle_area_by_cross():
    """
        已知三角形三点坐标，A，B，C坐标点。
        求三角形面积
    """
    print('----- 通过向量积求三角形面积 ----- ')
    A = np.array([1, 2, 3])
    B = np.array([3, 4, 5])
    C = np.array([2, 4, 7])
    AB = B - A
    AC = C - A
    # 面积
    AB_AC = 1/2 * np.cross(AB, AC)
    S = get_module(AB_AC)
    print(S)
    if S - np.sqrt(14) < 0.0001:
        print('向量积结果正确')

if __name__ == "__main__":
    get_mudule_len_by_dot()
    get_angle_by_cross()
    get_triangle_area_by_cross()


from sympy import *

x = symbols('x')

psi = [2*(x-Rational(1,2))*(x-1),
       4*x*(1-x),
       2*x*(x-Rational(1,2))]

dpsi = [ diff(psi[k], x) for k in range(3) ]

def a(p, q):
    return integrate(dpsi[q]*dpsi[p], (x, 0, 1))

def c(p,q):
    return integrate(psi[q]*psi[p], (x, 0, 1))

A = Matrix(3, 3, a)
C = Matrix(3, 3, c)

print('Quadratic element stiffness matrix is 1/(3h) times')
display(3*A)
print('Quadratic element mass matrix is h/30 times')
display(30*C)


# We are going to implement a linear regression with degree one.
# Simple equation would be y = mx + c
#
# Formula to find m,
#       mean(x).mean(y) - mean(xy)
#  m = ----------------------------
#       sqr(mean(x)) - mean(sqr(x))
#
# Formula to find c,
#  c = mean(y) - m.mean(x)

from statistics import mean
import numpy as np
import matplotlib.pyplot as plot
from matplotlib import style

x = [1,2,3,4,5,6,7,8]
y = [9,8,5,5,3,3,3,2]
x = np.array(x, dtype=float)
y = np.array(y, dtype=float)

style.use('fivethirtyeight')
plot.scatter(x, y)
# plot.show()
# plot.hold()

def best_fit(x, y):
    m = ((mean(x) * mean(y)) - mean(x * y)) / ((mean(x) ** 2) - mean(x ** 2))
    c = mean(y) - m * mean(x)
    return m, c

m, c = best_fit(x, y)
print(m,c)

regressionLine = [(m * x_ + c) for x_ in x]
# The above equation is equivalent to the following
# for x_ in x:
#     regressionLine.append(m*x_ + c)

predictX = 10
predictY = m * predictX + c

plot.scatter(predictX, predictY, edgecolors='g')

plot.plot(x, regressionLine)
plot.show()


import numpy
from scipy import stats
import matplotlib.pyplot as plt

speed = [99,86,87,88,111,86,103,87,94,78,77,85,86]

print(numpy.mean(speed))
print(numpy.median(speed))
print( numpy.median([99,86,87,88,86,103,87,94,78,77,85,86]) )
print(stats.mode(speed))
print(numpy.std(speed))
print(numpy.var(speed))
print(numpy.percentile(speed, 75))
print(numpy.percentile(speed, 90))

x = numpy.random.uniform(0.0, 5.0, 1000000)
#print(x)
#plt.hist(x, 100)
#plt.show()

#x = numpy.random.normal(5.0, 1.0, 10000000)
#plt.hist(x, 1000)
#plt.show()

x = [5,7,8,7,2,17,2,9,4,11,12,9,6]
y = [99,86,87,88,111,86,103,87,94,78,77,85,86]
#plt.scatter(x, y)
#plt.show()

x = numpy.random.normal(5.0, 1.0, 1000)
y = numpy.random.normal(10.0, 2.0, 1000)
#plt.scatter(x, y)
#plt.show()

# linear regression

x = [5,7,8,7,2,17,2,9,4,11,12,9,6]
y = [99,86,87,88,111,86,103,87,94,78,77,85,86]

slope, intercept, r, p, std_err = stats.linregress(x, y)
def myfunc(x):
  return slope * x + intercept

mymodel = list(map(myfunc, x))

#plt.scatter(x, y)
#plt.plot(x, mymodel)
#plt.show()

print(r)

# Polynomial Regression

import matplotlib.pyplot as plt

x = [1,2,3,5,6,7,8,9,10,12,13,14,15,16,18,19,21,22]
y = [100,90,80,60,60,55,60,65,70,70,75,76,78,79,90,99,99,100]

mymodel = numpy.poly1d(numpy.polyfit(x, y, 3))
myline = numpy.linspace(1, 22, 100)

#plt.scatter(x, y)
#plt.plot(myline, mymodel(myline))
#plt.show()

from sklearn.metrics import r2_score
print(r2_score(y, mymodel(x)))

# Multiple Regression

import pandas

df = pandas.read_csv("trainingW3C/cars.csv")
#print(df)
X = df[['Weight', 'Volume']]
y = df['CO2']

from sklearn import linear_model
regr = linear_model.LinearRegression()
regr.fit(X, y)

print(regr.coef_)

#predict the CO2 emission of a car where the weight is 2300kg, and the volume is 1300ccm:
predictedCO2 = regr.predict([[2300, 1300]])

print(predictedCO2)

# train/test

numpy.random.seed(2)

x = numpy.random.normal(3, 1, 100)
y = numpy.random.normal(150, 40, 100) / x
#plt.scatter(x, y)
#plt.show()

train_x = x[:80]
train_y = y[:80]
#plt.scatter(train_x, train_y)
#plt.show()

test_x = x[80:]
test_y = y[80:]
#plt.scatter(test_x, test_y)
#plt.show()

mymodel = numpy.poly1d(numpy.polyfit(train_x, train_y, 4))

#plt.scatter(train_x, train_y)
#myline = numpy.linspace(0, 6, 100)
#plt.plot(myline, mymodel(myline))
#plt.show()

r2 = r2_score(train_y, mymodel(train_x))
print(r2)

r2 = r2_score(test_y, mymodel(test_x))
print(r2)

# Decision Tree

import pandas
from sklearn import tree
import pydotplus
from sklearn.tree import DecisionTreeClassifier
import matplotlib.pyplot as plt
import matplotlib.image as pltimg

df = pandas.read_csv("trainingW3C/shows.csv")
#print(df)

d = {'UK': 0, 'USA': 1, 'N': 2}
df['Nationality'] = df['Nationality'].map(d)
d = {'YES': 1, 'NO': 0}
df['Go'] = df['Go'].map(d)

#print(df)

features = ['Age', 'Experience', 'Rank', 'Nationality']
X = df[features]
y = df['Go']
#print(X)
#print(y)

dtree = DecisionTreeClassifier()
dtree = dtree.fit(X, y)
data = tree.export_graphviz(dtree, out_file=None, feature_names=features)
graph = pydotplus.graph_from_dot_data(data)
graph.write_png('trainingW3C/mydecisiontree.png')

img = pltimg.imread('trainingW3C/mydecisiontree.png')
imgplot = plt.imshow(img)
plt.show()


# coding: utf-8

# # Regularization

# ## 1. Introduction
#
# ### 1.1 Libraries

# In[1]:

import numpy as np
import matplotlib.pyplot as plt
from sklearn import linear_model
from scipy.optimize import minimize
from numpy.random import permutation
from sympy import var, diff, exp, latex, factor, log, simplify
from IPython.display import display, Math, Latex
np.set_printoptions(precision=4)
get_ipython().magic('matplotlib inline')

# ### 1.2 Dataset
#
# The in-sample and out-of-sample data is retrieved from the following sources:
#
# ```bash
# curl -O http://work.caltech.edu/data/in.dta
# curl -O http://work.caltech.edu/data/out.dta
# ```
#
#

# ## 2. Non-linear Transforms
#
# ### 2.1 Hypothesis Set
#
# Consider the linear transform $z_i = \phi_i\left(\mathbf{x}\right)$ or $\mathbf{z} = \Phi\left(\mathbf{x}\right)$, with the following mapping:
#
# $$\mathbf{x} = \left(x_0, x_1, \dots, x_d\right) \rightarrow \mathbf{z} = \left(z_0, z_1, \dots, z_{\tilde d}\right)$$
#
# The final hypothesis, $\mathcal{X}$ space is:
#
# $$g\left(\mathbf{x}\right) = \mathbf{\tilde w^T} \Phi\left(\mathbf{x}\right)$$
#
# $$g\left(\mathbf{x}\right) = \left(w_0, w_1, w_2, w_3, w_4, w_5, w_6, w_7\right) \left(\begin{array}{c}1\\x_1\\x_2\\x_1^2\\x_2^2\\x_1 x_2\\|x_1 - x_2|\\|x_1 + x_2|\end{array}\right) = w_0 + w_1 x_1 + w_2 x_2 + \dots$$

# To implement the above hypothesis set on an arbitrary matrix $\mathbf{X}$,
# we augment the matrix $\mathbf{X}$ with a *ones column*, and
# non-linear features are added as new columns via the following two subroutines.

# In[2]:

def add_ones_column(X):
    n = X.shape[0]
    x0 = np.ones((n,1))
    return np.hstack((x0,X))

# In[3]:

def add_nonlinear_features(X,n):
    """
    Here 'n' can be 3 to 7, corresponding to one of the following ϕ_n:
    ϕ_0 = 1
    ϕ_1 = 1 + x_1
    ϕ_2 = 1 + x_1 + x_2
    ϕ_3 = 1 + x_1 + x_2 + x_1^2
    ϕ_4 = 1 + x_1 + x_2 + x_1^2 + x_2^2
    ϕ_5 = 1 + x_1 + x_2 + x_1^2 + x_2^2 + x_1 x_2
    ϕ_6 = 1 + x_1 + x_2 + x_1^2 + x_2^2 + x_1 x_2 + |x_1 - x_2|
    ϕ_7 = 1 + x_1 + x_2 + x_1^2 + x_2^2 + x_1 x_2 + |x_1 - x_2| + |x_1 + x_2|
    """
    m = X.shape[0]
    X = np.hstack((X,np.zeros((m,n-X.shape[1]+1))))
    if n >= 3: X[:,3] = X[:,1]**2
    if n >= 4: X[:,4] = X[:,2]**2
    if n >= 5: X[:,5] = X[:,1]*X[:,2]
    if n >= 6: X[:,6] = np.abs(X[:,1] - X[:,2])
    if n >= 7: X[:,7] = np.abs(X[:,1] + X[:,2])
    return(X)

# ### 2.2 Loading and Transforming the Data
#
# The data is loaded and transformed as follows:

# In[4]:

training = np.loadtxt("data/in.dta")
X_train = add_ones_column(training[:,:-1])
y_train = training[:,-1]
X_train = add_nonlinear_features(X_train,7)
n_train = y_train.shape[0]

testing  = np.loadtxt("data/out.dta")
X_test  = add_ones_column(testing[:,:-1])
y_test  = testing[:,-1]
X_test  = add_nonlinear_features(X_test,7)
n_test = y_test.shape[0]

n_features = X_train.shape[1]

# ### 2.2 Linear Regression (without regularization)
#
# The [ordinary least squares](https://en.wikipedia.org/wiki/Ordinary_least_squares) (OLS) estimator for the weights is:
#
# $$w = \left(\mathbf{X^T X}\right)^{-1}\mathbf{X^T y} = \mathbf{X^\dagger}y$$
#
# where $\mathbf{X^\dagger} = \left(\mathbf{X^T X}\right)^{-1}\mathbf{X^T}$ is the [Moore–Penrose pseudoinverse](https://en.wikipedia.org/wiki/Moore–Penrose_pseudoinverse).
#
# This is implemented via the following line of code in the subroutine below:
#
# `w_linreg = np.dot(np.linalg.pinv(X),y)`

# In[5]:

def train_without_regularization(X_train,y_train):
    w_no_reg = np.dot(np.linalg.pinv(X_train),y_train)
    return w_no_reg

def get_E_in_E_out(X_test,y_test,X_train,y_train,w,verbose=True):
    n_train = y_train.shape[0]
    n_test  = y_test.shape[0]
    y_train_eval = np.sign(np.dot(X_train,w)).astype(int)
    y_test_eval  = np.sign(np.dot(X_test,w)).astype(int)
    E_in = np.sum(y_train != y_train_eval) / n_train
    E_out = np.sum(y_test != y_test_eval) / n_test
    if verbose is True:
        print("E_in = {}".format(E_in))
        print("E_out = {}".format(E_out))
    return E_in,E_out

# The un-regularized in-sample and out-of-sample error can be calculated as follows (in the next section, the regularized in-sample and out-of-sample errors will be calculated):

# In[6]:

w_linreg = np.dot(np.linalg.pinv(X_train),y_train)
y_train_linreg = np.sign(np.dot(X_train,w_linreg))
y_test_linreg  = np.sign(np.dot(X_test,w_linreg))
E_in = np.sum(y_train != y_train_linreg) / n_train
E_out = np.sum(y_test != y_test_linreg) / n_test
print("E_in = {}".format(E_in))
print("E_out = {}".format(E_out))

# ### 2.3 Linear Regression (with Regularization)
#
# #### 2.3.1 Background
#
# In order to perform regularization, a "soft-order" constraint is employed for the weights:
#
# $$\sum\limits_{q=0}^Q w_q^2 \le C$$
#
# Subject to the above constraint, our aim is to minimize the following cost function on the in-sample points:
#
# $$E_{in}\left(\mathbf{w}\right) = \frac{1}{N} \sum\limits_{n=1}^N \left(\mathbf{w^T z_n - y_n}\right)^2 = \frac{1}{N}\left(\mathbf{Zw-y}\right)^T\left(\mathbf{Zw-y}\right)$$
#
# The minimum of the above function occurs when:
#
# $$\nabla E_{in}\left(\mathbf{w}\right) \propto -\mathbf{w_{reg}}$$
# $$\nabla E_{in}\left(\mathbf{w}\right) = -2\frac{\lambda}{N}\mathbf{w_{reg}}$$
#
# where in the last step, we have introduced $2\frac{\lambda}{N}$ as our constant of proportionality.

# Hence, we wish to solve:
#
# $$\nabla E_{in}\left(\mathbf{w_{reg}}\right) + 2\frac{\lambda}{N}\mathbf{w_{reg}} = 0$$
#
# Notice that solving the above equation corresponds to minimizing the following:
#
# $$E_{in}\left(\mathbf{w}\right) + \frac{\lambda}{N}\mathbf{w^T w}$$
#
# This new cost function can be viewed as an augmented error, which we can write as follows:
#
# $$E_{aug}\left(\mathbf{w}\right) = \frac{1}{N}\left(\mathbf{Zw-y}\right)^T \left(\mathbf{Zw-y}\right) + \frac{\lambda}{N}\mathbf{w^T w}$$
#
# Minimizing the above cost function is equivalent to minimizing $E_{in}\left(\mathbf{w}\right)$ subject to the constraint $\mathbf{w^T w} \le C$
#
# To determine the **regularized weights**, we minimize the augmented error as follows:
#
# $$\nabla E_{aug}\left(\mathbf{w}\right) = 0$$
#
# $$\mathbf{Z^T} \left(\mathbf{Zw-y}\right) + \lambda\mathbf{w} = 0$$
#
# $$\left(\mathbf{Z^T Z} + \lambda\mathbf{I}\right)\mathbf{w} - \mathbf{Z^T y} = 0$$
#
# $$\mathbf{w_{reg}} = \left(\mathbf{Z^T Z} + \lambda\mathbf{I}\right)^{-1}\mathbf{Z^T y}$$

# In[7]:

def train_with_regularization(lmbda,X_train,y_train,use_sklearn=False):
    n_samples = y_train.shape[0]
    if use_sklearn is True:
        reg = linear_model.Ridge(alpha = lmbda/n_samples, fit_intercept=True)
        reg.fit(X_train,y_train)
        w_reg = reg.coef_
        w_reg[0] = reg.intercept_
    else:
        n_features=X_train.shape[1]
        ZTy = np.dot(X_train.T,y_train)
        ZTZ = np.dot(X_train.T,X_train)
        w_reg = np.dot(np.linalg.inv(ZTZ + lmbda*np.identity(n_features)),ZTy)
    return w_reg

# In[8]:

def get_E_in_E_out(X_test,y_test,X_train,y_train,w):
    n_train = y_train.shape[0]
    n_test = y_test.shape[0]
    y_train_w = np.sign(np.dot(X_train,w))
    y_test_w  = np.sign(np.dot(X_test,w))
    E_in = np.sum(y_train != y_train_w) / n_train
    E_out = np.sum(y_test != y_test_w) / n_test
    return E_in, E_out

# #### 2.3.2 Checking the above subroutine.
#
# The data can be trained by either using the subroutine `train_without_regularization()` or setting $\lambda=0$ in the subroutine `train_with_regularization()`.  The weights obtained should be the same in either case, and this is easy to check with `numpy.allclose()`.

# In[9]:

w_no_reg = train_without_regularization(X_train,y_train)
w_reg = train_with_regularization(0.0,X_train,y_train,use_sklearn=False)
assert np.allclose(w_no_reg,w_reg)
E_in,E_out = get_E_in_E_out(X_test,y_test,X_train,y_train,w_no_reg)

# #### 2.3.3 Finding the best $\lambda$
#
# To find the best choice of $\lambda$, the values of $E_{in}$ and $E_{out}$ are plotted over a range of values of $\lambda$.

# In[10]:

def find_best_lambda(lmbd_arr,X_train,y_train,X_test,y_test,use_sklearn=False):
    E_in_arr = []
    E_out_arr = []
    w_arr = []
    for lmbda in lmbd_arr:
        w_reg = train_with_regularization(lmbda,X_train,y_train,use_sklearn=use_sklearn)
        E_in, E_out = get_E_in_E_out(X_test,y_test,X_train,y_train,w_reg)
        E_in_arr.append(E_in)
        E_out_arr.append(E_out)
        w_arr.append(w_reg)
    return np.array(E_in_arr), np.array(E_out_arr), np.array(w_arr)

# In[11]:

def plot_regularization(fig,plot_id,title,lmbd_arr,E_in_reg,E_out_reg):
    ax = fig.add_subplot(plot_id)
    ax.semilogx(lmbd_arr,E_in_reg,'x-',label="$E_{in}$")
    ax.semilogx(lmbd_arr,E_out_reg,'o-',label="$E_{out}$")
    ax.set_xlabel('$\lambda$')
    ax.grid(True)
    ax.legend(loc='best',frameon=False)
    ax.set_title(title)

# In[12]:

lmbd_arr1 = [0.0001, 0.0002, 0.0005, 0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1.0]
fig = plt.figure(figsize=(12,5))
E_in_reg1, E_out_reg1, w_reg1 = find_best_lambda(lmbd_arr1,X_train,y_train,X_test,y_test,False)
plot_regularization(fig,121,"",lmbd_arr1,E_in_reg1,E_out_reg1)
#E_in_sk, E_out_sk, w_sk = find_best_lambda(lmbd_arr,X_train,y_train,X_test,y_test,True)
#plot_regularization(fig,122,"sklearn",lmbd_arr,E_in_sk,E_out_sk)
lmbd_arr2 = [1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1000.0]
E_in_reg2, E_out_reg2, w_reg2 = find_best_lambda(lmbd_arr2,X_train,y_train,X_test,y_test,False)
plot_regularization(fig,122,"",lmbd_arr2,E_in_reg2,E_out_reg2)

# Here we can see that when regularization turned on ($\lambda \ne 0$):
#
# * $\lambda = 10^{-3}$ has an $E_{in}$ of about 0.03, and an $E_{out}$ of 0.08.
# * $\lambda = 10^{-1}$ has the best $E_{out}$ of about 0.06.
# * $\lambda = 10^{3}$ has an $E_{in}$ and $E_{out}$ of about 0.4.

# ### 2.4 Validation Sets
#
# The data may also be split into validation sets for training, and the new model evaluated on the final, held-out, test set.  In the following subroutine, the training set (`X_train_all,y_train_all`) is split one more time into a new training set (`X_train,y_train`) and a validation set (`X_valid,y_valid`).  By default, the first 25 points are designated as training points (`train_pts`).

# In[13]:

def training_validation_test_split(training,testing,train_pts=25,k=7,swap=False):
    X_train_all = add_ones_column(training[:,:-1])
    y_train_all = training[:,-1]
    X_train_all = add_nonlinear_features(X_train_all,k)

    X_test  = add_ones_column(testing[:,:-1])
    y_test  = testing[:,-1]
    X_test  = add_nonlinear_features(X_test,k)

    val_pts = X_train_all.shape[0]-train_pts
    assert val_pts == y_train_all.shape[0]-train_pts
    if swap is True:
        X_train = X_train_all[val_pts:,:]
        X_valid = X_train_all[:val_pts,:]
        y_train = y_train_all[val_pts:]
        y_valid = y_train_all[:val_pts]
    else:
        X_train = X_train_all[:train_pts,:]
        X_valid = X_train_all[train_pts:,:]
        y_train = y_train_all[:train_pts]
        y_valid = y_train_all[train_pts:]

    assert X_train.shape[0]==train_pts
    assert X_valid.shape[0]==val_pts
    assert y_train.shape[0]==train_pts
    assert y_valid.shape[0]==val_pts
    n_features = X_train.shape[1]
    return X_train_all,y_train_all,X_train,y_train,X_valid,y_valid,X_test,y_test

# In this example, the first 3,4,5,6, or 7 terms (corresponding to $\phi_3, \phi_4, \dots, \phi_7$) of the non-linear transform are used and the classification error of the validation set is evaluated.

# In[14]:

lmbd_arr = [0.0001, 0.0002, 0.0005, 0.001, 0.002, 0.005, 0.01, 0.02, 0.05,
            0.1, 0.2, 0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 500.0, 1000.0]
k_arr = np.array([3, 4, 5, 6, 7])
fig = plt.figure(figsize=(12,10))
plot_ids = [ 321, 322, 323, 324, 325 ]
for i,k in enumerate(k_arr):
    X_train_all,y_train_all,X_train,y_train,X_valid,y_valid,X_test,y_test =         training_validation_test_split(training,testing,k=k)
    E_in_reg, E_val_reg, w_reg_arr = find_best_lambda(lmbd_arr,X_train,y_train,X_valid,y_valid,False)
    plot_regularization(fig,plot_ids[i],"$\phi_{}$".format(k),lmbd_arr,E_in_reg,E_val_reg)

# Here we can see that for small $\lambda$, $\phi_6$ has the lowest classification error on the validation set.

# ### 2.5 Plotting
#
# The in-sample and out-of-sample error from the non-linear transforms can be visualized via contour plots.  In this case, we will use $\lambda = 0.1$ which had the lowest out-of-sample error.

# In[15]:

def plot_data_nonlinear(fig,plot_id,X,y,w_arr,w_colors,titles):
    p = 2.0
    x1 = np.linspace(-p,p,100)
    x2 = np.linspace(-p,p,100)
    X1,X2 = np.meshgrid(x1,x2)
    X1_sq= X1**2
    X2_sq= X2**2
    X1X2 = X1*X2
    X1mX2= np.abs(X1-X2)
    X1pX2= np.abs(X1+X2)

    for i,w in enumerate(w_arr):
        G = w[0] + w[1]*X1 + w[2]*X2 + w[3]*X1_sq + w[4]*X2_sq +             w[5]*X1X2 + w[6]*X1mX2 + w[7]*X1pX2
        ax = fig.add_subplot(plot_id[i])
        ax.plot(X[y > 0,1],X[y > 0,2],'b+',label='Positive (+)')
        ax.plot(X[y < 0,1],X[y < 0,2],'r_',label='Negative (-)')
        cp0 = ax.contour(X1,X2,G,1,linewidth=4, levels=[0.0],
                         colors=w_colors[i])
        ax.clabel(cp0, inline=True, fontsize=14)
        #cp1 = ax.contour(X1,X2,Y,N=1,linewidth=4, levels=[-1.0, 1.0],
        #                 linestyles='dashed', colors=w_colors[i], alpha=0.3)
        cp1 = ax.contourf(X1,X2,G,1,linewidth=4, linestyles='dashed', alpha=0.8)
        ax.clabel(cp1, inline=True, fontsize=14)

        plt.colorbar(cp1)
        ax.set_title(titles[i])
        #ax.set_axis_off()  #ax.axis('off')
        #ax.axes.xaxis.set_ticks([])
        #ax.axes.yaxis.set_ticks([])
        ax.set_xlim(-p,p)
        ax.set_ylim(-p,p)
    fig.tight_layout()

# In[16]:

w_no_reg = train_with_regularization(0.0,X_train_all,y_train_all,use_sklearn=False)
w_reg    = train_with_regularization(0.1,X_train_all,y_train_all,use_sklearn=False)
E_in_no_reg,E_out_no_reg = get_E_in_E_out(X_test,y_test,X_train_all,y_train_all,w_no_reg)
E_in_reg,   E_out_reg    = get_E_in_E_out(X_test,y_test,X_train_all,y_train_all,w_reg)

# In[17]:

def string_Ein(E_in):
    return '$E_{in}=' + '{}'.format(np.round(E_in,5)) + '$'

def string_Eout(E_out):
    return '$E_{out}=' + '{}'.format(np.round(E_out,5)) + '$'

# In[18]:

w_arr = [ w_no_reg, w_reg ]
w_colors = ['red', 'blue' ]
title_train = ['Training Set: (a) $\mathbf{w}$, ' + string_Ein(E_in_no_reg),
               'Training Set: (b) $\mathbf{w_{reg}}$, ' + string_Ein(E_in_reg)]
title_test  = ['Testing Set: (a) $\mathbf{w}$, ' + string_Eout(E_out_no_reg),
               'Testing Set: (b) $\mathbf{w_{reg}}$, ' + string_Eout(E_out_reg)]
fig = plt.figure(figsize=(14,12))
plot_data_nonlinear(fig,[ 221, 222 ],X_train,y_train,w_arr,w_colors,title_train)
plot_data_nonlinear(fig,[ 223, 224 ],X_test, y_test, w_arr,w_colors,title_test)

# ## 3. Hypothesis sets
#
# Consider the following hypothesis set
#
# $$\mathcal{H}_Q = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^Q w_q L_q(x) \right\}$$

# We thus have $\mathcal{H_4, H_3, H_2, H_1}$ as:
#
# $$\mathcal{H}_4 = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^4 w_q L_q(x) \right\}$$
#
# $$\mathcal{H}_3 = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^3 w_q L_q(x) \right\}$$
#
# $$\mathcal{H}_2 = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^2 w_q L_q(x) \right\}$$
#
# $$\mathcal{H}_1 = \left\{ h \; | \; h(x) = \mathbf{w^T z} = \sum\limits_{q=0}^1 w_q L_q(x) \right\}$$

# We also have:
#
# $$\mathcal{H}\left(Q, C, Q_0\right) = \left\{ h \; | \; h(x) = \mathbf{w^T z} \in \mathcal{H}_Q; w_Q = C \text{ for } q \ge Q_0 \right\}$$
#
# For $Q = 10, C = 0, Q_0 = 3$:
# $$\mathcal{H}\left(10, C, 3\right) = \left\{ h \; | \; h(x) = \mathbf{w^T z} \in \mathcal{H}_{10}; w_Q = 0 \text{ for } q \ge 3 \right\} = \mathcal{H}_2$$
#
# For $Q = 10, C = 0, Q_0 = 4$:
# $$\mathcal{H}\left(10, C, 4\right) = \left\{ h \; | \; h(x) = \mathbf{w^T z} \in \mathcal{H}_{10}; w_Q = 0 \text{ for } q \ge 4 \right\} = \mathcal{H}_3$$
#
# Hence we have:
#
# $$\mathcal{H}\left(10, C, 3\right) \cap \mathcal{H}\left(10, C, 4\right) = \mathcal{H}_2 \cap \mathcal{H}_3 = \mathcal{H}_2$$

# ## 4. Neural Networks
#
# ### Forward Propagation
#
# $$x_j^{(l)} = \theta \left( s_j^{(l)} \right) = \theta \left( \sum\limits_{i=0}^{d^{(l-1)}} w_{ij}^{(l)} x_i^{(l-1)} \right)$$
#
# where
#
# $$\theta\left(s\right) = \tanh\left(s\right)$$
#
# ### Backpropagation
#
# $$\delta w_{ij}^{(l)} = -\eta x_i^{(l-1)} \delta_j^{(l)}$$
#
# where
#
# $$\delta_j^{(l-1)} = \left[1 - \left(x_i^{(l-1)}\right)^2\right] \sum_{j=1}^{d^{(l)}} w_{ij}^{(l)} \delta_j^{(l)}$$

# ### Gradient Descent
#
# $$\mathbf{w}\left(t+1\right) = \mathbf{w}\left(t\right) - \eta \nabla E_{in}\left(\mathbf{w}\left(t\right)\right) - 2\eta \frac{\lambda}{N} \mathbf{w}\left(t\right)$$
#
# $$\mathbf{w}\left(t+1\right) = \mathbf{w}\left(t\right)\left(1 - 2\eta \frac{\lambda}{N}\right) - \eta \nabla E_{in}\left(\mathbf{w}\left(t\right)\right)$$
#
# where
#
# $$\mathbf{w^T w} = \sum\limits_{l=1}^L \sum\limits_{i=0}^{d^{(l-1)}} \sum\limits_{j=1}^{d^{(l)}} \left( w_{ij}^{(l)}\right)^2$$
#

# Consider a neural network with 5 inputs, 3 nodes at the hidden layer, and a single node in the output layer, i.e. $d^{(0)} = 5, d^{(1)} = 3, d^{(2)} = 1$:
#
# Further reading: http://home.agh.edu.pl/~vlsi/AI/backp_t_en/backprop.html

# In[19]:

def calculate_weights(layer_nodes):
    """
    Calculate the number of weights required for a neural network,
    where layer_nodes contains the number of nodes for each layer,
    excluding bias units.
    """
    n = len(layer_nodes)
    total = 0
    for i in range(n-1):
        prev = layer_nodes[i]+1  # Add 1 to include bias unit
        curr = layer_nodes[i+1]
        total += prev*curr
        print("Layer {}: d+1={} -> {} = {}".format(i+1,prev,curr,prev*curr))
    return total

# For a neural network with 5 inputs, 3 nodes in the hidden layer, and 1 node in the output layer, the total number of weights is calculated as follows:

# In[20]:

calculate_weights(layer_nodes = [5, 3, 1])

# Suppose we wish to calculate the minimum and maximum number of weights of a neural network with the restriction that it has 10 input units, 36 hidden units, and one output unit.  The minimum number of weights occurs when the 36 hidden units is split into 18 hidden layers (each with 1 bias node + 1 regular node).  The number of weights is calculated as follows:

# In[21]:

calculate_weights(layer_nodes = [9] + ([1] * 18) + [1])

# To calculate the maximum number of weights, a neural network with a single hidden layer would have the following number of weights:

# In[22]:

calculate_weights(layer_nodes = [9, 35, 1])

# If we had two hidden layers:

# In[23]:

calculate_weights([9, 17, 17, 1])

# In[24]:

calculate_weights([9, 18, 16, 1])

# In[25]:

calculate_weights([9, 19, 15, 1])

# In[26]:

calculate_weights([9, 20, 14, 1])

# In[27]:

calculate_weights([9, 21, 13, 1])

# If we had 3 hidden layers:

# In[28]:

calculate_weights([9, 11, 11, 11, 1])

# If we had 4 hidden layers:

# In[29]:

calculate_weights([9, 8, 8, 8, 8, 1])

# Hence, the maximum number of weights corresponds to the neural network [9, 21, 13, 1], with 10 units in the input layer, 22 units in the first hidden layer, and 14 units in the second hidden layer, and 1 unit in the final output layer.  The total number of weights for this network is 510.

import numpy,from numpy,from scipy,from sympy


import numpy as np
import matplotlib.pyplot as plt
import math

# 특정 서브플롯 axis에 벡터 v를 시작점 start에서 출발하도록 그림
# 화살표의 폭, 머리의 크기, 화살표 색상을 지정하고 선 스타일은 ls로 지정 가능
def draw_vector(axis, v, start = (0,0),
                width = 1, headsize=0.3, color='k', ls='-'):

    # 인자에 따라 적절한 화살표를 그린다.
    s = np.array(start)
    if headsize > 0:
        headsize = np.linalg.norm(v)/15
    axis.arrow(start[0], start[1],
               v[0], v[1],
               linewidth = width,
               head_width=headsize, head_length=headsize,
               fc=color, ec=color,
               length_includes_head = True, ls = ls)

def draw_vector3d(axis, v, start = (0,0,0) , color='k', ls='-', arrow=True):
    s = np.array(start)
    if arrow == True:
        axis.quiver(s[0], s[1], s[2],
                    v[0], v[1], v[2], color=color, ls=ls)
    else:
        axis.plot([s[0], v[0]+s[0]],
                  [s[1], v[1]+s[1]],
                  [s[2], v[2]+s[2]], color=color, ls=ls)

def draw_line(axis, start, end, color='k', ls='-'):
    if len(start) == 2:
        axis.plot([start[0], end[0]],
                  [start[1], end[1]], color=color, ls=ls)
    elif len(start) == 3:
        axis.plot([start[0], end[0]],
                  [start[1], end[1]],
                  [start[2], end[2]], color=color, ls=ls)

def draw_vectors(vector_list, start = (0,0,0), color_list = [], figsize=(10,10), linked=False, axis=None) :
    if vector_list != None and len( vector_list) > 0:
        if len(vector_list) != len(color_list) :
            if len(color_list) > 0:
                 colors = [color_list[0] for _ in range(len(vector_list))]
            else:
                 colors = ['k' for _ in range(len(vector_list))]
        else:
             colors = color_list
    dim = len(vector_list[0])
    # figure and axis preparation

    vecsum = np.zeros_like(vector_list[0])
    for v in vector_list:
        vecsum += v
    maxvalue = max(abs(vecsum.min()), abs(vecsum.max())) * 1.1

    if dim == 2:
        s = np.array([start[0], start[1]], dtype=float)
        if axis == None:
            ax = createAxis2d(-maxvalue, maxvalue,  figsize = figsize)
            print(ax)
        else:
            ax = axis
        for v, c in zip(vector_list, colors):
            draw_vector(ax, v, color=c, start=s)
            if linked: s += v
        if linked:
            draw_vector(ax, s, color='k', ls='--', headsize=0.0)
        if axis == None:
            plt.show()
    if dim == 3:
        s = np.array([start[0], start[1], start[2]], dtype=float)
        if axis == None:
            ax = createAxis3d(-maxvalue, maxvalue,  figsize = figsize)
            print(ax)
        else:
            ax = axis
        for v, c in zip(vector_list, colors):
            draw_vector3d(ax, v, color=c, start=s)
            if linked: s += v
        if linked:
            draw_vector3d(ax, s, color='k', ls='--', arrow=False)
        if axis == None:
            plt.show()

def draw_matrix(mat, figsize=(10, 10), axis = None):
    cols = mat.shape[1]
    color = ['r', 'g', 'b' ]
    vectors = []
    colors = []
    for i in range(cols):
        vectors.append(mat[:, i])
        colors.append(color[i%3])

    dim = vectors[0].shape[0]

    flatten_numbers = np.array(vectors).flatten()
    minvalue = flatten_numbers.min()
    maxvalue = flatten_numbers.max()
    d = maxvalue - minvalue
    minvalue -= d*0.25
    maxvalue += d*0.25

    if dim == 2:
        if axis == None:
            ax = createAxis2d(-maxvalue, maxvalue, figsize = figsize)
        else:
            ax = axis
        u = vectors[0]
        v = vectors[1]
        for vec, c in zip(vectors, colors):
            draw_vector(ax, vec, color=c, start=(0,0))

        vectors = [-u,-v]
        for vec, c in zip(vectors, colors):
            draw_vector(ax, vec, start=u+v, color='gray', ls='--', headsize=0)
        if axis==None: plt.show()

    if dim == 3:
        if axis == None:
            ax = createAxis3d(-maxvalue, maxvalue,  figsize = figsize)
        else:
            ax = axis
        u = vectors[0]
        v = vectors[1]
        w = vectors[2]
        for vec, c in zip(vectors, colors):
            draw_vector3d(ax, vec, color=c, start=(0,0,0))
        colors = ['r', 'g', 'b']
        vectors = [-u, v, -w]
        for vec, c in zip(vectors, colors):
            draw_vector3d(ax, vec, start=u+w, color='gray', ls='--', arrow=False)
        vectors = [-u,-v, w]
        for vec, c in zip(vectors, colors):
            draw_vector3d(ax, vec, start=u+v, color='gray', ls='--', arrow=False)
        vectors = [u, -v,-w]
        for vec, c in zip(vectors, colors):
            draw_vector3d(ax, vec, start=v+w, color='gray', ls='--', arrow=False)
        if axis==None: plt.show()

def draw_points(mat, figsize=(10, 10), color='k', ls='-', axis=None):

    dim = mat.shape[0]

    flatten_numbers = np.array(mat).flatten()
    minvalue = flatten_numbers.min()
    maxvalue = flatten_numbers.max()
    d = maxvalue - minvalue
    minvalue -= d * 0.25
    maxvalue += d * 0.25

    if dim == 2:
        if axis == None:
            ax = createAxis2d(-maxvalue, maxvalue, figsize=figsize)
        else:
            ax = axis

        ax.plot(mat[0,:], mat[1,:], color=color, ls=ls)
        if axis == None: plt.show()

    if dim == 3:
        if axis == None:
            ax = createAxis3d(-maxvalue, maxvalue, figsize=figsize)
        else:
            ax = axis
        ax.plot(mat[0,:], mat[1,:], mat[2, :], color=color, ls=ls)
        if axis == None: plt.show()

def draw_line(a, b, c, axis, ls = '-', color=None): # draw line: ax + by + c = 0
    if math.fabs(b) > 1e-5 :
        y = - (c / b)
        dir = np.array([1, -a / b])
        if math.fabs(a) > 1e-5:
            x = - (c + b*y) / a
        else:
            x = 0.0
    else:
        y = 0.0
        x = - (c / a)
        dir = np.array([0, 1])

    p = np.array([x, y])
    p0 = p - dir * 10000
    p1 = p + dir* 10000
    axis.plot([p0[0], p1[0]], [p0[1], p1[1]], ls = ls, color=color)

def createAxis2d(minvalue, maxvalue, subplot=(1,1), figsize=(10,10)):

    if subplot == (1,1):
        fig, ax = plt.subplots(figsize=figsize)
        ax.set_xlim([minvalue, maxvalue])
        ax.set_ylim([minvalue, maxvalue])
        ax.set_xlabel('$x$')
        ax.set_ylabel('$y$')
        ax.grid(True)
        return ax
    elif subplot[0] > 1 and subplot[1] > 1:
        fig, ax = plt.subplots(subplot[0], subplot[1], figsize=figsize)  # figure 크기를 결정
        for row in range(subplot[0]):
            for col in range(subplot[1]):
                ax[row, col].set_xlim([minvalue, maxvalue])
                ax[row, col].set_ylim([minvalue, maxvalue])
                ax[row, col].set_xlabel('$x$')
                ax[row, col].set_ylabel('$y$')
                ax[row, col].grid(True)
        return ax
    else:
        dim = max(subplot[0], subplot[1])
        fig, ax = plt.subplots(subplot[0], subplot[1], figsize=figsize)  # figure 크기를 결정
        for d in range(dim):
            ax[d].set_xlim([minvalue, maxvalue])
            ax[d].set_ylim([minvalue, maxvalue])
            ax[d].set_xlabel('$x$')
            ax[d].set_ylabel('$y$')
            ax[d].grid(True)
        return ax

def createAxis3d(minvalue, maxvalue, subplot=(1,1), figsize=(10,10)):

    if subplot == (1,1):
        fig = plt.figure(figsize = figsize)   # figure 크기를 결정
        ax = fig.gca(projection='3d')
        ax.set_xlim([minvalue, maxvalue])
        ax.set_ylim([minvalue, maxvalue])
        ax.set_zlim([minvalue, maxvalue])
        ax.set_xlabel('$x$')
        ax.set_ylabel('$y$')
        ax.set_zlabel('$z$')
        ax.grid(True)
        return ax
    elif subplot[0] > 1 and subplot[1] > 1:
        fig, ax = plt.subplots(subplot[0], subplot[1], figsize = figsize)   # figure 크기를 결정
        i = 1
        for row in range(subplot[0]):
            for col in range(subplot[1]):
                ax[row, col] = fig.add_subplot(subplot[0], subplot[1], i, projection='3d')
                ax[row, col].set_xlim([minvalue, maxvalue])
                ax[row, col].set_ylim([minvalue, maxvalue])
                ax[row, col].set_zlim([minvalue, maxvalue])
                ax[row, col].set_xlabel('$x$')
                ax[row, col].set_ylabel('$y$')
                ax[row, col].set_zlabel('$z$')
                ax[row, col].grid(True)
                i+=1
        return ax
    else:
        dim = max(subplot[0], subplot[1])
        fig, ax = plt.subplots(subplot[0], subplot[1], figsize=figsize)  # figure 크기를 결정
        i = 1
        for d in range(dim):
            ax[d] = fig.add_subplot(subplot[0], subplot[1], i, projection='3d')
            ax[d].set_xlim([minvalue, maxvalue])
            ax[d].set_ylim([minvalue, maxvalue])
            ax[d].set_zlim([minvalue, maxvalue])
            ax[d].set_xlabel('$x$')
            ax[d].set_ylabel('$y$')
            ax[d].set_zlabel('$z$')
            ax[d].grid(True)
            i += 1
        return ax


from pandas import DataFrame
from scipy.stats import chisqprob, chi2

"""
Chi2 Analysis should be used with Independent Test Subjects
"""
def calc_chi2_expected_val(original_d, marginal_col_totals, marginal_row_totals, total):
    """
    this calculates the expected value matrix;
    see 'chi2' to see how to calcualte chi2/kai 2 values using
    orignal/actual value and expected value matrices.
    """
    percentage_col = marginal_col_totals/total
    expected_val = marginal_row_totals.values * percentage_col.values
    return DataFrame(data = expected_val, index = original_d.index, columns = original_d.columns)

def chi2_val(original,expected):
    """
    Given 2 dataframes:
    original = DataFrame(data=[(7,16,6),(43,34,44)], index = ['YES','NO'], columns=['HIT', 'SMASHED', 'CONTROL'])

    expected = DataFrame(data=[[9.67]*3,[40.33]*3], index = ['YES','NO'], columns=['HIT', 'SMASHED', 'CONTROL'])
    """
    return ((original-expected)**2/expected).values.sum()

critical_chi2_val_from_p = lambda p, df: \
round(scipy.stats.chi2.isf(p, df), 3)

p_from_chi2_val = lambda chi2_val, df: \
scipy.stats.chi2.sf(chi2_val, df)

def chi2_cmp(chi2_val, df, alpha_val):
    """
    if the probability of getting chi2_val at df is < alpha_val is TRUE,
    then it is a significant val. Reject the Null.
    """
    return round(chisqprob(chi2_val, df), 4) < alpha_val

def kramers_phi_v(chi2_val, num_samples, d):
    """
    Independence Test: tests for strength between the two variables
    used when >= 2x2 matrix;
    d: original DataFrame
    """
    k = min((d.columns.size,d.index.size))
    return sqrt(float(chi2_val)/(num_samples*(k-1)))


"""
Understanding data error mathematics and analysis

This script contains...
Functions for calculating error and deviation between two or one data lists.
    All expressions use the standard Python library only, except rmse which also uses numpy.
Documentation attempting to illuminate the reasoning behind the math.
Standard Deviation can be calculated specifying form='rmse' and list2 == None

Written by Felicty Nielson, mostly for my own intituion,
but if you find this useful, then I am happy to oblige.
"""

def print_outlim(val1, val2):
    """ Helper for printing outliers
    """
    print("Outlier found:  |{} - {}| = {}".format(val1, val2, abs(val1 - val2)))
    print("Outlier has been excluded from calculations")

def zero_chk(list1):
    """ Some of these formulas don't handle 0's in certain places well (e.g. diving by 0 in MAPE).
        For this reason, we raise an exception if any are found.
    """
    for x in list1:
        if x == 0:
            print("0 found in list1. MAPE can not divide by 0.")

def float_chk(list1):
    """ In python 2 versions, at least one of the lists will need to be in floats (not ints)
        to ensure correct calculation.
    """
    for i, j in enumerate(list1):
        list1[i] = float(j)
    return list1

def mean_list(list1, N):
    """ Helper function calculates mean and returns list of N means
    """
    ave = 0
    for i in list1:
        ave += i
    ave /= N
    lista = [ave for x in range(0,N)]
    return lista

def errors(form, list1, list2=None, deviation=False, outlim=0):
    """ Function for calculating error and deviation between two or one data lists.
        Lists must be of same length and the elements of the first list corresponding
        to the elements of the second.
        Args:
          form  (string): Mathematical formula for calculating error.
                          Availiable formulas are either 'mae', 'mape', 'mse', or 'rmse'
          deviation  (bool): default False. If specifying only one list, deviation is automatic.
                             If specifying two lists, deviation of the second list is measured off of
                             the mean of the first.
          list1 (list of floats): The "true" values
          list2 (list of floats): The "observed" values. Default None.
                                  If None, the deviation of list1 is calculated.
          outlim (float): default 0 equivalent to None. If the difference between two elements
                          is greater than the outlier limit, print exception and exclude from
                          error calculation.
        Returns:
          err (int): The computed error or deviation.
    """
    import numpy as np

    list1 = float_chk(list1)

    outlim_count = 0
    error = 0
    N = len(list1)

    # Determine if calculating error or deviation. If calculating deviation, build mean list.
    if list2 != None and deviation == False:
        print("Calculating error...")
    else:
        print("Calculating deviation...")
        if list2 == None:
            list2 = mean_list(list1, N)
        elif deviation == True:
            list1 = mean_list(list1, N)

    for i in range(0, N):
        if form == 'mae':
            err = abs(list1[i]-list2[i])
            """ Mean Absolute Error (MAE)

            Sum(|y-y'|)/N

            The simpliest way to decide the error between measurements is to take the difference.
            Say you measure a mile and it was 4 inch off of a true mile. The 4 inch is your error.
            Whether this error is + or - often doesn't matter and can complicate analysis,
            so we take the absolute value.
            That is the error between two values. If we have two lists and we want a single value
            to represent the error of all data, we can average them
            (sum and divide by the total number of errors or data point comparisons).

            To calculate deviation, we modify the formula to subtract the mean of one list from the values
            of another list. This lets us know how far the observed values are deviating from the true average.
            Or we can subtract the values of one list from the mean of the same list. This measures how
            much the data deviates from its own average.

            Standard deviation uses RMSE (or rather, RMSD...). See the 'rmse' condition below.
            """

        elif form  == 'mape':
            zero_chk(list1)
            err = abs(list1[i]-list2[i])/list1[i]
            """ Mean Absolute Percent Error (MAPE)

            Sum(|y-y'|/y)/N*100

            We are looking at a percent on the original or true y value,
            rather than a value of the differences of the measurement (MAE),
            which can be meaningless to the user unless they understand the scale of their data.

            For example,
            say I measure the size of giant bacteria in cm I get
            y' = [1.59, 4.37, 10.95, 6.27, 8.04, 9.29, 3.48]
            And the true vales are
            y = [1.62, 4.10, 10.81, 6.28, 8.10, 9.50, 3.47]

            The MAE is 0.104285714285714

            Then I do the same thing but convert to mm first

            The MAE is 104.285714285714

            But if I use MAPE, I get the same value for both errors (1.828204390849)
            since the hard difference is taken as a percentage of the true measurement.
            Therefore if I use MAPE, I don't need to know what scale I'm working with,
            at least for this purpose.
            """

        elif form == 'mse' or form == 'rmse':
            err = (list1[i] - list2[i])**2
            """ Mean Squared Error (MSE)

            Sum((y - y')^2)/N

            Another way of removing the negative from errors is to square them.
            This will convert everything to positive values without having to take the
            absolute value.

            --------

            Root Mean Squared Error (RMSE)

            Sqrt(Sum((y - y')^2)/N)

            Specifying only one list will result in the Standard Deviation (SD).

            Warning: outliers will dramatically increase the error for MSE and RMSE.
            Suggest specifying an outlim (outlier limit) value.
            """

        else:
            print("form should be 'mae', 'mape', 'mse', or 'rmse'")

        if outlim != 0 and abs(list1[i]-list2[i]) > outlim:
            print_outlim(list1[i], list2[i])
            outlim_count += 1
        else:
            error += err

    error /= (N - outlim_count)
    if form == 'mape':
        error *= 100
    elif form == 'rmse':
        error = np.sqrt(error)

    return error


from random import randint
from tempfile import TemporaryFile
import numpy as np

def _inPlaceQuickSort(A, start, end):
    count = 0
    if start < end:
        pivot = randint(start, end)
        temp = A[end]
        A[end] = A[pivot]
        A[pivot] = temp

        p, count = _inPlacePartition(A, start, end)
        count += _inPlaceQuickSort(A, start, p - 1)
        count += _inPlaceQuickSort(A, p + 1, end)
    return count

def _inPlacePartition(A, start, end):

    count = 0
    pivot = randint(start, end)
    temp = A[end]
    A[end] = A[pivot]
    A[pivot] = temp
    newPivotIndex = start - 1
    for index in range(start, end):

        count += 1
        if A[index] < A[end]:  # check if current val is less than pivot value
            newPivotIndex = newPivotIndex + 1
            temp = A[newPivotIndex]
            A[newPivotIndex] = A[index]
            A[index] = temp

    temp = A[newPivotIndex + 1]
    A[newPivotIndex + 1] = A[end]
    A[end] = temp
    return newPivotIndex + 1, count

outfile = TemporaryFile()
p = 100  # 1000 elements are to be sorted

mu, sigma = 0, 1  # mean and standard deviation
X = np.random.normal(mu, sigma, p)
np.save(outfile, X)
print("The array is")
print(X)

outfile.seek(0)  # using the same array
M = np.load(outfile)
r = len(M) - 1
z = _inPlaceQuickSort(M, 0, r)

print(
    "No of Comparisons for 100 elements selected from a standard normal distribution"
    "is :"
)
print(z)


'''
Estimating an MA Model

You will estimate the MA(1) parameter, θ
θ
, of one of the simulated series that you generated in the earlier exercise. Since the parameters are known for a simulated series, it is a good way to understand the estimation routines before applying it to real data.

For simulated_data_1 with a true θ
θ
 of -0.9, you will print out the estimate of θ
θ
. In addition, you will also print out the entire output that is produced when you fit a time series, so you can get an idea of what other tests and summary statistics are available in statsmodels.

INSTRUCTIONS
100XP
Import the class ARMA in the module statsmodels.tsa.arima_model.
Create an instance of the ARMA class called mod using the simulated data simulated_data_1 and the order (p,q) of the model (in this case, for an MA(1)), is order=(0,1).
Fit the model mod using the method .fit() and save it in a results object called res.
Print out the entire summmary of results using the .summary() method.
Just print out an estimate of the constant and phi parameter using the .params attribute (no arguments).
'''
# Import the ARMA module from statsmodels
from statsmodels.tsa.arima_model import ARMA

# Fit an MA(1) model to the first simulated data
mod = ARMA(simulated_data_1, order=(0,1))
res = mod.fit()

# Print out summary information on the fit
print(res.summary())

# Print out the estimate for the constant and for theta
print("When the true theta=-0.9, the estimate of theta (and the consant) are:")
print(res.params)