from itertools import product

from numpy import einsum, zeros
from objects.grtensors.metrictensor import MetricTensor
from objects.simplifyobjects import Simplify
from sympy import Array, MutableDenseNDimArray, diff

def derivative_of_metric(coord_sys, metric_tensor, i, j, k):
    """
    Taking the partial derivative of a given metric tensor [g_jk] with respect to the
    [i]'th component

    Args:
        coord_sys [list]: The coordinate system given as a list (e.g., [t,x,y,z])
        metric_tensor [sympy.tensor]: Given metric tensor, g_jk
        i,j,k [int]: Coordinate indices; (0-ndim)
    """
    return Simplify(diff(metric_tensor[j, k], coord_sys[i]))

class ChristoffelSymbol(MetricTensor):
    def __init__(self, metric_tensor, coord_sys):
        """
        Creating the christoffel symbol object

        Args:
            metric_tensor [list]: The metric tensor, provided by the user
            coord_sys [list]: The coordinate system given as a list (e.g., [t,x,y,z])

        Returns:
            self.chris_type [str]: Type of the christoffel symbol. Default type is 'udd'
            self.chris_obj [sympy.tensor]: The christoffel symbol, Gamma^m_ij
        """
        MetricTensor.__init__(self, metric_tensor, coord_sys)
        self.chris_type = 'udd'
        chris_sym = MutableDenseNDimArray(zeros((self.ndim,)*3))
        for m, i, j in product(range(self.ndim), repeat=3):
            einstein_sum = 0
            for k in range(self.ndim):
                I1 = derivative_of_metric(
                    self.coord_sys, self.metric_obj, j, k, i)
                I2 = derivative_of_metric(
                    self.coord_sys, self.metric_obj, i, k, j)
                I3 = derivative_of_metric(
                    self.coord_sys, self.metric_obj, k, i, j)
                S = I1 + I2 - I3
                einstein_sum += 1/2 * self.inverse_metric_obj[m, k] * S
            chris_sym[m, i, j] = einstein_sum
        self.chris_obj = chris_sym

    def get_christoffelsymbol(self):
        """
        Returns the christoffel symbol object
        """
        return Simplify(self.chris_obj)

    def get_christoffelsymbol_type(self):
        """
        Returns the type of the christoffel symbol
        """
        return self.chris_type

    def lower_index(self, xchris_symbol):
        """
        Lowering the index of the christoffel symbol

        Args:
            xchris_symbol [sympy.tensor]: Given christoffel symbol
        """
        return Array(einsum('ijk,il->ljk', xchris_symbol, self.metric_obj, optimize='optimal'))

    def raise_index(self, xchris_symbol):
        """
        Raising the index of the christoffel symbol

        Args:
            xchris_symbol [sympy.tensor]: Given christoffel symbol
        """
        return Array(einsum('ikj,jm->imk', xchris_symbol, self.inverse_metric_obj, optimize='optimal'))

    def raise_index1(self, xchris_symbol):
        """
        Raising the second index of the christoffel symbol

        Args:
            xchris_symbol [sympy.tensor]: Given christoffel symbol
        """
        return Array(einsum('imk,kn->imn', xchris_symbol, self.inverse_metric_obj, optimize='optimal'))

    def vary_christoffelsymbol_type(self, xchris_symbol, new_type):
        """
        Varying the type of the christoffel symbol

        Args:
            xchris_symbol [sympy.tensor]: Given christoffel symbol
            new_type [str]: The new type of the christoffel symbol. It should be given
            in terms of 'u': contravariant (upper-indices) and 'd': covariant (lower-indices)

        Returns:
            The new christoffel symbol for a given type
        """
        self.chris_type = new_type
        if new_type == 'ddd':
            return Simplify(self.lower_index(xchris_symbol))
        elif new_type == 'udd':
            return Simplify(self.chris_obj)
        elif new_type == 'uud':
            return Simplify(self.raise_index(xchris_symbol))
        elif new_type == 'uuu':
            return Simplify(self.raise_index1(self.raise_index(xchris_symbol)))


"""
///////////////////////// TOP OF FILE COMMENT BLOCK ////////////////////////////
//
// Title:           CRR pricing model
// Course:          Master's thesis, 2020
//
// Author:          Peter Pommergård Lind
// Email:           ppl_peter@protonmail.com
// Encoding:        utf-8
///////////////////////////////// CITATIONS ////////////////////////////////////
//
// Option Pricing: A Simplified Approach
// by John C. Cox, Stephen A. Ross, and Mark Rubinstein
//
/////////////////////////////// 80 COLUMNS WIDE ////////////////////////////////
"""
#Binomial tree in Hull p 453
#packages
import numpy as np
import datetime

#Make tree dictionary
def binStockPath(stepSize, steps, spot, vol):
    STree = {}
    #stepSize = maturity/steps
    u = np.exp(vol*stepSize**0.5)
    d = np.exp(-vol*stepSize**0.5)
    for i in range(steps+1):
        if i==0:
            STree["{}".format(i*stepSize)] = spot
        else:
            list1 = []
            [list1.append(spot*u**j*d**(i-j)) for j in range(i+1)]

            STree["{}".format(i*stepSize)] = list1
    return STree

#intrinsic value dictionary for american put option
def findIntrinsicTree(stepSize, steps, spot, vol, strike):
    #stepSize = maturity/steps
    STree = binStockPath(stepSize, steps, spot, vol)
    intrinsicTree = {}
    for i in reversed(range(steps+1)):
        if i==0:
            intrinsicTree["{}".format(i*stepSize)] = max(strike-STree["{}".format(i*stepSize)],0)
        else:
            intrinsicTree["{}".format(i*stepSize)] = [max(strike-item,0) for item in STree["{}".format(i*stepSize)]]
    return intrinsicTree

#find condition expection
def findPayoff(steps, maturity, spot, vol, strike, r):
    start = datetime.datetime.now()
    stepSize = maturity/steps
    intrinsicTree = findIntrinsicTree(stepSize, steps, spot, vol, strike)
    u = np.exp(vol*stepSize**0.5)
    d = np.exp(-vol*stepSize**0.5)
    a = np.exp(r*stepSize)
    # risk neutral probabilities
    p = (a-d)/(u-d)
    expecTree = {}
    for i in reversed(range(steps+1)):
        if i==steps:
            expecTree["{}".format(i*stepSize)] = intrinsicTree["{}".format(i*stepSize)]
            continue
        else:
            list1 = [None]*(i+1)
            for count in range(i+1):
                condExp = 0
                for k in range(count, count+1):
                    condExp += (1-p)*expecTree["{}".format((i+1)*stepSize)][k]
                    k+=1
                    condExp += (p) * expecTree["{}".format((i+1)*stepSize)][k]
                    condExp = condExp*np.exp(-r*stepSize) # discount back
                if (i==0):
                    list1[count] = condExp
                elif (condExp > intrinsicTree["{}".format(i*stepSize)][count]):
                    list1[count] = condExp
                else:
                    list1[count] = intrinsicTree["{}".format(i*stepSize)][count]
        expecTree["{}".format(i*stepSize)]=list1
    finish = datetime.datetime.now()
    print (finish-start)
    return expecTree

if __name__ == '__main__':
    print(findPayoff(steps=50, maturity=1, spot=36, vol=0.2, strike=40, r=0.06)['0.0'])

    #print(findPayoff(steps=200, maturity=2, spot=S, vol=0.4, strike=40, r=0.06)['0.0'])


import numpy as np
import matplotlib.pyplot as plt
from pandas.io.parsers import read_csv

def load_csv(file_name):
    values = read_csv(file_name, header=None).values

    return values.astype(float)

def gradient_descent(X, Y, alpha, iterations):
    m = np.shape(X)[0]
    n = np.shape(X)[1]
    thetas = np.zeros((n, 1), dtype=float)
    costs = np.empty(iterations)
    thetasHistory = np.empty((iterations, np.shape(thetas)[0]))

    for i in range(iterations):
        H = h(X, thetas)
        thetas = thetas - (alpha * (1/float(m)) * (X.T.dot(H - Y)))

        thetasHistory[i] = thetas.T
        costs[i] = cost(X, Y, thetas)
        print("Cost [", i, "]: ", costs[i])

    return thetasHistory, costs

def cost(X, Y, thetas):
    m = np.shape(X)[0]
    H = h(X, thetas)
    aux = H-Y
    return np.dot(aux.T, aux) / (2*m)

def h(X, thetas):
    return np.dot(X, thetas)

def main():
    # DATA PREPROCESSING
    datos = load_csv("data/ex1data2.csv")
    X = datos[:, :-1]
    X = normalize(X)[0]
    Y = datos[:, -1][np.newaxis].T

    m = np.shape(X)[0]
    n = np.shape(X)[1]

    X = np.hstack([np.ones([m, 1]), X])
    alpha = 0.01

    # TRAINING
    thetas, costs = gradient_descent(X, Y, alpha, 500)

    # PREDICTION
    prediction = h(X, thetas[-1,:][np.newaxis].T)

    # i changes the case you output (for test purposes)
    i = 40
    print("Case %i | %f:%f = %f | Real value: %f" %(i, X[i, 1], X[i, 2], prediction[0], Y[i]))

    # RESULTS
    plt.figure()
    plt.plot(costs)
    plt.savefig('images/regresion_multi_gradient_descent.png')
    plt.show()
    plt.close()

def normalize(X):
    mean = np.mean(X, 0)
    deviation = np.std(X, 0)
    normalized = (X-mean)/deviation

    return (normalized, mean, deviation)

if __name__ == "__main__":
    main()


from __future__ import annotations
import numpy as np
from numpy.linalg import inv, det, slogdet

class UnivariateGaussian:
    """
    Class for univariate Gaussian Distribution Estimator
    """

    def __init__(self, biased_var: bool = False) -> UnivariateGaussian:
        """
        Estimator for univariate Gaussian mean and variance parameters

        Parameters
        ----------
        biased_var : bool, default=False
            Should fitted estimator of variance be a biased or unbiased estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `UnivariateGaussian.fit` function.

        mu_: float
            Estimated expectation initialized as None. To be set in `UnivariateGaussian.fit`
            function.

        var_: float
            Estimated variance initialized as None. To be set in `UnivariateGaussian.fit`
            function.
        """
        self.biased_ = biased_var
        self.fitted_, self.mu_, self.var_ = False, None, None

    def fit(self, X: np.ndarray) -> UnivariateGaussian:
        """
        Estimate Gaussian expectation and variance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Training data

        Returns
        -------
        self : returns an instance of self.

        Notes
        -----
        Sets `self.mu_`, `self.var_` attributes according to calculated estimation (where
        estimator is either biased or unbiased). Then sets `self.fitted_` attribute to `True`
        """
        m = X.shape[0]

        def sample_mean_estimator(X: np.ndarray):
            return np.sum(X) / m

        def sample_variance_estimator(X: np.ndarray):
            sum_elems = np.array(
                [np.power(X[i] - self.mu_, 2) for i in range(m)])
            divisor = m if self.biased_ else m + 1
            return (np.sum(sum_elems)) / divisor

        self.mu_ = sample_mean_estimator(X)
        self.var_ = sample_variance_estimator(X)

        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray) -> np.ndarray:
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, var_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")

        m = X.shape[0]

        def formula(x):
            exp = np.exp(-(np.power(x - self.mu_, 2) / (2 * self.var_)))
            return exp / (np.sqrt(2 * np.pi * self.var_))

        return np.array([formula(X[i]) for i in range(m)])

    @staticmethod
    def log_likelihood(mu: float, sigma: float, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : float
            Expectation of Gaussian
        sigma : float
            Variance of Gaussian
        X : ndarray of shape (n_samples, )
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated
        """
        m = X.shape[0]
        exp_sum = np.sum([np.power(X[i] - mu, 2) for i in range(m)])
        # following formula calculated in class
        return -(m / 2) * np.log(2 * np.pi) \
               - (m / 2) * np.log(sigma) - (exp_sum / (2 * sigma))

class MultivariateGaussian:
    """
    Class for multivariate Gaussian Distribution Estimator
    """

    def __init__(self):
        """
        Initialize an instance of multivariate Gaussian estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `MultivariateGaussian.fit` function.

        mu_: ndarray of shape (n_features,)
            Estimated expectation initialized as None. To be set in `MultivariateGaussian.fit`
            function.

        cov_: ndarray of shape (n_features, n_features)
            Estimated covariance initialized as None. To be set in `MultivariateGaussian.fit`
            function.
        """
        self.mu_, self.cov_ = None, None
        self.fitted_ = False

    def fit(self, X: np.ndarray) -> MultivariateGaussian:
        """
        Estimate Gaussian expectation and covariance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Training data

        Returns
        -------
        self : returns an instance of self

        Notes
        -----
        Sets `self.mu_`, `self.cov_` attributes according to calculated estimation.
        Then sets `self.fitted_` attribute to `True`
        """
        m = X.shape[0]

        def sample_mean_estimator(X: np.ndarray):
            return (np.sum(X, axis=0)) / m

        def sample_covariance_estimator(X: np.ndarray):
            # first subtract the empirical mean from each sample
            X_centered = np.array([X[i] - self.mu_ for i in range(m)])
            # then follow the definition of sample covariance matrix
            return np.dot(X_centered.T, X_centered) / (m - 1)

        self.mu_ = sample_mean_estimator(X)
        self.cov_ = sample_covariance_estimator(X)
        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray):
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, cov_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")

        m = X.shape[0]
        d = X.shape[1]

        def formula(x: np.ndarray):
            # following formula from course book - definition 1.2.5
            exp = np.exp(
                -(1 / 2) * np.dot(
                    (x - self.mu_).T,
                    np.dot(np.linalg.inv(self.cov_), (x - self.mu_))))
            divisor = np.sqrt(
                np.power((2 * np.pi), d) * np.linalg.det(self.cov_))
            return exp / divisor

        return np.array([formula(X[i]) for i in range(m)])

    @staticmethod
    def log_likelihood(mu: np.ndarray, cov: np.ndarray, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : ndarray of shape (n_features,)
            Expectation of Gaussian
        cov : ndarray of shape (n_features, n_features)
            covariance matrix of Gaussian
        X : ndarray of shape (n_samples, n_features)
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated over all input data and under given parameters of Gaussian
        """

        m, d = X.shape[0], X.shape[1]
        # following formula calculated in Ex01 Q09
        exp_sum = np.sum((X - mu) @ np.linalg.inv(cov) * (X - mu))
        return -(m * d / 2) * np.log(2 * np.pi) \
               - (m / 2) * np.log(np.linalg.det(cov)) \
               - (exp_sum / 2)


import numpy as np
import matplotlib.pyplot as plt

N = 1000
n = int(N/2)
D = 2

R_inner = 5
R_outer = 10

R1 = np.random.randn(n) + R_inner
theta = 2 * np.pi * np.random.random(n)
X_inner = np.concatenate([[R1 * np.cos(theta)], [R1 * np.sin(theta)]]).T

R2 = np.random.randn(n) + R_outer
theta = 2 * np.pi * np.random.random(n)
X_outer = np.concatenate([[R2 * np.cos(theta)], [R2 * np.sin(theta)]]).T

X = np.concatenate([X_inner, X_outer])
T = np.array([0] * (n) + [1] * (n))

plt.scatter(X[:, 0], X[:, 1], c=T)
plt.show()

ones = np.array([[1] * N]).T

r = np.zeros((N, 1))
for i in range(N):
    r[i] = np.sqrt(X[i, :].dot(X[i, :]))

Xb = np.concatenate((ones, r, X), axis=1)
w = np.random.randn(D + 2)
z = Xb.dot(w)

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

Y = sigmoid(z)

def cross_entropy(T, Y):
    E = 0
    for i in range(N):
        if T[i] == 1:
            E -= np.log(Y[i])
        else:
            E -= np.log(1 - Y[i])
    return E

learning_rate = 0.0001
error = []
for i in range(5000):
    e = cross_entropy(T, Y)
    error.append(e)
    if i % 100 == 0:
        print(e)

    w += learning_rate * (np.dot((T - Y).T, Xb) - 0.01 * w)

    Y = sigmoid(Xb.dot(w))

plt.plot(error)
plt.show()

print('Final w', w)
print('Final classification rate', 1 - np.abs(T - np.round(Y)).sum() / N)


import numpy as np

# Perform numerous numpy operations

def run():
    # create a column vector
    col_vec = np.array([[1], [2]])
    print "column vector"
    print col_vec

    # create a row vector
    row_vec = np.array([[1, 2]])
    print "row vector"
    print row_vec

    # create a matrix
    mat = np.array([[1, 2], [3, 4]])
    print "matrix"
    print mat

    # inspect dimensions
    print "row vector dimensions", row_vec.ndim
    shape = row_vec.shape
    print "row vector rows", shape[0], "columns", shape[1]

    print "matrix dimensions", mat.ndim
    shape = mat.shape
    print "matrix rows", shape[0], "columns", shape[1]

    # transpose
    vec_t = row_vec.transpose()  # or row_vec.T
    print "transposed vector"
    print vec_t

    mat_t = mat.transpose()  # or mat.T
    print "transposed matrix"
    print mat_t

    a = np.array([[2], [-4], [1]])
    b = np.array([[2], [1], [-2]])

    # addition
    print "a + b"
    print a + b

    # subtraction
    print "a - b"
    print a - b

    # scalar multiplication
    print "1.2 * a"
    print 1.2 * a

    # element wise multiplication
    print "a * b"
    print a * b

    # vector scalar product
    print "a . b"
    print np.dot(a.transpose(), b)

    # vector cross product
    print "a x b"
    print np.cross(a, b, axis=0)  # or np.cross(a.T, b.T).T

    identity = np.array([[1, 0], [0, 1]])

    # matrix vector product
    print "identity . col_vec"
    print np.dot(identity, col_vec)

    # matrix product
    print "identity . mat"
    print np.dot(identity, mat)

# this is executed when we run this script from commandline
if __name__ == "__main__":
    run()


#coding:utf-8
import numpy as np
import matplotlib.pyplot as plt

"""
勾配降下法で線形回帰による曲線フィッティング
PRML1章の多項式曲線フィッティングと同じだが、
正規方程式ではなく、勾配降下法で解く点が異なる
"""

def plotData(X, y):
    plt.scatter(X, y, c='red', marker='o', label="Training data")
    plt.xlabel("x")
    plt.ylabel("y")

def computeCost(X, y, theta):
    m = len(y)  # 訓練データ数
    tmp = np.dot(X, theta) - y
    J = 1.0 / (2 * m) * np.dot(tmp.T, tmp)
    return J

def gradientDescent(X, y, theta, alpha, iterations):
    m = len(y)      # 訓練データ数
    J_history = []  # 各更新でのコスト
    for iter in range(iterations):
        theta = theta - alpha * (1.0 / m) * np.dot(X.T, np.dot(X, theta) - y)
        J_history.append(computeCost(X, y, theta))
    return theta, J_history

if __name__ == "__main__":
    # 訓練データを作成
    # sin(2 * pi * x) の値にガウス分布に従う小さなランダムノイズを加える
    X = np.linspace(0, 1, 100)
    y = np.sin(2 * np.pi * X) + np.random.normal(0, 0.2, X.size)

    # 訓練データ数
    m = len(y)

    # 訓練データをプロット
    plt.figure(1)
    plotData(X, y)

    # 訓練データの1列目に1を追加
    # 元の変数の二乗、三乗の項も加える
    # 多項式回帰になり曲線フィッティングも可能
    X = X.reshape((m, 1))
    X = np.hstack((np.ones((m, 1)), X, X**2, X**3))

    # パラメータを0で初期化
    # yとthetaはわざわざ縦ベクトルにreshapeしなくてもOK
    # np.dot()では自動的に縦ベクトル扱いして計算してくれる
    theta = np.zeros(4)
    iterations = 100000
    alpha = 0.2

    # 初期状態のコストを計算
    initialCost = computeCost(X, y, theta)
    print "initial cost:", initialCost

    # 勾配降下法でパラメータ推定
    theta, J_history = gradientDescent(X, y, theta, alpha, iterations)
    print "theta:", theta
    print "final cost:", J_history[-1]

    # コストの履歴をプロット
    plt.figure(2)
    plt.plot(J_history)
    plt.xlabel("iteration")
    plt.ylabel("J(theta)")

    # 曲線をプロット
    plt.figure(1)
    plt.plot(X[:, 1], np.dot(X, theta), 'b-', label="Linear regression")
    plt.xlim((0, 1))
    plt.legend()
    plt.show()


# example of a stiff non-linear system -- this is the example defined in
# the VODE source code
#
# y1' = -0.04 y1 + 1.e4 y2 y3
# y2' =  0.04 y1 - 1.e4 y2 y3 - 3.e7 y2**2
# y3' =                         3.e7 y2**2
#
# y1(0) = 1
# y2(0) = 0
# y3(0) = 0
#
# long term behavior: y1 -> 0, y2 -> 0, y3 -> 1
#
# here we solve the system twice: once taking a small step initially
# and then increasing by 10x until we reach tmax -- this lets us see
# the intermediate behavior.  The second call has VODE handle the
# timestep throughout the entire integration and does 0 to tmax in one
# call.
#
# M. Zingale (2013-02-23)

import numpy
import pylab
import scipy
from scipy.integrate import ode

def rhs(t, Y):
    """ RHS of the system -- using 0-based indexing """
    y1 = Y[0]
    y2 = Y[1]
    y3 = Y[2]

    dy1dt = -0.04*y1 + 1.e4*y2*y3
    dy2dt =  0.04*y1 - 1.e4*y2*y3 - 3.e7*y2**2
    dy3dt =                         3.e7*y2**2

    return numpy.array([dy1dt, dy2dt, dy3dt])

def jac(t, Y):
    """ J_{i,j} = df_i/dy_j """

    y1 = Y[0]
    y2 = Y[1]
    y3 = Y[2]

    df1dy1 = -0.04
    df1dy2 = 1.e4*y3
    df1dy3 = 1.e4*y2

    df2dy1 = 0.04
    df2dy2 = -1.e4*y3 - 6.e7*y2
    df2dy3 = -1.e4*y2

    df3dy1 = 0.0
    df3dy2 = 6.e7*y2
    df3dy3 = 0.0

    return numpy.array([ [ df1dy1, df1dy2, df1dy3 ],
                         [ df2dy1, df2dy2, df2dy3 ],
                         [ df3dy1, df3dy2, df3dy3 ] ])

def VODEIntegrate(Y0, dt, tmax):
    """ integrate using the VODE method, start with a timestep of dt and
        increase by 10x after each call until we reach tmax.  This is
        the behavior used in the DVODE Fortran source. """

    r = ode(rhs, jac).set_integrator("vode", method="bdf",
                                     with_jacobian=True,
                                     atol=1.e-10, rtol=1.e-10,
                                     nsteps = 15000, order=5) #, min_step=dt)

    t = 0.0
    r.set_initial_value(Y0, t)

    tout = [t]
    y1out = [Y0[0]]
    y2out = [Y0[1]]
    y3out = [Y0[2]]

    while r.successful() and r.t < tmax:
        r.integrate(r.t+dt)

        tout.append(r.t)
        y1out.append(r.y[0])
        y2out.append(r.y[1])
        y3out.append(r.y[2])

        dt = 10.0*dt

    return numpy.array(tout), \
        numpy.array(y1out), numpy.array(y2out), numpy.array(y3out)

Y0 = numpy.array([1.0, 0.0, 0.0])
tmax = 4.e7

# first call -- give an initial small timestep and have our integrate
# routine increase it by 10x each call to VODE to get to tmax
dt_init = 4.e-6

t, y1, y2, y3 = VODEIntegrate(Y0, dt_init, tmax)

# second call -- give an initial timestep of tmax -- VODE will handle
# everything
t_single, y1_single, y2_single, y3_single = VODEIntegrate(Y0, tmax, tmax)

ax = pylab.gca()
ax.set_xscale('log')
ax.set_yscale('log')

pylab.plot(t, y1, label=r"$y_1$", color="r")
pylab.plot(t, y2, label=r"$y_2$", color="g")
pylab.plot(t, y3, label=r"$y_3$", color="b")

pylab.scatter(t_single, y1_single, color="r",
              label="r$y_1$ from single VODE call")

pylab.scatter(t_single, y2_single, color="g",
              label="r$y_2$ from single VODE call")

pylab.scatter(t_single, y3_single, color="b",
              label="r$y_3$ from single VODE call")

pylab.legend(loc=8, frameon=False, fontsize="small")

pylab.xlabel("$t$")
pylab.ylabel(r"$Y(t)$")
pylab.title("VODE reference stiff nonlinear system")

pylab.savefig("stiff-nonlinear.png")


#@ Model: Lasso
#@ Description: Machine Learning: Lasso Regression. Example from cvxpy.org/examples/machine_learning/lasso_regression.html  $$\min (\beta X - Y)+\lambda||\beta||$$ Find the regression coefficients (beta) that minimize the above regularized function

import cvxpy as cp
import numpy as np
import matplotlib.pyplot as plt
import json

np.random.seed(1)

#@ Helper Object: m
#@ Description: number of data points
m = 100

#@ Helper Object: n
#@ Description: number of features
n = 20

#@ Helper Object: sigma
#@ Description: $$\sigma$$ parameter for generating random dataset
sigma = 5

#@ Helper Object: density
#@ Description: density parameter for generating random dataset
density = 0.2

#@ Helper Object: X
#@ Description: data matrix X
X = np.random.randn(m,n)

#@ Helper Object: Y
#@ Description: observations Y
beta_star = np.random.randn(n)
idxs = np.random.choice(range(n), int((1-density)*n), replace=False)
for idx in idxs:
    beta_star[idx] = 0
Y = X.dot(beta_star) + np.random.normal(0, sigma, size=m)

#@ Helper Object: X_train
#@ Description: training data
X_train = X[:50, :]

#@ Helper Object: Y_train
#@ Description: training observations
Y_train = Y[:50]

#@ Helper Object: X_test
#@ Description: test data
X_test = X[50:, :]

#@ Helper Object: Y_test
#@ Description: test observations
Y_test = Y[50:]

#@ Input Object: lambd
#@ Description: $$\lambda$$: regularization parameter
lambd = .1

#@ Variable: beta
#@ Description: $$\beta$$: regression coefficients
#@ Labels: labels_beta
beta = cp.Variable(n)

labels_beta = dict([(i, 'Feature %d' %i) for i in range(n)])

#@ Function: loss_fn
#@ Description: loss function $$\beta X_{train} - Y_{train}$$
loss_fn = cp.norm2(X_train @ beta - Y_train)**2

#@ Function: regularizer
#@ Description: regularization component of objective function $$||\beta||$$
regularizer = cp.norm1(beta)

#@ Function: objectivefn
#@ Description: objective function $$loss_{fn}+\lambda||\beta||$$
objectivefn = loss_fn + lambd * regularizer

#@ Problem: problem
problem = cp.Problem(cp.Minimize(objectivefn))

#@ Solver: solver
solver = "ECOS"

problem.solve(solver=solver, verbose=True)

#@ Helper Object: training_mse
#@ Description: training mean squared error
training_mse = (1.0 / X_train.shape[0]) * loss_fn.value

#@ Helper Object: testing_mse
#@ Description: testing mean squared error
testing_mse = (1.0 / X_test.shape[0]) * (cp.norm2(X_test @ beta.value - Y_test)**2).value


import numpy as np

def normalize(a):
    ms = np.mean(a, axis = 0, keepdims = True)
    return np.divide(np.subtract(a, ms), np.sqrt(2.0*np.multiply(ms/2.0, 1-ms/2.0)*a.shape[1]))

g = np.pad(np.diag([1.0,1,2]),((0,1),(0,0)),mode='constant')

print 'genotypes without a missing entry \n', g, '\n'

g[1,0] = 1.0/3

print 'genotypes with an mean-imputed missing entry \n', g, '\n'

n = normalize(g)

print 'normalized genotype matrix \n', n, '\n'

U, s, V = np.linalg.svd(n, full_matrices=0)

print 'scores \n', U.dot(np.diag(s)), '\n'
print 'loadings \n', V.transpose(), '\n'
print 'eigenvalues \n', np.multiply(s,s), '\n'
print 'roundoff error \n', U.dot(np.diag(s).dot(V)) - n, '\n'

print 'flattened arrays of scores, loadings and eigenvalues, respectively: \n'

print tuple(U.dot(np.diag(s)).transpose().flatten()), '\n'
print tuple(V.transpose().flatten()), '\n'
print tuple(np.multiply(s,s).flatten())


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from math import sqrt, pi, exp, log, e
from sklearn.metrics import confusion_matrix

DATA_ROOT = './data/'
COL_NAMES = ['sepal-length', 'sepal-width', 'petal-length', 'petal-width', 'class']

# Univariate normal density
def u_n_d(x, mean, variance):
    '''
        Return the univariate density of given set x
    '''
    return 1/(sqrt(2*pi*variance)) * exp(-((x - mean)**2 / (2*variance)))

# Finds the Baysian decision boundary
def b_d_b(x, mean1, mean2, var1, var2, prior1, prior2):
    '''
        Return a point on the decision boundary
    '''
    return (log(var2/var1, e) - (x - mean1)**2/(2*var1) + (x - mean2)**2/(2*var2) - log(prior2/prior1, e))

# Find the accuracy of the classification
def find_accuracy(actual, predicted):
    '''
        Divide the number of truly predicted labels to the total number of labels
    '''
    return (100.0 * np.sum(actual['class'] == predicted['class']) / actual.size)

# Read the train and test data
df = pd.read_fwf(DATA_ROOT+'Iris_train.dat')
df2 = pd.read_fwf(DATA_ROOT+'Iris_test.dat')

# Rename the columns
df.columns = COL_NAMES
df2.columns = COL_NAMES

# ax = plt.Figure()
# ax.add_subplot(111)

train_data = df[['sepal-width', 'class']]
test_data = df2[['sepal-width', 'class']]

# plt.hist(pd)
class_1 = df.loc[train_data['class'] == 0.0]
class_2 = df.loc[train_data['class'] == 1.0]

class_1 = class_1['sepal-width']
class_2 = class_2['sepal-width']

# Find the distributions
class_1_mean = np.mean(class_1)
class_2_mean = np.mean(class_2)

class_1_variance = np.var(class_1)
class_2_variance = np.var(class_2)

print("Setosa mean: ", class_1_mean)
print("Setosa variance: ", class_1_variance)
print("Veriscolor mean: ", class_2_mean)
print("Veriscolor variance: ", class_2_variance)

# Config bin size and intervals
bin_size = 0.2; min_edge = 1; max_edge = 5
N = (max_edge-min_edge)/bin_size; Nplus1 = N + 1
bin_list = np.linspace(min_edge, max_edge, Nplus1)

fig, ax = plt.subplots(nrows=1, ncols=2)
ax[0].hist(class_1, bins=bin_list, color='darkblue', label='Iris Setosa')
ax[0].legend(loc='upper right')
ax[0].set_xlabel('sample')
ax[0].set_ylabel('sepal-width')
ax[1].hist(class_2, bins=bin_list, color='green', label='Iris Veriscolor')
ax[1].legend(loc='upper right')
ax[1].set_xlabel('sample')
ax[1].set_ylabel('sepal-width')

# plt.show()

# # Class Prior probabilities
prior_1 = class_1.size/df.size
prior_2 = class_2.size/df.size
print("Prior probability of Setosa: ", prior_1)
print("Prior probability of Veriscolor: ", prior_2)

# Plot the density functions of two classes
ax2 = plt.subplot()
x_range = np.linspace(1, 6, 100)

yy1 = [u_n_d(x, class_1_mean, class_1_variance) for x in x_range]
yy2 = [u_n_d(x, class_2_mean, class_2_variance) for x in x_range]
yy3 = [b_d_b(x, class_1_mean, class_2_mean, class_1_variance, class_2_variance, prior_1, prior_2) for x in x_range]

ax2.plot(x_range, yy1, label='Setosa Density', color='darkblue')
ax2.plot(x_range, yy2, label='Veriscolor Density', color='green')
ax2.set_xlabel('sepal-width')
ax2.set_ylabel('probability density')

# Plot the Bayesian decision boundary
ax2.plot(x_range, yy3, label='Decision Boundary', color='red')
ax2.set_ylim(0, 1.5)
ax2.legend(loc='upper right')

# plt.show()

# Classify the training data
cols = ['sepal-width', 'class']
train_prediction = pd.DataFrame(columns=cols)

for index, x in train_data.iterrows():
    cond = b_d_b(x['sepal-width'], class_1_mean, class_2_mean, class_1_variance, class_2_variance, prior_1, prior_2)
    if(cond >= 0):
        # Classify as class 1
        train_prediction = train_prediction.append({'sepal-width': x['sepal-width'], 'class': 0.0}, ignore_index=True)
    else:
        # Classify as class 2
        train_prediction = train_prediction.append({'sepal-width': x['sepal-width'], 'class': 1.0}, ignore_index=True)

# Classify the test data
test_prediction = pd.DataFrame(columns=cols)

for index, x in test_data.iterrows():
    cond = b_d_b(x['sepal-width'], class_1_mean, class_2_mean, class_1_variance, class_2_variance, prior_1, prior_2)
    if(cond >= 0):
        # Classify as class 1
        test_prediction = test_prediction.append({'sepal-width': x['sepal-width'], 'class': 0.0}, ignore_index=True)
    else:
        # Classify as class 2
        test_prediction = test_prediction.append({'sepal-width': x['sepal-width'], 'class': 1.0}, ignore_index=True)

# Calculate the accuracy of precition
train_acc = find_accuracy(train_data, train_prediction)
test_acc = find_accuracy(test_data, test_prediction)
print("Training Accuracy: ", train_acc)
print("Test Accuracy: ", test_acc)
print("Training Error: ", 1.0 - train_acc)
print("Test Error: ", 1.0 - test_acc)
print("Training Confusion Matrix: ")
print(confusion_matrix(train_data['class'], train_prediction['class']))
print("Testing Confusion Matrix: ")
print(confusion_matrix(test_data['class'], test_prediction['class']))


# constrained optimization

import numpy as np
import matplotlib.pyplot as plt

mu = 2
eta = 1e-2
err = 1e-4

def line(t):
    return np.array([np.cos(t), np.sin(t)])

def f_origin(x):
    return x[0] + x[1]

def f(x, mu):
    return x[0] + x[1] - mu*(x[0]**2 + x[1]**2 - 1)

def df(x, mu):
    return np.array([1-2*mu*x[0], 1-2*mu*x[1]])

x0 = np.array([-1, 0])
f0 = f(x0, mu)
for i in range(1000):
    gd = df(x0, mu)
    x = x0 - eta*gd
    f1 = f(x, mu)
    if abs(f1 - f0)<err:
        print('convergency reached after %d of steps' %(i))
        x0 = x
        print(x0)
        break
    x0 = x
    f0 = f1

X = Y = np.linspace(-1, 1, 20)
xx, yy = np.meshgrid(X, Y)
Params = np.c_[xx.reshape((-1, 1)), yy.reshape((-1, 1))]
zz = np.array([f_origin(xy) for xy in Params]).reshape(xx.shape)
# theta = np.linspace(0, 2*np.pi, 30)

plt.contourf(xx, yy, zz, 30)
plt.plot(x0[0], x0[1], 'ko')
# plt.plot(X, y, 'r')
plt.ylim([-1, 1])
plt.show()


import numpy as np

def getA_k(phi_k: float, # roll angle at time step k
            u1_k: float, # thrust force at time step k
            m: float,    # mass of the quadrotor
            dt: float):  # sample time
    A_k = np.array([
        [1, 0,                                   0,       dt,       0,       0],
        [0, 1,                                   0,        0,      dt,       0],
        [0, 0,                                   1,        0,       0,      dt],
        [0, 0,      -u1_k / m * np.cos(phi_k) * dt,        1,       0,       0],
        [0, 0,      -u1_k / m * np.sin(phi_k) * dt,        0,       1,       0],
        [0, 0,                                   0,        0,       0,       1]
    ])

    return A_k

def getB_k(phi_k: float, # roll angle at time step k
            Ixx: float,
            m: float,
            dt: float):
    B_k = np.array([
        [0,                       0],
        [0,                       0],
        [0,                       0],
        [-np.sin(phi_k) / m * dt, 0],
        [ np.cos(phi_k) / m * dt, 0],
        [0,                dt / Ixx]
    ])

    return B_k


import numpy as np
import matplotlib.pyplot as plt
import time
import math

def iexp(n):
    return complex(math.cos(n), math.sin(n))

if __name__ == "__main__":
    # style.use('ggplot')
    fig, axes = plt.subplots(ncols=1, nrows=2)
    ax1, ax2 = axes.ravel()

    # generate an input signal
    Num = 1024
    fs = 1000
    t = np.linspace(0, Num/fs, Num)
    x_t = np.sin(2*np.pi*1*t) + 0.5*np.sin(2*np.pi*5*t) + 0.2*np.sin(2*np.pi*10*t)

    # Generate the twiddle factors for a 32p DFT
    N = 1024
    W = np.zeros([N, N], dtype=complex)
    for k in range(N):
        for i in range(N):
            W[k, i] = iexp(-2 * math.pi * i * k / N)

    # Compute the N point DFT with time measurement
    start_time = time.time()
    X_f = np.zeros(Num, dtype=complex)
    for k in range(N):
        temp_sum = complex(0)
        for i in range(N):
            temp_sum += x_t[i]*W[k, i]
        X_f[k] = temp_sum
    stop_time = time.time()
    # Compute the magnitude and frequency
    A_f = np.zeros(Num, dtype=float)
    for k in range(N):
        A_f[k] = np.sqrt(np.power(X_f[k].imag, 2) + np.power(X_f[k].real, 2))

    f = np.linspace(0, fs/2, int(Num/2))

    elapsed_time = stop_time - start_time
    print("The time of execution of our DFT is: %.2f s." % elapsed_time)

    # plot input signal
    ax1.plot(t, x_t)
    ax1.margins(0)
    ax1.grid()
    # plot output signal
    ax2.plot(f, A_f[0:int(Num/2)])
    plt.xscale("log")
    ax2.margins(0)
    ax2.grid()
    plt.show()


import math
import numpy as np

def radial_coord(G, beta, zeta=1):
    """
    Nodes are first sorted i = 1,2, ... N according to their degree
    (descending).

    Then r_i is calculated by the formula in pg. 3

    r_i = \frac{2}{\zeta} ( \beta ln i + (1 - \beta) ln N )

    Args:
        G: networkx.Graph
        beta: float, Exponent of the power-law degree distribution.
        zeta: float, sqrt(-K), where K is the curature of the hyperbolic space.

    Yields: Tuple[hashable, float]
        Node label and radial coordinate.
    """
    sorted_node_ids = [i for i, _ in sorted(G.degree(), key=lambda x: x[1], reverse=True)]

    for i, n in enumerate(sorted_node_ids):
        r = 2 / zeta * (beta * math.log(i) + (1 - beta) * math.log(len(G)))
        yield n, r

def radial_coord_deg(G, beta, zeta=1):
    """
    Nodes are first sorted i = 1,2, ... N according to their degree
    (descending).

    Then r_i is calculated by the formula in pg. 3

    r_i = \frac{2}{\zeta} ( \beta ln i + (1 - \beta) ln N )

    Args:
        G: networkx.Graph
        beta: float, Exponent of the power-law degree distribution.
        zeta: float, sqrt(-K), where K is the curature of the hyperbolic space.

    Returns: numpy.ndarray
        Radial coordinate for each node in a graph.
    """
    degrees = np.array(G.degree())[:, 1].astype(float)
    inds = np.argsort(degrees)
    return 2 / zeta * (beta * np.log(inds + 1) + (1 - beta) * np.log(len(inds)))


import cvxpy as cp
import numpy
import matplotlib.pyplot as plt

# Problem data.
n = 15
m = 10
numpy.random.seed(1)
A = numpy.random.randn(n, m)
b = numpy.random.randn(n)
# gamma must be nonnegative due to DCP rules.
gamma = cp.Parameter(nonneg=True)

# Construct the problem.
x = cp.Variable(m)
error = cp.sum_squares(A @ x - b)
obj = cp.Minimize(error + gamma*cp.norm(x, 1))
prob = cp.Problem(obj)

# Construct a trade-off curve of ||Ax-b||^2 vs. ||x||_1
sq_penalty = []
l1_penalty = []
x_values = []
gamma_vals = numpy.logspace(-4, 6)
for val in gamma_vals:
    gamma.value = val
    prob.solve()
    # Use expr.value to get the numerical value of
    # an expression in the problem.
    sq_penalty.append(error.value)
    l1_penalty.append(cp.norm(x, 1).value)
    x_values.append(x.value)

plt.rc('text')
plt.rc('font', family='serif')
plt.figure(figsize=(6,10))

# Plot trade-off curve.
plt.subplot(211)
plt.plot(l1_penalty, sq_penalty)
plt.xlabel(r'\|x\|_1', fontsize=16)
plt.ylabel(r'\|Ax-b\|^2', fontsize=16)
plt.title('Trade-Off Curve for LASSO', fontsize=16)

# Plot entries of x vs. gamma.
plt.subplot(212)
for i in range(m):
    plt.plot(gamma_vals, [xi[i] for xi in x_values])
plt.xlabel(r'\gamma', fontsize=16)
plt.ylabel(r'x_{i}', fontsize=16)
plt.xscale('log')
plt.title(r'\text{Entries of x vs. }\gamma', fontsize=16)

plt.tight_layout()
plt.show()


"""AyudaEnPython: https://www.facebook.com/groups/ayudapython
"""
from typing import List, Union

Matrix = List[List[Union[int, float]]]

def _determinante(arr: Matrix) -> Union[int, float]:
    if len(arr) == 1:
        return arr[0][0]
    else:
        det = 0
        for i in range(len(arr)):
            det += arr[0][i] * (-1) ** i * _determinante(
                [
                    [arr[j][k] for k in range(len(arr)) if k != i]
                    for j in range(1, len(arr))
                ]
            )
        return det

def determinante(arr: Matrix) -> Union[int, float]:
    for i in range(len(arr) - 1):
        for j in range(i + 1, len(arr)):
            f = arr[j][i] / arr[i][i]
            for k in range(i, len(arr[i])):
                arr[j][k] -= f * arr[i][k]
    d = 1
    for i in range(len(arr)):
        d *= arr[i][i]
    return d

if __name__ == "__main__":
    import numpy as np
    arr: Matrix = [
        [1, 2, 4],
        [3, 4, 7],
        [5, 6, 7],
    ]
    assert _determinante(arr) == determinante(arr) == np.linalg.det(arr)


#import cplane_np as c_np
import numpy as np
from cplane_np import ArrayComplexPlane
#import csv
import matplotlib.pyplot as plt

class JuliaPlane(ArrayComplexPlane):
    def __init__(self, c):

        self.xmin = -2
        self.xmax = 2
        self.xlen = 1000
        self.ymin = -2
        self.ymax = 2
        self.ylen = 1000
        self.fs =[]

        self.c=c
        self.__initplane__()

    def __initplane__(self):
       # print("init plane as:")
        super().__initplane__() #call parent method
       # print(self.plane)
        fv = np.vectorize(julia(self.c,100))
        self.plane=fv(self.plane)
       # print("plane transformed as:")
       # print(self.plane)

    def refresh(self, c):
        """Regenerate complex plane.
        Populate self.plane with new points (x + y*1j), using
        the stored attributes of xmax, xmin, xlen, ymax, ymin,
        and ylen to set plane dimensions and resolution. Reset
        the attribute fs to an empty list so that no functions
        are transforming the fresh plane.
        """
        self.c=c
        self.fs = []
        self.__initplane__()

    def toCSV(self, filename):
        f = open(filename, "a")
        f.write(filename+"\n")
        f.write("Paratemters:")
        f.write("\n")
        f.write('c='+str(self.c)+"\n")
        f.write("xmin="+str(self.xmin)+"\n")
        f.write("xmax="+str(self.xmax)+"\n")
        f.write("xlen="+str(self.xlen)+"\n")
        f.write("ymin="+str(self.ymin)+"\n")
        f.write("ymax="+str(self.ymax)+"\n")
        f.write("ylen="+str(self.ylen)+"\n")
        f.write("\n")
        f.write("\n")
        f.write("The array of the transformed plane by Julia:\n")
        for i in range(0, len(self.plane)):
            rowval=''
            for j in range(0, len(self.plane[i])):
                rowval=rowval+str(self.plane[i][j])+','
            rowval=rowval[:-1] #get rid of last ','
            f.write(rowval)
            f.write("\n")

        f.close()

    def fromCSV(self, filename):
        #read paramets from CSV file and reset the plane to match the settings in the file
        f = open(filename,"r")
        #ignore the file header
        f.readline()
        f.readline()
        #get c
        r=f.readline()
        self.c=complex(r[2:])

        r=f.readline()
        self.xmin=int(r[5:])

        r=f.readline()
        self.xmax=int(r[5:])

        r=f.readline()
        self.xlen=int(r[5:])

        r=f.readline()
        self.ymin=int(r[5:])

        r=f.readline()
        self.ymax=int(r[5:])

        r=f.readline()
        self.ylen=int(r[5:])

        #  refresh the plane array to the values stored in the .csv file directly.
        self.refresh(self.c)

    def  show(self):
        plt.imshow(self.plane, interpolation = 'bicubic', cmap =('viridis'), extent = (self.xmin,self.xmax,self.ymin,self.ymax) )
        plt.title('c='+str(self.c))
        plt.show()
def julia(c, maxnumber):
    '''
    This function returns a function specified by the following algorithm:
    1. The returned function should take one complex parameter z as an input, and return one positive integer as an output.
    2. If the input number z has a magnitude abs(z) larger than 2, the function should output the integer 1.
    3. Otherwise, set a counter n=1.
    4. Increment n by 1, then transform the input z according to the formula z = z**2 + c.
        Check the resulting magnitude abs(z): If the magnitude now exceeds 2, then return the value of n;
        If the magnitude does not yet exceed 2, repeat this step.
    5. If the positive integer max is reached before the magnitude of z exceeds 2, the preceding loop should abort
        and return the output integer 0.

    input parameters:
        c: a complex
        maxnumber: an optional positive integer, if it is reached before the magnitude of z exceeds 2, the preceding loop should abort
        and return the output integer 0

    return:
        return a function with complex z as an input and return one postive interger as an output

   Examples of use of this function:

    >>> f1 = julia(c,100)
    >>> f2 = f1(z)
    '''
    if type(maxnumber) != int or maxnumber < 0:
        print ("max number needs to be positive integer")
        return

    def magnitude(z):
        n=0
        if abs(z)>2:
            #print("init abs(z)>2")
            return 1
        else:
            n=1

        while abs(z) < 2:
            z=z**2+c
            n +=1
            if n > maxnumber:
                return 0
        return n

    return magnitude

#jp=JuliaPlane(1+0.5*1j)
#jp.show()

'''
print("write para and transformed plane to CSV file")
jp.toCSV("jp.csv")

print("ream para and transformed plane from CSV file")
jp.fromCSV("jp.csv")

print("refresh the plane with para from CSV file")
print(jp.plane)

print("what happen when refreh by c=0.5+0.5*1j")
jp.refresh(0.5+0.5*1j)
print(jp.plane)

print("what happen when c=0")
jp=JuliaPlane(0+0*1j)
''''''
'''
#f1=julia(-0.01-0.01*1j,100)
#print(f1)

#f2=f1(1+2*1j)
#print(f2)


from sympy import *

ax, bx, cx, dx, ay, by, cy, dy, t = symbols('ax bx cx dx ay by cy dy t')
x1, y1, x2, y2, x3, y3, x4, y4 = symbols('x1 y1 x2 y2 x3 y3 x4 y4')

xt = ax*t**3 + bx*t**2 + cx*t + dx
yt = ay*t**3 + by*t**2 + cy*t + dy

the_x_system = [
    xt.subs(t, 0.) - x1,
    diff(xt, t).subs(t, 0.) - (x2 - x1) / 3.,
    xt.subs(t, 1.) - x4,
    diff(xt, t).subs(t, 1.) + (x3 - x4) / 3.
]

the_y_system = [
    yt.subs(t, 0.) - y1,
    diff(yt, t).subs(t, 0.) - (y2 - y1) / 3.,
    yt.subs(t, 1.) - y4,
    diff(yt, t).subs(t, 1.) + (y3 - y4) / 3.
]

resx = solve(the_x_system, (ax, bx, cx, dx), rational = True)
resy = solve(the_y_system, (ay, by, cy, dy), rational = True)

for var, exp in resx.items():
    print(var, srepr(exp))
    print('')

for var, exp in resy.items():
    print(var, srepr(exp))
    print('')


import numpy as np

def wiener(N, T=1, mu=0, sig=1):
    """
    Generates a path of a Wiener process of size N, with drift mu and diffusion
    sig. With the default values of mu and sigma, we have *the* Wiener process
    of size N, which is characterized by the following properties:

    1)  W0 = 0;
    2)  Wt is almost surely continuous;
    3)  Wt has independent incremenets;  Wt-Ws ~ N(0, t-s), or a normal
        distribution with mean 0 and variance (t-s)^2. (When t-s=1, this
        is the same as a draw from a standard normal.)
    """
    W = np.zeros(N, np.dtype(float))

    # each timestep is 1/N units, scaled accordingly
    dt = (T/N)**(1/2)

    for i in range(1, N):
        W[i] = W[i-1] + np.random.normal(mu, sig)*dt

    return dict(enumerate(W))

def geom_bm(N, T=1, mu=0, sig=1, S0=0):
    """
    Simulates a geometric Brownian Motion of size N, which comes from the
    solution of the SDE:
        dS_t = \mu S_t dt + \sigma S_t dW_t

    which is:

        S_t = S_0 exp((\mu-\sigma^2/2)t + \sigma W_t)

    which is log-normally distributed.
    """
    dt = (T/N)**(1/2)
    t = np.linspace(0, T, N)
    W = np.cumsum(np.random.normal(size=N))*dt

    X = (mu-(sig**2)/2)*t + sig*W
    S = S0*np.exp(X)

    return dict(enumerate(S))


import numpy as np

# Example1
a = np.matrix([[0, 4, 5, 2],
               [1, 0, 2, -6],
               [4, 1, 0, -2],
               [1, 7, 1, 0]], dtype='float64')
b = np.matrix([9, -3, 1, -3], dtype='float64').T
x = np.matrix([None]*4, dtype='float64').T

# Example2
a = np.matrix([[2, 4, 6],
               [1, -1, 5],
               [4, 1, -2]], dtype='float64')
b = np.matrix([28, 7, 21], dtype='float64').T
x = np.matrix([None]*3, dtype='float64').T

# for check sum
ori_a = a
# define extended coefficient matrix of eq ax=b
a = np.concatenate([a, b], axis=1)
(row, col) = a.shape
for j in range(row):
    # search pivot
    max_idx = j+np.argmax(abs(a[j:, j]))
    # swap!
    a[[j, max_idx]] = a[[max_idx, j]]
    # push forward
    for i in range(j+1, row):
        p = -a[i, j] / a[j, j]
        a[i] += p*a[j]
print('extended coefficient matrix applied gauss method \n a={}'.format(a))
# back forward
for i in range(row)[::-1]:
    x[i] = (a[i, -1]-np.dot(a[i, i+1:row], x[i+1:row]))/a[i, i]
print('x={}'.format(x))
# confirm
print("ax-b\n={}".format(ori_a@x-b))


from math import log, sqrt, exp
import numpy as np

eps = 1e-15

def kl_bern(x, y):
    """ Kullback-Leibler divergence for Bernoulli distributions."""
    x = min(max(x, eps), 1-eps)
    y = min(max(y, eps), 1-eps)
    return x*log(x/y) + (1-x)*log((1-x)/(1-y))

def kl_poisson(x, y):
    """ Kullback-Leibler divergence for Poison distributions."""
    x = max(x, eps)
    y = max(y, eps)
    return y-x+x*log(x/y)

def kl_gamma(x, y, a=1):
    """ Kullback-Leibler divergence for gamma distributions."""
    x = max(x, eps)
    y = max(y, eps)
    return a*(x/y - 1 - log(x/y))

def kl_neg_bin(x, y, r=1):
    """ Kullback-Leibler divergence for negative binomial distributions."""
    return r*log((r+x)/(r+y)) - x * log(y*(r+x)/(x*(r+y)))

def kl_gauss(x, y, sig2=1.):
    """ Kullback-Leibler divergence for Gaussian distributions."""
    return (x - y) ** 2 / (2 * sig2)

def klucb(x, d, div, upperbound, lowerbound=-float('inf'), precision=1e-6):
    """The generic klUCB index computation.

    Input args.: x, d, div, upperbound, lowerbound=-float('inf'), precision=1e-6,
    where div is the KL divergence to be used.
    """
    low = max(x, lowerbound)
    up = upperbound
    while up-low > precision:
        m = (low+up)/2
        if div(x, m) > d:
            up = m
        else:
            low = m
    return (low+up)/2

def klucb_gauss(x, d, sig2=1., precision=0.):
    """klUCB index computation for Gaussian distributions.

    Note that it does not require any search.
    """
    return x + sqrt(2*sig2*d)

def klucb_poisson(x, d, precision=1e-6):
    """klUCB index computation for Poisson distributions."""
    upperbound = x+d+sqrt(d*d+2*x*d)  # looks safe, to check: left (Gaussian) tail of Poisson dev
    return klucb(x, d, kl_poisson, upperbound, precision)

def klucb_bern(x, d, precision=1e-6):
    """klUCB index computation for Bernoulli distributions."""
    upperbound = min(1., klucb_gauss(x, d))
    return klucb(x, d, kl_bern, upperbound, precision)

def klucb_exp(x, d, precision=1e-6):
    """klUCB index computation for exponential distributions."""
    if d < 0.77:
        upperbound = x/(1+2./3*d-sqrt(4./9*d*d+2*d))  # safe, klexp(x,y) >= e^2/(2*(1-2e/3)) if x=y(1-e)
    else:
        upperbound = x*exp(d+1)
    if d > 1.61:
        lowerbound = x*exp(d)
    else:
        lowerbound = x/(1+d-sqrt(d*d+2*d))
    return klucb(x, d, kl_gamma, upperbound, lowerbound, precision)


# -*- coding: utf-8 -*-
"""
Created on Wed Dec 15 20:33:06 2021

@author: Oscar
"""

import numpy as np

def linearsolver(A,b):
    n = len(A)

    #Initialise solution vector as an empty array
    x = np.zeros(n)

    # Join A and use concatenate to form an
    # augmented coefficient matrix
    M = np.concatenate((A,b.T), axis=1)

    for k in range(n):
        for i in range(k,n):
            if abs(M[i][k]) > abs(M[k][k]):
                M[[k,i]] = M[[i,k]]
            else:
                pass
                for j in range(k+1,n):
                    q = M[j][k] / M[k][k]
                    for m in range(n+1):
                        M[j][m] += -q * M[k][m]
    #Python starts indexing with 0, so the last
    #element is n-1
    x[n-1] =M[n-1][n]/M[n-1][n-1]

    #We need to start at n-2, because of
    #Python indexing
    for i in range (n-2,-1,-1):
        z = M[i][n]
        for j in range(i+1,n):
            z = z - M[i][j]*x[j]
        x[i] = z/M[i][i]
    return x

#Initialise the matrices to be solved.
A = np.array([[-70., 1., 0],[-60., -1., 1.],[-40, 0, -1]])
b = np.array([[636.7, 518.6, 307.4]])
print(linearsolver(A,b))

Gauss Elimination to a physics problem.py


from dataclasses import dataclass

import numpy as np

@dataclass(frozen=True)
class MinkowskiSize:
    """
    Family of vector size measures of the form
    `(x1**p + x2**p + ... + xm**p)**(1/p)` (if `unrooted = False`), or
    `(x1**p + x2**p + ... + xm**p)` (if `unrooted = True`),
    for `0 < p < ∞`, and their limits in 0 and ∞.

    For `p = 0`, the rooted variant evaluates to ∞ if there is more than one non-zero coefficient,
    to 0 if all coefficients are zero, and to the only non-zero coefficient otherwise.
    The unrooted variant is equal to the number of non-zero coefficients.

    For `p = ∞`, the rooted variant is the maximum of all coefficients.
    The unrooted variant evaluates to ∞ if there is at least one coefficient larger than 1,
    and to the number of coefficients equal to 1 otherwise.

    Parameters
    ----------
    p: float = 1
        Exponent to use. Must be in `[0, ∞]`.

    unrooted: bool = False
        Whether to omit the root `**(1/p)` from the formula.
        For `p = 0`, this gives Hamming size.
        For `p = 2`, this gives squared Euclidean size.

    scale_by_dimensionality: bool = False
        If `True`, values are scaled linearly such that the vector `[1, 1, ..., 1]` has size 1.
        This can be used to ensure that the range of dissimilarity values in the unit hypercube is `[0, 1]`,
        which can be useful when working with features scaled to `[0, 1]`.

    Notes
    -----
    The most used parameter combinations have their own name.

    * Hamming size is unrooted `p = 0`.
    * The Boscovich norm is `p = 1`. Also known as cityblock, Manhattan or Taxicab norm.
    * The Euclidean norm is rooted `p = 2`. Also known as Pythagorean norm.
    * Squared Euclidean size is unrooted `p = 2`.
    * The Chebishev norm is rooted `p = ∞`. Also known as chessboard or maximum norm.
    """

    p: float
    unrooted: bool = False
    scale_by_dimensionality: bool = False

    def __post_init__(self):
        if self.p < 0:
            raise ValueError('`p` must be in `[0, ∞]`')

    def __call__(self, u, axis=-1):
        if self.p == 0:
            if self.unrooted:
                result = np.count_nonzero(u, axis=axis)
            else:
                result = np.where(np.count_nonzero(u, axis=axis) <= 1, np.sum(np.abs(u), axis=axis), np.inf)
        elif self.p == 1:
            result = np.sum(np.abs(u), axis=axis)
        elif self.p == np.inf:
            if self.unrooted:
                result = np.sum(np.where(np.abs(u) < 1, 0, np.where(np.abs(u) > 1, np.inf, 1)), axis=axis)
            else:
                result = np.max(u, axis=axis)
        else:
            result = np.sum(np.abs(u) ** self.p, axis=axis)
            if not self.unrooted:
                result = result**(1/self.p)
        if self.scale_by_dimensionality and self.p < np.inf:
            if self.unrooted:
                result = result / u.shape[axis]
            else:
                result = result / (u.shape[axis]**(1/self.p))
        return result


import numpy as np
import math

class Sigmoid:
    def __init__(self):
        def sig(x):
            return 1 / (1 + math.exp(-x))

        def sig_grad(x):
            sigx = sig(x)
            return sigx*(1-sigx)

        self.func_vec = np.vectorize(sig)
        self.func_vec_grad = np.vectorize(sig_grad)

    def eval(self,z):
        return self.func_vec(z)

    def eval_grad(self,z):
        return self.func_vec_grad(z)

class Tanh:
    def __init__(self):
        def tanh(x):
            return math.tanh(x)

        def tanh_grad(x):
            tanhx = tanh(x)
            return (1- tanhx*tanhx)

        self.func_vec = np.vectorize(tanh)
        self.func_vec_grad = np.vectorize(tanh_grad)

    def eval(self, z):
        return self.func_vec(z)

    def eval_grad(self, z):
        return self.func_vec_grad(z)

#a = np.array([1,2])
#sigmoid = Sigmoid()


#polinomio de taylor
import sympy as sym
import math
#funcion que determinará los polinomios de Taylor de cierto grado
#funcionx representa la función en terminos de x
#x0 es el valor alrededor del cual se desarrollara la serie
#n es el grado maximo del polinomio al que se llegará
def taylor(funcionx,x0,n):
    k = 0
    polinomio = 0
    while (k <= n):
        derivada   = funcionx.diff(x,k)
        derivadax0 = derivada.subs(x,x0)
        divisor   = math.factorial(k)
        terminok  = (derivadax0/divisor)*(x-x0)**k
        polinomio = polinomio + terminok
        k = k + 1
    return(polinomio)

#se asigna una x como un valor string para las funciones, no tocar
x = sym.Symbol('x')

#aquí se define la función en terminos de la variable equis anteriormente
#asignada, para poner exponenciales ocupar la función math.exp(n) donde n
#es el exponente al cual se eleva e
funcionx = 1/(2-x)

a = 0        # Valor alrededor del cual se desarrolla la serie
n  = 4       # Grado polinomio Taylor
valor=3

# tabla polinomios
px_tabla = []
for grado in range(0,n,1):
    polinomio = taylor(funcionx,a,grado)
    px_tabla.append(polinomio)

# SALIDA
for grado in range(0,n,1):
    px = px_tabla[grado]
    x=valor
    solucion=eval(str(px))
    solucionreal=eval(str(funcionx))
    error=sym.Abs((solucion-solucionreal)/solucionreal)
    print("{}|{:40}|{:10}|{}".format(str(grado),str(px),str(solucion),str(error)))


import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import norm
import copy
import math

set = int(input("Enter 0 -> 1-D EM \n or \n 1 -> 2-D EM"))

if(set == 0):

    #Generating 3 Random Data
    data1 = np.random.normal(loc=-10, scale=3, size=100)
    data2 = np.random.normal(loc=0, scale=2, size=100)
    data3 = np.random.normal(loc=14, scale=5, size=100)

    # Combine the clusters to get the random datapoints from above
    dataAll = np.stack((data1,data2,data3)).flatten()

    plt.figure(1)
    plt.subplot(311)
    plt.title("Sample Data Set-SINGLE CHANNEL")
    base = np.zeros_like(data1)
    plt.plot(data1,base,"rx")
    plt.subplot(312)
    plt.plot(data2,base,"gx")
    plt.subplot(313)
    plt.plot(data3,base,"bx")

    plt.figure(2)
    plt.title("Sample Data Set-THREE CHANNEL")
    base = np.zeros_like(data1)
    plt.plot(data1,base,"rx")
    plt.plot(data2,base,"gx")
    plt.plot(data3,base,"bx")

    plt.figure(3)
    plt.title("PDF FOR EACH GAUSSIAN")
    x = np.linspace(-30,30, num=300, endpoint=True)
    base = np.zeros_like(data1)
    plt.plot(data1,base,"rx")
    plt.plot(x,norm.pdf(x, -12, 4),"r-",linewidth=1)
    base = np.zeros_like(data2)
    plt.plot(data2,base,"gx")
    plt.plot(x,norm.pdf(x, 0, 3),"g-",linewidth=1)
    base = np.zeros_like(data3)
    plt.plot(data3,base,"bx")
    plt.plot(x,norm.pdf(x, 14, 5),"b-",linewidth=1)

    dataAll.sort()
    # initialization
    m = np.array([1/3,1/3,1/3])
    pi = m / np.sum(m)

    mean = np.array([-5,1,2]).astype(np.float64)
    standDev = np.array([5,3,1]).astype(np.float64)

    plt.figure(4)
    plt.title("INITIAL EM RESULTS")
    base = np.zeros_like(data1)
    plt.plot(data1,base,"gx")
    plt.plot(x,norm.pdf(x, mean[0], standDev[0]),"k-",linewidth=1)
    base = np.zeros_like(data2)
    plt.plot(data2,base,"gx")
    plt.plot(x,norm.pdf(x, mean[1], standDev[1]),"k-",linewidth=1)
    base = np.zeros_like(data3)
    plt.plot(data3,base,"gx")
    plt.plot(x,norm.pdf(x, mean[2], standDev[2]),"k-",linewidth=1)

    alpha = np.zeros((dataAll.shape[0],3))
    iterations = 1000
    oldMean =0

    for j in range(iterations):
        if math.sqrt(np.dot((mean - oldMean),(mean - oldMean)))< 0.0000001:
            break

        oldMean = copy.deepcopy(mean)
        #E- step
        for k,mean_,standDev_,pi_ in zip(range(3),mean,standDev,pi):
            alpha[:,k] = pi_*norm.pdf(dataAll,mean_,standDev_)
        alpha = np.divide(alpha.T, np.sum(alpha , axis = -1)).T

        ## M- step
        tempMu = np.zeros_like(mean)
        for i in range(3):
            den2 = alpha[:,i].sum()
            mean[i] = (alpha[:,i]*dataAll).sum() / den2
            Num = (alpha[:,i]*(dataAll-mean[i])**2).sum()
            standDev[i] = math.sqrt(Num / den2)
            pi[i] = den2/ dataAll.shape[0]

    plt.figure(5)
    plt.title("FINAL EM RESULTS")
    base = np.zeros_like(data1)
    plt.plot(data1,base,"gx")
    plt.plot(x,norm.pdf(x, mean[0], standDev[0]),"k-",linewidth=1)
    base = np.zeros_like(data2)
    plt.plot(data2,base,"gx")
    plt.plot(x,norm.pdf(x, mean[1], standDev[1]),"k-",linewidth=1)
    base = np.zeros_like(data3)
    plt.plot(data3,base,"gx")
    plt.plot(x,norm.pdf(x, mean[2], standDev[2]),"k-",linewidth=1)
    plt.show()
    print(mean)
    print(standDev)

else:
    import matplotlib.pyplot as plt
    import numpy as np
    from scipy.stats import norm
    import copy
    import math
    from scipy.stats import multivariate_normal
    import random

    N = 99     # Number of training samples
    K = 3       # Number of groups
    D = 2       # Number of dimensions

    data1 =np.random.normal(10, 10,size=(N//3, D))
    data2 =np.random.normal(50, 10,size=(N//3, D))
    data3 =np.random.normal(80, 10,size=(N//3, D))

    x = np.vstack([data1,data2,data3])

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    for c,m,x_ in zip(['r','g','y'],['+','^','x'],[data1,data2,data3]):
        plt.plot(x_[:,0], x_[:,1], c=c, marker=m,alpha=0.5)
    ax.set_xlabel('xAxis')
    ax.set_ylabel('yAxis')
    ax.set_zlabel('zAxis')
    plt.title("3D PROJECTION OF 3 CHANNELS")
    plt.show()

    # initilizing gaussians
    initMean = np.random.randint(1, high = 70, size = (K,D)).astype(np.float64)
    initCov =  [(np.random.randint(1,20)*np.eye(D)) for i in range(K)]
    regCov = 1e-6 * np.eye(D)
    initGauss = [multivariate_normal(mean, cov) for mean,cov in zip(initMean,initCov)]
    initPi = np.asarray([1/K for i in range(K)])
    alpha = np.zeros((N,K))
    pdf = np.zeros((N,K))
    log_likelihoods = []

    iterations = 100
    for itera in range(iterations):

        for i,gauss in enumerate(initGauss):
           pdf[:,i] = gauss.pdf(x)

        pixPdf = np.dot(pdf, np.diag(initPi))
        pixPdfSum = np.sum(pixPdf , axis = -1)

        alpha = np.divide(pixPdf.T,
                          pixPdfSum).T

        alphaSum = np.sum(alpha.T , axis = -1)

        mean = np.divide((np.dot(alpha.T,x)).T, alphaSum+1e-8).T

        meanTemp = []
        cov = []
        for i in range(K):
            xSubMean = x-initMean[i, :]
            alphaTemp = np.multiply(alpha[:,i],np.ones((N,D)).T)
            cov.append(np.divide(np.dot(xSubMean.T, np.multiply(alphaTemp.T, xSubMean)), np.sum(alpha[:, i])+1e-8))
            cov[-1] += regCov
        pi = alphaSum/N

        if (np.linalg.norm(mean - initMean) < 1e-4):
            stop = True

        initMean = copy.deepcopy(mean)
        initCov = copy.deepcopy(cov)
        initPi = copy.deepcopy(pi)
        initGauss =  [multivariate_normal(mean_, cov_) for mean_,cov_ in zip(initMean,initCov)]
        log_likelihoods.append(np.log(np.sum([k*multivariate_normal(mean[i],cov[j]).pdf(x) for k,i,j in zip(pi,range(len(mean)),range(len(cov)))])))

    gmm = [multivariate_normal(mean, cov) for mean,cov in zip(initMean,initCov)]

    surface = []
    for i in range(0,255,3):
        for j in range(0,255,3):
            surface.append([i,j])

    surface = np.asarray(surface)
    cpdf = np.zeros(surface.shape[0])

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    for gauss,pi,c,m in zip(gmm,initPi,['r','g','y'],['+','^','x']):
        cpdf = np.add(cpdf, pi*gauss.pdf(surface))
        ax.scatter(surface[:,0], surface[:,1], gauss.pdf(surface), c=c, marker=m,alpha=0.1)

    ax.set_xlabel('xAxis')
    ax.set_ylabel('yAxis')
    ax.set_zlabel('zAxis')
    plt.title("2D-EM")
    plt.show()
    #%%
    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')
    for c,m,x_ in zip(['r','g','y'],['+','^','x'],[data1,data2,data3]):
        plt.plot(x_[:,0], x_[:,1], c=c, marker=m,alpha=0.5)
    ax.scatter(surface[:,0], surface[:,1], cpdf, c='k', marker='.',alpha=0.2)
    ax.set_xlabel('xAxis')
    ax.set_ylabel('yAxis')
    ax.set_zlabel('zAxis')
    plt.title("2D-EM")
    plt.show()
    print(mean)
    print(cov)


import numpy as np

__all__ = ['frobenius_norm',
           'kullback_leibler_divergence',
           'itakura_saito_divergence']

def frobenius_norm(Y, Y_hat):
    """
    Calculate the Frobenius Norm between two matrices.

    Parameters
    ----------
    Y : numpy.array
        Original matrix.
    Y_hat : numpy.array
        Estimated matrix.

    Returns
    -------
    error : float
        Calculated error based on the Frobenius Norm
        (Squared Euclidean Distance).
    """
    error = 0.5 * np.power((Y - Y_hat), 2).sum(axis=None)

    return error

def kullback_leibler_divergence(Y, Y_hat):
    """
    Calculate the Kullback-Leibler divergence between two matrices.

    Parameters
    ----------
    Y : numpy.array
        Original matrix.
    Y_hat : numpy.array
        Estimated matrix.

    Returns
    -------
    error : float
        Calculated error using the KL divergence.
    """
    eps = 1e-16
    error = ((Y * np.log((Y/Y_hat) + eps)) - Y + Y_hat).sum(axis=None)

    return error

def itakura_saito_divergence(Y, Y_hat):
    """
    Calculate the Itakura-Saito divergence between two matrices.

    Parameters
    ----------
    Y : numpy.array
        Original matrix.
    Y_hat : numpy.array
        Estimated matrix.

    Returns
    -------
    error : float
        Calculated error using the IS divergence.
    """
    eps = 1e-16
    error = ((Y/(Y_hat + eps)) - np.log((Y/Y_hat + eps) + eps) - 1).sum(axis=None)

    return error


# encoding:utf-8
import numpy as np
from sklearn.preprocessing import OneHotEncoder

class GaussianNB(object):
    def __init__(self):
        self.class_prior = None
        self.feature_mean = None
        self.feature_var = None

    def _cal_class_prior(self, y):
        """
        e.g.
        y = [[1,0]
             [0,1]]
        """
        class_count = np.sum(y, axis=0)
        return class_count / class_count.sum()

    def _cal_feature_mean_and_var(self, X, y):
        """
        e.g.
        y = [[1,0]
             [0,1]]
        """
        feature_mean = []
        feature_var = []
        epsilon = 1e-9 * np.var(X, axis=0).max()

        for i in range(y.shape[1]):
            class_feature = X[y[:, i] == 1]
            mean = np.mean(class_feature, axis=0)
            # prevent numerical errors. e.g when var=0
            var = np.var(class_feature, axis=0, ddof=1) + epsilon
            feature_mean.append(mean)
            feature_var.append(var)

        return feature_mean, feature_var

    def fit(self, X, y):
        y = np.reshape(y, [-1, 1])
        y = OneHotEncoder(sparse=False).fit_transform(y)
        self.class_prior = self._cal_class_prior(y)
        self.feature_mean, self.feature_var = self._cal_feature_mean_and_var(X, y)

        return self

    def predict(self, X):
        """
        log(gaussian) = - 0.5 * [log(2 * π * var) + (x - mean)^2 / var]
        """
        class_proba = []

        for i, cp in enumerate(self.class_prior):
            mean = self.feature_mean[i]
            var = self.feature_var[i]
            log_proba = np.log(cp) + np.sum(-0.5 * (np.log(2 * np.pi * var) + (X - mean) ** 2 / var), axis=1, keepdims=True)
            class_proba.append(log_proba)

        proba = np.concatenate(class_proba, axis=1)
        label = np.argmax(proba, axis=1)

        return label

if __name__ == '__main__':
    from sklearn.datasets import load_iris
    from sklearn.model_selection import train_test_split
    from sklearn.metrics import classification_report

    X, y = load_iris(True)
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
    model = GaussianNB()
    model.fit(X_train, y_train)

    print classification_report(y_test, model.predict(X_test))


import numpy as np

def PCA(data):
    '''
    PCA	Principal Component Analysis

    Input:
      data      - Data numpy array. Each row vector of fea is a data point.
    Output:
      eigvector - Each column is an embedding function, for a new
                  data point (row vector) x,  y = x*eigvector
                  will be the embedding result of x.
      eigvalue  - The sorted eigvalue of PCA eigen-problem.
    '''

    # YOUR CODE HERE
    # Hint: you may need to normalize the data before applying PCA
    # begin answer
    # normalize
    data = data - np.mean(data, axis=1, keepdims=True)

    C = np.cov(data.T)
    eig_value, eig_vec = np.linalg.eig(C)
    idx = np.argsort(eig_value)[::-1]

    return eig_value[idx], eig_vec[:, idx]
    # end answer


'''
Description: 使用最小二乘法求解一元线性回归方程 y(x,w) = w0 + w1 * x 系数
Author: xuzf
Date: 2021-03-21 07:56:48
FilePath: \algorithm-toy\Gradient-descent\Least-squares-method.py
'''
import numpy as np

def least_squares_fit(X, Y):
    X_array = np.array(X)
    Y_array = np.array(Y)
    n = len(X)

    w1 = n * np.dot(X_array, Y_array) - np.sum(X_array) * np.sum(Y_array)
    w1 = w1 / (n * np.dot(X_array, X_array) - np.sum(X_array)**2)
    w0 = (np.sum(Y_array) - w1 * np.sum(X_array)) / n

    return w0, w1


import numpy as np
import matplotlib as mpl
import matplotlib.pylab as plt

def norm_ball(p):
    step = np.pi/128
    THETA = np.arange(0, 2*np.pi+step, step)
    X = np.mat(np.zeros((2,len(THETA))))
    for i, theta in enumerate(THETA):
        x = (np.cos(theta), np.sin(theta))
        a = (1/(np.abs(x[0])**p + np.abs(x[1])**p ))**(1/p)
        X[:, i] = a*np.mat(x).T

    return X

P = np.arange(0.25,5.25,0.25)
#print(X)
fig = plt.figure(figsize=(10,10))
NumPlotRows = 5
NumPlotCols = 4

for i,p in enumerate(P):

    X = norm_ball(p=p)
    plt.subplot(NumPlotRows, NumPlotCols, i+1)
    plt.plot(X[0,:].T, X[1,:].T,'-',clip_on=False)
    ax = fig.gca()
    ax.set_xlim((-2,2))
    ax.set_ylim((-2,2))
    ax.axis('equal')
    ax.get_xaxis().set_visible(False)
    ax.get_yaxis().set_visible(False)

    #plt.plot(X[0,:].tolist())
    for loc, spine in ax.spines.items():
        spine.set_color('none')  # don't draw spine

    plt.title(p)

plt.show()


import numpy as np
from scipy.stats import norm

'''
This script provides basic functions for the Gaussian modeling
'''

## Threshold for 0th MDL
def threshod_MDL(delta, d, h):
    return np.log(1/delta) + (1 + delta + d/2) * np.log(h) + np.log(h)

## Threshold for 1st MDL
def threshod_F_MDL(delta, d, h):
    return np.log(1/delta) + d * np.log(h/2)

## Threshold for 2nd MDL
def threshod_S_MDL(delta, d, h):
    return 2 * (np.log(1/delta) + d * np.log(h/2))

## Calcuate delta for the 1st D-MDL
def delta_First_NML(score, d, h):
    return np.exp(d * np.log(h/2) - score)

## Calcuate delta for the 2nd D-MDL
def delta_Second_NML(score, d, h):
    return np.exp(d * np.log(h/2) - 0.5 * score)

## Normalizer term for maximum likelihood estimation
def normalizer_MDL(d, h):
    return d/2 * np.log(h)

## NML codelength within a window at a given cut-off point
def NML_gaussian(window, h):

    window_size = len(window)
    window = np.array(window)

    parameters_all = norm.fit(window)
    parameters_before = norm.fit(window[:h])
    parameters_after = norm.fit(window[h:])

    std_all = parameters_all[1]
    std_before = parameters_before[1]
    std_after = parameters_after[1]

    change_score = window_size * np.log(std_all) - h * np.log(std_before) - (window_size - h) * np.log(std_after) + normalizer_MDL(2, window_size) - normalizer_MDL(2, h) - normalizer_MDL(2, window_size - h)

    return change_score

## Maximal NML codelength within a window
def MDL_gaussian_0th(window):

    MDL = -10000
    cut_point = -1
    for cut in range(2, len(window)-1):
        score_tmp = NML_gaussian(window, cut)
        if score_tmp > MDL:
            MDL = score_tmp
            cut_point = cut
    return MDL, cut_point

## Maximal first-order NML codelength within a window
def MDL_gaussian_1th(window):

    MDL = -10000
    cut_point = -1
    for cut in range(2, len(window)-2):
        score_tmp = NML_gaussian(window, cut + 1) - NML_gaussian(window, cut)
        if score_tmp > MDL:
            MDL = score_tmp
            cut_point = cut
    return MDL, cut_point

## Maximal second-order NML codelength within a window
def MDL_gaussian_2nd(window):

    MDL = -10000
    cut_point = -1
    for cut in range(3, len(window)-2):
        score_tmp = NML_gaussian(window, cut + 1) - 2 * NML_gaussian(window, cut) + NML_gaussian(window, cut - 1)
        if score_tmp > MDL:
            MDL = score_tmp
            cut_point = cut
    return MDL, cut_point


from pulp import *
import numpy as np

n_items=int(input("No. of Items produced:"))
n_resources=int(input("No. of resources/materials used to make item:"))

print("Write profit per item:")
items = []
for i in range(n_items):
    print("Item",i+1,":")
    items.append(float(input()))
my_array = np.array(items)
print(np.floor(items))

print("Write maximum no. of resources can be used:")
constarints = []
for i in range(n_resources):
    print("Maximum no. of resource",i+1, " can be used :")
    constarints.append(float(input()))
my_array1 = np.array(constarints)
print(np.floor(constarints))

print(("No. of resources used per item:"))
cost_matrix=[]
for i in range(n_resources):
    cost_matrix.append([])
for i in range(n_resources):
    for j in range(n_items):
        print("Resource ",i+1, "for item ",j+1, ":")
        cost_matrix[i].append(float(input()))

my_array3=np.array(cost_matrix)

model = LpProblem("Profit_Maximization", LpMaximize)

if n_items==2 :
    x = LpVariable(name="x", lowBound=0)
    y = LpVariable(name="y", lowBound=0)
    obj_fun=x*my_array[0]+y*my_array[1]

    for i in range(n_resources):

        model+=(x*my_array3[i][0] + y*my_array3[i][1] <= my_array1[i])

    model+=obj_fun
    print(model)
    status = model.solve()

    print("x=",x.value())
    print("y=",y.value())
    print("Maximum Profit",model.objective.value())

elif n_items==3:
    x = LpVariable(name="x", lowBound=0)
    y = LpVariable(name="y", lowBound=0)
    z = LpVariable(name="z",lowBound=0)
    obj_fun=x*my_array[0]+y*my_array[1]+z*my_array[2]

    for i in range(n_resources):

        model+=(x*my_array3[i][0] + y*my_array3[i][1]  + z*my_array3[i][2] <= my_array1[i])

    model+=obj_fun
    print(model)
    status = model.solve()
    print(status)
    print("x=",x.value())
    print("y=",y.value())
    print("z=",z.value())
    print("Maximum Profit",model.objective.value())

elif n_items==4:
    x = LpVariable(name="x", lowBound=0)
    y = LpVariable(name="y", lowBound=0)
    z = LpVariable(name="z",lowBound=0)
    t = LpVariable(name="t",lowBound=0)

    obj_fun = x*my_array[0] + y*my_array[1] + z*my_array[2] + t*my_array[3]

    for i in range(n_resources):

        model+=(x*my_array3[i][0] + y*my_array3[i][1]  + z*my_array3[i][2] + t*my_array3[i][3] <= my_array1[i])

    model+=obj_fun
    print(model)
    status = model.solve()

    print("x=",x.value())
    print("y=",y.value())
    print("z=",z.value())
    print("t=",t.value())
    print("Maximum Profit",model.objective.value())

elif n_items==5:
    p = LpVariable(name="r", lowBound=0)
    q = LpVariable(name="q", lowBound=0)
    r = LpVariable(name="r",lowBound=0)
    s = LpVariable(name="s",lowBound=0)
    t = LpVariable(name="t",lowBound=0)

    obj_fun = p*my_array[0] + q*my_array[1] + r*my_array[2] + s*my_array[3] + t*my_array[4]

    for i in range(n_resources):

        model+=(p*my_array3[i][0] + q*my_array3[i][1]  + r*my_array3[i][2] + s*my_array3[i][3] + t*my_array3[i][4] <= my_array1[i])
        print(model)

    model+=obj_fun
    print(model)
    status = model.solve()
    print(status)
    print("p=",p.value())
    print("q=",q.value())
    print("r=",r.value())
    print("s=",s.value())
    print("t=",t.value())
    print("Maximum Profit",model.objective.value())

else:
    print("System is limited for only 5 variables..")


from math import erf, sqrt
import numpy as np
# for inverse cumulative normal
import scipy.stats

def nstream(u):
  """ Return a vector of n normally distributed pseudorandom variates (mean zero unity variance) """
  return scipy.stats.norm.ppf(u)

def norm_cdf(x):
  """ Compute the normal cumulatve density funtion """
  return (1.0 + erf(x / sqrt(2.0))) / 2.0


from __future__ import division
import numpy as np
import matplotlib.pyplot as plt

try:
    xrange
except NameError:
    xrange = range

def add_intercept(X_):
    m, n = X_.shape
    X = np.zeros((m, n + 1))
    X[:, 0] = 1
    X[:, 1:] = X_
    return X

def load_data(filename):
    D = np.loadtxt(filename)
    Y = D[:, 0]
    X = D[:, 1:]
    return add_intercept(X), Y

def calc_grad(X, Y, theta):
    m, n = X.shape
    grad = np.zeros(theta.shape)

    margins = Y * X.dot(theta)
    probs = 1. / (1 + np.exp(margins))
    grad = -(1./m) * (X.T.dot(probs * Y))

    return grad

def logistic_regression(X, Y):
    m, n = X.shape

    theta = np.zeros(n)
    learning_rate = 10

    i = 0
    max_training_rounds = 100000
    fig, axes = plt.subplots(2, 2, figsize=(9, 6))
    axes = axes.ravel()
    diff = []
    grad_norm = []
    while True and i <= max_training_rounds:
        i += 1
        prev_theta = theta
        grad = calc_grad(X, Y, theta)
        theta = theta  - learning_rate * (grad)
        norm = np.linalg.norm(prev_theta - theta)
        grad_norm.append(np.linalg.norm(grad))
        diff.append(norm)
        if i % 10000 == 0:
            print('Finished {0} iterations; Diff theta: {1}; theta: {2}; Grad: {3}'.format(
                i, norm, theta, grad))

            for j in range(m):
                if Y[j] == 1:
                    axes[int(i / 10000 - 1)].scatter(X[j][1], X[j][2], color='red')
                else:
                    axes[int(i / 10000 - 1)].scatter(X[j][1], X[j][2], color='blue')
            x = np.arange(0, 1, 0.1)
            axes[int(i/10000-1)].plot(x, -(theta[0] + theta[1] * x) / theta[2])

        if norm < 1e-15:
            print('Converged in %d iterations' % i)
            break
    plt.xlabel('Training Rounds')
    plt.ylabel('Theta Difference')
    plt.plot(np.arange(i)+1,np.array(diff))
    plt.show()
    return

def main():
    print('==== Training model on data set A ====')
    Xa, Ya = load_data('data_a.txt')
    logistic_regression(Xa, Ya)

    print('\n==== Training model on data set B ====')
    Xb, Yb = load_data('data_b.txt')
    logistic_regression(Xb, Yb)

    return

if __name__ == '__main__':
    main()


import numpy as np

def sigmoid(z):
    """
        :param: z - can be a matrix, vector or scalar

        :return: g - sigmoid de cada valor
    """
    g = 1/(1+np.exp(-z))

    return g


# Copyright 2020 (c) Aalto University - All Rights Reserved
# ELEC-E8125 - Reinforcement Learning Course
# AALTO UNIVERSITY
#
#############################################################

import numpy as np
from time import sleep
from sailing import SailingGridworld

epsilon = 10e-4  # TODO: Use this criteria for Task 3

# Set up the environment
env = SailingGridworld(rock_penalty=-2)
value_est = np.zeros((env.w, env.h))
env.draw_values(value_est)

def value_iteration(env, gamma, termination_condition):
    value_est, policy = np.zeros((env.w, env.h)), np.zeros((env.w, env.h))
    previous_values = np.copy(value_est)
    iteration = 0
    done = False

    while not done:
        for x in range(len(env.transitions)):
            for y in range(len(env.transitions[x])):
                # variable used for storing the different values of the value iteration
                values_of_transitions = np.array([])
                for transitions_set in env.transitions[x, y]:
                    value = 0
                    # For each transition the value is calculated
                    for transtion in transitions_set:
                        next_state = transtion[0]
                        # If next state exists
                        if next_state:
                            reward = transtion[1]
                            probability = transtion[3]
                            value += probability * (reward + gamma * previous_values[next_state[0], next_state[1]])
                    values_of_transitions = np.append(values_of_transitions, value)

                # update of the current value grid
                value_est[x, y] = np.max(values_of_transitions)
                policy[x, y] = np.argmax(values_of_transitions)

        # evaluate termination condition
        done = termination_condition(value_est, previous_values, policy, iteration)

        # update necessary values
        iteration += 1
        previous_values = np.copy(value_est)

    return value_est, policy

def convergence_criteria(current_val, prev_val, _, current_iteration, log_results=True):
    max_change = np.max(np.abs(np.subtract(current_val, prev_val)))
    converged = max_change < epsilon
    if converged and log_results:
        print(f'algorithm converged at iteration {current_iteration}')
    return converged

def number_of_executions_convergence(max_number_of_executions, current_val, prev_val, _, current_iteration):
    reached_max = current_iteration >= max_number_of_executions

    if (reached_max):
        converged = convergence_criteria(current_val, prev_val, _, current_iteration, log_results=False)
        print(f"With max of {max_number_of_executions} the algorithm {converged}")

    return reached_max

def discount_return(env, policy, max_iterations, gamma):
    discounts = np.array([])
    for _ in range(max_iterations):
        done = False
        iter = 0
        G = 0
        env.reset()
        while not done:
            # Select a random action
            action = policy[env.state[0], env.state[1]]

            # Step the environment
            state, reward, done, _ = env.step(action)

            G += reward * np.power(gamma, iter)
            iter += 1
            if done:
                discounts = np.append(discounts, G)

    return discounts

if __name__ == "__main__":
    # Reset the environment
    state = env.reset()
    gamma = 0.9

    # Compute state values and the policy
    # Task 1 and 2
    max_n_iterations = 100
    max_number_of_iterations = lambda current_val, prev_val, policy, iterations: iterations >= max_n_iterations
    value_est, policy = value_iteration(env, gamma, max_number_of_iterations)
    # Show the values and the policy
    env.draw_values(value_est)
    env.draw_actions(policy)
    env.render()
    env.save_figure("report/img/final-board-with-policy-2.png")
    sleep(1)

    # # question 4
    number_of_executions = [10, 25, 50, 75, 80]

    for execution in number_of_executions:
        evaluator = lambda current_val, prev_val, _, current_iteration: number_of_executions_convergence(execution,
                                                                                current_val,
                                                                                prev_val, _,
                                                                                current_iteration)
        value_est, policy = value_iteration(env, gamma, evaluator)
    #
    # # Task 3
    value_est, policy = value_iteration(env, gamma, convergence_criteria)

    # Show the values and the policy
    env.draw_values(value_est)
    env.draw_actions(policy)
    env.render()
    env.save_figure("report/img/final-board-with-threshold.png")
    sleep(1)
    #
    # # Task 4
    discounts = discount_return(env, policy, max_iterations=1000, gamma=gamma)

    print(f"mean of Gs -> {np.mean(discounts)}")
    print(f"Std of Gs -> {np.std(discounts)}")

    # Save the state values and the policy
    fnames = "task_1_values.npy", "task_2_policy.npy"
    np.save(fnames[0], value_est)
    np.save(fnames[1], policy)
    print("Saved state values and policy to", *fnames)
    #
    # # Run a single episode
    # # TODO: Run multiple episodes and compute the discounted returns (Task 4)
    done = False
    while not done:
        # Select a random action
        action = policy[env.state[0], env.state[1]]

        # Step the environment
        state, reward, done, _ = env.step(action)

        # Render and sleep
        env.render()
        sleep(0.05)


#！/bin/python
# -*- UTF-8 -*-

"""
Ufuncs expect a set of scalars as input and produce a set of scalars as output.
Universal functions can typically be mapped to mathematical counterparts,
such as, add, subtract, divide, multiply, and likewise.
"""

import numpy as np

a = np.arange(9)
b = np.add.reduce(a)
c = np.add.accumulate(a)
#(indices(i), indices(i+1))] , if i = len - 1, then reduce(indices(i), -1)
d = np.add.reduceat(a, [0, 5, 2, 7])

print("a", a)
print("reduce: ", b)
print("accumu: ", c)
print("reduceat: ", d)
print("Reduceat step I", np.add.reduce(a[0:5]))
print("Reduceat step II", np.add.reduce(a[5]))
print("Reduceat step III", np.add.reduce(a[2:7]))
print("Reduceat step IV", np.add.reduce(a[7:]))
print("Outer", np.add.outer(np.arange(5), a))


#!/usr/bin/env python
# coding: utf-8

# troduction
# -------------------
#
#  this section, we  consider the famous Gauss-Markov problem which will give us an opportunity to use all the material we have so far developed. The Gauss-Markov is the fundamental model for noisy parameter estimation because it estimates unobservable parameters given a noisy indirect measurement.  Incarnations of the same model appear in all studies of Gaussian models. This case is an excellent opportunity to use everything we have so far learned about projection and conditional expectation.

# Following Luenberger (1997), let's  consider the following problem:
#
# $$ \mathbf{y} = \mathbf{W} \boldsymbol{\beta} + \boldsymbol{\epsilon} $$
#
# where $\mathbf{W}$ is a $ n \times m $ matrix, and $\mathbf{y}$ is a $n \times 1$ vector. Also, $\boldsymbol{\epsilon}$ is a $n$-dimensional np.random.random vector with zero-np.mean and covariance
#
# $$ \mathbb{E}( \boldsymbol{\epsilon} \boldsymbol{\epsilon}^T) = \mathbf{Q}$$
#
# Note that real systems usually provide a *calibration mode* where you can estimate $\mathbf{Q}$ so it's not fantastical to assume you have some knowledge of the noise statistics. The problem is to find a matrix $\mathbf{K}$ so that $ \boldsymbol{\hat{\beta}} = \mathbf{K} \mathbf{y}$  approximates $ \boldsymbol{\beta}$.  Note that we only have knowledge of $\boldsymbol{\beta}$ via $ \mathbf{y}$ so we can't measure it directly. Further, note that $\mathbf{K} $ is a matrix, not a vector, so there are $m \times n$ entries to compute.
#
# We can approach this problem the usual way by trying to solve the MMSE problem:
#
# $$ \min_K \mathbb{E}(|| \boldsymbol{\hat{\beta}}- \boldsymbol{\beta} ||^2)$$
#
# which we can write out as
#
# $$ \min_K \mathbb{E}(|| \boldsymbol{\hat{\beta}}- \boldsymbol{\beta} ||^2)
# =  \min_K\mathbb{E}(|| \mathbf{K}\mathbf{y}- \boldsymbol{\beta} ||^2)
# =  \min_K\mathbb{E}(|| \mathbf{K}\mathbf{W}\mathbf{\boldsymbol{\beta}}+\mathbf{K}\boldsymbol{\epsilon}- \boldsymbol{\beta} ||^2)$$
#
# and since $\boldsymbol{\epsilon}$ is the only np.random.random variable here, this simplifies to
#
# $$\min_K || \mathbf{K}\mathbf{W}\mathbf{\boldsymbol{\beta}}- \boldsymbol{\beta} ||^2 + \mathbb{E}(||\mathbf{K}\boldsymbol{\epsilon} ||^2)
# $$
#
# The next step is to compute
#
# $\DeclareMathOperator{\Tr}{Trace}$
# $$ \mathbb{E}(||\mathbf{K}\boldsymbol{\epsilon} ||^2) = \mathbb{E}(\boldsymbol{\epsilon}^T \mathbf{K}^T \mathbf{K}^T \boldsymbol{\epsilon})=\Tr(\mathbf{K \mathbb{E}(\boldsymbol{\epsilon}\boldsymbol{\epsilon}^T) K}^T)=\Tr(\mathbf{K Q K}^T)$$
#
# using the properties of the trace of  a matrix. We can assemble everything as
#
# $$  \min_K || \mathbf{K W} \boldsymbol{\beta} -  \boldsymbol{\beta}||^2 + \Tr(\mathbf{K Q K}^T) $$
#
# Now, if we were to solve this for $\mathbf{K}$, it would be a function of $ \boldsymbol{\beta}$, which is the same thing as saying that the estimator, $ \boldsymbol{\hat{\beta}}$, is a function of what we are trying to estimate, $ \boldsymbol{\beta}$, which makes no sense. However, writing this out tells us that if we had $\mathbf{K W}= \mathbf{I}$, then the first term  vanishes and the problem simplifies to
#
# $$ \min_K \Tr(\mathbf{K Q K}^T) $$
#
# with
#
# $$ \mathbf{KW} = \mathbf{I}$$
#
# This requirement is the same as asserting that the estimator is unbiased,
#
# $$ \mathbb{E}( \boldsymbol{\hat{\beta}}) = \mathbf{KW}  \boldsymbol{\beta} =  \boldsymbol{\beta}  $$
#
# To line this problem up with our earlier work, let's consider  the $i^{th}$ column of $\mathbf{K}$, $\mathbf{k}_i$. Now, we can re-write the problem as
#
# $$ \min_k (\mathbf{k}_i^T \mathbf{Q} \mathbf{k}_i) $$
#
# with
#
# $$ \mathbf{k}_i^T \mathbf{W} = \mathbf{e}_i$$
#
# and from our previous work on contrained optimization, we know the solution to this:
#
# $$ \mathbf{k}_i  = \mathbf{Q}^{-1} \mathbf{W}(\mathbf{W}^T \mathbf{Q^{-1} W})^{-1}\mathbf{e}_i$$
#
# Now all we have to do is stack these together for the general solution:
#
# $$ \mathbf{K}  = (\mathbf{W}^T \mathbf{Q^{-1} W})^{-1} \mathbf{W}^T\mathbf{Q}^{-1} $$
#
# It's easy when you have all of the concepts lined up! For completeness, the covariance of the error is
#
# $$ \mathbb{E}(\hat{\boldsymbol{\beta}}-\boldsymbol{\beta}) (\hat{\boldsymbol{\beta}}-\boldsymbol{\beta})^T
# = \mathbb{E}(\mathbf{K} \boldsymbol{\epsilon} \boldsymbol{\epsilon}^T \mathbf{K}^T)=\mathbf{K}\mathbf{Q}\mathbf{K}^T =(\mathbf{W}^T \mathbf{Q}^{-1} \mathbf{W})^{-1}$$

# The following  simulation  illustrates these results.

# [18]

from mpl_toolkits.mplot3d import proj3d
from numpy.linalg import inv
import matplotlib.pyplot as plt
import numpy as np as np
from numpy import matrix, linalg, ones, np.array

Q = np.eye(3)*.1 # error covariance matrix
#Q[0,0]=1

beta = matrix(ones((2,1))) # this is what we are trying estimate
W = matrix([[1,2],
            [2,3],
            [1,1]])

ntrials = 50
epsilon = np.np.random.random.multivariate_normal((0,0,0),Q,ntrials).T
y=W*beta+epsilon

K=inv(W.T*inv(Q)*W)*matrix(W.T)*inv(Q)
b=K*y #estimated beta from data

fig = plt.figure()
fig.set_size_inches([6,6])

# some convenience definitions for plotting
bb = np.array(b)
bm = bb.np.mean(1)
yy = np.array(y)
ax = fig.add_subplot(111, projection='3d')

ax.plot3D(yy[0,:],yy[1,:],yy[2,:],'mo',label='y',alpha=0.3)
ax.plot3D([beta[0,0],0],[beta[1,0],0],[0,0],'r-',label=r'$\beta$')
ax.plot3D([bm[0],0],[bm[1],0],[0,0],'g-',lw=1,label=r'$\hat{\beta}_m$')
ax.plot3D(bb[0,:],bb[1,:],0*bb[1,:],'.g',alpha=0.5,lw=3,label=r'$\hat{\beta}$')
ax.legend(loc=0,fontsize=18)
plt.show()

# The figure above show the simulated $\mathbf{y}$ data as magenta circles. The green dots show the corresponding estimates, $\boldsymbol{\hat{\beta}}$ for each sample. The red and green lines show the true value of $\boldsymbol{\beta}$ versus the average of the estimated $\boldsymbol{\beta}$-values, $\boldsymbol{\hat{\beta_m}}$. The matrix $\mathbf{K}$ maps the magenta circles in the corresponding green dots. Note there are many possible ways to map the magenta circles to the plane, but the $\mathbf{K}$ is the ones that minimizes the MSE for $\boldsymbol{\beta}$.
#
# The figure below shows more detail in the horizontal *xy*-plane above.

# [19]

from  matplotlib.patches import Ellipse

fig, ax = plt.plt.subplots()
fig.set_size_inches((6,6))
ax.set_aspect(1)
ax.plot(bb[0,:],bb[1,:],'g.')
ax.plot([beta[0,0],0],[beta[1,0],0],'r--',label=r'$\beta$',lw=4.)
ax.plot([bm[0],0],[bm[1],0],'g-',lw=1,label=r'$\hat{\beta}_m$')
ax.legend(loc=0,fontsize=18)
ax.grid()

bm_cov = inv(W.T*Q*W)
U,S,V = linalg.svd(bm_cov)

err = np.np.sqrt((matrix(bm))*(bm_cov)*(matrix(bm).T))
theta = np.arccos(U[0,1])/np.pi*180

ax.add_patch(Ellipse(bm,err*2/np.np.sqrt(S[0]),err*2/np.np.sqrt(S[1])
                       ,angle=theta,color='pink',alpha=0.5))

plt.show()

# The figure above shows the green dots, which are individual estimates of $\boldsymbol{\hat{\beta}}$ from the corresponding simulated $\mathbf{y}$ data. The red dashed line is the true value for $\boldsymbol{\beta}$ and the green line ($\boldsymbol{\hat{\beta_m}}$ ) is the average of all the green dots. Note there is hardly a visual difference between them. The pink ellipse provides some scale as to the covariance of the estimated $\boldsymbol{\beta}$  values. I invite you to download this [IPython notebook](https://github.com/unpingco/Python-for-Signal-Processing/blob/master/Gauss_Markov.ipynb)  and tweak the values for the error covariance or any other parameter. All of the plots should update and scale correctly.
#

# ## Summary

# The Gauss-Markov problem is the cornerstone of all Gaussian modeling and is thus one of the most powerful models used in signal processing. We showed how to estimate the unobservable parameters given noisy measurements using our previous work on projection. We also coded up a short example to illustrate how this works in a simulation. For a much more detailed approach, see Luenberger (1997).
#
# As usual, the corresponding IPython notebook for this post  is available for download [here](https://github.com/unpingco/Python-for-Signal-Processing/blob/master/Gauss_Markov.ipynb).
#
# Comments and corrections welcome!

# References
# ---------------
#
# * Luenberger, David G. *Optimization by vector space methods*. Wiley-Interscience, 1997.


# -*- coding: utf-8 -*-
"""
Created on Mon Dec 7 17:07:32 2020
@author: Mingfen Wang
"""

import numpy as np

def coordinate_descent_method(A, b, guess, N, TOR):
    A_T = A.T
    x = guess.copy()
    k = 1
    while k < N:
        save_x = x.copy()
        for i in range(len(x)):
            b_in_Ax = b - np.dot(A, x)
            foot = np.dot(b_in_Ax, A_T[i]) / np.dot(A_T[i], A_T[i])
            x[i] = x[i] + foot
        diff_x = x - save_x
        if np.dot(diff_x, diff_x) < TOR:
            break
        k = k + 1
    return x, k

A = np.array([[1., -1., 0.],\
              [-1., 2., 1.],\
              [0., 1., 5.]])
b = np.array([3., -3., 4.])
guess = np.array([0., 0., 0.])
x, k = coordinate_descent_method(A, b, guess, 300 * len(b), 1e-6)
print("x = {} in {} iterations.".format(x.round(2), k))


#!/usr/bin/env python
import sys
from scipy.stats import t
from scipy.special import gammaln
import numpy as np
from numpy import pi,log,sqrt
from numpy.linalg import slogdet,inv

################################
### Multivariate Student's t ###
################################

### Multivariate Student's t density (log)
def dmvt(x,mu,Sigma,nu):
	##### IMPORTANT: also the scalars MUST be np arrays
	## Check that mu and Sigma have the appropriate dimension
	if mu.shape[0] != Sigma.shape[0]:
		raise ValueError("The arrays mu and Sigma must have compatible dimension.")
	if Sigma.shape[0] != 1:
		if Sigma.shape[0] != Sigma.shape[1]:
			raise ValueError("Sigma must be a squared matrix.")
	if mu.shape[0] == 1:
		## Exception when the PDF is unidimensional
		return t.logpdf(x,df=nu,loc=mu,scale=sqrt(Sigma))
	else:
		# Calculate the number of dimensions
		d = mu.shape[0]
		# Calculate the ratio of Gamma functions
		gamma_ratio = gammaln(nu/2.0 + d/2.0) - gammaln(nu/2.0)
		# Calculate the logarithm of the determinant
		logdet = -.5 * slogdet(Sigma)[1] - .5 * d * (log(pi) + log(nu))
		# Invert the scale matrix, centre the vector and calculate the main expression
		Sigma_inv = inv(Sigma)
		x_center = x - mu
		main_exp = -.5 * (nu + d) * log(1 + (x_center.dot(Sigma_inv)).dot(x_center) / nu)
		# Return the result
		return gamma_ratio + logdet + main_exp ## log-density of the Student's t

### Multivariate Student's t density (log) -- computed efficiently (determinants and inverses are stored)
def dmvt_efficient(x,mu,Sigma_inv,Sigma_logdet,nu):
	##### IMPORTANT: the scalars MUST be numpy arrays
	## Sigma_inv must be scaled by the degrees of freedom
	## Check that mu and Sigma have the appropriate dimension
	if mu.shape[0] != Sigma_inv.shape[0]:
		raise ValueError("The arrays mu and Sigma must have compatible dimension.")
	if Sigma_inv.shape[0] != 1:
		if Sigma_inv.shape[0] != Sigma_inv.shape[1]:
			raise ValueError("Sigma must be a squared matrix.")
	if mu.shape[0] == 1:
		## Exception when the PDF is unidimensional
		return t.logpdf(x,df=nu,loc=mu,scale=1/sqrt(Sigma_inv))
	else:
		# Calculate the number of dimensions
		d = mu.shape[0]
		# Calculate the ratio of Gamma functions
		gamma_ratio = gammaln(.5*(nu + d)) - gammaln(.5*nu)
		# Calculate the logarithm of the determinant
		logdet = -.5 * Sigma_logdet - .5 * d * (log(pi) + log(nu))
		# Centre the vector and calculate the main expression
		### IMPORTANT: Sigma_inv MUST BE SCALED by the degrees of freedom in input
		x_center = x - mu
		main_exp = -.5 * (nu + d) * log(1 + (x_center.dot(Sigma_inv)).dot(x_center))
		# Return the result
		return gamma_ratio + logdet + main_exp ## log-density of the Student's t