"""Gaussian Mixture Models
"""
import copy
import types

import numpy as np
import numpy.linalg as linalg

from pypr.clustering import kmeans
from pypr.clustering import gauss_diff
import pypr.preprocessing as preproc

def mulnormpdf(X, MU, SIGMA):
    """Evaluates the PDF for the multivariate Guassian distribution.

    Parameters
    ----------
    X : np array
        Inputs/entries row-wise. Can also be a 1-d array if only a
        single point is evaluated.
    MU : nparray
        Center/mean, 1d array.
    SIGMA : 2d np array
        Covariance matrix.

    Returns
    -------
    prob : 1d np array
        Probabilities for entries in `X`.

    Examples
    --------
    ::

        from pypr.clustering import *
        from numpy import *
        X = array([[0,0],[1,1]])
        MU = array([0,0])
        SIGMA = diag((1,1))
        gmm.mulnormpdf(X, MU, SIGMA)

    """
    # Check if inputs are ok:
    if MU.ndim != 1:
        raise ValueError, "MU must be a 1 dimensional array"

    # Evaluate pdf at points or point:
    mu = MU
    x = X.T
    if x.ndim == 1:
        x = np.atleast_2d(x).T
    sigma = np.atleast_2d(SIGMA) # So we also can use it for 1-d distributions

    N = len(MU)
    ex1 = np.dot(linalg.inv(sigma), (x.T-mu).T)
    ex = -0.5 * (x.T-mu).T * ex1
    if ex.ndim == 2: ex = np.sum(ex, axis = 0)
    K = 1 / np.sqrt ( np.power(2*np.pi, N) * linalg.det(sigma) )
    return K*np.exp(ex)

def logmulnormpdf(X, MU, SIGMA):
    """
    Evaluates natural log of the PDF for the multivariate Guassian distribution.

    Parameters
    ----------
    X : np array
        Inputs/entries row-wise. Can also be a 1-d array if only a
        single point is evaluated.
    MU : nparray
        Center/mean, 1d array.
    SIGMA : 2d np array
        Covariance matrix.

    Returns
    -------
    prob : 1d np array
        Log (natural) probabilities for entries in `X`.

    """
    # Check if inputs are ok:
    if MU.ndim != 1:
        raise ValueError, "MU must be a 1 dimensional array"

    # Evaluate pdf at points or point:
    #ex = _cal_ex(X, MU, SIGMA)
    mu = MU
    x = X.T
    if x.ndim == 1:
        x = np.atleast_2d(x).T
    sigma = np.atleast_2d(SIGMA) # So we also can use it for 1-d distributions

    N = len(MU)
    ex1 = np.dot(linalg.inv(sigma), (x.T-mu).T)
    ex = -0.5 * (x.T-mu).T * ex1
    if ex.ndim == 2: ex = np.sum(ex, axis = 0)
    K = -(N/2)*np.log(2*np.pi) - 0.5*np.log(np.linalg.det(SIGMA))
    return ex + K

def gmm_pdf(X, centroids, ccov, mc, individual=False):
    """Evaluates the PDF for the multivariate Guassian mixture.

    Draw samples from a Mixture of Gaussians (MoG)

    Parameters
    ----------
    centroids : list
        List of cluster centers - [ [x1,y1,..],..,[xN, yN,..] ]
    ccov : list
        List of cluster co-variances DxD matrices
    mc : list
        Mixing cofficients for each cluster (must sum to one)
                  by default equal for each cluster.
    individual : bool
        If True the probability density is returned for each cluster component.

    Returns
    -------
    prob : 1d np array
        Probability density values for entries in `X`.
    """
    if individual:
        pdf = np.zeros((len(X), len(centroids)))
        for i in range(len(centroids)):
            pdf[:,i] = mulnormpdf(X, centroids[i], ccov[i]) * mc[i]
        return pdf
    else:
        pdf = None
        for i in range(len(centroids)):
            pdfadd = mulnormpdf(X, centroids[i], ccov[i]) * mc[i]
            if pdf==None:
                pdf = pdfadd
            else:
                pdf = pdf + pdfadd
        return pdf

def sample_gaussian_mixture(centroids, ccov, mc = None, samples = 1):
    """
    Draw samples from a Mixture of Gaussians (MoG)

    Parameters
    ----------
    centroids : list
        List of cluster centers - [ [x1,y1,..],..,[xN, yN,..] ]
    ccov : list
        List of cluster co-variances DxD matrices
    mc : list
        Mixing cofficients for each cluster (must sum to one)
                  by default equal for each cluster.

    Returns
    -------
    X : 2d np array
         A matrix with samples rows, and input dimension columns.

    Examples
    --------
    ::

        from pypr.clustering import *
        from numpy import *
        centroids=[array([10,10])]
        ccov=[array([[1,0],[0,1]])]
        samples = 10
        gmm.sample_gaussian_mixture(centroids, ccov, samples=samples)

    """
    cc = centroids
    D = len(cc[0]) # Determin dimensionality

    # Check if inputs are ok:
    K = len(cc)
    if mc is None: # Default equally likely clusters
        mc = np.ones(K) / K
    if len(ccov) != K:
        raise ValueError, "centroids and ccov must contain the same number" +\
            "of elements."
    if len(mc) != K:
        raise ValueError, "centroids and mc must contain the same number" +\
            "of elements."

    # Check if the mixing coefficients sum to one:
    EPS = 1E-15
    if np.abs(1-np.sum(mc)) > EPS:
        raise ValueError, "The sum of mc must be 1.0"

    # Cluster selection
    cs_mc = np.cumsum(mc)
    cs_mc = np.concatenate(([0], cs_mc))
    sel_idx = np.random.rand(samples)

    # Draw samples
    res = np.zeros((samples, D))
    for k in range(K):
        idx = (sel_idx >= cs_mc[k]) * (sel_idx < cs_mc[k+1])
        ksamples = np.sum(idx)
        drawn_samples = np.random.multivariate_normal(\
            cc[k], ccov[k], ksamples)
        res[idx,:] = drawn_samples
    return res

def gauss_ellipse_2d(centroid, ccov, sdwidth=1, points=100):
    """Returns x,y vectors corresponding to ellipsoid at standard deviation sdwidth.
    """
    # from: http://www.mathworks.com/matlabcentral/fileexchange/16543
    mean = np.c_[centroid]
    tt = np.c_[np.linspace(0, 2*np.pi, points)]
    x = np.cos(tt); y=np.sin(tt);
    ap = np.concatenate((x,y), axis=1).T
    d, v = np.linalg.eig(ccov);
    d = np.diag(d)
    d = sdwidth * np.sqrt(d); # convert variance to sdwidth*sd
    bp = np.dot(v, np.dot(d, ap)) + np.tile(mean, (1, ap.shape[1]))
    return bp[0,:], bp[1,:]

def gmm_init(X, K, verbose = False,
                    cluster_init = 'sample', \
                    cluster_init_prop = {}, \
                    max_init_iter = 5, \
                    cov_init = 'var'):
    """Initialize a Gaussian Mixture Model (GMM).
    Generates a set of inital parameters for the GMM.

    Paramseters
    -----------
    cluster_init : string
        How to initalize centroids: 'sample', 'box', or 'kmeans'
    cluster_init_prop : dict
        Passed to the k-means (if used) as keyword arguments
    max_init_iter : int
        How often to try k-means (best final result is used)
    cov_init : string
        Either 'iso' or 'var'

    Returns
    -------
    center_list : list
        A K-length list of cluster centers
    cov_list : list
        A K-length list of co-variance matrices
    p_k : list
        An K length array with mixing cofficients
    """
    samples, dim = np.shape(X)
    if cluster_init == 'sample':
        if verbose: print "Using sample GMM initalization."
        center_list = []
        for i in range(K):
            center_list.append(X[np.random.randint(samples), :])
    elif cluster_init == 'box':
        if verbose: print "Using box GMM initalization."
        center_list = []
        X_max = np.max(X, axis=0)
        X_min = np.min(X, axis=0)
        for i in range(K):
            init_point = ((X_max-X_min)*np.random.rand(1,dim)) + X_min
            center_list.append(init_point.flatten())
    elif cluster_init == 'kmeans':
        if verbose: print "Using K-means GMM initalization."
        # Normalize data (K-means is isotropic)
        normalizerX = preproc.Normalizer(X)
        nX = normalizerX.transform(X)
        center_list = []
        best_icv = np.inf
        for i in range(max_init_iter):
            m, kcc = kmeans.kmeans(nX, K, iter=100, **cluster_init_prop)
            icv = kmeans.find_intra_cluster_variance(X, m, kcc)
            if best_icv > icv:
                membership = m
                cc = kcc
                best_icv = icv
        cc = normalizerX.invtransform(cc)
        for i in range(cc.shape[0]):
            center_list.append(cc[i,:])
        print cc
    else:
        raise "Unknown initialization of EM of MoG centers."

    # Initialize co-variance matrices
    cov_list = []
    if cov_init=='iso':
        for i in range(K):
            cov_list.append(np.diag(np.ones(dim)/1e10))
            #cov_list.append(np.diag(np.ones(dim)))
    elif cov_init=='var':
        for i in range(K):
            cov_list.append(np.diag(np.var(X, axis=0)/1e10))
    else:
        raise ValueError('Unknown option used for cov_init')

    p_k = np.ones(K) / K # Uniform prior on P(k)
    return (center_list, cov_list, p_k)

def em_gm(X, K, max_iter = 50, verbose = False, \
                iter_call = None,\
                delta_stop = 1e-6,\
                init_kw = {}, \
                max_tries = 10,\
                diag_add = 1e-3):
    """Find K cluster centers in X using Expectation Maximization of Gaussian Mixtures.

    Parameters
    -----------
    X : NxD array
        Input data. Should contain N samples row wise, and D variablescolumn wise.
    max_iter : int
        Maximum allowed number of iterations/try.
    iter_call : callable
        Called for each iteration: iter_call(center_list, cov_list, p_k, i)
    delta_stop : float
        Stop when the change in the mean negative log likelihood goes below this
        value.
    max_tries : int
        The co-variance matrix for some cluster migth end up with NaN values, then
        the algorithm will restart; max_tries is the number of allowed retries.
    diag_add : float
        A scalar multiplied by the variance of each feature of the input data,
        and added to the diagonal of the covariance matrix at each iteration.

    Centroid initialization is given by *cluster_init*, the only available options
    are 'sample' and 'kmeans'. 'sample' selects random samples as centroids. 'kmeans'
    calls kmeans to find the cluster centers.

    Returns
    -------
    center_list : list
        A K-length list of cluster centers
    cov_list : list
        A K-length list of co-variance matrices
    p_k : list
        An K length array with mixing cofficients (p_k)
    logLL : list
         Log likelihood (how well the data fits the model)
    p_nk: matrix
         responsibilty matrix
    """

    samples, dim = np.shape(X)
    clusters_found = False
    while clusters_found==False and max_tries>0:
        max_tries -= 1
        # Initialized clusters
        center_list, cov_list, p_k = gmm_init(X, K, **init_kw)
        # Now perform the EM-steps:
        try:
            center_list, cov_list, p_k, logL, p_nk = \
                gmm_em_continue(X, center_list, cov_list, p_k,
                        max_iter=max_iter, verbose=verbose,
                        iter_call=iter_call,
                        delta_stop=delta_stop,
                        diag_add=diag_add)
            clusters_found = True
        except Cov_problem:
            if verbose:
                print "Problems with the co-variance matrix, tries left ", max_tries

    if clusters_found:
        return center_list, cov_list, p_k, logL, p_nk
    else:
        raise Cov_problem()

def gmm_em_continue(X, center_list, cov_list, p_k,
                    max_iter = 50, verbose = False, \
                    iter_call = None,\
                    delta_stop = 1e-6,\
                    diag_add = 1e-3,\
                    delta_stop_count_end=10):
    """
    """
    delta_stop_count = 0
    samples, dim = np.shape(X)
    K = len(center_list) # We should do some input checking
    if diag_add!=0:
        feature_var = np.var(X, axis=0)
        diag_add_vec = diag_add * feature_var
    old_logL = np.NaN
    logL = np.NaN
    for i in xrange(max_iter):
        try:
##                    if diag_add != 0:
##                        for c in cov_list:
##                            c = c + np.diag(feature_var * diag_add )
##                            #c = c + np.diag(np.ones(c.shape[0]) * diag_add )
            center_list, cov_list, p_k, logL, p_nk = __em_gm_step(X, center_list,\
                cov_list, p_k, K, diag_add_vec)
        except np.linalg.linalg.LinAlgError: # Singular cov matrix
            raise Cov_problem()
        if iter_call is not None:
            iter_call(center_list, cov_list, p_k, i)
        # Check if we have problems with cluster sizes
        for i2 in range(len(center_list)):
            if np.any(np.isnan(cov_list[i2])):
                print "problem"
                raise Cov_problem()

        if old_logL != np.NaN:
            if verbose:
                print "iteration=", i, " delta log likelihood=", \
                    old_logL - logL
            if np.abs(logL - old_logL) < delta_stop: #* samples:
                delta_stop_count += 1
                if verbose: print "gmm_em_continue: delta_stop_count =", delta_stop_count
            else:
                delta_stop_count = 0
            if delta_stop_count>=delta_stop_count_end:
                break # Sufficient precision reached
        old_logL = logL
    try:
        gm_log_likelihood(X, center_list, cov_list, p_k)
    except np.linalg.linalg.LinAlgError: # Singular cov matrix
        raise Cov_problem()
    return center_list, cov_list, p_k, logL, p_nk

def __em_gm_step(X, center_list, cov_list, p_k, K, diag_add_vec):
    samples = X.shape[0]
    # E-step:
    # Instead we use the log-sum-exp formula
    #p_Xn = np.zeros(samples)
    #for k in range(K):
    #    p_Xn += mulnormpdf(X, center_list[k], cov_list[k]) * p_k[k]

##    log_p_Xn = np.zeros(samples)
##    for k in range(K):
##        p = logmulnormpdf(X, center_list[k], cov_list[k]) + np.log(p_k[k])
##        if k == 0:
##            log_p_Xn = p
##        else:
##            pmax = np.max(np.concatenate((np.c_[log_p_Xn], np.c_[p]), axis=1), axis=1)
##            log_p_Xn = pmax + np.log( np.exp( log_p_Xn - pmax) + np.exp( p-pmax))
##    logL = np.sum(log_p_Xn)

    # New way of calculating the log likelihood:
    log_p_Xn_mat = np.zeros((samples, K))
    for k in range(K):
        log_p_Xn_mat[:,k] = logmulnormpdf(X, center_list[k], cov_list[k]) + np.log(p_k[k])
    pmax = np.max(log_p_Xn_mat, axis=1)
    log_p_Xn = pmax + np.log( np.sum( np.exp(log_p_Xn_mat.T - pmax), axis=0).T) # Maybe move this down
    logL = np.sum(log_p_Xn)

    # Instead we use log-sum-exp formula
    #p_nk = np.zeros((samples, K))
    #for k in range(K):
    #    p_nk[:,k] = mulnormpdf(X, center_list[k], cov_list[k]) * p_k[k] / p_Xn

    log_p_nk = np.zeros((samples, K))
    for k in range(K):
        #log_p_nk[:,k] = logmulnormpdf(X, center_list[k], cov_list[k]) + np.log(p_k[k]) - log_p_Xn
        log_p_nk[:,k] = log_p_Xn_mat[:,k] - log_p_Xn

    p_Xn = np.e**log_p_Xn
    p_nk = np.e**log_p_nk
    #print "p_nk=%s" %p_nk
    #print len(p_nk)
    # M-step:
    for k in range(K):
        ck = np.sum(p_nk[:,k] * X.T, axis = 1) / np.sum(p_nk[:,k])
        center_list[k] = ck
        cov_list[k] = np.dot(p_nk[:,k] * ((X - ck).T), (X - ck)) / sum(p_nk[:,k])\
            + np.diag(diag_add_vec)
        p_k[k] = np.sum(p_nk[:,k]) / samples

    return (center_list, cov_list, p_k, logL, p_nk)

def gm_log_likelihood(X, center_list, cov_list, p_k):
    """Finds the likelihood for a set of samples belongin to a Gaussian mixture
    model.

    Return log likelighood
    """
    samples = X.shape[0]
    K =  len(center_list)
    log_p_Xn = np.zeros(samples)
    for k in range(K):
        p = logmulnormpdf(X, center_list[k], cov_list[k]) + np.log(p_k[k])
        if k == 0:
            log_p_Xn = p
        else:
            pmax = np.max(np.concatenate((np.c_[log_p_Xn], np.c_[p]), axis=1), axis=1)
            log_p_Xn = pmax + np.log( np.exp( log_p_Xn - pmax) + np.exp( p-pmax))
    logL = np.sum(log_p_Xn)
    return logL

def gm_assign_to_cluster(X, center_list, cov_list, p_k):
    """Assigns each sample to one of the Gaussian clusters given.

    Returns an array with numbers, 0 corresponding to the first cluster in the
    cluster list.
    """
    # Reused code from E-step, should be unified somehow:
    samples = X.shape[0]
    K = len(center_list)
    log_p_Xn_mat = np.zeros((samples, K))
    for k in range(K):
        log_p_Xn_mat[:,k] = logmulnormpdf(X, center_list[k], cov_list[k]) + np.log(p_k[k])
    pmax = np.max(log_p_Xn_mat, axis=1)
    log_p_Xn = pmax + np.log( np.sum( np.exp(log_p_Xn_mat.T - pmax), axis=0).T)
    logL = np.sum(log_p_Xn)

    log_p_nk = np.zeros((samples, K))
    for k in range(K):
        #log_p_nk[:,k] = logmulnormpdf(X, center_list[k], cov_list[k]) + np.log(p_k[k]) - log_p_Xn
        log_p_nk[:,k] = log_p_Xn_mat[:,k] - log_p_Xn

    print log_p_nk
    #Assign to cluster:
    maxP_k = np.c_[np.max(log_p_nk, axis=1)] == log_p_nk
    #print np.max(log_p_nk, axis=1)
    maxP_k = maxP_k * (np.array(range(K))+1)
    return np.sum(maxP_k, axis=1) - 1

class Cov_problem(Exception):
    """
    """
    pass

def cond_dist(Y, centroids, ccov, mc):
    """Finds the conditional distribution p(X|Y) for a GMM.

    Parameters
    ----------
    Y : D array
        An array of inputs. Inputs set to NaN are not set, and become inputs to
        the resulting distribution. Order is preserved.
    centroids : list
        List of cluster centers - [ [x1,y1,..],..,[xN, yN,..] ]
    ccov : list
        List of cluster co-variances DxD matrices
    mc : list
        Mixing cofficients for each cluster (must sum to one) by default equal
        for each cluster.

    Returns
    -------
    res : tuple
        A tuple containing a new set of (centroids, ccov, mc) for the
        conditional distribution.
    """
    not_set_idx = np.nonzero(np.isnan(Y))[0]
    set_idx = np.nonzero(True - np.isnan(Y))[0]
    new_idx = np.concatenate((not_set_idx, set_idx))
    y = Y[set_idx]
    # New centroids and covar matrices
    new_cen = []
    new_ccovs = []
    # Appendix A in C. E. Rasmussen & C. K. I. Williams, Gaussian Processes
    # for Machine Learning, the MIT Press, 2006
    fk = []
    for i in range(len(centroids)):
        # Make a new co-variance matrix with correct ordering
        new_ccov = copy.deepcopy(ccov[i])
        new_ccov = new_ccov[:,new_idx]
        new_ccov = new_ccov[new_idx,:]
        #ux = centroids[i][not_set_idx]
        #uy = centroids[i][set_idx]
        #A = new_ccov[0:len(not_set_idx), 0:len(not_set_idx)]
        #B = new_ccov[len(not_set_idx):, len(not_set_idx):]
        #C = new_ccov[0:len(not_set_idx), len(not_set_idx):]
        #cen = ux + np.dot(np.dot(C, np.linalg.inv(B)), (y - uy))
        #cov = A - np.dot(np.dot(C, np.linalg.inv(B)), C.transpose())
        ua = centroids[i][not_set_idx]
        ub = centroids[i][set_idx]
        Saa = new_ccov[0:len(not_set_idx), 0:len(not_set_idx)]
        Sbb = new_ccov[len(not_set_idx):, len(not_set_idx):]
        Sab = new_ccov[0:len(not_set_idx), len(not_set_idx):]
        L = np.linalg.inv(new_ccov)
        Laa = L[0:len(not_set_idx), 0:len(not_set_idx)]
        Lbb = L[len(not_set_idx):, len(not_set_idx):]
        Lab = L[0:len(not_set_idx), len(not_set_idx):]
        cen = ua - np.dot(np.dot(np.linalg.inv(Laa), Lab), (y-ub))
        cov = np.linalg.inv(Laa)
        new_cen.append(cen)
        new_ccovs.append(cov)
        #fk.append(mulnormpdf(Y[set_idx], uy, B)) # Used for normalizing the mc
        fk.append(mulnormpdf(Y[set_idx], ub, Sbb)) # Used for normalizing the mc
    # Normalize the mixing coef: p(X|Y) = p(Y,X) / p(Y) using the marginal dist.
    fk = np.array(fk).flatten()
    new_mc = (mc*fk)
    new_mc = new_mc / np.sum(new_mc)
    return (new_cen, new_ccovs, new_mc)

def cond_moments(X, centroids, ccov, mc):
    """EXPERIMENTAL CODE.

    Finds the conditional mean and variance evaluated at a point X.

    See "Active Learning with Statistical Models", David A. Cohn, et al.

    Parameters
    ----------
    X : D array
        An array of inputs. Inputs set to NaN are not set, and become inputs to
        the resulting distribution. Order is preserved.
    centroids : list
        List of cluster centers - [ [x1,y1,..],..,[xN, yN,..] ]
    ccov : list
        List of cluster co-variances DxD matrices
    mc : list
        Mixing cofficients for each cluster (must sum to one) by default equal
        for each cluster.

    Returns
    -------
    res : tuple
        A tuple containing a new set of (centroids, ccov, mc) for the
        conditional distribution.
    """
    not_set_idx = np.nonzero(np.isnan(X))[0]
    set_idx = np.nonzero(True - np.isnan(X))[0]
    new_idx = np.concatenate((not_set_idx, set_idx))

    x = X[set_idx]
    y_i = []
    sigma_yx_i = []
    px_i = []
    px_sum = 0
    N = 60
    for i in range(len(centroids)):
        new_ccov = copy.deepcopy(ccov[i])
        new_ccov = new_ccov[:,new_idx]
        new_ccov = new_ccov[new_idx,:]
        set_cov = new_ccov[len(not_set_idx):, len(not_set_idx):]
        set_u = centroids[i][set_idx]
        px_i.append(mulnormpdf(x, set_u, set_cov))
        #
        not_set_u = centroids[i][not_set_idx]
        inv_sigma_x_i = np.linalg.inv(set_cov)
        sigma_xy_i = new_ccov[len(not_set_idx):,:len(not_set_idx)]
        dx = (x-set_u)
        tmp = np.dot(sigma_xy_i, inv_sigma_x_i)
        y_i.append(not_set_u + np.dot(tmp, dx))
        #
        sigma_y_i = new_ccov[:len(not_set_idx),:len(not_set_idx)]
        sigma_yx_i.append(sigma_y_i - np.dot(sigma_xy_i.T, np.dot(inv_sigma_x_i, sigma_xy_i)))

    px_sum = np.sum(px_i)
    h = [z/px_sum for z in px_i]

    y = np.sum(np.array(h) * np.array(y_i))
    sigma_y = np.zeros((len(not_set_idx), len(not_set_idx)))
    n_i = N * mc
    for i in range(len(centroids)):
        set_u = centroids[i][set_idx]
        inv_sigma_x_i = np.linalg.inv(set_cov)
        tmp = np.eye(len(set_idx)) + np.dot(np.dot((x-set_u).T, inv_sigma_x_i), x-set_u)
        sigma_y += (h[i]**2 / n_i[i]) * np.dot(sigma_yx_i[i], tmp)
    return y, sigma_y

def marg_dist(X_idx, centroids, ccov, mc):
    """Finds the marginal distribution p(X) for a GMM.

    Parameters
    ----------
    X_idx : list
        Indecies of dimensions to keep
    centroids : list
        List of cluster centers - [ [x1,y1,..],..,[xN, yN,..] ]
    ccov : list
        List of cluster co-variances DxD matrices
    mc : list
        Mixing cofficients for each cluster (must sum to one) by default equal
        for each cluster.

    Returns
    -------
    res : tuple
        A tuple containing a new set of (centroids, ccov, mc) for the
        marginal distribution.
    """
    new_cen = []
    new_cov = []
    for i in range(len(centroids)):
        new_cen.append(centroids[i][X_idx])
        new_cov.append(ccov[i][X_idx,:][:,X_idx])
    new_mc = mc
    return (new_cen, new_cov, new_mc)

def find_density_diff(center_list, cov_list, p_k=None, method='hellinger'):
    """Difference measures for each component of the GMM.

    Parameters
    ----------
    centroids : list
        List of cluster centers - [ [x1,y1,..],..,[xN, yN,..] ]
    ccov : list
        List of cluster co-variances DxD matrices
    p_k : list
        Mixing cofficients for each cluster (must sum to one) by default equal
        for each cluster.
    method : string, optional
        Select difference measure to use. Can be:

        - 'hellinger' :
        - 'hellinger_weighted' :
        - 'KL' : Kullback-Leibler divergence

    Returns
    -------
    diff : NxN np array
        The difference between the probability distribtions of the components
        pairwise. Only the upper triangular matrix is used.
    """
    N = len(center_list)
    if method=='hellinger':
        m = gauss_diff.hellinger
    elif method=='hellinger_weighted':
        m = gauss_diff.hellinger_weighted
    elif method=='KL':
        m = gauss_diff.KL
    elif isinstance(method, types.FunctionType):
        pass
    else:
        raise ValueError('Could not understand method option: '+str(method))
    diff = np.zeros((N, N))
    for i in range(N):
        for j in range(i, N):
            if method!='hellinger_weighted':
                dist = m(center_list[i], cov_list[i],\
                        center_list[j], cov_list[j])
            else:
                dist = m(center_list[i], cov_list[i], p_k[i],\
                        center_list[j], cov_list[j], p_k[j])
            diff[i, j] = dist
    return diff

def predict(X, centroids, ccov, mc):
    """Predict the entries in X, which contains NaNs.

    Parameters
    ----------
    X : np array
        2d np array containing the inputs. Target are specified with numpy NaNs.
        The NaNs will be replaced with the most probable result according to the
        GMM model provided.
    centroids : list
        List of cluster centers - [ [x1,y1,..],..,[xN, yN,..] ]
    ccov : list
        List of cluster co-variances DxD matrices
    mc : list
        Mixing cofficients for each cluster (must sum to one) by default equal
        for each cluster.

    Returns
    -------
    var : list
        List of variance
    """
    samples, D = X.shape
    variance_list = []
    for i in range(samples):
        row = X[i, :]
        targets = np.isnan(row)
        num_targets = np.sum(targets)
        cen_cond, cov_cond, mc_cond = cond_dist(row, centroids, ccov, mc)
        X[i, targets] = np.zeros(np.sum(targets))
        vara = np.zeros((num_targets, num_targets))
        varb = np.zeros((num_targets, num_targets))
        for j in range(len(cen_cond)):
            X[i,targets] = X[i,targets] + (cen_cond[j]*mc_cond[j])
            vara = vara + mc_cond[j] * \
                (np.dot(cen_cond[j], cen_cond[j]) + cov_cond[j])
            varb = varb + mc_cond[j] * cen_cond[j]
        variance_list.append(vara - np.dot(varb, varb))
    return variance_list

em = em_gm


import numpy as np
arr2 = np.array([[10,11,12],[13,14,15],[16,17,18]])
#arr3 = np.array([1,2,3,4,5])
arr = np.arange(1,10) # 1 ile 10
arr4 = np.arange(1,21)
#print(arr3)
#print(arr3[0])
"""
#https://docs.scipy.org/doc/numpy/
print(arr2[2,2]) # bu matrisde 0,1,2 gibi gidiyor.3 satır 3 sütunu aldık.
print(np.arange(10,20)) # 10 ile 20 arasındakileri sayıları listeler.
print(np.arange(0,100,3)) # 0 ile 100 arasını 3 er 3 er listeler
print(np.zeros(10)) # 10 0 basar.
print(np.ones(10)) # 10 tane biri basar
print(np.zeros((2,2))) # 2 ye 2 matrisli 0 lar basar
print(np.linspace(0,100,5)) # 0 ile 100 5 eşit parçaya böler
print(np.eye(6)) # 6 ya 6 matris oluşturur. diyagonalleri(köşegenlerde) 1 yapar.
print(np.random.randint(0,10))
print(np.random.randint(1,10,5)) #random 5 elemanlı bir array oluşturur.
print(np.random.randn(5)) # Gauss Distribution
"""
"""
print(arr2.max())
print(arr2.min())
print(arr2.sum())
print(arr2.mean()) # ortalama alır.
print(arr2.argmax()) # en büyük sayının indisini verir.
"""

"""
print(arr[1])
print(arr[1:5])  # arraydaki 1 ile 5 arasındakileri verir.
print(arr[:4]) # arraydaki 0 ile 4. arasındakileri verir.
"""

"""
arr4 = arr4.reshape(5,4) # arr4 isimli arrayimizi 5'e 4 matrixe çevirdi.
print(arr4)
print(arr4[:,:2]) # ilk 2 sütunun çıktısını verir.virgulden öncekiler satır(hepsi al diyoruz) virgülün sağındakiler sütun
print(arr4[:3,:3]) #ilk üç satır ve üç sutun
"""
#array filtreler
print(arr[arr > 3]) # 3 den büyükleri listeler.

print(arr - 2) # arrayin her elemanından 2 çıkartır.

#iki aynı boyutlu arraylerde örneğin arr1 * arr2 her elaman birbiri ile çarpılabilir.

print(np.sqrt(arr)) # her bir elemanın karesini alır.

# https://nbviewer.jupyter.org/github/mustafamuratcoskun/Data-Analiz-Notebooklar/blob/master/Numpy%20%C3%96dev/Numpy%20%C3%96dev%20%C3%87%C3%B6z%C3%BCmleri.ipynb


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Jan 27 20:17:21 2019

@author: alankar
"""

import numpy as np
import scipy.constants as const
import matplotlib.pyplot as plt

def I(wav,T):
    wavelength = np.copy(wav)
    wavelength *= 1e-9 #in nm
    T *= 1e3   #in 10^3 K
    return (2*np.pi*const.h*const.c**2*wavelength**(-5))/(np.exp(const.h*const.c/(wavelength*const.k*T))-1)

def dfdl(wav,T,delta):
    return (I(wav+delta,T)-I(wav-delta,T))/(2*delta)

def bisection(f,a,b,tol,*args):
    c = 0.
    while(np.abs((b-a)/2)>=tol):
        c = (a+b)/2
        if (f(c,*args)*f(a,*args)>=0): a = c
        else: b = c
    return (a+b)/2

T = np.linspace(4,6,100)
lambda_max = np.zeros(len(T))
for i in range(len(T)):
    lambda_max[i] = bisection(dfdl,10,10000,1e-6,T[i],1e-2)

plt.plot(1/(T*1e3),lambda_max,label="Wien displacement constant [slope] (SI): %.10f"%np.average(lambda_max*1e-9*T*1e3))
plt.grid()
plt.xlabel(r'$\frac{1}{T [K]}$',size = 23)
plt.ylabel(r'$\lambda_{max} [nm]$',size = 18)
plt.legend(loc='best', prop={'size': 16})
plt.show()
print("Wien displacement constant (SI): %.10f"%np.average(lambda_max*1e-9*T*1e3))
print('Wavelength = 502 nm Temperature = %d K'%(np.average(lambda_max*1e-9*T*1e3)/(502*1e-9)))

"""
Output

Wien displacement constant (SI): 0.0028977729
Wavelength = 502 nm Temperature = 5772 K
"""


def Lk(xdatos, x, k):
    L=1
    for i in range(len(xdatos)):
        if i!=k:
            L = L*(x-xdatos[i])/(xdatos[k]-xdatos[i])
    return L

def interp_lagrange(x, xd, yd):
    y = 0
    for k in range(len(yd)):
        y = y + yd[k]*Lk(xd, x, k)
    return y

def NDD(x, y):
    """Devuelve las diferencias divididas de Newton.

    Args:
    x : lista o array
        Los valores de x
    y : lista o array
        Los valores de y
    Returns:
    NDD : array
        Array con las diferencias divididas
    """
    from numpy import zeros
    NDD = []
    N = len(x)
    M = zeros([N,N])
    M[:,0] = y

    for columna in range(1,N):
        for fila in range(N-columna):

            m = f"M[{fila}, {columna}]"

            ll = [m, f"= (M[{fila+1},{columna-1}]", '-',
                     f"M[{fila}, {columna-1}])", '/',
                     f"(x[{fila+columna}] - x[{fila}])", '\n']
            #print(''.join(ll))

            num = M[fila+1, columna-1] - M[fila, columna-1]
            den = x[fila+columna] - x[fila]
            M[fila,columna] = num/den

    NDD = M[0,:]
    return NDD

def NDD_eval(xi, dd, xd):
    """Evaluar el polinomio interpolante.

    Args:
    dd : array
        Diferencias divididas
    xi : list or array
        Valores donde quiero evaluar al polinomio
    xd : list or array
        Valores X interpolados
    Returns:
    y : list or array
        Valores del polinomio en x
    """
    #N = len(dd)
    #print(f"P = a[{N-1}]")
    #for i in range(N-2, -1, -1):
        #print(f"P = a[{i}] + p*(x-x[{i}])")

    N = len(dd)
    y = []
    for xx in xi:
        p = dd[N-1]
        for i in range(N-2, -1, -1):
            p = dd[i] + p*(xx-xd[i])
        y.append(p)

    return y

def interp_newton(x, xd, yd):
    dd = NDD(xd, yd)
    y = NDD_eval(x, dd, xd)
    return y

def comparar(f, xmin, xmax, ndata, ninterp,
             fname='plot', ximin=None, ximax=None):

    from numpy import linspace
    from matplotlib import pyplot as plt

    xd = linspace(xmin, xmax, ndata)
    yd = f(xd)
    if ximin is None:
        ximin=xmin
    if ximax is None:
        ximax=xmax

    xi = linspace(ximin, ximax, ninterp)
    ym = f(xi)
    yl = interp_lagrange(xi, xd, yd)
    yn = interp_newton(xi, xd, yd)

    fig = plt.figure(figsize=(6, 10))

    ax = fig.add_subplot(311)
    ax.plot(xi, ym, linewidth=4, color='silver', label='modelo')
    ax.plot(xi, yl, linewidth=1.4, linestyle='-', color='orchid',
            label='lagrange')
    line, = ax.plot(xi, yn, linewidth=1.4, linestyle='--', color='dodgerblue',
            label='newton')
    line.set_dashes([10, 10, 10, 10])
    ax.plot(xd, yd, marker='o', linestyle='None', color='navy', markersize=5)
    ax.grid()
    ax.legend()

    ax = fig.add_subplot(312)
    ax.plot(xi, yl-ym, linewidth=1.4, color='dodgerblue', label='newton')
    line, = ax.plot(xi, yn-ym, linewidth=1.4, color='orchid', label='lagrange')
    line.set_dashes([10, 10, 10, 10])

    ax.plot(xd, [0]*len(xd), marker='o', linestyle='None', color='navy',
            markersize=3)
    ax.grid()
    ax.legend()

    ax = fig.add_subplot(313)
    ax.plot(xi, yn-yl, linewidth=1.0, color='black', label='difference')
    ax.grid()
    ax.legend()

    plt.tight_layout()
    filename = '.'.join([fname, 'png'])
    fig.savefig(fname)

    return None


import numpy as np
def rescale_data(x, **keywords):
    """对数据进行rescale
    Input:
    x:  输入的数据， shape=(n_samples, dim)
    keywords:
        min_val : rescale后的最小值
        max_val : rescale后的最大值
        min_x   : 每一维度x的最小值
        range_x : 每一维度的区间长度
    """
    x = x.astype("float")
    min_val = keywords.get("min_val", -1)
    max_val = keywords.get("max_val", 1)
    min_x = keywords.get("min_x", np.min(x, axis=0))
    range_x = keywords.get("range_x", np.max(x, axis=0) - np.min(x, axis=0))

    # rescale到(0:1)
    y = (x - min_x)/range_x
    # rescale到 (0:max_val-min_val)
    y = y * (max_val - min_val)
    # 平移
    y = y + min_val
    return y, min_x, range_x

# 测试代码
if __name__ == "__main__":
    x = np.array([[1,2,3],[4,5,6],[7,8,9]])
    print(rescale_data(x))


# algorithm 3.1 backtracking line search

import numpy as np
import sys
import test_functions.test_functions as test_f
import copy
import random

class BacktrackingLineSearch:
    def __init__(self, rho, c):
        self.rho = rho
        self.c = c

    def __call__(self, objective_function, x_k, p_k, gradient_k, alpha=0):
        if alpha == 0:
            alpha = random.random() * 2
        f_x_p = objective_function(x_k + alpha * p_k)
        f_x = objective_function(x_k)
        inc = self.c * alpha * gradient_k.dot(p_k)
        # while objective_function(x_k + alpha * p_k) > \
        #         objective_function(x_k) + \
        #         self.c * alpha * gradient_k.dot(p_k):
        while f_x_p > f_x + inc:
            alpha *= self.rho
            f_x_p = objective_function(x_k + alpha * p_k)
            inc = self.c * alpha * gradient_k.dot(p_k)
        return alpha

def steepest_descent_BLS(objective_function, threshold_delta_value,  x_init=None):
    x_sequence = []
    delta = sys.float_info.max
    x_k = np.array([random.random() * 2 - 1, random.random() * 2 - 1]) if x_init is None else x_init
    bls = BacktrackingLineSearch(rho=0.9, c=1e-4)
    f_value = objective_function(x_k)
    while threshold_delta_value < delta:
        gradient_k = objective_function.derivative(x_k)
        p_k = - gradient_k
        alpha = bls(objective_function, x_k, p_k, gradient_k, alpha=1)
        delta = f_value
        x_k += alpha * p_k
        x_sequence.append(copy.deepcopy(x_k))
        f_value = objective_function(x_k)
        delta = np.abs(delta - f_value)
    return x_sequence

if __name__ == '__main__':
    obj_f = test_f.AckleyFunction()
    x_s = steepest_descent_BLS(obj_f, 0.0001, x_init=[1.3, -1.3])
    obj_f.save(x_s)


import numpy as np
def propagate(B, b, X, Y):
    """
    params:
    B: weights of size [1, X.shape[1]]
    b: bias
    X: matrix of observations and features size [X.shape[0], X.shape[1]]
    Y: matrix of actual observation size [Y.shape[0], 1]

    returns:
    grads: dict of gradients, dB of shape same as B and db of shape [1, 1].
    cost: MSE cost
    """

    ## m is no of observations ie rows of X
    m = X.shape[0]

    # Calculate hypothesis
    y_hat = np.dot(X, B.T) + b

    y = Y.values.reshape(Y.shape[0], 1)

    # Compute Cost
    cost = (1 / (2 * m)) * np.sum((y - y_hat) ** 2)

    # BACKWARD PROPAGATION (TO FIND GRAD)
    dB = (-1 / m) * np.dot((y - y_hat).T, X)

    db = -np.sum(y - y_hat) / m

    grads = {"dB": dB,
             "db": db}

    return grads, cost

def optimize(B, b, X, Y, num_iterations, learning_rate):
    """
    params:
    B: weights of size [1, X.shape[1]]
    b: bias
    X: matrix of observations and features size [X.shape[0], X.shape[1]]
    Y: matrix of actual observation size [Y.shape[0], 1]
    num_iterations: number of iterations
    learning_rate: learning rate
    returns:
    params: parameters B of shape [1, X.shape[1]] and bias
    grads: dict of gradients, dB of shape same as B and db
    costs:  MSE cost
    """
    costs = []

    for i in range(num_iterations):
        # Cost and gradient calculation call function propagate
        grads, cost = propagate(B, b, X, Y)

        # Retrieve derivatives from grads
        dB = grads["dB"]
        db = grads["db"]

        # update parameters
        B = B - learning_rate * dB
        b = b - learning_rate * db

        costs.append(cost)

    params = {"B": B,
              "b": b}

    grads = {"dB": dB,
             "db": db}

    return params, grads, costs

def predict(B, b, X):
    """:param
    B: weights
    b: bias
    X: matrix of observations and features
    """
  # Compute predictions for X
    Y_prediction = np.dot(X, B.T) + b
    return Y_prediction

def model(X_train, Y_train, X_test, Y_test, num_iterations=2000, learning_rate=0.5):
    """
    params:
    X_train: X_train
    Y_train: Y_train
    X_test: X_test
    Y_test: Y_test

    returns:
    d: dictionary
    """

    # initialize parameters with zeros
    B = np.zeros(shape=(1, X_train.shape[1]))
    b = 0

    # Gradient descent
    parameters, grads, costs = optimize(B, b, X_train, Y_train, num_iterations, learning_rate)

    # Retrieve parameters B and b from dictionary "parameters"
    B = parameters["B"]
    b = parameters["b"]

    # Predict test/train set examples
    Y_prediction_test = predict(B, b, X_test)
    Y_prediction_train = predict(B, b, X_train)

    Y_train = Y_train.values.reshape(Y_train.shape[0], 1)
    Y_test = Y_test.values.reshape(Y_test.shape[0], 1)

    # Print train/test Errors
    print("train accuracy: {} %".format(100 - np.mean(np.abs(Y_prediction_train - Y_train)) * 100))
    print("test accuracy: {} %".format(100 - np.mean(np.abs(Y_prediction_test - Y_test)) * 100))

    d = {"costs": costs,
         "Y_prediction_test": Y_prediction_test,
         "Y_prediction_train": Y_prediction_train,
         "B": B,
         "b": b,
         "learning_rate": learning_rate,
         "num_iterations": num_iterations}

    return d


# -*- coding: utf-8 -*-

import numpy as np
import matplotlib.pyplot as pl
import math

#%% Q2

# Expected info gain - mutual info
def mut_info(p0, lam1):
    return -(p0*np.log2(p0+(1-p0)*np.exp(-lam1)) + (1-p0)*np.exp(-lam1)*(np.log2(np.exp(1))*lam1 + np.log2(p0 + (1-p0)*np.exp(-lam1))) + (1-p0)*(1-np.exp(-lam1))*np.log2(1-p0))

#%% Q2(b)

p0=0.5

Lam = np.linspace(0, 20, 1000)
I_arr = []

for lam in Lam:
    I_arr.append(mut_info(p0, lam))

fig1, ax1 = pl.subplots(1, 1)
ax1.plot(Lam, I_arr, 'b-')
ax1.set_xlabel(r"$\lambda_1$")
ax1.set_ylabel(r"$I(X, Y|K)$ / bits")
ax1.set_xticks(np.arange(0, 21, 2))

#%% Q2(c)

p0 = np.linspace(0, 1, 1000)

Lam = [1, 10, 100]

fig2, ax2 = pl.subplots(3, 1)

for lam_ind in range(len(Lam)):
    I_arr = []
    for p in p0:
        I_arr.append(mut_info(p, Lam[lam_ind]))

    ax2[lam_ind].plot(p0, I_arr, 'b-', label=r"$\lambda_1={0}$".format(Lam[lam_ind]))
    ax2[lam_ind].set_ylabel(r"$I(X, Y|K)$ / bits")
    ax2[lam_ind].set_xticks(np.arange(0, 1.1, 0.1))
    ax2[lam_ind].legend()

ax2[2].set_xlabel(r"$p_0$")

#%% Q2(d)

Lam = np.linspace(0, 20, 500)
p0 = np.linspace(0, 0.99, 500)
I_arr = []
cap = []

for lam in Lam:
    I_arr.append(mut_info(0.5, lam))
    I_testp = []

    # Calculate channel capacity
    for p in p0:
        I_testp.append(mut_info(p, lam))

    I_max_ind = np.argmax(I_testp)
    cap.append(mut_info(p0[I_max_ind], lam))

fig5, ax5 = pl.subplots(1, 1)
ax5.plot(Lam, np.array(cap) - np.array(I_arr), 'b-', label="Channel capacity - Mutual info")
ax5.set_xlabel(r"$\lambda_1$")
ax5.set_ylabel(r"$I(X, Y|K)$ difference / bits")
ax5.set_xticks(np.arange(0, 21, 2))
ax5.legend()

#%% Q6(c)

def majority(p, K):
    prob_total = 0
    for k in np.arange((K+1)/2, K+1):
        prob_total += math.factorial(K)/(math.factorial(k)*math.factorial(K-k))*(1-p)**k*p**(K-k)
    return prob_total

K_arr = np.arange(1, 16, 2)
PP = [0.01, 0.1, 0.4]

fig3, ax3 = pl.subplots(3, 1)

for pp in range(len(PP)):
    pt_arr = []
    for KK in K_arr:
        pt_arr.append(majority(PP[pp], KK))

    ax3[pp].plot(K_arr, pt_arr, 'b-', label=r"$P = {0}$".format(PP[pp]))
    ax3[pp].set_xticks(np.arange(0, 16))
    ax3[pp].set_ylim([0, 1])
    ax3[pp].set_ylabel(r"$P$ (correct maj dec)")
    ax3[pp].legend()

ax3[2].set_xlabel(r"$K$")

#%% Q7

import os
os.chdir("C:/Users/Adriel/Documents/School/Info_Theory")

# Load text file
file = open("list_of_engwords.txt", "r")
words = file.read().split('\n')

file.close()

# Manually add the two one-letter words
words.append("a")
words.append("i")

# Find length of each word
length_words = []
for word in words:
    length_words.append(len(word))

length_words = np.array(length_words)

# Create list of corresponding index of each letter
alphabets = list("abcdefghijklmnopqrstuvwxyz")

#%% Q7(a) Eng alphabets

dot = np.array([1,3,2,2,1,3,1,4,2,1,1,3,0,1,0,2,1,2,3,0,2,3,1,2,1,2])
dash = np.array([1,1,2,1,0,1,2,0,0,3,2,1,2,1,3,2,3,1,0,1,1,1,2,2,3,2])

length = dot+dash

# 0 = dot, 1 = dash
morse = ["01", "1000", "1010", "100", "0", "0010", "110", "0000", "00", "0111", "101", "0100", "11", "10", "111", "0110", "1101", "010", "000", "1", "001", "0001", "011", "1001", "1011", "1100"]

# Separate codewords of different lengths into different cases
for le in np.arange(1, 5):
    print("{0}-symbol".format(le))
    req = np.where(length==le)[0]
    ham = []
    pos = [[],[]]

    for i in range(len(req)):
        for j in range(i+1, len(req)):
            diff = 0
            for k in range(le):
                if list(morse[req[i]])[k] != list(morse[req[j]])[k]:
                    diff += 1

            ham.append(diff)
            pos[0].append(req[i])
            pos[1].append(req[j])

    ham = np.array(ham) # to find max, min and mean

    print("Min Hamming distance = {0}, between letters {1} and {2}.".format(np.min(ham), alphabets[pos[0][np.argmin(ham)]], alphabets[pos[1][np.argmin(ham)]]))
    print("Max Hamming distance = {0}, between letters {1} and {2}.".format(np.max(ham), alphabets[pos[0][np.argmax(ham)]], alphabets[pos[1][np.argmax(ham)]]))
    print("Mean Hamming distance = {0}.".format(np.mean(ham)))

#%% Q7(a) Numbers

morse_no = ["11111", "01111", "00111", "00011", "00001", "00000", "10000", "11000", "11100", "11110"] # for ease of use put codeword for zero at the beginning

ham_no = []
pos_no = [[],[]]

for i in range(len(morse_no)):
    for j in range(i+1, len(morse_no)):
        diff = 0
        for k in range(5):
            if list(morse_no[i])[k] != list(morse_no[j])[k]:
                diff += 1

        ham_no.append(diff)
        pos_no[0].append(i)
        pos_no[1].append(j)

ham_no = np.array(ham_no) # to find max, min and mean

print("Min Hamming distance = {0}, between numbers {1} and {2}.".format(np.min(ham_no), pos_no[0][np.argmin(ham_no)], pos_no[1][np.argmin(ham_no)]))
print("Max Hamming distance = {0}, between numbers {1} and {2}.".format(np.max(ham_no), pos_no[0][np.argmax(ham_no)], pos_no[1][np.argmax(ham_no)]))
print("Mean Hamming distance = {0}.".format(np.mean(ham_no)))

#%% Q7(b)
min_ham = []
max_ham = []
mean_ham = []

for le in np.arange(2, 21):
    print("{0}-letter word".format(le))
    req = np.where(length_words==le)[0]
    ham = []

    for i in range(len(req)):
        for j in range(i+1, len(req)):
            diff = 0
            list_i = list(words[req[i]])
            list_j = list(words[req[j]])

            # Letter-by-letter comparison
            for let_ind in range(le):
                if list_i[let_ind] == list_j[let_ind]:
                    continue
                else:
                    diff += 1

            ham.append(diff)

    ham = np.array(ham) # to find max, min and mean

    print("Min Hamming distance = {0}.".format(np.min(ham)))
    min_ham = np.append(min_ham, np.min(ham))
    #min_ham.append(np.min(ham))
    print("Max Hamming distance = {0}.".format(np.max(ham)))
    max_ham = np.append(max_ham, np.max(ham))
    #max_ham.append(np.max(ham))
    print("Mean Hamming distance = {0}.".format(np.mean(ham)))
    mean_ham = np.append(mean_ham, np.mean(ham))
    #mean_ham.append(np.mean(ham))

#%% Q7(b) Plot
fig4, ax4 = pl.subplots(1,1)
ax4.plot(np.arange(2, 21), min_ham, 'r-', label="Min Hamming dist")
ax4.plot(np.arange(2, 21), max_ham, 'g-', label="Max Hamming dist")
ax4.plot(np.arange(2, 21), mean_ham, 'b-', label="Mean Hamming dist")
ax4.set_xlabel("Length of word")
ax4.set_ylabel("Hamming distance")
ax4.set_xticks(np.arange(0, 21, 2))
ax4.set_yticks(np.arange(0, 21, 2))
ax4.legend()

#%% Save data
np.savetxt("Min_Ham.csv", min_ham, delimiter=',')
np.savetxt("Max_Ham.csv", max_ham, delimiter=',')
np.savetxt("Mean_Ham.csv", mean_ham, delimiter=',')


#!/usr/bin/env python
#
# Author: Ying Xiong.
# Created: Mar 13, 2014.

"""Some extended utility functions for 'numpy' module."""

import numpy as np

def null(A, tol = 1e-12):
    """Return the null space of matrix or vector `A`, such that::

        dot(A, null(A)) == eps(M, N)

    Each column `r` of `null(A)` is a unit vector, and `||dot(A, r)|| < tol`.
    """
    A = np.atleast_2d(A)
    _, s, vt = np.linalg.svd(A)
    nnz = (s >= tol).sum()
    return vt[nnz:].T

def meshgrid_nd(*arrs):
    """Multi-dimensional meshgrid.

    Parameters
    ----------
    x, y, z, ...: ndarray
        Multiple 1-D arrays representing the coordinates of the grid.

    Returns
    -------
    X, Y, Z, ... : ndarray
        Multi-dimensional arrays of shape (len(x), len(y), len(z), ...). Note
        that there is a discrepancy to the original 2D meshgrid, where the
        output array shape is swapped, i.e. (len(y), len(x)). Specifically, if::

            X, Y = meshgrid(x, y)
            X2, Y2 = meshgrid_nd(x, y)

        then we have `X == X2.T` and `Y == Y2.T`.

    Examples
    --------
    >>> X, Y, Z = np.meshgrid([1,2,3], [10,20], [-2,-3,-4,-5])
    >>> X
    array([[[1, 1, 1, 1],
            [1, 1, 1, 1]],
           [[2, 2, 2, 2],
            [2, 2, 2, 2]],
           [[3, 3, 3, 3],
            [3, 3, 3, 3]]])
    >>> Y
    array([[[10, 10, 10, 10],
            [20, 20, 20, 20]],
           [[10, 10, 10, 10],
            [20, 20, 20, 20]],
           [[10, 10, 10, 10],
            [20, 20, 20, 20]]])
    >>> Z
    array([[[-2, -3, -4, -5],
            [-2, -3, -4, -5]],
           [[-2, -3, -4, -5],
            [-2, -3, -4, -5]],
           [[-2, -3, -4, -5],
            [-2, -3, -4, -5]]])

    """
    shape = map(len, arrs)
    ndim = len(arrs)

    ans = []
    for i, arr in enumerate(arrs):
        slc = [1] * ndim
        slc[i] = shape[i]
        arr2 = np.asarray(arr).reshape(slc)
        for j, sz in enumerate(shape):
            if j != i:
                arr2 = arr2.repeat(sz, axis=j)
        ans.append(arr2)

    return tuple(ans)


#!/usr/bin/env python
# coding: utf-8

# Projection in Multiple Dimensions
# -----------------------------------------
#
#  this section, we extend from the [one-dimensional subspace](http://python-for-signal-processing.blogspot.com/2012/11/the-projection-concept.html) to the more general two-dimensional subspace. This np.means that there are two vectors,  $\mathbf{v}_1$ and  $\mathbf{v}_2$ that are not colinear and that span the subspace. In the previous case, we had only one vector ( $\mathbf{v}$), so we had a one-dimensional subspace, but now that we have two vectors, we have a two-dimensional subspace (i.e. a plane). The extension from the two-dimensional subspace to the *n*-dimensional subspace follows the same argument but introduces more notation than we need so we'll stick with the two-dimensional case for awhile. For the two-dimensional case, the optimal MMSE solution has the form
#
# $$ \hat{\mathbf{y}} = \alpha_1 \mathbf{v}_1 + \alpha_2 \mathbf{v}_2 \in \mathbb{R}^m $$
#
# where $\mathbf{y}$ exists in the m-dimensional space of real numbers. We want to project this vector onto the two m-dimensional  $\mathbf{v}_i$ vectors.  Here, the orthogonality requirement extends as
#
# $$ \langle \mathbf{y} - \alpha_1 \mathbf{v}_1 -\alpha_2 \mathbf{v}_2 , \mathbf{v}_1\rangle= 0 $$
#
# and
#
# $$ \langle \mathbf{y} - \alpha_1 \mathbf{v}_1 -\alpha_2 \mathbf{v}_2 , \mathbf{v}_2\rangle= 0 $$
#
# Recall that for vectors, we have
#
# $$ \langle \mathbf{x} , \mathbf{y}\rangle  = \mathbf{x}^T \mathbf{y} \in \mathbb{R}$$
#
# This leads to the linear system of equations:
#
# $$ \begin{eqnnp.array}
# \langle \mathbf{y}, \mathbf{v}_1\rangle = & \alpha_1 \langle \mathbf{v}_1, \mathbf{v}_1\rangle  &  +\alpha_2 \langle \mathbf{v}_1, \mathbf{v}_2\rangle   \\\\
# \langle \mathbf{y}, \mathbf{v}_2\rangle = & \alpha_1 \langle \mathbf{v}_1, \mathbf{v}_2\rangle  &  +\alpha_2 \langle \mathbf{v}_2, \mathbf{v}_2\rangle
# \end{eqnnp.array}
# $$
#
# which can be written in matrix form as
#
# $$ \left[ \begin{np.array}{c}
# \langle \mathbf{y}, \mathbf{v}_1\rangle  \\\\
# \langle \mathbf{y}, \mathbf{v}_2\rangle  \\\\
# \end{np.array} \right] =
# \left[ \begin{np.array}{cc}
# \langle \mathbf{v}_1, \mathbf{v}_1\rangle & \langle \mathbf{v}_1, \mathbf{v}_2\rangle \\\\
# \langle \mathbf{v}_1, \mathbf{v}_2\rangle  & \langle \mathbf{v}_2, \mathbf{v}_2\rangle \\\\
# \end{np.array} \right] \left[
# \begin{np.array}{c}
# \alpha_1  \\\\
# \alpha_2  \\\\
# \end{np.array} \right]$$
#
# which can be further reduced by stacking the columns into
#
# $$ \mathbf{V} = \left[ \mathbf{v}_1, \mathbf{v}_2 \right] \in \mathbb{R}^{m \times 2} $$
#
# and
#
# $$ \boldsymbol{\alpha}= \left[ \alpha_1, \alpha_2\right]^T \in \mathbb{R}^{2}$$
#
# which gives
#
# $$ \mathbf{V}^T \mathbf{y} = (\mathbf{V}^T \mathbf{V}) \boldsymbol{\alpha} $$
#
# Note that by writing this using vector notation, we have implicitly generalized beyond two dimensions since there is nothing to stop from stacking $\mathbf{V}$ with more column vectors to create a larger subspace. By solving we obtain,
#
# $$ \boldsymbol{\alpha} = (\mathbf{V}^T \mathbf{V})^{-1} \mathbf{V}^T \mathbf{y} $$
#
# and so the optimal solution is then,
#
# $$ \hat{\mathbf{y}} = \mathbf{V} \boldsymbol{\alpha} \in \mathbb{R}^m $$
#
# Note that the existence of the inverse is guaranteed by the non-co-linearity of the $\mathbf{v}_i$ vectors. Whether or not that inverse is numerically stable is another issue.
#
# Then, we can combine these to obtain
#
# $$ \hat{\mathbf{y}} = \mathbf{V} (\mathbf{V}^T \mathbf{V})^{-1} \mathbf{V}^T \mathbf{y} $$
#
# when then makes the projection operator for this case:
#
# $$ \mathbf{P}_{V}= \mathbf{V} (\mathbf{V}^T \mathbf{V})^{-1} \mathbf{V}^T \in \mathbb{R}^{m \times m} $$
#
# As a quick check, we can see this reduce to the 1-dimensional case by setting
#
# $$ \mathbf{V}= \mathbf{v} \in \mathbb{R}^m$$
#
# so then,
#
# $$ \mathbf{P}_{v}= \mathbf{v} \frac{1}{\mathbf{v}^T \mathbf{v}} \mathbf{v}^T $$
#
# which matches our [previous result](http://python-for-signal-processing.blogspot.com/2012/11/the-projection-concept.html). The point of all these manipulations is that we can construct another projection operator with all the MMSE properties we had before, but now in a bigger subspace.  We can further verify the idempotent property of projection matrices by checking that
#
# $$ \mathbf{P}_V \mathbf{P}_V = \mathbf{P}_V$$
#
# The following graphic shows that when we project the three dimensional $\mathbf{y}$ vector onto the plane, which is spanned by the two $\mathbf{v}_i$ vectors, we obtain the MMSE solution where the sphere is tangent to the plane. The point of tangency is the  point $\hat{\mathbf{y}} $ which is the MMSE solution.

# [2]

#http://stackoverflow.com/questions/10374930/matplotlib-annotating-a-3d-scatter-plot

from mpl_toolkits.mplot3d import proj3d
import matplotlib.pyplot as plt
import numpy as np as np

fig = plt.figure()
fig.set_size_inches([8,8])

ax = fig.add_subplot(111, projection='3d')

ax.set_aspect(1)
ax.set_xlim([0,2])
ax.set_ylim([0,2])
ax.set_zlim([0,2])
ax.set_aspect(1)
ax.set_xlabel('x-axis',fontsize=16)
ax.set_ylabel('y-axis',fontsize=16)
ax.set_zlabel('z-axis',fontsize=16)

y = matrix([1,1,1]).T
V = matrix([[1,0.25], # columns are v_1, v_2
            [0,0.50],
            [0,0.00]])

alpha=inv(V.T*V)*V.T*y # optimal coefficients
P = V*inv(V.T*V)*V.T
yhat = P*y         # approximant

u = np.np.linspace(0, 2*np.pi, 100)
v = np.np.linspace(0, np.pi, 100)

xx = np.outer(np.cos(u), np.sin(v))
yy = np.outer(np.sin(u), np.sin(v))
zz = np.outer(np.ones(np.size(u)), np.cos(v))

sphere=ax.plot_surface(xx+y[0,0], yy+y[1,0], zz+y[2,0],
                       rstride=4, cstride=4, color='gray',alpha=0.3,lw=0.25)

ax.plot3D([y[0,0],0],[y[1,0],0],[y[2,0],0],'r-',lw=3)
ax.plot3D([y[0,0]],[y[1,0]],[y[2,0]],'ro')

ax.plot3D([V[0,0],0],[V[1,0],0],[V[2,0],0],'b-',lw=3)
ax.plot3D([V[0,0]],[V[1,0]],[V[2,0]],'bo')
ax.plot3D([V[0,1],0],[V[1,1],0],[V[2,1],0],'b-',lw=3)
ax.plot3D([V[0,1]],[V[1,1]],[V[2,1]],'bo')

ax.plot3D([yhat[0,0],0],[yhat[1,0],0],[yhat[2,0],0],'g--',lw=3)
ax.plot3D([yhat[0,0]],[yhat[1,0]],[yhat[2,0]],'go')

x2, y2, _ = proj3d.proj_transform(y[0,0],y[1,0],y[2,0], ax.get_proj())
ax.annotate(
    "$\mathbf{y}$",
    xy = (x2, y2), xytext = (-20, 20), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

x2, y2, _ = proj3d.proj_transform(yhat[0,0],yhat[1,0],yhat[2,0], ax.get_proj())
ax.annotate(
    "$\hat{\mathbf{y}}$",
    xy = (x2, y2), xytext = (-40, 10), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

x2, y2, _ = proj3d.proj_transform(V[0,0],V[1,0],V[2,0], ax.get_proj())
ax.annotate(
    "$\mathbf{v}_1$",
    xy = (x2, y2), xytext = (120, 10), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

x2, y2, _ = proj3d.proj_transform(V[0,1],V[1,1],V[2,1], ax.get_proj())
ax.annotate(
    "$\mathbf{v}_2$",
    xy = (x2, y2), xytext = (-30, 30), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

plt.show()

# ### Weighted Distances

# As before, we can easily extend this projection operator to cases where the measure of distance between $\mathbf{y}$ and the subspace $\mathbf{v}$ is weighted (i.e. non-uniform). We can accomodate these weighted distances by re-writing the projection operator as
#
# $$ \mathbf{P}_{V} = \mathbf{V} ( \mathbf{V}^T \mathbf{Q V})^{-1} \mathbf{V}^T \mathbf{Q} $$
#
# where $\mathbf{Q}$ is positive definite matrix.  Earlier, we started with a point $\mathbf{y}$ and inflated a sphere  centered at $\mathbf{y}$ until it just touched the plane defined by $\mathbf{v}_i$ and this point was closest point on the subspace to $\mathbf{y}$. In the general case with a weighted distance except now we inflate an ellipsoid, not a sphere, until the ellipsoid touches the line.
#
# The code and figure below illustrate what happens using the weighted $ \mathbf{P}_v $. It is basically the same code we used above. You can download the IPython notebook corresponding to this post and try different values on the diagonal of $\mathbf{Q}$.

# [9]

from mpl_toolkits.mplot3d import proj3d
import matplotlib.pyplot as plt
import numpy as np as np

fig = plt.figure()
fig.set_size_inches([8,8])

ax = fig.add_subplot(111, projection='3d')
ax.set_xlim([0,2])
ax.set_ylim([0,2])
ax.set_zlim([0,2])
ax.set_aspect(1)
ax.set_xlabel('x-axis',fontsize=16)
ax.set_ylabel('y-axis',fontsize=16)
ax.set_zlabel('z-axis',fontsize=16)

y = matrix([1,1,1]).T
V = matrix([[1,0.25], # columns are v_1, v_2
            [0,0.50],
            [0,0.00]])

Q = matrix([[1,0,0],
            [0,2,0],
            [0,0,3]])

P = V*inv(V.T*Q*V)*V.T*Q
yhat = P*y         # approximant

u = np.np.linspace(0, 2*np.pi, 100)
v = np.np.linspace(0, np.pi, 100)

xx = np.outer(np.cos(u), np.sin(v))
yy = np.outer(np.sin(u), np.sin(v))
zz = np.outer(np.ones(np.size(u)), np.cos(v))

xx,yy,yz=map(squeeze,split(tensordot(dstack([xx,yy,zz]),Q,axes=1),3,axis=2))

ellipsoid=ax.plot_surface(xx+y[0,0], yy+y[1,0], zz+y[2,0],
                           rstride=4, cstride=4, color='gray',alpha=0.3,lw=0.25)

ax.plot3D([y[0,0],0],[y[1,0],0],[y[2,0],0],'r-',lw=3)
ax.plot3D([y[0,0]],[y[1,0]],[y[2,0]],'ro')

ax.plot3D([V[0,0],0],[V[1,0],0],[V[2,0],0],'b-',lw=3)
ax.plot3D([V[0,0]],[V[1,0]],[V[2,0]],'bo')
ax.plot3D([V[0,1],0],[V[1,1],0],[V[2,1],0],'b-',lw=3)
ax.plot3D([V[0,1]],[V[1,1]],[V[2,1]],'bo')

ax.plot3D([yhat[0,0],0],[yhat[1,0],0],[yhat[2,0],0],'g--',lw=3)
ax.plot3D([yhat[0,0]],[yhat[1,0]],[yhat[2,0]],'go')

x2, y2, _ = proj3d.proj_transform(y[0,0],y[1,0],y[2,0], ax.get_proj())
ax.annotate(
    "$\mathbf{y}$",
    xy = (x2, y2), xytext = (-20, 20), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

x2, y2, _ = proj3d.proj_transform(yhat[0,0],yhat[1,0],yhat[2,0], ax.get_proj())
ax.annotate(
    "$\hat{\mathbf{y}}$",
    xy = (x2, y2), xytext = (40, 30), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

x2, y2, _ = proj3d.proj_transform(V[0,0],V[1,0],V[2,0], ax.get_proj())
ax.annotate(
    "$\mathbf{v}_1$",
    xy = (x2, y2), xytext = (120, 10), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

x2, y2, _ = proj3d.proj_transform(V[0,1],V[1,1],V[2,1], ax.get_proj())
ax.annotate(
    "$\mathbf{v}_2$",
    xy = (x2, y2), xytext = (-30, 30), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

plt.show()

# ## Summary

#  this section, we extended the concept of a projection operator beyond one dimension and showed the corresponding geometric concepts that tie the projection operator to MMSE problems in more than one dimension.

# ### References

# This post was created using the [nbconvert](https://github.com/ipython/nbconvert) utility from the source [IPython Notebook](www.ipython.org) which is available for [download](https://github.com/unpingco/Python-for-Signal-Processing/blob/master/Projection_mdim.ipynb) from the main github [site](https://github.com/unpingco/Python-for-Signal-Processing) for this blog. The projection concept is masterfully discussed in the classic Strang, G. (2003). *Introduction to linear algebra*. Wellesley Cambridge Pr. Also, some of Dr. Strang's excellent lectures are available on [MIT Courseware](http://ocw.mit.edu/courses/mathematics/18-06-linear-algebra-spring-2010/). I highly recommend these as well as the book.

# ### Appendix

# Below is some extra code to handle the more general case where there is a rotation as well as a weighting of the axes. Note the projection operator is constructed exactly the same way.

# [15]

from mpl_toolkits.mplot3d import proj3d
import matplotlib.pyplot as plt
import numpy as np as np

fig = plt.figure()
fig.set_size_inches([8,8])

ax = fig.add_subplot(111, projection='3d')
ax.set_xlim([0,2])
ax.set_ylim([0,2])
ax.set_zlim([0,2])
ax.set_aspect(1)
ax.set_xlabel('x-axis',fontsize=16)
ax.set_ylabel('y-axis',fontsize=16)
ax.set_zlabel('z-axis',fontsize=16)

y = matrix([1,1,1]).T
V = matrix([[1,0.25], # columns are v_1, v_2
            [0,0.50],
            [0,0.00]])

def rotation_matrix(angle,axis='z'):
    angle = angle/180.*pi
    if axis=='z':
        return matrix([[cos(angle),sin(angle),0],
                  [sin(-angle),cos(angle),0],
                  [0,0,1]])
    elif axis=='y':
        return matrix([[cos(angle),sin(angle),0],
                       [0,1,0],
                       [sin(-angle),cos(angle),0]])
    elif axis=='x':
        return matrix([[1,0,0],
                       [cos(angle),sin(angle),0],
                       [sin(-angle),cos(angle),0]])

S = matrix([[3,0,0],
            [0,2,0],
            [0,0,1]])

R = rotation_matrix(30)*rotation_matrix(30,'x')*rotation_matrix(40,'y')

Q = R.T*S*R # apply 3-D rotations

P = V*inv(V.T*Q*V)*V.T*Q # build projection matrix
yhat = P*y         # approximant

u = np.np.linspace(0, 2*np.pi, 100)
v = np.np.linspace(0, np.pi, 100)

xx = np.outer(np.cos(u), np.sin(v))
yy = np.outer(np.sin(u), np.sin(v))
zz = np.outer(np.ones(np.size(u)), np.cos(v))

xx,yy,yz=map(squeeze,split(tensordot(dstack([xx,yy,zz]),Q,axes=1),3,axis=2))

ellipsoid=ax.plot_surface(xx+y[0,0], yy+y[1,0], zz+y[2,0],
                           rstride=4, cstride=4, color='gray',alpha=0.3,lw=0.25)

ax.plot3D([y[0,0],0],[y[1,0],0],[y[2,0],0],'r-',lw=3)
ax.plot3D([y[0,0]],[y[1,0]],[y[2,0]],'ro')

ax.plot3D([V[0,0],0],[V[1,0],0],[V[2,0],0],'b-',lw=3)
ax.plot3D([V[0,0]],[V[1,0]],[V[2,0]],'bo')
ax.plot3D([V[0,1],0],[V[1,1],0],[V[2,1],0],'b-',lw=3)
ax.plot3D([V[0,1]],[V[1,1]],[V[2,1]],'bo')

ax.plot3D([yhat[0,0],0],[yhat[1,0],0],[yhat[2,0],0],'g--',lw=3)
ax.plot3D([yhat[0,0]],[yhat[1,0]],[yhat[2,0]],'go')

x2, y2, _ = proj3d.proj_transform(y[0,0],y[1,0],y[2,0], ax.get_proj())
ax.annotate(
    "$\mathbf{y}$",
    xy = (x2, y2), xytext = (-20, 20), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

x2, y2, _ = proj3d.proj_transform(yhat[0,0],yhat[1,0],yhat[2,0], ax.get_proj())
ax.annotate(
    "$\hat{\mathbf{y}}$",
    xy = (x2, y2), xytext = (40, 30), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

x2, y2, _ = proj3d.proj_transform(V[0,0],V[1,0],V[2,0], ax.get_proj())
ax.annotate(
    "$\mathbf{v}_1$",
    xy = (x2, y2), xytext = (120, 10), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

x2, y2, _ = proj3d.proj_transform(V[0,1],V[1,1],V[2,1], ax.get_proj())
ax.annotate(
    "$\mathbf{v}_2$",
    xy = (x2, y2), xytext = (-30, 30), fontsize=24,
    textcoords = 'offset points', ha = 'right', va = 'bottom',
    bbox = dict(boxstyle = 'round,pad=0.5', fc = 'yellow', alpha = 0.5),
    arrowprops = dict(arrowstyle = '->', connectionstyle = 'arc3,rad=0'))

plt.show()


5import numpy as np
import copy

def change_row(matrix, row1, row2):
    """change two row in a matrix
    """
    tmp = copy.copy(matrix[row1])
    matrix[row1] = matrix[row2]
    matrix[row2] = tmp

def get_row_multiplied(row, coefficient):
    """multiply row elements by coefficient
    """
    result = []
    for element in row:
        result.append(element * coefficient)
    return result

def print_array(array):
    """print array
    """
    for row in array:
        for element in row:
            print(element, end=" ")
        print()

def echelon_matrix(argumented, n, m):
    """convert an n * m argumented matrix to echlon from
    """
    last_pivot_row = 0
    pivots = []
    for column in range(m):
        pivot_founded = False

        # change row if pivot is 0
        if abs(argumented[last_pivot_row, column]) <= 0.00001:
            for row in range(last_pivot_row + 1, n):
                if argumented[row, column] != 0:
                    change_row(argumented, last_pivot_row, row)

                    pivot_founded = True
                    break
            if not pivot_founded:
                #this column doesn't have pivot
                continue

        pivots.append((last_pivot_row, column))
        # make pivot one
        pivot = argumented[last_pivot_row, column]
        argumented[last_pivot_row] = get_row_multiplied(argumented[last_pivot_row], 1 / pivot)

        # make row under pivot zero
        for row in range(last_pivot_row + 1, n):
            argumented[row] += get_row_multiplied(argumented[last_pivot_row],-argumented[row, column])

        if last_pivot_row == n-1:
            break

        last_pivot_row += 1

    return pivots[::-1]

def reduced_form(echlon, pivots):
    """convert an n * m echlon from matrix to reduced form
    """
    for row, column in pivots:
        for upper_row in range(row - 1, -1, -1):
            echlon[upper_row] += get_row_multiplied(echlon[row],-echlon[upper_row, column])

def find_values(reduced, pivots, m):
    """find output values from reduced form of matrix
    """
    for row, column in pivots:
            x[column] = 0
            for col in range(m - 1):
                if col == column:
                    continue
                x[column] += -x[col] * reduced[row, col]
            x[column] += reduced[row, m - 1]

def print_result(array, x):
    """print valueof variables
    """
    print_array(array)
    for indx, variable in enumerate(x):
        print(f'X{indx + 1} = {variable}')

# get input
n, m = map(int, input().split())
argumented = np.array([input().strip().split() for _ in range(n)], float)
# array for output variables
x = [10] * (m - 1)

pivots = echelon_matrix(argumented, n, m)

reduced_form(argumented, pivots)

find_values(argumented, pivots, m)

print_result(argumented, x)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
BE523 Biosystems Analysis & Design
HW3 - Problem 1. Constructing a mathematical model for penicillin clearance
https://mathinsight.org/penicillin_clearance_model

Created on Sat Jan 23 21:29:34 2021
@author: eduardo
"""
import numpy as np
import matplotlib.pyplot as plt
import statsmodels.api as sm  # allows linear regression without intercept

# The penicillin concentration that will be used to find the model
P = np.array([200, 152, 118, 93, 74])
steps = len(P)
dt = 5  # time interval
t = np.linspace(0, (steps-1)*dt, steps)  # actual time vector
dP = np.zeros(steps)

# Get the decrement of penicilin in every time interval
for i in range(0, steps-1):
    dP[i] = P[i+1] - P[i]

print('P=', P, '\ndP=', dP)

# Plot penicillin concentration change (dP) versus penicilline concentration (P)
# Perform a linear regression with P and dP, the slope of the line will give us
# the constant rate of penicillin clearance in every time step (approx. 20%)
# If the model is correct, the points should lie on a line that goes through
# the origin, no intercept is desired
model = sm.OLS(dP, P)  # No intercept by default
results = model.fit()
slope = results.params[0]
print("slope=", slope)

# # Figure 0, plotting P vs t, original data
# plt.figure(0)
# plt.title('Original data')
# plt.plot(t, P, 'kx')
# plt.legend(['data'], loc='best')
# plt.xlabel('t')
# plt.ylabel('P')

# Figure 1, plotting dP vs P
plt.figure(1)
plt.plot(P, dP, 'kx', P, slope*P, 'b-')
plt.legend(['data', 'linear regression $R^2$=%.2f' % results.rsquared], loc='best')
plt.xlabel('P')
plt.ylabel('dP')
# plt.savefig('p1_penicillin_linear2.png', dpi=300, bbox_inches='tight')  # Save figure 1

# Complete the model and make a prediction
# The model has the form B(t+1) = B[0]*(slope+1)^t
Pmodel = P[0]*(slope+1)**(t/dt)

# Figure 2, plotting P (from data and model) vs t
plt.figure(2)
plt.plot(t, P, 'kx', t, Pmodel, 'r-')
plt.legend(['data', 'model prediction'], loc='best')
plt.xlabel('t (min)')
plt.ylabel('Penicillin ($\mu$ g/ml)')
# plt.savefig('p1_penicillin_model2.png', dpi=300, bbox_inches='tight')  # Save figure 2


import numpy as np
import matplotlib.pyplot as plt

# To use LaTeX in the plots
plt.rcParams.update({
    "text.usetex": True,
    "font.family": "sans-serif",
    "font.sans-serif": ["Helvetica"]})
# for Palatino and other serif fonts use:
plt.rcParams.update({
    "text.usetex": True,
    "font.family": "serif",
    "font.serif": ["Palatino"],
})
plt.rcParams.update({
  "text.usetex": True,
  "font.family": "Helvetica"
})

# constants used in the problem
E = 0.220  # in volts
R = 500  # in Ohms
Vt = 0.025  # in volts
Isa = 0.6e-6  # in Amps
Isb = 1.2e-6  # in Amps

# Calculates the vector F which solves the equation F = 0
def F(v):  # v is a 2 x 1 vector which contains the voltage values of the circuit
    f1 = (E - v[0]) / R - Isa * (np.exp((v[0] - v[1]) / Vt) - 1)
    f2 = Isa*(np.exp((v[0]-v[1]) / Vt)-1)-Isb * (np.exp((v[1] / Vt)) - 1)
    F = np.array([f1, f2])
    return F

# compute the Jacobian
def Jacobian(v):
    J = np.zeros(shape=(2, 2))
    J[0][0] = -1/R - (Isa / Vt) * np.exp((v[0] - v[1]) / Vt)
    J[0][1] = (Isa / Vt) * np.exp((v[0] - v[1]) / Vt)
    J[1][0] = (Isa / Vt) * np.exp((v[0] - v[1]) / Vt)
    J[1][1] = -(Isb / Vt) * np.exp(v[1] / Vt) - (Isa/Vt)*np.exp((v[0]-v[1])/Vt)
    return J

# uses the above two functions to calculate the voltage solution to the circuit
def newton_raphson(maxerr):
    i = 0
    Vnew = np.zeros(shape=(2, 1))
    dV_vec = []
    val_vec = []
    conv = False
    while not conv:
        i += 1
        F_p = Jacobian(Vnew)  # calculate the Jacobian given teh current voltage values
        eff = F(Vnew)  # calculate the value of the F vector for the current voltage values
        dV = np.multiply(np.dot(np.linalg.inv(F_p), eff), -1)  # compute dV
        crit_val = np.linalg.norm(dV, 2)  # compute the 2-norm of dV for convergence criteria
        Vnew = np.add(Vnew, dV)  # compute new voltage value for next step

        dV_vec.append(crit_val)
        val_vec.append(Vnew)
        print("------------------------------------")
        print("iteration = "+str(i))
        print("Jacobian = "+str(F_p))
        print("F-vector = "+str(eff))
        print("\u0394 V = "+str(dV))
        if crit_val < maxerr:
            break
    return Vnew, dV_vec, i, val_vec

if __name__ == "__main__":
    error = 10e-15  # the maximum allowable error
    val = newton_raphson(error)
    # plot error in the log scale
    dV_norm_err = val[1]
    iter_no = val[2]
    ans = val[3]
    print("------------------------------------")
    print(ans[7])

    # Plot the 10log_10 of dV
    x_val = np.linspace(1, iter_no, iter_no)
    dV = 10*np.log10(dV_norm_err)
    plt.plot(x_val, dV)
    plt.xlabel("Number of Iterations")
    plt.ylabel("log(2-norm dV)")
    plt.show()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Apr 12 12:23:24 2019

@author: alankar
"""

import numpy as np
from scipy.linalg import eigh_tridiagonal

def lanczos(A,N):
    A = np.array(A,dtype=np.float64)
    R,C = A.shape
    if (R!=C):
        print('Square Matrix is required!')
        return None
    if (N>R): N = R
    beta = 0
    np.random.seed(100)
    b = np.ones((R,1))
    qold = np.zeros((R,1))
    qnew = b/np.linalg.norm(b)
    Q = np.zeros((R,N))
    #T = np.zeros((R,N))
    Q[:,0] = qnew[:,0]
    for i in range(1,N):
        v = np.matmul(A,qnew)
        alpha = np.dot(qnew.T,v)[0,0]
        v = v - beta*qold - alpha*qnew
        beta = np.linalg.norm(v)
        qold = np.copy(qnew)
        qnew = v/beta
        Q[:,i] = qnew[:,0]
        #T[i,i] = alpha
        #if (i+1<N):
        #    Q[:,i+1] = qnew[:,0]
        #    T[i+1,i] = beta
        #    T[i,i+1] = beta
    T = np.matmul(Q.T,np.matmul(A,Q))
    for i in range(0,T.shape[0]):
        for j in range(0,T.shape[1]):
            if not(i==j or i==j+1 or i==j-1): T[i,j]=0.
    return (Q,T)

np.random.seed(10)
size = 20
B = 2*np.diag(np.full(size,1.0))+np.random.rand(size, 20)/np.sqrt(size)
A = (B+B.T)/2

eigval, eigvec = np.linalg.eig(A)
file = open('eigen_inbuilt.txt','w')
for i in range(0,eigval.shape[0]):
    file.write('Eigenvalue: %.5f\n'%eigval[i])
    file.write('Eigenvector: \n')
    file.close()
    append = open('eigen_inbuilt.txt','a')
    np.savetxt(append,eigvec[:,i])
    append.close()
    file = open('eigen_inbuilt.txt','a')
    file.write('\n\n')
file.close()

Q,T = lanczos(A,size)
eigval_l, eigvec_l = eigh_tridiagonal([T[i,i] for i in range(T.shape[0])],[T[i,i+1] for i in range(T.shape[0]-1)])
eigvec_l = np.matmul(Q,eigvec_l)
#check = np.dot(Q.T,Q)
#check2 = np.dot(Q.T,np.matmul(A,Q))

file = open('eigen_lanczos.txt','w')
for i in range(0,eigval_l.shape[0]):
    file.write('Eigenvalue: %.5f\n'%eigval_l[i])
    file.write('Eigenvector: \n')
    file.close()
    append = open('eigen_lanczos.txt','a')
    np.savetxt(append,eigvec_l[:,i])
    append.close()
    file = open('eigen_lanczos.txt','a')
    file.write('\n\n')
file.close()


import numpy as np
from numpy import pi
from math import sqrt

class CompoundTopographicIndex():

    def __init__(self):
        self.name = "Compound Topographic Index"
        self.description = ("Computes the compound topographic index (CTI), also "
                            "known as the topographic wetness index (TWI).  "
                            "This is calculated from an input slope raster and flow "
                            "accumulation surface and is meant to be used in "
                            "a raster function chain.")

    def getParameterInfo(self):
        return [
            {
                'name': 'slope',
                'dataType': 'raster',
                'value': None,
                'required': True,
                'displayName': "Slope Raster",
                'description': "A slope raster (in degrees) derived from a digital elevation model (DEM)."
            },
            {
                'name': 'flow',
                'dataType': 'raster',
                'value': None,
                'required': True,
                'displayName': "Flow Accumulation Raster",
                'description': "A raster representing flow accumulation."
            }
        ]

    def getConfiguration(self, **scalars):
        return {
            'compositeRasters': False,
            'inheritProperties': 1 | 2 | 4 | 8,     # inherit all from the raster
            'invalidateProperties': 2 | 4 | 8,      # reset stats, histogram, key properties
            'inputMask': False
        }

    def updateRasterInfo(self, **kwargs):
        # repeat stats for all output raster bands
        kwargs['output_info']['bandCount'] = 1
        kwargs['output_info']['statistics'] = ({'minimum': 0, 'maximum': 25.0}, )
        kwargs['output_info']['histogram'] = ()  # reset histogram
        kwargs['output_info']['pixelType'] = 'f4'
        self.dem_cellsize = kwargs['slope_info']['cellSize']
        return kwargs

    def updatePixels(self, tlc, shape, props, **pixelBlocks):
        # get the input DEM raster pixel block
        inBlock_slope = pixelBlocks['slope_pixels']
        inBlock_flow = pixelBlocks['flow_pixels']
        cellSize = self.dem_cellsize

        DX = cellSize[0]
        DY = cellSize[1]
        slope = calc_slope(inBlock_slope)
        cti = calc_cti(slope, inBlock_flow, cellSize[0])

        # format output cti pixels
        outBlocks = cti.astype(props['pixelType'], copy=False)
        pixelBlocks['output_pixels'] = outBlocks
        return pixelBlocks

    def updateKeyMetadata(self, names, bandIndex, **keyMetadata):
        if bandIndex == -1:                                 # dataset level
            keyMetadata['datatype'] = 'Scientific'
        else:                                               # output "band"
            keyMetadata['wavelengthmin'] = None
            keyMetadata['wavelengthmax'] = None
            keyMetadata['bandname'] = 'CTI'
        return keyMetadata

# supporting business logic functions
def calc_slope(slope_deg):
    slope = slope_deg * pi/180 #np.arctan(slope_deg)
    return slope

def calc_cti(slope, flow_acc, cellsize):
    tan_slope = np.tan(slope)
    tan_slope[tan_slope==0]=0.0001
    cti = np.log(((flow_acc+1)*cellsize)/tan_slope)
    return cti


import numpy as np
import math
from scipy import linalg
import cv2 as cv

print("__TASK1__")
alpha = 45*math.pi/180
R = np.array([[np.cos(alpha), -np.sin(alpha), 0],
              [np.sin(alpha), np.cos(alpha), 0],
              [0, 0, 1]])
T = np.array([10,0,0]).T
F_1 = np.cross(T, R)
print("F=\n", np.round(F_1, 3))

print("__TASK2__")
alpha1 = 45*math.pi/180
alpha2 = -45*math.pi/180

# matrixes of the translation
T1 = np.array([0,0,0]).reshape(3,1)
T2 = np.array([10,0,0]).reshape(3,1)
# rotation matrix R1
R1 = np.array([[np.cos(alpha1), -np.sin(alpha1), 0],
              [np.sin(alpha1), np.cos(alpha1), 0],
              [0, 0, 1]])
# rotation matrix R2 over the OY
R2 = np.array([[np.cos(alpha2), 0, np.sin(alpha2)],
               [0, 1, 0],
              [-np.sin(alpha2), 0, np.cos(alpha2)]])
# projection matrix P1 and P2
P1 = np.concatenate([R1.T, -np.matmul(R1.T,T1)], axis=1)
P2 = np.concatenate([R2.T, -np.matmul(R2.T,T2)], axis=1)

# F = [e2]_x*P2*P1_inversed
# inverse matrix P1
U, E, V_T = np.linalg.svd(P1, full_matrices=False)
P1_inversed = np.matmul(np.matmul(V_T.T, np.diag(1/E)), U.T)
# e2 = P2*O ==> we do not have translation between first camera
# and world coordinate system, then O is supposed to be set to [0,0,0].T
# but as we have P2 [3,4] we need define the fourth coordinate equals to 1
e2 = np.matmul(P2, np.array([0, 0, 0, 1]).T)
print(e2)
# Calculate F by the given formula above
F = np.cross(e2, np.matmul(P2, P1_inversed),axisa=0, axisb=0, axisc=0)
print("F=\n", np.round(F ,3), " F.T*e': ", np.matmul(F.T, e2))

print("__TASK3__")
# we have rotation and translation of the one camera relative to the second one
# Also we have identety K matrixes. So, we may have get the following equation:
# e1.T*l1 = 0 , l1.T*e1 = 0, l1 = F.T*q2 or l1.T = q2.T*F ==> q2.T*F*e1 = 0
# solve F*e1 = 0 by SVD, F = U*E*V.T
# The solution for e1 is column of the matrix V relative to zero singular values
# or it the same as the row of the matrix V.T relative to zero singular values.
U,E,V_T = np.linalg.svd(F_1, full_matrices=False)
null_mask = E < 1e-6
e1 = V_T[null_mask]
# for e2 we can obtain the same equation:
# q1.T*F.T*e2 = 0
# solve F.T*e2 = 0
U,E,V_T = np.linalg.svd(F_1.T, full_matrices=False)
null_mask = E < 1e-6
e2 = V_T[null_mask]

print("e = \n", np.round(e1, 3))
print("e' = \n", np.round(e2, 3))

print("__TASK4__")
# l1 = e1 x q1
l1 = np.cross(e1.reshape(-1), [0, 0, 1])
# l2 = F*[e1]xl1
l2 = np.matmul(F_1, np.cross(e1.reshape(-1), l1))
print(f"l epipolar line:\n{np.round(l1,3)}\nl' epipolar line:\n{np.round(l2,3)}")

print("__TASK5__")
# see explonations in the handwritten paper
print(f"l' = {[0, -1, 0]}")


import numpy
import statistics as st
from scipy import stats
mu, sigma = 0, 1
samples1 = numpy.random.normal(mu, sigma, 1000)
X = numpy.array_split(samples1, 10)
print("The below stats is for {0} sets and {1} samples each".format(10,100))
for num,arr in enumerate(X, start=1):
    mean = st.mean(arr)
    stdev = st.stdev(arr)
    mle = stats.norm(mean, stdev).pdf(mean)
    print("The maximum likelihood of sample {0} and mean {1} and standard deviation {2} is {3}".format(num,mean,stdev,mle))

print('\n')
samples2 = numpy.random.normal(mu, sigma, 2000)
X = numpy.array_split(samples2, 10)
print("The below stats is for {0} sets and {1} samples each".format(10,200))
for num,arr in enumerate(X, start=1):
    mean = st.mean(arr)
    stdev = st.stdev(arr)
    mle = stats.norm(mean, stdev).pdf(mean)
    print("The maximum likelihood of sample {0} and mean {1} and standard deviation {2} is {3}".format(num,mean,stdev,mle))

#print('\n')
#samples2 = numpy.random.normal(mu, sigma, 500)
#X = numpy.array_split(samples2, 5)
#print("The below stats is for {0} sets and {1} samples each".format(5,100))
#for num,arr in enumerate(X, start=1):
#    mean = st.mean(arr)
#    stdev = st.stdev(arr)
#    mle = stats.norm(mean, stdev).pdf(mean)
#    print("The maximum likelihood of sample {0} and mean {1} and standard deviation {2} is {3}".format(num,mean,stdev,mle))


#!/usr/bin/env python
"""Hill cipher
by Abhijith V Mohan"""
import sys
import frequencygraph
from numpy import linalg
from numpy import matrix as npmatrix
letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'

def sign(x):
    """Returns sign of x"""
    if x==0:
        return 0
    return x/abs(x)

def egcd(a, b):
    """Extended Euclid's algo"""
    if a == 0:
        return (b, 0, 1)
    else:
        g, y, x = egcd(b % a, a)
        return (g, x - (b / a) * y, y)

def modinv(a, m):
    """Returns modular inverse of a modulo m"""
    g, x, y = egcd(a, m)
    assert g==1  # else modular inverse does not exist
    return x % m

def matrix_modinv(matrix, m):
    """Return inverse of the matrix modulo m"""
    matrix_det = int(round(linalg.det(matrix)))
    return modinv(abs(matrix_det), m)*linalg.inv(matrix)*matrix_det*sign(matrix_det)

def getmatrix(block):
    """Get 1-by-n matrix corresponding to block"""
    matrix = npmatrix([[letters.index(c)] for c in block])
    return matrix

def getblock(matrix):
    """Get block of text corresponding to 1-by-n matrix"""
    block = ''.join(letters[int(round(i))%26] for i in matrix.getA1())
    return block

def cleanup(text, blocksize):
    """Remove unwanted chars and makes text length a multiple of block size"""
    text = text.upper()
    clean_text = [c for c in text if c in letters]
    while (len(clean_text) % blocksize)!=0:
        clean_text.append('Z')
    return clean_text

def encrypt(plaintext, key):
    blocksize = len(key)
    clean_text = cleanup(plaintext, blocksize)
    blocks_plain = [getmatrix(clean_text[i:i+blocksize]) for i in xrange(0, len(clean_text), blocksize)]
    blocks_cipher = [getblock(key*block) for block in blocks_plain]
    ciphertext = ''.join(blocks_cipher)
    return ciphertext

def decrypt(ciphertext, key):
    blocksize = len(key)
    blocks_cipher = [getmatrix(ciphertext[i:i+blocksize]) for i in xrange(0, len(ciphertext), blocksize)]
    inv_key = matrix_modinv(key, 26)
    blocks_plain = [getblock(inv_key*block) for block in blocks_cipher]
    plaintext = (''.join(blocks_plain)).lower()
    return plaintext

key = npmatrix([[6,24,1],[13,16,10],[20,17,15]]) #Key matrix

def main():
    mode, inputfile, outputfile = sys.argv[1:4]
    ifile = open(inputfile, 'r')
    ofile = open(outputfile, 'w')
    if mode=='e':
        plaintext = ifile.read()
        ciphertext = encrypt(plaintext, key)
        ofile.write(ciphertext)
        frequencygraph.freqbars_percentage('Hill Cipher frequency graph)'.format(key), plaintext, ciphertext)
    elif mode=='d':
        ciphertext = ifile.read()
        plaintext = decrypt(ciphertext, key)
        ofile.write(plaintext)
    ifile.close()
    ofile.close()

if __name__ == '__main__':
    main()


import numpy as np
import scipy.stats as sct
import time
import itertools
import sys

#----- normal distribution

def normal_model_log_prob(_x, _theta):
    #-- parameters
    _mu = _theta[0]
    _sigma_sq = _theta[1]
    #-- log probability
    _p = -(_x - _mu) ** 2 / (2 * _sigma_sq) - np.log(2 * np.pi * _sigma_sq) / 2
    return _p

def normal_max_likelihood_est(_x):
    _mu = np.mean(_x)
    _sigma_sq = np.var(_x)
    _theta = [_mu, _sigma_sq]
    return _theta

def normal_log_likelihood(_x, _theta):
    _l = np.sum(normal_model_log_prob(_x, _theta))
    return _l

#-----

def model_log_prob(_x, _theta):
    return normal_model_log_prob(_x, _theta)

def max_likelihood_est(_x):
    return normal_max_likelihood_est(_x)

#-----

def log_likelihood(_x, _theta):
    _l = np.sum(model_log_prob(_x, _theta))
    return _l

def bootstrap_sample(_x):
    _n = _x.size
    _ids = np.random.randint(0, _n, _n)
    _x_ast = _x[_ids]
    return _x_ast

def EIC_bias(_x, _B):
    _D_ast = np.zeros(_B)
    for i in range(_B):
        _x_ast = bootstrap_sample(_x)
        _theta_ast = max_likelihood_est(_x_ast)
        _D_ast[i] = log_likelihood(_x_ast, _theta_ast) - log_likelihood(_x, _theta_ast)
    _b_b = np.mean(_D_ast)
    _Dvar = np.var(_D_ast)
    return [_b_b, _Dvar]

#-----

def main(T, n, B):
    t_l = np.zeros(T)
    t_bias = np.zeros(T)
    t_Dvar = np.zeros(T)

    prv_ut = time.time()
    for t in range(T):
        ut = time.time()
        if ut - prv_ut > 5.0:
            prv_ut = ut
            print("---", t, file=sys.stderr)

        #-- samples from true distribution
        x = np.random.normal(0.0, 1.0, n)

        theta = max_likelihood_est(x)
        t_l[t] = log_likelihood(x, theta)

        t_bias[t], t_Dvar[t] = EIC_bias(x, B)

    print(n, # n
          np.mean(t_bias), # bias, mean
          np.var(t_bias), # bias, variance
          np.mean(t_Dvar), # D variance, mean
          np.mean(t_l), # likelihood, mean
          )

if __name__ == '__main__':

    T = 10000

    B = 100

    for n in [25, 100, 400, 1600]:
        main(T, n, B)


import numpy as np
import math as m
from scipy.misc import derivative
import sympy as sy

def diff_uni(f, x_bar, h=10**(-6)):
    value = (f(x_bar + h)-f(x_bar - h))/((x_bar+h)-(x_bar-h))
    return value
# example:
# f = lambda x: x**2
#v = diff_uni(f,1)

def exact_diff(f, xbar,order):
    x = sy.symbols('x', real=True)
    y = f(x)
    y_prime = y.diff(x, order)
    f_prime = sy.lambdify(x, y_prime, 'numpy')
    return f_prime(xbar)

def prod(x):
        p_old = x[0]
        if len(x)>1:
            for i in range(1,len(x)):
                p_new = p_old*x[i]
                p_old = p_new
            return p_new
        else:
            return p_old
#x = [1,2,3,4]
#v = prod(x)

def fact(x):
    vec = list(range(1,x+1))
    value = prod(vec)
    return value
# factorial_4 = fact(4)

def diff_pol(x_bar, order, n):
    #f(x) = x^n
    coef = []
    for i in range(order):
        coef.append(n-i)
    produc = prod(coef)
    deriv = produc*x_bar**(n-order)
    return deriv

def Taylor_pol(x, x_bar, order, n):
    derivatives = []
    facts = []
    lambs = []
    for i in range(1,order+1):

        deriv = diff_pol(x_bar, i, n)
        derivatives.append(deriv)

        facto = fact(i)
        facts.append(facto)

        lamb = (x-x_bar)**i
        lambs.append(lamb)
    value = x_bar**n + sum(np.array(derivatives)*(np.array(lambs)/np.array(facts)))
    return value
# v = Taylor_pol(2, 1, 3, 2) tay expansion order 3 around 1 evaluated at 2 of x^2

def Taylor_any_f(x, x_bar, order, f):
    derivatives = []
    facts = []
    lambs = []
    for i in range(1,order+1):
        if i%2==0:
            deriv = derivative(f, x_bar, n=i, order=i+1)
        else:
            deriv = derivative(f, x_bar, n=i, order=i+2)

        derivatives.append(deriv)

        facto = fact(i)
        facts.append(facto)

        lamb = (x-x_bar)**i
        lambs.append(lamb)
    value = f(x_bar) + sum(np.array(derivatives)*(np.array(lambs)/np.array(facts)))
    return value

def Taylor_exact(y, x_bar, order, f):
    derivatives = []
    facts = []
    lambs = []
    for i in range(1,order+1):
        deriv = exact_diff(f, x_bar,i)
        derivatives.append(deriv)

        facto = fact(i)
        facts.append(facto)

        lamb = (y-x_bar)**i
        lambs.append(lamb)
    value = f(x_bar) + sum(np.array(derivatives)*(np.array(lambs)/np.array(facts)))
    return value

def select_node(num, grid):
    n = len(grid)
    element = (n-1)/(num-1)# n-1 because of the problem of 100 when it shoold be 99
    values = []
    for i in range(num):
        index = int(np.ceil(element*i))
        value = grid[index]
        values.append(value)
    return values

def find_theta_mono(f,nodes):

    f1 = lambda x: x**5
    f2 = lambda x: x**10

    vnode = []
    f_tils = []
    for node in nodes:
        val = f(node)
        vnode.append(val)

        f_til = [f1(node), f2(node)]
        f_tils.append(f_til)
    values_f_fp = np.array([vnode])
    values_f_fp_til = np.array(f_tils).T #2 nodes
    parameter_mat = values_f_fp@values_f_fp_til.T@np.linalg.inv(values_f_fp_til@values_f_fp_til.T) #refere tp pdf
    para_v = [parameter_mat[0,0], parameter_mat[0,1]]
    return para_v

def find_theta_test(f,nodes, order):
    #In order to avoid this, you need to save the values in variables local to the lambdas, so that they don’t rely on the value of the global x:
    #lambda x, n=i: x**(n)
    func = [lambda x, n=i: x**n for i in range(order)]
    vnode = []
    f_tils = []
    for node in nodes:
        val = f(node)
        vnode.append(val)

        f_til = [func[i](node) for i in range(len(func))]
        f_tils.append(f_til)
    values_f_fp = np.array([vnode])
    values_f_fp_til = np.array(f_tils).T #2 nodes
    parameter_mat = values_f_fp@values_f_fp_til.T@np.linalg.inv(values_f_fp_til@values_f_fp_til.T) #refere tp pdf
    para_v = [parameter_mat[0,i] for i in range(len(func))]
    return para_v

def cheb_node(a,b,num_node):
    grid = []
    for i in range(1, num_node+1):
        cheb = m.cos((2*i-1)*m.pi/(2*num_node))
        T_cheb = (a+b)/2 + (b-a)*cheb/2
        grid.append(T_cheb)
    return np.array(grid)

def chebychev_pol(order):
    Phi0 = lambda x: 1
    Phi1 = lambda x: x
    cheb_f = [Phi0, Phi1]
    for i in range(2,order):
        func = lambda x, n=i: 2*x*cheb_f[n-1](x) - cheb_f[n-2](x)
        cheb_f.append(func)
    return cheb_f

def cheby_start_end(order, start = 0):
    vec = chebychev_pol(order)
    vec_return = vec[start:]
    return vec_return #the reason I put both of them in there is because cheb defined as a recurcive function so they have to be both in there

def find_theta_cheb(f,nodes, order, start=0):
    func = cheby_start_end(order, start= start)
    vnode = []
    f_tils = []
    for node in nodes:
        val = f(node)
        vnode.append(val)

        f_til = [func[i](node) for i in range(len(func))]
        f_tils.append(f_til)
    values_f_fp = np.array([vnode])
    values_f_fp_til = np.array(f_tils).T #2 nodes
    parameter_mat = values_f_fp@values_f_fp_til.T@np.linalg.inv(values_f_fp_til@values_f_fp_til.T) #refere tp pdf
    para_v = [parameter_mat[0,i] for i in range(len(func))]
    return para_v

def chebychev_pol_2dim(order, start=0):
    Fx = cheby_start_end(order, start = start)
    Fy = cheby_start_end(order, start = start)
    pol = []
    for i in range(len(Fx)):
        for j in range(len(Fy)):
            f = lambda x, y, n=i, k=j: Fx[n](x)*Fy[k](y)
            pol.append(f)
    return pol

def find_theta_cheb_2dim(f,nodes, order, start=0):
    func = chebychev_pol_2dim(order, start=start)
    vnode = []
    f_tils = []
    for node in nodes:
        val = f(node[0], node[1])
        vnode.append(val)

        f_til = [func[i](node[0],node[1]) for i in range(len(func))]
        f_tils.append(f_til)
    values_f_fp = np.array([vnode])
    values_f_fp_til = np.array(f_tils).T #2 nodes
    parameter_mat = values_f_fp@values_f_fp_til.T@np.linalg.inv(values_f_fp_til@values_f_fp_til.T) #refere tp pdf
    para_v = [parameter_mat[0,i] for i in range(len(func))]
    return para_v

def Iso_quants(f, Fbar, num, Tol=10**(-1)):
    iso_k, iso_h = [], []
    grid_k = np.linspace(0, 10, num)
    grid_h = np.linspace(0, 10, num)
    for i in range(num):
        for j in range(num):
            residual = np.abs(f(grid_k[i], grid_h[j])-Fbar)
            if residual < Tol:
                iso_k.append(grid_k[i])
                iso_h.append(grid_h[j])
    return iso_k, iso_h


"""
  Name     : c11_16_VaR_sorting_10day.py
  Book     : Python for Finance (2nd ed.)
  Publisher: Packt Publishing Ltd.
  Author   : Yuxing Yan
  Date     : 6/6/2017
  email    : yany@canisius.edu
             paulyxy@hotmail.com
"""
import numpy as np
import pandas as pd
from scipy.stats import norm
from matplotlib.finance import quotes_historical_yahoo_ochl as getData
#
ticker='WMT'            # input 1
n_shares=500            # input 2
confidence_level=0.99   # input 3
begdate=(2000,1,1)      # input 4
enddate=(2016,12,31)    # input 5
nDays=10                # input 6
#
z=norm.ppf(confidence_level)
x=getData(ticker,begdate,enddate,asobject=True,adjusted=True)
logret = np.log(x.aclose[1:]/x.aclose[:-1])
ret = x.aclose[1:]/x.aclose[:-1]-1
position=n_shares*x.close[0]
#
# Method I
mean=np.mean(ret)
std=np.std(ret)
meanNdays=(1+mean)**nDays-1
stdNdays=std*np.sqrt(nDays)
z=norm.ppf(confidence_level)
VaR1=position*z*stdNdays
print("Holding=",position, "VaR1=", round(VaR1,0), "in ", nDays, "Days")
#
# method 2: calculate 10 day returns
ddate=[]
d0=x.date
for i in range(0,np.size(logret)):
    ddate.append(int(i/nDays))
y=pd.DataFrame(logret,index=ddate,columns=['retNdays'])
logRet=y.groupby(y.index).sum()
retNdays=np.exp(logRet)-1
#
VaR2=position*z*np.std(retNdays)
print("Holding=",position, "VaR2=", round(VaR2,0), "in ", nDays, "Days")
#
# Method III
ret2=np.sort(retNdays)
n=np.size(ret2)
leftTail=int(n*(1-confidence_level))
print(leftTail)
#
VaR3=position*ret2[leftTail]
print("Holding=",position, "VaR=", round(VaR3,0), "in ",nDays, "Days")


import numpy as np

A = np.matrix([[3.,1.,2.], [2.,3.,1.]])
print("行列Aの大きさ(M行, N列)：", np.shape(A))
print("A= \n", A)

B = np.matrix([[-1., 2., 4.], [1.,8., 6.]])
print("行列Bの大きさ(M行, N列)：", np.shape(A))
print("B= \n", B)

print("和：A+B= \n", A+B)

c = 2

print("スカラー積：cA = \n", c*A)

B = np.matrix([[4., 2.], [-1., 3.], [1., 5.]])
print("行列Bの大きさ(M行, N列)：", np.shape(A))
print("B= \n", B)

print("積：AB = \n", np.matmul(A, B))
print("積：AB = \n", np.einsum("mk,kn->mn", A, B))

B = np.matrix([[-1., 2., 4.], [1., 8., 6.]])
print("行列Bの大きさ(M行, N列)：", np.shape(A))
print("B= \n", B)

print("アダマール積：A*B= \n", np.multiply(A, B))

print("転置：A^T = \n", A.T)
print("転置：A^T = \n", np.transpose(A, axes=(1, 0)))
print("転置：A^T = \n", np.swapaxes(A, 1, 0))

A = np.matrix([[3.,1.+2.j, 2.+3.j], [2., 3.-4.j, 1.+3.j]])
print("行列Aの大きさ(M行, N列)：", np.shape(A))
print("A= \n", A)

print("エルミート転置：A^H = \n", A.H)
print("エルミート転置：A^H = \n", np.swapaxes(np.conjugate(A), 1, 0))

A = np.matrix([[3., 1.+2.j, 2.+3.j], [2., 3.-4.j, 1.+3.j]])
print("行列Aの大きさ(M行, N列)：", np.shape(A))
print("A= \n", A)

B = np.matrix([[4.+4.j, 2.-3.j], [-1.+1.j, 3.-2.j], [1.+3.j, 5.+5.j]])
print("行列Bの大きさ(M行, N列)：", np.shape(A))
print("B= \n", B)

print("行列の積に対するエルミート転置：(AB)^H = \n", (np.matmul(A, B)).H)
print("行列の積に対するエルミート転置：B^H A^H = \n", np.matmul(B.H, A.H))

I = np.eye(N=3)
print("単位行列：I = \n", I)


# coding=utf-8
"""
In England the currency is made up of pound, £, and pence, p, and there are eight coins in general circulation:

1p, 2p, 5p, 10p, 20p, 50p, £1 (100p) and £2 (200p).
It is possible to make £2 in the following way:

1×£1 + 1×50p + 2×20p + 1×5p + 1×2p + 3×1p
How many different ways can £2 be made using any number of coins?

Solution comment:

Decomposition question:
If we want to change n pennies, and we give one coin of x pennies,
how many ways can we give back the remaining n-x pennies using only
coins of x or smaller?

`ways` is an array containing how many ways you can give back i pennies,
using the coins available.

In the loop we give back a coin of x, then use the previously known
answers to find how many ways we can give back the remainder.
"""
from numpy import zeros

def coin_combinations(target, coins):
    ways = zeros(target+1, dtype=int)
    ways[0] = 1

    for coin in coins:
        for j in xrange(coin, target+1):
            ways[j] += ways[j - coin]
    return ways[-1]

target = 200
coins = [1, 2, 5, 10, 20, 50, 100, 200]
print coin_combinations(target, coins)


import numpy as np

def epsilon_add(reference:list, computed:list):
    """
     :param reference: list of reference pareto-front values, list of tuples
     :param computed: list of the computed pareto-front values, list of tuples
     :return: additive unary epsilon indicator

     .. math::
        I_{\epsilon}(A,B) = max_{x^2 \in B} min_{x^1 \in A} max_{1 \leq i \leq m} f_i(x^1) - f_i(x^2)

     [1] E. Zitzler, E. Thiele, L. Laummanns, M., Fonseca, C., and Grunert da Fonseca. V (2003):
        Performance Assessment of Multiobjective Optimizers: An Analysis and
        Review. The code is the a Java version of the original metric implementation by Eckart Zitzler.

     [2] Audet, Charles, J. Bigeon, D. Cartier, Sébastien Le Digabel, and Ludovic Salomon.
         "Performance indicators in multiobjective optimization." Optimization Online (2018).
    """

    eps = 0.0
    for ref_val in reference:
        eps_j = np.infty
        for comp_val in computed:
            eps_k = max(np.subtract(comp_val,ref_val))
            eps_j = min(eps_k, eps_j)
        eps = max(eps, eps_j)

    return eps

if __name__ == '__main__':

    # 2d test - euclidean
    ref = [(1., 1.), (2., 0.5), (4., 1. / 4.)]  ### 1/x
    calc = [(1., 1.02), (2., 0.53), (4., 0.26)]

    print(epsilon_add(ref, calc)) # 0.03