# %%
import numpy as np
import numpy.linalg as la
import time
import scipy.optimize as sciopt
from mrina.solver_l1_norm import sft, project_l1_ball, MinimizeSumOfSquares

# %%
# Projection onto the nuclear norm ball / tall matrices
def project_nuclear_ball(X, t, eps=1E-5):
    # Compute right singular vectors
    s, V = la.eigh(X.conj().T @ X)
    s = np.sqrt(np.abs(s))

    if np.sum(s) <= t:
        return X

    idx = np.where(s > eps)[0]
    V = V[:, idx]
    s = s[idx]

    st = project_l1_ball(s, t)
    return X @ ((V * (st/s) ) @ V.conj().T)

# Singular value thresholding
def svt(X, t, eps=1E-6):
    # Compute right singular vectors
    s, V = la.eigh(X.conj().T @ X)
    s = np.sqrt(np.abs(s))

    idx = np.where(s >= t)[0]
    if idx.size == 0:
        return np.zeros(shape=X.shape, dtype=X.dtype)

    idx = np.where(s > eps)[0]
    V = V[:, idx]
    s = s[idx]

    st = sft(s, t)
    return X @ ((V * (st/s) ) @ V.conj().T)

# Minimize Sum-of-Squares subject to nuclear norm constraints
#   min_X || A(X) - y ||_2^2 s.t. ||X||_* <= t
def MinimizeSumOfSquaresNuclearBall(t, y, A, Xinit=None, L=None,
                                        maxItns=1E4,
                                        dxAbsTol=1E-4, dxRelTol=1E-6,
                                        dpAbsTol=1E-5, dpRelTol=1E-8,
                                        disp=False, printEvery=10,
                                        restart=True):
    # Project initial iterate on the nuclear-ball
    if Xinit is None:
        X = np.zeros(A.inShape, dtype=np.complex)
    else:
        X = Xinit
    X = project_nuclear_ball(X, t)
    # Lipschitz constant
    if L is None:
        L = 2.05 * A.norm() ** 2
    # Variables
    rX = A.eval(X) - y
    fX = la.norm(rX.ravel()) ** 2
    gX = 2.0 * A.adjoint(rX)
    rpX = X - project_nuclear_ball(X - gX, t)
    rpXNrm = la.norm(rpX.ravel())
    # Initialize variables
    XNrm = la.norm(X.ravel())
    dXNrm = np.inf
    dfX = np.inf
    itn = 0
    s = 1.0
    Z = X
    # Optimization loop
    stop = False
    if disp:
        print('[PGD-SoS-NuclearBall]')
        print(' Initial objective:   {:1.6e}'.format(fX))
        print(' Initial residual:    {:1.6e}'.format(rpXNrm))
    while not stop:
        itn = itn + 1
        gZ = 2.0 * A.adjoint(A.eval(Z) - y)
        Xp = project_nuclear_ball(Z - gZ/L, t)
        sp = 0.5 * (1.0 + np.sqrt(1.0 + 4.0 * s ** 2))
        Zp = Xp + ((s - 1.0)/sp) * (Xp - X)

        # Update
        rXp = A.eval(Xp) - y
        fXp = la.norm(rXp.ravel()) ** 2

        dfX = fXp - fX
        dXNrm = la.norm((Xp - X).ravel())

        if restart and dfX > 1E-12:
            Z = X
            s = 1.0
            dfX = np.inf
            dXNrm = np.inf
        else:
            X = Xp
            gX = 2.0 * A.adjoint(rXp)
            rpX = X - project_nuclear_ball(X - gX, t)
            rpXNrm = la.norm(rpX.ravel())

            Z = Zp
            fX = fXp
            s = sp

        XNrm = la.norm(X.ravel())
        dXTol = np.max([ dxAbsTol, dxRelTol * XNrm ])
        rpXTol = np.max([ dpAbsTol, dpRelTol * XNrm ])
        if (dXNrm < dXTol and rpXNrm < rpXTol) or itn > maxItns:
            stop = True
        if disp and (itn == 1 or np.mod(itn, printEvery) == 0 or stop):
            print(' {:04d} | fX: {:1.3E} | dfX: {:+1.3E} | dXNrm: {:1.3E} | rpXNrm: {:5.3E}'.format(itn, fX, dfX, dXNrm, rpXNrm))
    return X, fX

# Minimize nuclear norm regularized Sum-of-Squares
#   min_X || A(X) - y ||_2^2 + t ||X||_*
def MinimizeNNDN(t, y, A, Xinit=None, L=None,
                    maxItns=1E4,
                    dxAbsTol=1E-4, dxRelTol=1E-6,
                    dpAbsTol=1E-5, dpRelTol=1E-8,
                    disp=False, printEvery=10,
                    restart=True):
    # Project initial iterate on the nuclear-ball
    if Xinit is None:
        X = np.zeros(A.inShape, dtype=np.complex)
    else:
        X = Xinit
    # Lipschitz constant
    if L is None:
        L = 2.05 * A.norm() ** 2
    # Variables
    rX = A.eval(X) - y
    fX = la.norm(rX.ravel()) ** 2 + t * la.norm(np.sqrt(la.eigvalsh(X.conj().T @ X)), 1)
    gX = 2.0 * A.adjoint(rX)
    rpX = X - svt(X - gX, t)
    rpXNrm = la.norm(rpX.ravel())
    # Initialize variables
    XNrm = la.norm(X.ravel())
    dXNrm = np.inf
    dfX = np.inf
    itn = 0
    s = 1.0
    Z = X
    # Optimization loop
    stop = False
    if disp:
        print('[PGD-NNDN]')
        print(' Initial objective:   {:1.6e}'.format(fX))
        print(' Initial residual:    {:1.6e}'.format(rpXNrm))
    while not stop:
        itn = itn + 1
        gZ = 2.0 * A.adjoint(A.eval(Z) - y)
        Xp = svt(Z - gZ/L, t/L)
        sp = 0.5 * (1.0 + np.sqrt(1.0 + 4.0 * s ** 2))
        Zp = Xp + ((s - 1.0)/sp) * (Xp - X)

        # Update
        rXp = A.eval(Xp) - y
        fXp = la.norm(rXp.ravel()) ** 2 + t * la.norm(np.sqrt(la.eigvalsh(Xp.conj().T @ Xp)), 1)

        dfX = fXp - fX
        dXNrm = la.norm((Xp - X).ravel())
        if restart and dfX > 1E-12:
            Z = X
            s = 1.0
            dfX = -np.inf
            dXNrm = np.inf
        else:
            X = Xp
            gX = 2.0 * A.adjoint(rXp)
            rpX = X - svt(X - gX, t)
            rpXNrm = la.norm(rpX.ravel())

            Z = Zp
            fX = fXp
            s = sp

        dXTol = np.max([ dxAbsTol, dxRelTol * XNrm ])
        rpXTol = np.max([ dpAbsTol, dpRelTol * XNrm ])
        if (dXNrm < dXTol and rpXNrm < rpXTol) or itn > maxItns:
            stop = True

        if disp and (itn == 1 or np.mod(itn, printEvery) == 0 or stop):
            print(' {:04d} | obj: {:1.3E} | dfx: {:+1.3E} | dxNrm: {:1.3E} | rpxNrm: {:5.3E}'.format(itn, fX, dfX, dXNrm, rpXNrm))
    return X, fX

class RootSolverNuclearNormNoisy():
    def __init__(self, eta, y, A,
                    maxItns=1E4,
                    dxAbsTol=1E-4, dxRelTol=1E-6,
                    dpAbsTol=1E-5, dpRelTol=1E-8,
                    disp=False, printEvery=10,
                    restart=True,
                    method='NNDN'):
        # Problem parameters
        self.eta = eta
        self.y = y
        self.A = A

        self.ANrm = A.norm()
        self.L = 2.05 * self.ANrm ** 2

        # For bracketing
        self._itn = 0
        self._t = [ None, None ]
        self._ft = [ None, None ]
        self._Xinit = [ None, None ]
        if method == 'SoS-NucBall':
            # Lower bound
            self._t[0] = 0.0
            self._ft[0] = la.norm(y.ravel()) - eta
            self._Xinit[0] = np.zeros(A.inShape, dtype=np.complex)
            # Upper bound
            _Xopt, _fopt = MinimizeSumOfSquares(y, A,
                                                maxItns=maxItns,
                                                dxAbsTol=dxAbsTol, dxRelTol=dxRelTol,
                                                gradAbsTol=dpAbsTol, gradRelTol=dpRelTol,
                                                disp=False)
            self._t[1] = la.norm(np.sqrt(la.eigvalsh(_Xopt.conj().T @ _Xopt)), 1)
            self._ft[1] = la.norm((A.eval(_Xopt) - y).ravel(), 2) - self.eta
            self._Xinit[1] = _Xopt
        elif method == 'NNDN':
            # Lower bound
            _Xopt, _fopt = MinimizeSumOfSquares(y, A,
                                                maxItns=maxItns,
                                                dxAbsTol=dxAbsTol, dxRelTol=dxRelTol,
                                                gradAbsTol=dpAbsTol, gradRelTol=dpRelTol,
                                                disp=False)
            self._t[0] = 0.0
            self._ft[0] = -eta
            self._Xinit[0] = _Xopt
            # Upper bound
            _Z = A.adjoint(y)
            self._t[1] = la.norm(np.sqrt(la.eigvalsh(_Z.conj().T @ _Z)), np.inf)
            self._ft[1] = la.norm(y.ravel(), 2) - self.eta
            self._Xinit[1] = np.zeros(A.inShape, dtype=np.complex)
        else:
            raise ValueError('Method can be either SoS-NucBall or NNDN.')
        self.method = method
        # Solver options
        self.restart = restart
        # Solver parameters
        self.maxItns = maxItns
        self.dxAbsTol = dxAbsTol
        self.dxRelTol = dxRelTol
        self.dpAbsTol = dpAbsTol
        self.dpRelTol = dpRelTol
        # Display options
        self.disp = disp
        self.printEvery = printEvery

    def solve(self, t):
        self._itn = self._itn + 1
        if self.disp:
            print(' {:02d} | t* = {:1.4E} in [{:1.4E}, {:1.4E}] | f: [{:+1.4E}, {:+1.4E}]'.format(self._itn, t, self._t[0], self._t[1], self._ft[0], self._ft[1]))

        if np.abs(t - self._t[0]) < np.abs(t - self._t[1]):
            tNear = 0
        else:
            tNear = 1

        theta = np.abs(t - self._t[0]) / (np.abs(t - self._t[0]) + np.abs(t - self._t[1]))
        Xinit = theta * self._Xinit[0] + (1.0 - theta) * self._Xinit[1]

        if self.method == 'SoS-NucBall':
            Xopt, _ = MinimizeSumOfSquaresNuclearBall(t, y, A, Xinit=Xinit, L=self.L,
                                                        maxItns=self.maxItns,
                                                        dxAbsTol=self.dxAbsTol, dxRelTol=self.dxRelTol,
                                                        dpAbsTol=self.dpAbsTol, dpRelTol=self.dpRelTol,
                                                        disp=False, printEvery=self.printEvery,
                                                        restart=self.restart)
        elif self.method == 'NNDN':
            Xopt, _ = MinimizeNNDN(t, y, A, Xinit=Xinit, L=self.L,
                                    maxItns=self.maxItns,
                                    dxAbsTol=self.dxAbsTol, dxRelTol=self.dxRelTol,
                                    dpAbsTol=self.dpAbsTol, dpRelTol=self.dpRelTol,
                                    disp=False, printEvery=self.printEvery,
                                    restart=self.restart)

        if t > self._t[1]:
            self._t[0] = self._t[1]
            self._ft[0] = self._ft[1]
            self._Xinit[0] = self._Xinit[1]

            self._t[1] = t
            self._ft[1] = la.norm((A.eval(Xopt) - y).ravel(), 2) - self.eta
            self._Xinit[1] = Xopt
        elif t < self._t[0]:
            self._t[1] = self._t[0]
            self._ft[1] = self._ft[0]
            self._Xinit[1] = self._Xinit[0]

            self._t[0] = t
            self._ft[0] = la.norm((A.eval(Xopt) - y).ravel(), 2) - self.eta
            self._Xinit[0] = Xopt
        else:
            self._t[tNear] = t
            self._ft[tNear] = la.norm((A.eval(Xopt) - y).ravel(), 2) - self.eta
            self._Xinit[tNear] = Xopt

        return la.norm((A.eval(Xopt) - y).ravel(), 2) - self.eta

def RecoveryNuclearNormNoisy(eta, y, A,
                    maxItns=1E4,
                    dxAbsTol=1E-4, dxRelTol=1E-6,
                    dpAbsTol=1E-5, dpRelTol=1E-8,
                    disp=False, printEvery=100,
                    method='NNDN'):

    # Create solver object
    T = RootSolverNuclearNormNoisy(eta, y, A,
                                    maxItns=maxItns,
                                    dxAbsTol=dxAbsTol, dxRelTol=dxRelTol,
                                    dpAbsTol=dpAbsTol, dpRelTol=dpRelTol,
                                    disp=disp, printEvery=printEvery,
                                    restart=True, method=method)
    if disp:
        print('[RECOVERY: NUCLEAR NORM-NOISY]' )
        print(' Method:          {:s}'.format(method) )
        print(' Bound:           {:1.4E}'.format(eta) )
        print(' Initial values:')
        print('     t           [{:1.4E},   {:1.4E}]'.format(T._t[0], T._t[1]))
        print('     f(t)        [{:+1.4E}, {:+1.4E}]'.format(T._ft[0], T._ft[1]))
    # Root finding via TOMS748
    tStart = time.time()
    tMin = T._t[0]
    tMax = T._t[1]
    topt, rnfo = sciopt.toms748(lambda t: T.solve(t), tMin, tMax, xtol=5E-4, full_output=disp, disp=disp)
    if np.abs(topt - T._t[0]) < np.abs(topt - T._t[1]):
        Xopt = T._Xinit[0]
    else:
        Xopt = T._Xinit[1]
    tStop = time.time() - tStart
    if disp:
        print('  Summary');
        print('      Iterations:          {:d}'.format(rnfo.iterations) )
        print('      Function calls:      {:d}'.format(rnfo.function_calls) )
        print('      Optimal value:       {:5.3E}'.format(topt) )
        print('      Elapsed time:        {:8.4f} seconds'.format(tStop) )
    return Xopt, la.norm(Xopt.ravel(), 1)

# %%
if __name__ == '__main__':
    # Import
    from maps import OperatorFourierLowRank

    print('[TESTING SOLVER ROUTINES]')
    # *** EXPERIMENTS FOR SINGLE IMAGE AND SUBSAMPLING - L1-NORM
    # Set up map
    imShape = (128, 128)
    samplingSet = np.where(np.random.uniform(size=imShape) < 0.75, True, False)
    A = OperatorFourierLowRank(imShape, samplingSet=samplingSet)
    xinit = np.random.normal(size=A.inShape) + 1j * np.random.normal(size=A.inShape)
    y = A.eval(xinit)
    print('[SINGLE IMAGE - RANDOM UNDERSAMPLING - L1-NORM RECOVERY]')
    print('    Input shape:    {:d} x {:d}'.format(A.inShape[0], A.inShape[1]))
    print('    Output shape:   {:d} x {:d}'.format(A.outShape[0], 1))
    print('    Operator norm:  {:1.3e}'.format(A.norm()))
    # Tests
    print('\n[TEST: MinimizeSumOfSquares]')
    xsol, fmin = MinimizeSumOfSquares(y, A, disp=True)
    print('\n[TEST: MinimizeSumOfSquaresNuclearBall]')
    print('  ---> with no restart...')
    xsol, fmin = MinimizeSumOfSquaresNuclearBall(1.0, y, A, disp=True, restart=False, printEvery=10)
    print('  ---> with restart...')
    _xsol, _fmin = MinimizeSumOfSquaresNuclearBall(1.0, y, A, disp=True, restart=True)
    print('  ---> Difference:     {:1.5e}'.format(la.norm((xsol - _xsol).ravel())))
    print('\n[TEST: RecoveryL1Noisy - SoS-NucBall]')
    xsol, fmin = RecoveryNuclearNormNoisy(2.0, y, A, maxItns=1E4, disp=True, printEvery=100, method='SoS-NucBall')
    print('\n[TEST: RecoveryL1Noisy - NNDN]')
    _xsol, _fmin = RecoveryNuclearNormNoisy(2.0, y, A, maxItns=1E4, disp=True, printEvery=100, method='NNDN')
    print('  ---> Difference:')
    print('         Objective:    {:1.5e}'.format(np.abs(fmin - _fmin)))
    print('         Solution:     {:1.5e}'.format(la.norm((xsol - _xsol).ravel())))

# %%


# used for manipulating directory paths
import os

# Scientific and vector computation for python
import numpy as np

def computeCostMulti(X, y, theta):
    """
    Compute cost for linear regression with multiple variables.
    Computes the cost of using theta as the parameter for linear regression to fit the data points in X and y.

    Parameters
    ----------
    X : array_like
        The dataset of shape (m x n+1).

    y : array_like
        A vector of shape (m, ) for the values at a given data point.

    theta : array_like
        The linear regression parameters. A vector of shape (n+1, )

    Returns
    -------
    J : float
        The value of the cost function.

    Instructions
    ------------
    Compute the cost of a particular choice of theta. You should set J to the cost.
    """
    # Initialize some useful values
    m = y.shape[0]  # number of training examples

    # You need to return the following variable correctly
    J = 0

    J = (np.dot((np.transpose(np.dot(X, theta)) - y), (np.dot(X, theta) - y))) / (2 * m)

    return J

if __name__ == '__main__':
    data = np.loadtxt(os.path.join("/home/thelichking/Desktop/ml-coursera-python-assignments/Exercise1/Data",
                                   "ex1data2.txt"), delimiter=',')
    X = data[:, :3]  # the first two rows
    y = data[:, 2]  # the actual 3rd row
    m = y.size

    print(computeCostMulti(X,y,theta=np.array([-1, 2, 3])))


import unittest
import numpy as np
from nptest import nptest

class StatisticsTests(unittest.TestCase):

    #region amin/amax
    def test_amin_1(self):

        a = np.arange(1,5).reshape((2,2))
        print(a)
        print("*****")

        b = np.amin(a)           # Minimum of the flattened array
        print(b)
        print("*****")

        c = np.amin(a, axis=0)   # Minima along the first axis
        print(c)
        print("*****")

        d = np.amin(a, axis=1)   # Minima along the second axis
        print(d)
        print("*****")

        e = np.arange(5, dtype=float)
        e[2] = np.NaN
        f = np.amin(e)
        print(f)
        print("*****")

        g = np.nanmin(b)
        print(g)

    def test_amin_2(self):

        a = np.arange(30.25,46.25).reshape((4,4))
        print(a)
        print("*****")

        b = np.amin(a)           # Minimum of the flattened array
        print(b)
        print("*****")

        c = np.amin(a, axis=0)   # Minima along the first axis
        print(c)
        print("*****")

        d = np.amin(a, axis=1)   # Minima along the second axis
        print(d)
        print("*****")

        e = np.arange(5, dtype=float)
        e[2] = np.NaN
        f = np.amin(e)
        print(f)
        print("*****")

        g = np.nanmin(b)
        print(g)

    def test_amax_1(self):

        a = np.arange(4).reshape((2,2))
        print(a)
        print("*****")

        b = np.amax(a)           # Maximum of the flattened array
        print(b)
        print("*****")

        c = np.amax(a, axis=0)   # Maxima along the first axis
        print(c)
        print("*****")

        d = np.amax(a, axis=1)   # Maxima along the second axis
        print(d)
        print("*****")

        e = np.arange(5, dtype=float)
        e[2] = np.NaN
        f = np.amax(e)
        print(f)
        print("*****")

        g = np.nanmax(b)
        print(g)

    def test_amax_2(self):

        a = np.arange(30.25,46.25).reshape((4,4))
        print(a)
        print("*****")

        b = np.amax(a)           # Maximum of the flattened array
        print(b)
        print("*****")

        c = np.amax(a, axis=0)   # Maxima along the first axis
        print(c)
        print("*****")

        d = np.amax(a, axis=1)   # Maxima along the second axis
        print(d)
        print("*****")

        e = np.arange(5, dtype=float)
        e[2] = np.NaN
        f = np.amax(e)
        print(f)
        print("*****")

        g = np.nanmax(b)
        print(g)
    #endregion

    #region nanmin/nanmax
        # see NANFunctionsTests
    #endregion

    #region ptp
    def test_ptp_1(self):

        a = np.arange(4).reshape((2,2))
        print(a)
        print("*****")

        b = np.ptp(a, axis=0)
        print(b)
        print("*****")

        c = np.ptp(a, axis=1)
        print(c)

        d = np.ptp(a)
        print(d)

    #endregion

    #region percentile/quantile

    def test_percentile_1(self):

        a = np.array([[10.0, 7.0, 4.0], [3.0, 2.0, 1.0]])

        b = np.percentile(a, 50)
        print(b)

        c = np.percentile(a, 50, axis=0)
        print(c)

        d = np.percentile(a, 50, axis=1)
        print(d)

        e = np.percentile(a, 50, axis=1, keepdims=True)
        print(e)

        m = np.percentile(a, 50, axis=0)
        n = np.zeros_like(m)
        o = np.percentile(a, 50, axis=0, out=n)
        print(o)
        print(n)

        b = a.copy()
        c = np.percentile(b, 50, axis=1, overwrite_input=True)
        print(c)

        assert not np.all(a == b)

        return

    def test_percentile_2(self):

        a = np.array([[10, 7, 4], [3, 2, 1]])

        b = np.percentile(a, [50, 75])
        print(b)

        c = np.percentile(a, [50, 75], axis=0)
        print(c)

        d = np.percentile(a, [50, 75], axis=1)
        print(d)

        e = np.percentile(a, [50, 75], axis=1, keepdims=True)
        print(e)

        m = np.percentile(a, [50, 75], axis=0)
        n = np.zeros_like(m)
        o = np.percentile(a, [50, 75], axis=0, out=n)
        print(o)
        print(n)

        b = a.copy()
        c = np.percentile(b, [50, 75], axis=1, overwrite_input=True)
        print(c)

        assert not np.all(a == b)

        return

    def test_quantile_1(self):

        a = np.array([[10, 7, 4], [3, 2, 1]])

        b = np.quantile(a, 0.5)
        print(b)

        c = np.quantile(a, 0.5, axis=0)
        print(c)

        d = np.quantile(a, 0.5, axis=1)
        print(d)

        e = np.quantile(a, 0.5, axis=1, keepdims=True)
        print(e)

        m = np.quantile(a, 0.5, axis=0)
        out = np.zeros_like(m)
        np.quantile(a, 0.5, axis=0, out=out)

        print(out)
        print(m)

        b = a.copy()
        c = np.quantile(b, 0.5, axis=1, overwrite_input=True)
        print(c)

        assert not np.all(a == b)
        return

    #endregion

    #region nanpercentile/nanquantile
        # see NANFunctionsTests
    #endregion

    #region median/average/mean

    def test_median_1(self):

        a = np.array([[10, 7, 4], [3, 2, 1]])

        b = np.median(a)
        print(b)

        c = np.median(a, axis=0)
        print(c)

        d = np.median(a, axis=1)
        print(d)

        m = np.median(a, axis=0)
        out = np.zeros_like(m)
        n = np.median(a, axis=0, out=m)
        print(n)
        print(m)

        b = a.copy()
        c = np.median(b, axis=1, overwrite_input=True)
        print(c)

        assert not np.all(a==b)

        b = a.copy()
        c = np.median(b, axis=None, overwrite_input=True)
        print(c)

        assert not np.all(a==b)

        return

    def test_median_2(self):

        shape = [1,2,3,4,5,6,7,8]
        shape2 = shape[:4]

        a = np.arange(0,64,1).reshape(4,4,4)
        #nd = a.ndim
        #axis = [0,2]

        #keep = set(range(nd)) - set(axis)
        #nkeep = len(keep)
        ## swap axis that should not be reduced to front
        #for i, s in enumerate(sorted(keep)):
        #    a = a.swapaxes(i, s);
        ## merge reduced axis
        #a = a.reshape(a.shape[:nkeep] + (-1,))

        #keepdim = tuple(keepdim)

        b = np.median(a,axis= [0,2], keepdims = True)
        print(b)

        c = np.median(a, axis= [0,1], keepdims = True)
        print(c)

        d = np.median(a, axis=[1,2], keepdims = True)
        print(d)

        return

    def test_median_3(self):

        a = np.array([[10.0, 7.2, 4.2], [3.2, 2.2, 1.2]])

        b = np.median(a)
        print(b)

        c = np.median(a, axis=0)
        print(c)

        d = np.median(a, axis=1)
        print(d)

        m = np.median(a, axis=0)
        out = np.zeros_like(m)
        n = np.median(a, axis=0, out=m)
        print(n)
        print(m)

        b = a.copy()
        c = np.median(b, axis=1, overwrite_input=True)
        print(c)

        assert not np.all(a==b)

        b = a.copy()
        c = np.median(b, axis=None, overwrite_input=True)
        print(c)

        assert not np.all(a==b)

        return

    def test_average_1(self):

        x = np.array([10,15,25,45,78,90,10,15,25,45,78,90], dtype= np.uint32).reshape(3, 2, -1)
        x = x * 3
        y = np.average(x);

        print(x)
        print(y)

        return

    def test_average_2(self):

        a = np.array([1,2,3,4,5,6,7,8,9,10]);
        w = [10,9,8,7,6,5,4,3,2,1]

        x = np.average(a)
        print(x)
        print("********")

        x = np.average(a, weights=w)
        print(x)
        print("********")

        x = np.average(a.reshape(2,-1), weights=np.array(w).reshape(2,-1))
        print(x)
        print("********")

        x = np.average(a.reshape(2,-1), axis = 0, weights=np.array(w).reshape(2,-1))
        print(x)
        print("********")

        x = np.average(a.reshape(2,-1), axis = 1, weights=np.array(w).reshape(2,-1))
        print(x)
        print("********")

        x = np.average(a.reshape(1,2, -1,1), axis = 1, weights=np.array(w).reshape(1,2, -1,1))
        print(x)
        print("********")

        x = np.average(a.reshape(1,-1,2,1), axis = 1, weights=np.array(w).reshape(1,-1,2,1))
        print(x)
        print("********")

        x = np.average(a.reshape(2,-1,1,1), axis = 1, weights=np.array(w).reshape(2,-1,1,1))
        print(x)

        return

    def test_average_3(self):

        a = np.array([1,2,3,4,5,6,7,8,9,10]);
        w = [10,9,8,7,6,5,4,3,2,1]

        x = np.average(a, returned = True)
        print(x)
        print("********")

        x = np.average(a, weights=w, returned = True)
        print(x)
        print("********")

        x = np.average(a.reshape(2,-1), weights=np.array(w).reshape(2,-1), returned = True)
        print(x)
        print("********")

        x = np.average(a.reshape(2,-1), axis = 0, weights=np.array(w).reshape(2,-1), returned = True)
        print(x)
        print("********")

        x = np.average(a.reshape(2,-1), axis = 1, weights=np.array(w).reshape(2,-1), returned = True)
        print(x)
        print("********")

        x = np.average(a.reshape(1,2, -1,1), axis = 1, weights=np.array(w).reshape(1,2, -1,1), returned = True)
        print(x)
        print("********")

        x = np.average(a.reshape(1,-1,2,1), axis = 1, weights=np.array(w).reshape(1,-1,2,1), returned = True)
        print(x)
        print("********")

        x = np.average(a.reshape(2,-1,1,1), axis = 1, weights=np.array(w).reshape(2,-1,1,1), returned = True)
        print(x)

        return

    def test_mean_1(self):

        x = np.array([10,15,25,45,78,90,10,15,25,45,78,90], dtype= np.uint32).reshape(3, 2, -1)
        x = x * 3
        print(x)

        y = np.mean(x);
        print(y)

        y = np.mean(x, axis=0);
        print(y)

        y = np.mean(x, axis=1);
        print(y)

        y = np.mean(x, axis=2);
        print(y)

        return

    def test_mean_2(self):

        a = np.zeros((2, 512*512), dtype=np.float32)
        a[0, :] = 1.0
        a[1, :] = 0.1
        b = np.mean(a)
        print(b)

        c = np.mean(a, dtype=np.float64)
        print(c)

    #endregion

    #region std/var

    def test_std_1(self):

        a = np.array([[1, 2], [3, 4]])
        b = np.std(a)
        print(b)

        c = np.std(a, axis=0)
        print(c)

        d = np.std(a, axis=1)
        print(d)

        #In single precision, std() can be inaccurate:

        a = np.zeros((2, 512*512), dtype=np.float32)
        a[0, :] = 1.0
        a[1, :] = 0.1
        b = np.std(a)
        print(b)

        # Computing the standard deviation in float64 is more accurate:

        c = np.std(a, dtype=np.float64)
        print(c)

    def test_var_1(self):

        a = np.array([[1, 2], [3, 4]])
        b = np.var(a)
        print(b)

        c = np.var(a, axis=0)
        print(c)

        d = np.var(a, axis=1)
        print(d)

        #In single precision, std() can be inaccurate:

        a = np.zeros((2, 512*512), dtype=np.float32)
        a[0, :] = 1.0
        a[1, :] = 0.1
        b = np.var(a)
        print(b)

        # Computing the standard deviation in float64 is more accurate:

        c = np.var(a, dtype=np.float64)
        print(c)

    #endregion

    #region nanmedian/nanmean
        # see NANFunctionsTests
    #endregion

    #region nanstd/nanvar
        # see NANFunctionsTests
    #endregion

    #region Correlating

    def test_correlate_1(self):

        a = np.correlate([1, 2, 3], [0, 1, 0.5])
        print(a)

        b = np.correlate([1, 2, 3], [0, 1, 0.5], "same")
        print(b)

        c = np.correlate([1, 2, 3], [0, 1, 0.5], "full")
        print(c)

        return

    def test_correlate_BIGINT(self):

        a = np.correlate([1, 2, 3], [0, 1, 5])
        print(a)

        b = np.correlate([1, 2, 3], [0, 1, 5], "same")
        print(b)

        c = np.correlate([1, 2, 3], [0, 1, 5], "full")
        print(c)

        return

    def test_cov_1(self):

        x1 = np.array([[0, 2], [1, 1], [2, 0]]).T
        print(x1)

        # Note how  increases while  decreases. The covariance matrix shows this clearly:

        a = np.cov(x1)
        print(a)

        x = [-2.1, -1,  4.3]
        y = [3,  1.1,  0.12]
        X = np.stack((x, y), axis=0)
        a = np.cov(X)
        print(a)

        b = np.cov(x, y)
        print(b)

        c = np.cov(x)
        print(c)

        d = np.cov(X, rowvar=False)
        print(d)

        e = np.cov(X, rowvar=False, bias=True)
        print(e)

        f = np.cov(X, rowvar=False, bias=True, fweights = [1,2])
        print(f)

        g = np.cov(X, rowvar=False, bias=True, fweights = [1,2], aweights = [1,2])
        print(g)

        return

    def test_cov_1_BIGINT(self):

        x1 = np.array([[0, 2], [1, 1], [2, 0]]).T
        print(x1)

        # Note how  increases while  decreases. The covariance matrix shows this clearly:

        a = np.cov(x1)
        print(a)

        x = [-21, -1,  43]
        y = [3,  11,  12]
        X = np.stack((x, y), axis=0)
        a = np.cov(X)
        print(a)

        b = np.cov(x, y)
        print(b)

        c = np.cov(x)
        print(c)

        d = np.cov(X, rowvar=False)
        print(d)

        e = np.cov(X, rowvar=False, bias=True)
        print(e)

        f = np.cov(X, rowvar=False, bias=True, fweights = [1,2])
        print(f)

        g = np.cov(X, rowvar=False, bias=True, fweights = [1,2], aweights = [1,2])
        print(g)

        return

    def test_corrcoef_1(self):

        x1 = np.array([[0, 2], [1, 1], [2, 0]]).T
        print(x1)

        # Note how  increases while  decreases. The covariance matrix shows this clearly:

        a = np.corrcoef(x1)
        print(a)

        x = [-2.1, -1,  4.3]
        y = [3,  1.1,  0.12]
        X = np.stack((x, y), axis=0)
        a = np.corrcoef(X)
        print(a)

        b = np.corrcoef(x, y)
        print(b)

        c = np.corrcoef(x,y, rowvar=False)
        print(c)

        return

    #endregion

    #region Histograms

    #endregion

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python
# coding: utf-8

# # Linear regression using scikit-learn
#
# In the previous notebook, we presented the parametrization of a linear model.
# During the exercise, you saw that varying parameters will give different models
# that will fit better or worse the data. To evaluate quantitatively this
# goodness of fit, you implemented a so-called metric.
#
# When doing machine learning, you are interested in selecting the model which
# will minimize the error on the data available the most.
# From the previous exercise, we could implement a brute-force approach,
# varying the weights and intercept and select the model with the lowest error.
#
# Hopefully, this problem of finding the best parameters values (i.e. that
# result in the lowest error) can be solved without the need to check every
# potential parameter combination. Indeed, this problem has a closed-form
# solution: the best parameter values can be found by solving an equation. This
# avoids the need for brute-force search. This strategy is implemented in
# scikit-learn.

# In[1]:

import pandas as pd

penguins = pd.read_csv("../datasets/penguins_regression.csv")
feature_names = "Flipper Length (mm)"
target_name = "Body Mass (g)"
data, target = penguins[[feature_names]], penguins[target_name]

#

#
Note

#
If you want a deeper overview regarding this dataset, you can refer to the
# Appendix - Datasets description section at the end of this MOOC.

#

# In[2]:

from sklearn.linear_model import LinearRegression

linear_regression = LinearRegression()
linear_regression.fit(data, target)

# The instance `linear_regression` will store the parameter values in the
# attributes `coef_` and `intercept_`. We can check what the optimal model
# found is:

# In[3]:

weight_flipper_length = linear_regression.coef_[0]
weight_flipper_length

# In[4]:

intercept_body_mass = linear_regression.intercept_
intercept_body_mass

# We will use the weight and intercept to plot the model found using the
# scikit-learn.

# In[5]:

import numpy as np

flipper_length_range = np.linspace(data.min(), data.max(), num=300)
predicted_body_mass = (
    weight_flipper_length * flipper_length_range + intercept_body_mass)

# In[6]:

import matplotlib.pyplot as plt
import seaborn as sns

sns.scatterplot(x=data[feature_names], y=target, color="black", alpha=0.5)
plt.plot(flipper_length_range, predicted_body_mass)
_ = plt.title("Model using LinearRegression from scikit-learn")

# In the solution of the previous exercise, we implemented a function to
# compute the goodness of fit of a model. Indeed, we mentioned two metrics: (i)
# the mean squared error and (ii) the mean absolute error. These metrics are
# implemented in scikit-learn and we do not need to use our own implementation.
#
# We can first compute the mean squared error.

# In[7]:

from sklearn.metrics import mean_squared_error

inferred_body_mass = linear_regression.predict(data)
model_error = mean_squared_error(target, inferred_body_mass)
print(f"The mean squared error of the optimal model is {model_error:.2f}")

# A linear regression model minimizes the mean squared error on the training
# set. This means that the parameters obtained after the fit (i.e. `coef_` and
# `intercept_`) are the optimal parameters that minimizes the mean squared
# error. In other words, any other choice of parameters will yield a model with
# a higher mean squared error on the training set.
#
# However, the mean squared error is difficult to interpret. The mean absolute
# error is more intuitive since it provides an error in the same unit as the
# one of the target.

# In[8]:

from sklearn.metrics import mean_absolute_error

model_error = mean_absolute_error(target, inferred_body_mass)
print(f"The mean absolute error of the optimal model is {model_error:.2f} g")

# A mean absolute error of 313 means that in average, our model make an error
# of +/- 313 grams when predicting the body mass of a penguin given its flipper
# length.

# In this notebook, you saw how to train a linear regression model using
# scikit-learn.


import torch
import numpy as np

def euclidean_distance(x1, x2):
    """Squared distance

    We are ignoring the sqrt here to since it
    does not affect k-means, and we gain a bit
    of speed.

    Note:
        While k-means is unaffected by using the
        squared distance, other clustering
        algorithms, like hierarchical clustering,
        are affected. Instead, you should use the
        regular Euclidean distance.
    """
    return np.sum(np.square(x1 - x2))

class Kmeans:
    """Clustering around k centroids"""

    def __init__(self, k=2, num_iterations=100):
        self.k = k
        self.num_iterations = num_iterations

    def initialize_centroids(self, data):
        """Randomly initialize centroids

        Randomly choose points in our data to be the
        positions of our centroids
        """
        num_samples, _ = data.shape

        idx = np.random.choice(
            num_samples, (self.k,), replace=False
        )

        return data[idx, :]

    def nearest_centroid(self, sample, centroids):
        """Find the nearest centroid for a data point

        Args:
            sample: np.array of shape (1, num_features),
                    a single data point
            centroids: current centroid positions
        """
        nearest = None
        nearest_distance = np.inf

        for idx, centroid in enumerate(centroids):
            distance = euclidean_distance(sample, centroid)
            if distance < nearest_distance:
                nearest = idx
                nearest_distance = distance

        return nearest

    def assign_clusters(self, data, centroids):
        """Assign the data to the nearest centroids"""
        num_samples, _ = data.shape

        clusters = [[] for _ in range(self.k)]

        for idx, sample in enumerate(data):
            nearest_centroid = self.nearest_centroid(sample, centroids)
            clusters[nearest_centroid].append(idx)

        return clusters

    def cluster_assignments(self, data, clusters):
        """Get which cluster each sample is assigned to"""
        num_samples, _ = data.shape

        assignments = np.zeros(num_samples)
        for idx, cluster in enumerate(clusters):
            for sample in cluster:
                assignments[sample] = idx

        return assignments

    def update_centroids(self, data, clusters):
        """Set the centroids to be the mean of the samples in each cluster"""
        _, num_features = data.shape

        centroids = np.zeros((self.k, num_features))

        for idx, cluster in enumerate(clusters):
            centroid = np.mean(data[cluster], axis=0)
            centroids[idx] = centroid

        return centroids

    def predict(self, data):
        """K-means clustering over data"""
        centroids = self.initialize_centroids(data)

        for idx in range(self.num_iterations):
            clusters = self.assign_clusters(data, centroids)
            centroids = self.update_centroids(data, clusters)

        cluster_assignments = self.cluster_assignments(data, clusters)

        return cluster_assignments, centroids

def kmeans_save(data, target, predictions, centroids, path):
    """Save both the data and our predictions"""
    torch.save({
        "data": data,
        "target": target,
        "predictions": predictions,
        "centroids": centroids
    }, path)

if __name__ == "__main__":
    from sklearn import datasets

    data, _ = datasets.make_blobs(n_features=10)
    print(f"blobs: {data.shape}")

    model = Kmeans(k=10)
    assignments, _ = model.predict(data)

    print(assignments.shape)
    print(assignments)


"""
newton_algorithm.py

Returns the minimizer of the function
X0 - initial guess
func - anonimous function
func_1st_der - anonimous function 1st derivative
func_2nd_der - anonimous function 2nd derivative
N_iter_max - max number of iterations
"""

import numpy as np

def newton_algorithm(X0, func, func_1st_der, func_2nd_der, options):

    epsilon = 10e-6
    report = {};
    N_iter_max = options['N_iter_max'];
    tolerance_x = options['tolerance_x'];
    tolerance_y = options['tolerance_y'];
    progress_x = np.zeros((1, N_iter_max + 1));
    progress_y = np.zeros((1, N_iter_max + 1));
    progress_x[:, [0]] = X0;
    progress_y[0, [0]] = func(X0);
    X_old = X0;

    for iter_no in range(1, N_iter_max + 1):
        grad = func_1st_der(X_old);

        if (np.linalg.norm(grad) < epsilon):
            print('norm(grad) < epsilon in %d iterations, exit..' % (iter_no));
            break;

        X = X_old - grad / func_2nd_der(X_old);
        progress_x[:, [iter_no]] = X;
        progress_y[0, [iter_no]] = func(X);

        if (np.abs(X - X_old) < tolerance_x * np.abs(X_old)):
            print('Tolerance in X is reached in %d iterations, exit..' % (iter_no));
            break;

        if (np.abs(progress_y[0, [iter_no]] - progress_y[0, [iter_no - 1]]) < tolerance_y * np.abs(progress_y[0, [iter_no - 1]])):
            print('Tolerance in Y is reached in %d iterations, exit..' % (iter_no));
            break;

        X_old = X;

    report = {'N_iter_max' : N_iter_max, 'iter_no' : iter_no, 'X0' : X0, 'X' : X, 'progress_x' : progress_x, 'progress_y' : progress_y};
    return (X, report);


# Fit linear and quadratic surfaces to data
# Based on https://github.com/probml/pmtk3/blob/master/demos/surfaceFitDemo.m

import superimport

import numpy as np
import matplotlib.pyplot as plt
import pyprobml_utils as pml
from scipy.io import loadmat
import requests
from io import BytesIO
from mpl_toolkits.mplot3d import Axes3D

url = 'https://raw.githubusercontent.com/probml/probml-data/main/data/moteData/moteData.mat'
response = requests.get(url)
#rawdata = response.text
rawdata = BytesIO(response.content)
data = loadmat(rawdata)

X = data['X']
y = data['y']

n = len(y)
X_pad = np.column_stack((np.ones(n), X))

for use_quad in (False, True):
  phi = X_pad

  if use_quad:
    phi = np.column_stack((X_pad, X**2))

  fig = plt.figure()
  ax = Axes3D(fig)
  ax.set_zlim(15, 19)
  ax.scatter(X[:,0], X[:,1], y, color='r')

  n = 10
  xrange = np.linspace(min(X[:,0]), max(X[:,0]), n)
  yrange = np.linspace(min(X[:,1]), max(X[:,1]), n)
  xx, yy = np.meshgrid(xrange, yrange)
  flatxx = xx.reshape((n**2, 1))
  flatyy = yy.reshape((n**2, 1))
  w = np.linalg.lstsq(phi, y)[0]

  z = np.column_stack((flatxx, flatyy))
  z = np.column_stack((np.ones(n**2), z))
  if use_quad:
    z = np.column_stack((z, flatxx**2, flatyy**2))

  f = np.dot(z , w)
  ax.plot_surface(xx, yy, f.reshape(n, n),
                  rstride=1, cstride=1, cmap='jet')

  name = 'linregSurfaceLinear.pdf'
  if use_quad:
    name = 'linregSurfaceQuad.pdf'

  pml.savefig(name)
  plt.show()


# Central Limit Theorem
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] = (20,10)
from distributionLib import Dist
import os
import seaborn as sns
import warnings
warnings.filterwarnings("ignore")
np.random.seed(404)

N      = 20
p      = 0.8
N5,N25 = 5,25
temp5  = []
temp25 = []

data = Dist.binomialDist(N,p)
data = np.array(data)

xBar = np.mean(data)
var  = np.var(data)
sd   = np.sqrt(var)

plt.subplot(2,2,(1,3))
plt.plot([0,len(data)-1],[xBar,xBar],color='k',linestyle='--',label='mean={:.4f}'.format(xBar))
plt.bar(np.arange(len(data)),data)
plt.title('Random Distribution')
plt.legend()
plt.grid()

ax5 = plt.subplot(2,2,2)
ax25 = plt.subplot(2,2,4)

filenames = []
for ii in range(500):
    ax5.clear()
    ax25.clear()

    ind5  = np.random.randint(0, len(data), size=(N5,1))
    ind25 = np.random.randint(0, len(data), size=(N25,1))
    tempo5, tempo25 = np.mean(data[ind5]),np.mean(data[ind25])

    temp5.append(np.round(tempo5,4))
    temp25.append(np.round(tempo25,4))

    plt.subplot(2,2,2)
    sns.distplot(temp5,label='{} Samples mean={:.4f}'.format(N5,np.mean(temp5)))#, fit='norm', kde=False)
    plt.title('{} samples'.format(N5))
    plt.legend()
    plt.grid()

    plt.subplot(2,2,4)
    sns.distplot(temp25,label='{} Samples mean={:.4f}'.format(N25,np.mean(temp25)))
    plt.legend()
    plt.grid()
    plt.title('{} samples'.format(N25))

    plt.pause(0.0001)
    plt.draw()
    if not ii%10:
        filename = f'{ii}.png'
        filenames.append(filename)
        plt.savefig(filename)
plt.close()
Dist.gif(filenames,'CLF.gif')

print("{} samples mean = {:.4f}".format(N5,np.mean(temp5)))
print("{} samples mean = {:.4f}".format(N25,np.mean(temp25)))


"""
Example usage of functions provided by beam module.

Simply supported beam of total length 1.2m with rectangular
cross-section.

The loads are:
* Constant continuous load q = 50kN/m between coordinates
  a = 0.5m and a+b = 0.8m.
* Two point loads F = -20kN (i.e. in the upwards direction)
  at a = 0.5m and a+b = 0.8m.

Dimensions of the cross-section (height H and width B) are to be determined so that maximum stress is 120MPa. H/B = 2 holds.

Deflection and angle of deflection are to be determined at point
C at the location a+b=0.8m.
"""
import os, sys
import numpy as np
import matplotlib.pyplot as plt
sys.path.insert(1, os.path.join(sys.path[0], '..'))
from beam import Beam, ConstantContinuousLoad, PointLoad

# known quantities
q = 50e3
F = 20e3
a, b, c = .5, .3, .4
C = a + b
E = 2e11
sigD = 120e6

### computation of reactions: ###

# loads - list of known loads
loads = [
    ConstantContinuousLoad(q, a, a+b),
    PointLoad(-F, a),
    PointLoad(-F, a+b)
]
# reactions - list of unknown loads
#  All must have magnitude = 1, otherwise the results of
#  Beam.get_reactions would be wrong.
reactions = [PointLoad(1, 0), PointLoad(1, a+b+c)]

# Initialize the beam object.
# For now we don't need the actual value of Jz since we are
# going to compute the actual dimensions based on the values
# of reaction forces.
beam = Beam(loads, reactions,
            l = a+b+c,
            E = E,
            Jz = 1.,
)

Rs = beam.get_reactions()
print('reactions:\n R_A = {:.2e}N\n R_B = {:.2e}N'.format(Rs[0], Rs[1]))

### computation and plots of bending force and moment ###
n = 12*5+1
x = np.linspace(0, a+b+c, n)

plt.plot(x, [beam.force(xi) for xi in x], '.-', label='force')
Mo = np.array([beam.moment(xi) for xi in x])
plt.plot(x, Mo, '.-', label='moment')

plt.legend(loc='best')
plt.grid()

### actual dimensions of the cross-section ###

# sigma = Mmax/Wo = Mmax/(Jz/(H/2)) <= sigD
# Jz >= H/2*Mmax/sigD
#   [rectangular cross-section]
#   Jz = 1/12*B*H**3 = (H/B=2) =
#      = 1/24*H**4
# H**4 >= 24*H/2*Mmax/sigD
# H >= (12*Mmax/sigD)**(1/3)

Mmax = max(abs(Mo))
print('Mmax = {:.4}N'.format(Mmax))

H = (12*Mmax/sigD)**(1./3)
print('H = {:.4}m'.format(H))
print('B = {:.4}m'.format(.5*H))

Jz = H**4/24
beam.Jz = Jz

### computation and plots of deflection and angle ###

v = np.array([beam.deflection(xi) for xi in x]).T
v_C = beam.deflection(C)
print('v_C = {:.4}m\nphi_C = {:.4}rad'.format(v_C[0], v_C[1]))

plt.figure()
plt.plot(x, v[0], '.-', label='deflection')
plt.plot(x, v[1], '.-', label='angle')
plt.grid()
plt.legend(loc='best')
plt.show()


import numpy as np
import pandas as pd

def preprocess(path):
    dataset = pd.read_csv(path, names=['x1', 'x2', 'x3', 'x4', 'y'], delim_whitespace=True)
    dataset.insert(0, 'x0', 1)
    training_set = dataset.iloc[:, :-1].values
    labels_set = dataset.iloc[:, -1].values
    return training_set, labels_set

def sign(x):
    if x <= 0:
        return -1
    else:
        return 1

def pla(training_set, labels_set):
    update_count = 0
    correct_count = 0
    weights = np.zeros(training_set.shape[1])
    converge = False
    while not converge:
        for i in range(training_set.shape[0]):
            x = training_set[i]
            y = labels_set[i]
            if sign(np.dot(weights, x)) != y:
                weights = weights + y * x
                update_count += 1
                correct_count = 0
            else:
                correct_count += 1
                if correct_count == labels_set.shape[0]:
                    converge = True
                    break
    return weights, update_count

training_set, labels_set = preprocess("train.dat")
weights, count = pla(training_set, labels_set)
print("Number of Updates:", count)

def unison_shuffled_copies(arr1, arr2):
    assert len(arr1) == len(arr2)
    p = np.random.permutation(len(arr1))
    return arr1[p], arr2[p]

def pla_random(training_set, labels_set, times_repeat):
    total = 0
    for _ in range(times_repeat):
        xs, ys = unison_shuffled_copies(training_set, labels_set)
        _, count = pla(xs, ys)
        total += count
    return total / times_repeat

avg = pla_random(training_set, labels_set, 2000)
print(f"Average Number of Updates in 2000 Experiments:", avg)

def error_counter(wt, xs, ys):
    count = 0
    for i in range(len(xs)):
        if sign(np.dot(wt, xs[i])) != ys[i]:
            count += 1
    return count

def pocket_pla(training_set, labels_set, time_repeat):
    weights = np.zeros(training_set.shape[1])
    weights_updated = weights.copy()
    error = error_counter(weights, training_set, labels_set)
    t = 0
    while t < time_repeat:
        i = np.random.randint(0, len(training_set))
        if sign(np.dot(weights, training_set[i])) != labels_set[i]:
            weights += training_set[i] * labels_set[i]
            new_error = error_counter(weights, training_set, labels_set)
            t += 1
            if new_error < error:
                weights_updated = weights.copy()
                error = new_error
    return weights_updated

training_set_new, labels_set_new = preprocess("train18.dat")
weights = pocket_pla(training_set_new, labels_set_new, 50)

testing_set, testing_label_set = preprocess("test.dat")

def pocket_test(weights, testing_set, testing_label_set):
    err_cnt = error_counter(weights, testing_set, testing_label_set)
    err_rate = err_cnt / len(testing_set)
    return err_rate

error_rate = pocket_test(weights, testing_set, testing_label_set)
print(f"The error rate is {error_rate}.")

Machine Learning Foundations/Perceptron Learning Algorithm/pla.py


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: sunluelectric@github
This program demonstrates using sklearn to solve a linear regression problem.
It also calculates the distance of the fitting curve to the origin.
"""
import numpy as np
from sklearn.linear_model import LinearRegression
import self_error

class LinearRegressionDemo:
    """
    LinearRegressionDemo solves a linear regression problem using sklearn.
    """
    def __init__(self, aaf_sample):
        self.aaf_sample = aaf_sample
        self.i_number_of_sample = len(self.aaf_sample)
        af_x = np.array([])
        af_y = np.array([])
        for iter_sample in self.aaf_sample:
            af_x = np.append(af_x, iter_sample[0])
            af_y = np.append(af_y, iter_sample[1])
        self.af_x = af_x.reshape((-1,1))
        self.af_y = af_y
        self.af_xinv = af_x
        self.af_yinv = af_y.reshape((-1,1))
        self.f_k = None
        self.f_b = None
        self.f_penalty = None
        self.f_kinv = None
        self.f_binv = None
        self.f_penalty_inv = None

    def solvelinearregression(self):
        """
        Use sklearn to solve the linear regression problem.
        """
        lr_model = LinearRegression().fit(self.af_x, self.af_y)
        self.f_k = lr_model.coef_[0]
        self.f_b = lr_model.intercept_
        self.f_penalty = np.sum(
            (self.af_y - (self.f_k * self.af_x + self.f_b))**2)
    def solvelinearregressioninv(self):
        """
        Use sklearn to solve the linear regression problem.
        """
        lr_model = LinearRegression().fit(self.af_yinv, self.af_xinv)
        self.f_kinv = lr_model.coef_[0]
        self.f_binv = lr_model.intercept_
        self.f_penalty_inv = np.sum(
            (self.af_xinv - (self.f_kinv * self.af_yinv + self.f_binv))**2)
    def calculatedistance(self):
        """
        calculatedistance calculates the distance (and direction)
        of the LR result to the origin.
        """
        if self.i_number_of_sample == 0:
            self_error.GeneralErrorMessage(
                "There is no input samples to the LR problem.")
            return np.array([0,0])
        if self._checkallequal():
            return self.aaf_sample[0]
        self.solvelinearregression()
        self.solvelinearregressioninv()
        if self.f_penalty <= self.f_penalty_inv:
            return np.array(
                [-(self.f_b*self.f_k)/(self.f_k**2+1),
                 self.f_b/(self.f_k**2+1)])
        return np.array(
            [self.f_binv/(self.f_kinv**2+1),
             -(self.f_binv*self.f_kinv)/(self.f_kinv**2+1)])
    def _checkallequal(self):
        """
        Check whether all samples are identical.
        """
        if self.i_number_of_sample == 1:
            return True
        b_flag = True
        for iter_sample in self.aaf_sample:
            if not np.array_equal(self.aaf_sample[0], iter_sample):
                b_flag = False
                break
        return b_flag
    def _calculateaverage(self):
        average_vector = np.zeros((1,2))
        for iter_sample in self.aaf_sample:
            average_vector += iter_sample
        average_vector = average_vector/self.i_number_of_sample


import sys
import numpy as np
import matplotlib

print("Python", sys.version)
print("Numpy", np.__version__)
print("Matplotlib", matplotlib.__version__)

inputs = [1.0, 2.0, 3.0, 2.5]

weights = [[0.2, 0.8, -0.5, 1.0],
           [0.5, -0.91, 0.26, -0.5],
           [-0.26, -0.27, 0.17, 0.87]]

bias = [2.0,3.0,0.5]

layer_output = []
for n_weight, n_bias in zip(weights, bias) :
    neuron_output = 0
    for neu_input,neu_weight in zip(inputs,n_weight) :
        neuron_output = neuron_output+ neu_input*neu_weight
    neuron_output = neuron_output+n_bias
    layer_output.append (neuron_output)

print("layer_output from Scratch Python", layer_output)

output = np.dot(weights,inputs)+ bias
print("layer_output using Mumpy", output)


import numpy as np
import matplotlib.pyplot as plt
import random
import math

# define objective function

def f(x):
    x1 = x[0]
    x2 = x[1]
    obj = -20*math.exp(-0.2*math.sqrt(0.5*(x1*x1*x2*x2)))-math.exp(0.5*(math.cos(2*math.pi*x1)+math.cos(2*math.pi*x2))) + math.e + 20
    return obj

#İstenilen test fonksiyonu, f isimli fonksiyonda obj olarak kullanılabilir.

#ackley    obj = -20*math.exp(-0.2*math.sqrt(0.5*(x1*x1*x2*x2)))-math.exp(0.5*(math.cos(2*math.pi*x1)+math.cos(2*math.pi*x2))) + math.e + 20
#beale     obj = (1.5 - x[0] + x[0]*x[1])**2 + (2.25 - x[0] + x[0]*x[1]**2)**2 + (2.625 - x[0] + x[0]*x[1]**3)**2
#goldstein obj = ((1 + (x[0] + x[1] + 1) ** 2 * (19 - 14 * x[0] + 3 * (x[0] ** 2) - 14 * x[1] + 6 * x[0] * x[1] + 3 * (x[1] ** 2))) * (30 + (2 * x[0] - 3 * x[1]) ** 2 * (18 - 32 * x[0] + 12 * (x[0] ** 2) + 48 * x[1] - 36 * x[0] * x[1] + 27 * (x[1] ** 2))))
#levi      obj = (math.sin(3*x[0]*math.pi))**2 + ((x[0] - 1)**2)*(1 + math.sin(3*x[1]*math.pi)**2) + ((x[1] - 1)**2)*(1 + math.sin(2*x[1]*math.pi)**2)

# Start location
x_start = [0.8, -0.5]

##################################################
# Simulated Annealing
##################################################
# Number of cycles
n = 100
# Number of trials per cycle
m = 100
# Number of accepted solutions
na = 0.0
# Probability of accepting worse solution at the start
p1 = 0.7
# Probability of accepting worse solution at the end
p50 = 0.001
# Initial temperature
t1 = -1.0/math.log(p1)
# Final temperature
t50 = -1.0/math.log(p50)
# Fractional reduction every cycle
frac = (t50/t1)**(1.0/(n-1.0))
# Initialize x
x = np.zeros((n+1,2))
x[0] = x_start
xi = np.zeros(2)
xi = x_start
na = na + 1.0
# Current best results so far
xc = np.zeros(2)
xc = x[0]
fc = f(xi)
fs = np.zeros(n+1)
fs[0] = fc
# Current temperature
t = t1
# DeltaE Average
DeltaE_avg = 0.0
for i in range(n):
    print('Cycle: ' + str(i) + ' with Temperature: ' + str(t))
    for j in range(m):
        # Generate new trial points
        xi[0] = xc[0] + random.random() - 0.5
        xi[1] = xc[1] + random.random() - 0.5
        # Clip to upper and lower bounds
        xi[0] = max(min(xi[0],1.0),-1.0)
        xi[1] = max(min(xi[1],1.0),-1.0)
        DeltaE = abs(f(xi)-fc)
        if (f(xi)>fc):
            # Initialize DeltaE_avg if a worse solution was found
            #   on the first iteration
            if (i==0 and j==0): DeltaE_avg = DeltaE
            # objective function is worse
            # generate probability of acceptance
            p = math.exp(-DeltaE/(DeltaE_avg * t))
            # determine whether to accept worse point
            if (random.random()<p):
                # accept the worse solution
                accept = True
            else:
                # don't accept the worse solution
                accept = False
        else:
            # objective function is lower, automatically accept
            accept = True
        if (accept==True):
            # update currently accepted solution
            xc[0] = xi[0]
            xc[1] = xi[1]
            fc = f(xc)
            # increment number of accepted solutions
            na = na + 1.0
            # update DeltaE_avg
            DeltaE_avg = (DeltaE_avg * (na-1.0) +  DeltaE) / na
    # Record the best x values at the end of every cycle
    x[i+1][0] = xc[0]
    x[i+1][1] = xc[1]
    fs[i+1] = fc
    # Lower the temperature for next cycle
    t = frac * t

# print solution
print('Best solution: ' + str(xc))
print('Best objective: ' + str(fc))

plt.plot(x[:,0],x[:,1],'y-o')
plt.savefig('contour.png')

fig = plt.figure()
ax1 = fig.add_subplot(211)
ax1.plot(fs,'r.-')
ax1.legend(['Objective'])
ax2 = fig.add_subplot(212)
ax2.plot(x[:,0],'b.-')
ax2.plot(x[:,1],'g--')
ax2.legend(['x1','x2'])

# Save the figure as a PNG
plt.savefig('iterations.png')

plt.show()


import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import style
from IPython.display import display

%config InlineBackend.figure_format = 'retina'

tau = 160.0/100.0
def response(w,z):

    Kp = 1.0
    a = 1- (tau**2)*(w**2)
    b = 2*tau*z*w
    sol = Kp/np.sqrt(a**2+b**2)
    sol1 = np.arctan(b/a)
    return sol
w = np.linspace(0.01,100,100000)

zeta = [0.2,1,2]
plt.figure();
plt.plot(np.log10(w), np.log10(response(w,zeta[0])), 'r', label = r"$\zeta$ = %.1f" %(zeta[0]))
plt.plot(np.log10(w), np.log10(response(w,zeta[1])), 'b', label = r"$\zeta$ = %.1f" %(zeta[1]))
plt.plot(np.log10(w), np.log10(response(w,zeta[2])), 'g', label = r"$\zeta$ = %.1f" %(zeta[2]))
plt.xlabel("Time (hours)", fontsize=12)
#plt.xlabel(r"{log}_{10}(\omega )")
#plt.ylabel(r"{log}_{10}(\frac{AR}{K_P} )")
plt.title(r"$\frac{AR}{K_P}$"+ "for 2nd Order System (" + r"$\tau =$ %.2f)"% (tau))
plt.legend();


import numpy as np

def entropy(x):
    x = x[x > 0]
    return -np.sum(x * np.log(x))

def information_gain(y, y_left, y_right):
    n = y.shape[0]
    n_l = y_left.shape[0]
    if n_l == 0:
        H_l = 0.0
    else:
        H_l = entropy(y_left.mean(axis=0))

    n_r = y_right.shape[0]
    if n_r == 0:
        H_r = 0
    else:
        H_r = entropy(y_right.mean(axis=0))

    H = entropy(y.mean(axis=0))
    return H - n_l / n * H_l - n_r / n * H_r

def predict_node_label(values, num_distinct_classes):
    probabilities = predict_node_probability(values, num_distinct_classes)
    return np.argmax(probabilities)

def predict_node_probability(values, num_distinct_classes):
    return np.bincount(values, minlength=num_distinct_classes) / values.shape[0]


import numpy

'''
compute angle (in degrees) for p0p1p2 corner
Inputs:
    p0,p1,p2 - points in the form of [x,y]
'''

def calc_angle(p0, p1, p2):
    v0 = numpy.array(p0) - numpy.array(p1)
    v1 = numpy.array(p2) - numpy.array(p1)

    angle = numpy.math.atan2(numpy.linalg.det([v0, v1]), numpy.dot(v0, v1))
    return numpy.degrees(angle)


from math import sqrt, pow, pi
from pylab import *
import scipy
from scipy.integrate.odepack import odeint

R0 = 6378137
mu = 3.9860047e14

"""Расчет текущей высоты"""

def h_x_y(x, y):
    return sqrt(pow(x, 2) + pow(y, 2)) - R0

"""Расчет периода обращения"""

def T(R):
    return 2 * pi * sqrt(pow(R, 3) / mu)

"""Правые части"""

def right_part(t, StateVector):
    R = sqrt(StateVector[0] * StateVector[0] + StateVector[1] * StateVector[1])
    R3 = pow(R, 3)

    result = [0, 0, 0, 0]
    result[0] = StateVector[2]
    result[1] = StateVector[3]
    result[2] = -mu / R3 * StateVector[0]
    result[3] = -mu / R3 * StateVector[1]
    return result

"""Интегрированния методом Рунге-Кутта (4-го порядка)"""

def runge_kutta_t_state_vector(t, params, t_step, f):
    k1, k2, k3, k4 = [], [], [], []

    h = t_step

    k1 = f(t, params)

    new_params = []

    for i in range(0, len(params)):
        p = params[i]
        new_params.append(p + h / 2 * k1[i])

    k2 = f(t + h / 2, new_params)

    new_params = []

    for i in range(0, len(params)):
        p = params[i]
        new_params.append(p + h / 2 * k2[i])

    k3 = f(t + h / 2, new_params)

    new_params = []

    for i in range(0, len(params)):
        p = params[i]
        new_params.append(p + h * k3[i])

    k4 = f(t + h, new_params)

    for i in range(0, len(params)):
        p = params[i]
        params[i] = p + h / 6 * (k1[i] + 2 * k2[i] + 2 * k3[i] + k4[i])

    return params

"""Обычный расчет (с выводом вектора состояния в файл output.tr)"""

def regular_calculation(Vx, Vy, t, t_end, t_step, x, y):
    file = open("output.tr", mode="w")
    print('t, x, y, Vx, Vy', file=file)

    while t <= t_end:
        print("%(t).3f %(x)d %(y)d %(Vx).3f %(Vy).3f" % {"t": t, "x": x, "y": y, "Vx": Vx, "Vy": Vy},
              file=file)

        t += t_step

        x, y, Vx, Vy = runge_kutta_t_state_vector(t, [x, y, Vx, Vy], t_step, right_part)
    file.close()

"""Тест производительности (без вывода вектора состояния в файл output.tr)"""

def benchmark_test(Vx, Vy, t, t_end, t_step, x, y):
    while t <= t_end:
        t += t_step
        x, y, Vx, Vy = runge_kutta_t_state_vector(t, [x, y, Vx, Vy], t_step, right_part)

"""Тест на корректность результатов
(с проверкой высоты и проверкой соответствия вектора состояния с момента начала движения
на момент завершения первого витка)"""

def test(Vx, Vy, t, t_end, t_step, x, y, R):
    T_period = T(R)

    parameters = [x, y, Vx, Vy]

    x_data = []
    y_data = []

    while t <= t_end:
        x_data.append(x)
        y_data.append(y)

        t += t_step
        x, y, Vx, Vy = runge_kutta_t_state_vector(t, [x, y, Vx, Vy], t_step, right_part)

        k = round(T_period, 1)
        # print(round(t, 1) % k)
        if round(t, 1) % k == 0:
            print("Виток № ", round(t / T_period))
            print("T = ", t)

            for text, x0, x_period in zip(["x", "y", "Vx", "Vy"], parameters, [x, y, Vx, Vy]):
                x1, x2 = round(x0), round(x_period)
                if x1 != x2:
                    print("Разница в значениях:", text, x1, x2)

        assert (round(h_x_y(x, y), 5) == 1000000)

    plot(x_data, y_data)
    show()

"""Начальные условия и выбор режима расчета (обычный, тест производительности, тест на корректность полученных данных)"""

def main(type, step):
    t_end = 10000  # Конец времени движения
    t_step = float(step)  # Шаг интегрирования
    h0 = 1000000
    R = R0 + h0
    t, x, y, Vx, Vy = 0, 0, R, sqrt(mu / R), 0

    if type == "regular":
        regular_calculation(Vx, Vy, t, t_end, t_step, x, y)
    elif type == "benchmark":
        benchmark_test(Vx, Vy, t, t_end, t_step, x, y)
    elif type == "test":
        test(Vx, Vy, t, t_end, t_step, x, y, R)

def run_calculation_and_measure_time(run_type, step, file):
    timeit_timeit = timeit.timeit("main('" + run_type + "','" + str(step) + "')", setup="from __main__ import main",
                                  number=1)
    print(run_type, "t_step =", step, "\n",
          timeit_timeit,
          file=file)  # Замер затраченного времени (вывод в файл)
    print(run_type, "t_step =", step, "\n",
          timeit_timeit,
          )  # Замер затраченного времени (вывод на экран)

if __name__ == '__main__':
    import timeit

    file = open("results.txt", "w")

    for run_type in ["benchmark", "regular", "test"]:
        if run_type != "test":
            for step in [1, 0.1, 0.01]:
                run_calculation_and_measure_time(run_type, step, file)
        else:
            step = 0.1
            run_calculation_and_measure_time(run_type, step, file)

    file.close()

# odeint()


from scipy.stats import beta
from matplotlib import pyplot as plt
import numpy as np

def samples(a, b, success, trials, num_episodes=100):
    '''

    :param a: the shape param for prior dist
    :param b: the shape param for prior dist
    :param success: num success in the experiments
    :param trials:  num trails conducted
    :param num_episodes:  num samples to draw from this distribution
    :return:
    '''
    dist = beta(a+success, b+trials-success)
    episodes = num_episodes
    nums = [dist.rvs() for _ in range(episodes)]
    return nums

def stats(nums):
    avg = sum(nums)/len(nums)
    var = sum(
        [pow((x - avg), 2) for x in nums]
    )
    print(avg, var)

def plots(data, bin_size=20):
    bins = np.arange(0, bin_size, 1) # fixed bin size
    bins = bins/bin_size # normalize bins
    num_plots = len(data)
    for i, nums in enumerate(data):
        plt.subplot(num_plots, 1, i+1)
        # plot histogram
        plt.hist(nums, bins=bins, alpha=0.5)
        # hist = np.histogram(nums, bin_size)

    plt.show()

'''
The conclusion is better prior requires less trails to converge.
Worse prior requires more trails to converge.
'''

successes = 3
trials =10
# alpha, beta defines the shape of beta dist, success and trials is number of experiments.
a, b = 1, 1  # uniform
num_episodes = 2000 # num samples sampled from distribution in order to draw distribution
bin_size = 100
container = []
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

a, b = 0.5, 0.5  # convex shape prior
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

a, b = 1.1, 30  # 0-0.2 prior
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

a, b = 2, 5  # .0-0.8 prior
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

a, b = 2, 2  # bell shape between 0,1
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

plots(container, bin_size=bin_size)


import numpy as np
import matplotlib.pyplot as plt

# numpy.linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None)
#
# 在指定的间隔内返回均匀间隔的数字。
#
# 返回num均匀分布的样本，在[start, stop]。
#
# 这个区间的端点可以任意的被排除在外。

X = np.linspace(-np.pi, np.pi, 256, endpoint=True)
print(X)
C, S = np.cos(X), np.sin(X)

plt.plot(X, C)
plt.plot(X, S)

plt.show()


import numpy as np
import numpy as np
import pylab as pl
import scipy.special as ss

def beta(a, b, mew):
    e1 = ss.gamma(a + b)
    e2 = ss.gamma(a)
    e3 = ss.gamma(b)
    e4 = mew ** (a - 1)
    e5 = (1 - mew) ** (b - 1)
    return (e1/(e2*e3)) * e4 * e5

def plot_beta(a, b):
    Ly = []
    Lx = []
    mews = np.mgrid[0:1:100j]
    for mew in mews:
        Lx.append(mew)
        Ly.append(beta(a, b, mew))
    pl.plot(Lx, Ly, label="a=%f, b=%f" %(a,b))
 o
def main():
    plot_beta(0.1, 0.1)
    plot_beta(1, 1)
    plot_beta(2, 3)
    plot_beta(8, 4)
    pl.xlim(0.0, 1.0)
    pl.ylim(0.0, 3.0)
    pl.legend()
    pl.show()

if __name__ == "__main__":
    main()


import numpy as np
import statsmodels.api as sm
import matplotlib.pyplot as plt
from scipy.fftpack import rfft
from scipy.fftpack import fftshift

data_loader = sm.datasets.sunspots.load_pandas()
sunspots = data_loader.data["SUNACTIVITY"].values

t = np.linspace(-2 * np.pi, 2 * np.pi, len(sunspots))
mid = np.ptp(sunspots)/2
sine = mid + mid * np.sin(np.sin(t))

sine_fft = np.abs(fftshift(rfft(sine)))
print "Index of max sine FFT", np.argsort(sine_fft)[-5:]

transformed = np.abs(fftshift(rfft(sunspots)))
print "Indices of max sunspots FFT", np.argsort(transformed)[-5:]

plt.subplot(311)
plt.plot(sunspots, label="Sunspots")
plt.plot(sine, lw=2, label="Sine")
plt.grid(True)
plt.legend()
plt.subplot(312)
plt.plot(transformed, label="Transformed Sunspots")
plt.grid(True)
plt.legend()
plt.subplot(313)
plt.plot(sine_fft, lw=2, label="Transformed Sine")
plt.grid(True)
plt.legend()
plt.show()

import numpy,from scipy,import statsmodels


import numpy as np
import matplotlib.pyplot as plt

n = 1
m = 97
theta = np.zeros((2, 1))

alpha = 0.01
iterations = 1500

x1, y = np.loadtxt("data/OneFeature.txt", delimiter=",", unpack = True)  # Ouput is 1D. Vectors are 2D m x 1 dimension so need to change to 2D using reshape

x1 = np.reshape(x1, (x1.shape[0], 1))  # Makes x1 a vector with each new line different training set
y = np.reshape(y, (y.shape[0], 1))  # Makes y a vector with each new line different training set

x1_mean = np.mean(x1)
x1_std = np.std(x1)
x1_scaled = (x1 - x1_mean) / x1_std
onesVector = np.ones((x1.shape[0], 1))

X = np.concatenate((onesVector,x1_scaled),axis=1)  # Creates matrix X, where each new line is a different training example, where x0 on each line is 1

iterationsX = np.arange(1, iterations + 1, dtype= int)
costVector = np.zeros([iterations, 1])  # Vector of cost for each iteration

for j in range(0, iterations):  # Repeat many times to optimise theta
    sumCost = 0
    sumTheta = 0
    for i in range(0, m):  # Iterate through dataset to find sigma (the cost's derivative for current theta values)
        # Change shape of feature so it's 2D and a vector
        currentFeatures = np.reshape(X[i, :], (X[i, :].shape[0], 1))  # Current features is a vector of x0 followed by x1, ... xn
        h = np.matmul(theta.transpose(), currentFeatures)  # Calculate new hypothesis based on current training set (vectorised h(x) version)

        sumCost += (h - y[i, 0]) ** 2
        sumTheta += (h - y[i, 0]) * currentFeatures  # Add on the cost functions derivative associated with this training set

    cost = (1 / (2 * m)) * sumCost  # Cost for current theta parameters
    costVector[j, 0] = cost  # Add cost for current theta parameters to costVector
    sigma = (1/m) * sumTheta
    theta += - alpha * sigma  # update theta vector to more optimal values

#unscale theta
theta[0, 0] = theta[0,0] - (x1_mean / x1_std) * theta[1, 0]
theta[1, 0] = theta[1, 0] / x1_std

print("Cost function = ", cost)
print("Theta =", theta)

# Plot graph of x against y
plt.plot(x1, y, 'rx')  # need x and y values separate or pyplot autogenerates x values.

# Plot line of best fit
xline = np.linspace(0,20,2000)
yline = theta[0, 0] + xline * theta[1, 0]
plt.plot(xline, yline, '-b')

plt.xlabel('Population of City in 10,000s')
plt.ylabel('Profit in $10,000s')
plt.show()

# Plot graph of cost function against iterations
plt.plot(iterationsX, costVector, '-r')  # need x and y values separate or pyplot autogenerates x values.
plt.xlabel('Iterations')
plt.ylabel('Cost Function')
plt.xlim((1, iterations))
plt.show()

One variable with feature scaling (inefficient).py


import numpy as np

# Use "new-style" classes for easier inheritance
__metaclass__ = type

class transformation:
  def __init__(self):
    self.matrix = np.zeros((4,4), dtype = np.float64)
    self.invmatrix = np.zeros((4,4), dtype = np.float64)

  def getMatrix(self):
    return self.matrix

  def getInvMatrix(self):
    return self.invmatrix

  def transformContraVector(self, v):
    """
    Transforms column vectors (1-times contravariant tensors).

    x'^i = T^i_j * x^j  <=>  x' = Tx

    Matrix index convention is row-column
    """
    # See transformation rule for x^j above
    return np.einsum('ij,j', self.matrix, v)

  def transformCoMatrix(self, m):
    """
    Transform matrices representing bilinear forms (2-times covariant tensors).

    We start with using the transformation T for contravariant vectors,

    g(x',x') = x'^i * g'_i_j * x'^j
             = T^i_k * x^k * g'_i_j * T^j_l * x^l
             = x^k * (T^i_k * g'_i_j * T^j_l) * x^l

    We thus get

    g_k_l = T^i_k * g'_i_j * T^j_l

    Applying the inverse matrices on both sides,

    T^i_k * (T^-1)^k_j = delta^i_j

    where delta is the Kronecker symbol, we get the transformation rule

    g'_i_j = (T^-1)^k_i * g_k_l * (T^-1)^l_j

    The bilinear form thus transforms with the inverse matrix T^-1, hence
    its covariant transformation behaviour. To write this as a matrix product,
    we need to use the matrix transpose (the order of indices is important!),

    (T^-1)^k_i = (T^-1^t)_i^k

    which results in

    g' = T^-1^t g T^-1
    """
    # See transformation rule for g_k_l above
    return np.einsum('ki,kl,lj', self.invmatrix, m, self.invmatrix)

# -----------------------------------------------------------------------

class lorentzRotation(transformation):
  """
  Rotations of contravariant vectors in counter-clockwise direction (for angle > 0)
  in a right-handed coordinate system; the inverse matrix is used for covariant
  vectors
  """
  def __init__(self, axis, angle):
    assert (isinstance(axis, int)), "Axis parameter must be an integer number"
    assert (axis > 0 and axis < 4), "Axis must be in range 1..3"
    assert (isinstance(angle, (int, float))), "Angle parameter must be a number"
    self.axis = axis
    self.angle = angle

    super(lorentzRotation, self).__init__()

    self.matrix[0,0] = 1
    if axis == 1:
      self.matrix[1,1] = 1
      self.matrix[2,2] = self.matrix[3,3] = np.cos(self.angle)
      self.matrix[2,3] = -np.sin(self.angle)
      self.matrix[3,2] = np.sin(self.angle)
    elif axis == 2:
      self.matrix[2,2] = 1
      self.matrix[1,1] = self.matrix[3,3] = np.cos(self.angle)
      self.matrix[1,3] = np.sin(self.angle)
      self.matrix[3,1] = -np.sin(self.angle)
    elif axis == 3:
      self.matrix[3,3] = 1
      self.matrix[1,1] = self.matrix[2,2] = np.cos(self.angle)
      self.matrix[1,2] = -np.sin(self.angle)
      self.matrix[2,1] = np.sin(self.angle)

    # Rotation matrices are orthogonal
    self.invmatrix = np.transpose(self.matrix)

# -----------------------------------------------------------------------

class lorentzBoost(transformation):
  """
  Boosts of contravariant vectors, the inverse matrix is used for covariant
  vectors
  """
  def __init__(self, axis, beta):
    assert (isinstance(axis, int)), "Axis parameter must be an integer number"
    assert (axis > 0 and axis < 4), "Axis must be in range 1..3"
    assert (isinstance(beta, (int, float))), "Angle parameter must be a number"
    assert (np.abs(beta) < 1), "Beta must be > -1 and < 1"
    self.axis = axis
    self.beta = beta
    self.rapidity = np.arctanh(self.beta)

    super(lorentzBoost, self).__init__()

    if axis == 1:
      self.matrix[0,0] = np.cosh(self.rapidity)
      self.matrix[0,1] = -np.sinh(self.rapidity)
      self.matrix[1,0] = -np.sinh(self.rapidity)
      self.matrix[1,1] = np.cosh(self.rapidity)
      self.matrix[2,2] = 1
      self.matrix[3,3] = 1
    elif axis == 2:
      self.matrix[0,0] = np.cosh(self.rapidity)
      self.matrix[0,2] = -np.sinh(self.rapidity)
      self.matrix[1,1] = 1
      self.matrix[2,0] = -np.sinh(self.rapidity)
      self.matrix[2,2] = np.cosh(self.rapidity)
      self.matrix[3,3] = 1
    elif axis == 3:
      self.matrix[0,0] = np.cosh(self.rapidity)
      self.matrix[0,3] = -np.sinh(self.rapidity)
      self.matrix[1,1] = 1
      self.matrix[2,2] = 1
      self.matrix[3,0] = -np.sinh(self.rapidity)
      self.matrix[3,3] = np.cosh(self.rapidity)

    # Off-diagonal elements of inverse matrix change sign
    self.invmatrix = -self.matrix
    self.invmatrix[0,0] *= -1
    self.invmatrix[1,1] *= -1
    self.invmatrix[2,2] *= -1
    self.invmatrix[3,3] *= -1


import numpy as np
import scipy.optimize as opt
import matplotlib.pyplot as plt

from plotData import plot_data
import costFunction as cf
import plotDecisionBoundary as pdb
from sigmoid import sigmoid
from predict import predict
"""
Logistic Regression 进行分类
"""

# ====== 1.读取数据并绘制散点图 ======
dataset = np.loadtxt('ex2data1.txt',delimiter =',')
X = dataset[:,0:2]
Y = dataset[:,2]

plot_data(X,Y)
plt.legend(['Admitted', 'Not admitted'])
plt.xlabel('Exam 1 score')
plt.ylabel('Exam 2 score')
# plt.show()

# ====== 2.计算代价和梯度 ======
(m,n) = X.shape
X = np.c_[np.ones(m),X]	# 添加偏置维度
# 初始theta值和lambda值
init_theta = np.zeros(X.shape[1])
lmd = 1
# 计算代价和梯度
cost,grad = cf.cost_Function_Reg(X,Y,init_theta,lmd)
print('Cost at initial theta (zeros): {:0.3f}'.format(cost))
print('Expected cost (approx): 0.693')
print('Gradient at initial theta (zeros): \n{}'.format(grad))
print('Expected gradients (approx): \n-0.1000\n-12.0092\n-11.2628')

# 使用非0的theta测试函数
test_theta = np.array([-24,0.2,0.2])	#生成的是列向量
cost, grad = cf.cost_Function_Reg(X,Y,test_theta,0)
print('Cost at test theta (zeros): {:0.3f}'.format(cost))
print('Expected cost (approx): 0.218')
print('Gradient at test theta: \n{}'.format(grad))
print('Expected gradients (approx): \n0.043\n2.566\n2.647')

# ======= 3.使用优化函数来优化求解 ========
def cost_func(t):
	return cf.cost_Function_Reg(X,Y,t,0)[0]

def grad_func(t):
	return cf.cost_Function_Reg(X,Y,t,0)[1]
# 使用opt.fmin_bfgs()来获得最优解
theta, cost, *unused = opt.fmin_bfgs(f=cost_func, fprime=grad_func, x0=init_theta, maxiter=400, full_output=True, disp=False)
print('Cost at theta found by fmin: {:0.3f}'.format(cost))
print('Expected cost (approx): 0.203')
print('theta: \n{}'.format(theta))
print('Expected Theta (approx): \n-25.161\n0.206\n0.201')

# 使用计算出的theta来绘制分界线
pdb.plot_Decision_Boundary(X,Y,theta)
plt.xlabel('Exam 1 score')
plt.ylabel('Exam 2 score')
# plt.show()

# ====== 4.预测和计算准确率 ======
prob = sigmoid(np.array([1, 45, 85]).T.dot(theta))
print('For a student with scores 45 and 85, we predict an admission probability of {:0.4f}'.format(prob))
print('Expected value : 0.775 +/- 0.002')

P = predict(X,theta)
print('Train accuracy:{}'.format(np.mean(P == Y)*100))
print('Expected accuracy (approx): 89.0')


from __future__ import annotations
import numpy as np
from numpy.linalg import inv, det, slogdet

pdfLambda = lambda mu,var,x: (1.0 / (var * ((2*np.pi)**0.5))) * np.exp(-0.5*((x - mu) / float(var)) ** 2)

def multivarPdf(X, mu, cov):
    size = len(X)
    if size == len(mu) and (size, size) == cov.shape:
        deter = det(cov)
        if deter == 0:
            raise NameError("The covariance matrix can't be singular")

        norm_const = 1.0 / ((2 * np.pi)**(float(size) / 2) * (deter**(1.0 / 2)))
        x_mu = np.matrix(X - mu)
        invar = inv(cov)
        result = np.exp(-0.5 * (x_mu * invar * x_mu.T))
        return norm_const * result
    else:
        raise NameError("The dimensions of the input don't match")

class UnivariateGaussian:
    """
    Class for univariate Gaussian Distribution Estimator
    """
    def __init__(self, biased_var: bool = False) -> UnivariateGaussian:
        """
        Estimator for univariate Gaussian mean and variance parameters

        Parameters
        ----------
        biased_var : bool, default=False
            Should fitted estimator of variance be a biased or unbiased estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `UnivariateGaussian.fit` function.

        mu_: float
            Estimated expectation initialized as None. To be set in `UnivariateGaussian.fit`
            function.

        var_: float
            Estimated variance initialized as None. To be set in `UnivariateGaussian.fit`
            function.
        """
        self.biased_ = biased_var
        self.fitted_, self.mu_, self.var_ = False, None, None

    def fit(self, X: np.ndarray) -> UnivariateGaussian:
        """
        Estimate Gaussian expectation and variance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Training data

        Returns
        -------
        self : returns an instance of self.

        Notes
        -----
        Sets `self.mu_`, `self.var_` attributes according to calculated estimation (where
        estimator is either biased or unbiased). Then sets `self.fitted_` attribute to `True`
        """
        self.mu_ = np.mean(X)
        if self.biased_:
            self.var_ = np.mean(abs(X-self.mu_))
        else:
            self.var_ = X.var(ddof=1)

        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray) -> np.ndarray:
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, var_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")

        # denom = (2*np.pi*self.var_)**0.5
        # num = np.exp(-(X-self.mu_)**2/(2*self.var_))
        # pdf = np.exp(-1*(((X-self.mu_)**2)/(2*self.var_)))/((2*np.pi*self.var_)**0.5)
        return np.array(np.array([pdfLambda(self.mu_, self.var_, v) for v in X]))

    @staticmethod
    def log_likelihood(mu: float, sigma: float, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : float
            Expectation of Gaussian
        sigma : float
            Variance of Gaussian
        X : ndarray of shape (n_samples, )
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated
        """
        likelihoodSum = 0
        constant = (-1.0) / (2*sigma)
        for xi in X:
            likelihoodSum += pow(xi-mu, 2)
        # denom = (2*np.pi*var)**0.5
        # num = np.exp(-(X-mu)**2/(2*var))
        # pdf = num/denom
        # logLikelihood = ((1/(2*np.pi*sigma**2)**0.5)**len(X))*np.exp((-1*(1/2*sigma**2)*np.sum((X-mu)**2)))
        # #return sum([np.log(x) for x in pdf])

        return (- len(X) / 2) * np.log(2*np.pi) + np.log(sigma) + constant * likelihoodSum

class MultivariateGaussian:
    """
    Class for multivariate Gaussian Distribution Estimator
    """
    def __init__(self):
        """
        Initialize an instance of multivariate Gaussian estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `MultivariateGaussian.fit` function.

        mu_: ndarray of shape (n_features,)
            Estimated expectation initialized as None. To be set in `MultivariateGaussian.fit`
            function.

        cov_: ndarray of shape (n_features, n_features)
            Estimated covariance initialized as None. To be set in `MultivariateGaussian.fit`
            function.
        """
        self.mu_, self.cov_ = None, None
        self.fitted_ = False

    def fit(self, X: np.ndarray) -> MultivariateGaussian:
        """
        Estimate Gaussian expectation and covariance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Training data

        Returns
        -------
        self : returns an instance of self

        Notes
        -----
        Sets `self.mu_`, `self.cov_` attributes according to calculated estimation.
        Then sets `self.fitted_` attribute to `True`
        """
        self.mu_ = np.mean(X, 0)
        self.cov_ = np.cov(X.T)

        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray):
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, cov_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")

        return np.array([multivarPdf(xi, self.mu_, self.cov_) for xi in X])

    @staticmethod
    def log_likelihood(mu: np.ndarray, cov: np.ndarray, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : ndarray of shape (n_features,)
            Expectation of Gaussian
        cov : ndarray of shape (n_features, n_features)
            covariance matrix of Gaussian
        X : ndarray of shape (n_samples, n_features)
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated over all input data and under given parameters of Gaussian
        """
        residuals = X-mu

        calcLogLikelihood = lambda resid, invariant, logDet, dim: -0.5 * (logDet + (resid.T @ invariant @ resid) + (dim * np.log(2 * np.pi)))
        invariantMat = inv(cov)
        logDeter = slogdet(cov)
        d = cov.shape[0]
        logPDF = np.apply_along_axis(lambda xi: calcLogLikelihood(xi, invariantMat, logDeter, d), 1, residuals)
        return logPDF.sum()

if __name__ == "__main__":
    quizX = np.array([1, 5, 2, 3, 8, -4, -2, 5, 1, 10, -10, 4, 5, 2, 7, 1, 1, 3, 2, -1, -3, 1, -4, 1, 2, 1,
          -4, -4, 1, 3, 2, 6, -6, 8, 3, -6, 4, 1, -2, 3, 1, 4, 1, 4, -2, 3, -1, 0, 3, 5, 0, -2])
    univar = UnivariateGaussian().fit(quizX)
    print(UnivariateGaussian.log_likelihood(10,1,quizX))
    pass


import random
import numpy as np
from scipy.stats import multivariate_normal

class LinearRegressionMixturesEM():
    def __init__(self, K, max_iter=100, tol=1e-4):
        self.K = K
        self.max_iter = max_iter
        self.tol = tol

    def fit(self, X, y):
        N = len(X)
        K = self.K

        # initialize parameters
        pi = np.array([1 / K] * K)
        w = np.random.randn(K, 1 + X.shape[1])
        beta = 1 / np.std(y)
        rnk = np.zeros(shape=(N, K))

        for iter in range(self.max_iter):

            # E step calc rnk
            phi = np.hstack((np.ones(shape=(X.shape[0], 1)), X))
            for i in range(N):
                norm_const = 0
                xi = phi[i]
                for j in range(K):
                    wjXi = xi.dot(w[j])
                    norm_const += pi[j] * multivariate_normal(wjXi, np.diag([1 / beta])).pdf(y[i])
                for k in range(K):
                    wkXi = xi.dot(w[k])
                    rnk[i, k] = pi[k] * multivariate_normal(wkXi, np.diag([1 / beta])).pdf(y[i]) / norm_const

            # M step
            nk = np.sum(rnk, axis=0)
            pi = nk / N

            # calc w
            for k in range(K):
                atrka = np.dot(phi.T, np.diag(rnk[:, k])).dot(phi)
                atrky = np.dot(phi.T, np.diag(rnk[:, k])).dot(y)
                w[k] = np.linalg.inv(atrka).dot(atrky).flatten()

            # calc beta
            beta = 0
            for k in range(K):
                beta += phi.dot(w[k]).dot(np.diag(rnk[:, k])).dot(phi.dot(w[k]))
            beta = N * (1 / beta)

            # calc likelihood
            likelihood = 0
            for i in range(N):
                likelihood_i = 0
                xi = phi[i]
                for j in range(K):
                    wjXi = xi.dot(w[j])
                    likelihood_i += pi[j] * multivariate_normal(wjXi, np.diag([1 / beta])).pdf(y[i])
                likelihood += np.log(likelihood_i)

            if iter != 0:
                if abs(prev_likelihood - likelihood) < self.tol:
                    break
            prev_likelihood = likelihood

            if iter == self.max_iter:
                break

        self.w = w
        self.beta = beta
        self.pi = pi
        self.rnk = rnk

    def predict(self, X):
        phi = np.hstack((np.ones(shape=(X.shape[0], 1)), X))
        y = np.zeros(shape=len(X))
        for k in range(self.K):
            y += self.pi[k] * phi.dot(self.w[k].T)

        return y

    def predictive_distribution(self, x):

        w, beta = self.w, self.beta
        phi = np.hstack((np.ones(shape=(1, 1)), x.reshape(1, len(x))))
        f_list = []
        for k in range(self.K):
            f_list.append(multivariate_normal(phi.dot(w[k]), np.diag([1 / beta])).pdf)

        return lambda x: sum([p*f(x) for p, f in zip(self.pi, f_list)])


import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn import datasets as skdata
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans
# We will use the iris dataset
iris_dataset = skdata.load_iris()
X = iris_dataset.data # (150, 4)
y = iris_dataset.target

kmeans = KMeans(n_clusters=3)
kmeans.fit(X)
y_hat = kmeans.labels_

# Visualize by projecting to lower dimensions
Z = PCA(n_components=3).fit_transform(X)

data_split = (Z[np.where(y_hat == 0)[0], :],
              Z[np.where(y_hat == 1)[0], :], Z[np.where(y_hat == 2)[0], :])
colors = ('blue', 'red', 'green')
labels = ('Setosa', 'Versicolour', 'Virginica')
markers = ('o', '^', '+')

fig = plt.figure()
ax = fig.add_subplot(1, 1, 1, projection='3d')
for z, c, l, m in zip(data_split, colors, labels, markers):
    ax.scatter(z[:, 0], z[:, 1], z[:, 2], c=c, label=l, marker=m)
    ax.legend(loc='upper right')

# Let’s compare it to groundtruth labels
data_split_kmeans = (Z[np.where(y_hat == 0)[0], :],
                     Z[np.where(y_hat == 1)[0], :], Z[np.where(y_hat == 2)[0], :])
data_split_gt = (Z[np.where(y == 0)[0], :],
                 Z[np.where(y == 1)[0], :], Z[np.where(y == 2)[0], :])

data_splits = [data_split_kmeans, data_split_gt]
plot_titles = ['Partition by k-Means', 'Groundtruth']
fig = plt.figure()
for i in range(len(data_splits)):
    ax = fig.add_subplot(1, 2, i+1, projection='3d')
    ax.set_title(plot_titles[i])
    for z, c, l, m in zip(data_splits[i], colors, labels, markers):
        ax.scatter(z[:, 0], z[:, 1], z[:, 2], c=c, label=l, marker=m)
        ax.legend(loc='upper right')
plt.show()

# Let’s look at what happens if we set K to 5
kmeans = KMeans(n_clusters=5)
kmeans.fit(X)
y_hat = kmeans.labels_ # Can be one of {0, 1, 2, 3, 4}

data_split_kmeans = [
 Z[np.where(y_hat == i)[0], :] for i in range(5)]
data_split_gt = [Z[np.where(y == 0)[0], :],
 Z[np.where(y == 1)[0], :], Z[np.where(y == 2)[0], :]]

colors_gt = ('blue', 'red', 'green')
colors_kmeans = ('blue', 'red', 'green', 'yellow', 'magenta')

labels_gt = ('Setosa', 'Versicolour', 'Virginica')
labels_kmeans = ('0', '1', '2', '3', '4')

markers_gt = ('o', '^', '+')
markers_kmeans = ('o', '^', '+', '*', 'd')

data_splits = [data_split_kmeans, data_split_gt]
plot_titles = ['k-Means k=5', 'Groundtruth']
colors = [colors_kmeans, colors_gt]
labels = [labels_kmeans, labels_gt]
markers = [markers_kmeans, markers_gt]
fig = plt.figure()

for i in range(len(data_splits)):
    ax = fig.add_subplot(1, 2, i+1, projection='3d')
    ax.set_title(plot_titles[i])
    for z, c, l, m in zip(data_splits[i], colors[i], labels[i], markers[i]):
        ax.scatter(z[:, 0], z[:, 1], z[:, 2], c=c, label=l, marker=m)
        ax.legend(loc='upper right')

plt.show()

# Let’s see what happens if we try different initializations
X = np.random.rand(1500, 2)

data_splits = []
for i in range(4):
    kmeans = KMeans(n_clusters=2, init='random', n_init=1)
    kmeans.fit(X)
    y_hat = kmeans.labels_ # Can be one of {0, 1}
    data_splits.append([X[np.where(y_hat == i)[0], :] for i in range(2)])
colors = ('blue', 'red')
labels = ('0', '1')
markers = ('o', '^')

plot_titles = ['k-Means rand1', 'k-Means rand2', 'k-Means rand3', 'k-Means rand4']
fig = plt.figure()
for i in range(len(data_splits)):
    ax = fig.add_subplot(1, 4, i+1)
    ax.set_title(plot_titles[i])
    for x, c, l, m in zip(data_splits[i], colors, labels, markers):
        ax.scatter(x[:, 0], x[:, 1], c=c, label=l, marker=m)
        ax.legend(loc='upper right')

plt.show()


# Code by Desh Iyer

# ----------------------------------------------------------------

# PROBLEM STATEMENT

# Import the z table as a data structure in python. Create some 5 demo examples to find the area under the curve and report the answer from the z-table.

# ----------------------------------------------------------------

# SOLUTION

# Consider the situation:
# The scores - the number of which are determined by user input - on an exam are normally distributed with a mean of 65 and a std of 9
# Find the percentage of scores,
    # (i). less than 54
    # (ii). less than 20
    # (iii). at least 80
    # (iv). at least 90
    # (v.) between 40 and 50

# Import libs
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

# Function to plot the graphs of both generated lists of data
def pltGraphs(normallyDistributedData, zDistributedData):
    # Plot two graphs
    plt.title("Normal distribution w/ Mean: 65 & Std. Deviation: 9")
    plt.xlabel('x')
    plt.ylabel('Normal Distribution')
    plt.grid()
    plt.hist(normallyDistributedData)
    plt.show()

    plt.title("Z - Distribution for above graph")
    plt.xlabel('x')
    plt.ylabel('Standard Normal / Z - Distribution')
    plt.grid()
    plt.hist(zDistributedData)
    plt.show()

# Function to generate the lists of normalized and z-converted data
def generateNormalDataAndConvertToStdNormal():
    # Take user input for number of samples
    numberSamples = int(input("Enter number of samples: "))

    # Generate a normal distribution with arbitrary mean 65 and std 9
    normallyDistributedData = np.random.normal(65, 9, numberSamples)

    # Calculate summary stats
    normalMean = np.mean(normallyDistributedData)
    normalStd = np.std(normallyDistributedData)

    # Convert the above generated normally distributed data to Z-distributed data using list comprehension
    zDistributedData = [calcZScore(i, normalMean, normalStd) for i in normallyDistributedData]

    # Print the lists
    print("Normally Distributed Data:")
    print(normallyDistributedData)

    print("\nStandardized Normally Distributed Data (Z):")
    print(zDistributedData)

    # Plot lists using aforementioned function
    pltGraphs(normallyDistributedData, zDistributedData)

# Simple function to return the z-Score of a given value
def calcZScore(x, mean, std):
    return ((x - mean) / std)

# The main meat of the program
# A function to return the value in the z-Table provided the true value as per the question
# This function first converts the value into a z-Score and then finds the cell in the z-Table correspoinding to the z-Score calculated
def returnAreaUnderCurve(x, zTable):
    # Calc z-Score
    zScore = np.round(calcZScore(x, 65, 9), 2)
    displayZScore = np.round(calcZScore(x, 65, 9), 5)

    # Calculate z-Table indices
    rowIndex = np.round(zScore, 1)
    strRowIndex = str(rowIndex)

    colIndex = ((zScore * 100) % 10) / 100 if zScore > 0 else ((-zScore * 100) % 10) / 100
    strColIndex = str(colIndex)

    # Attain cell and store its value
    zTableValue = float(zTable.at[strRowIndex, strColIndex])

    # Print data
    print(f"Z-Score = {displayZScore}")
    print(f"Reading at ({rowIndex}, {colIndex}) = {zTableValue}")

    # Return the cell value
    return zTableValue

# Function that processes questions of "Score < " type
def calculateAndPrintLessThan(reqValue, zTable):
    reqArea = returnAreaUnderCurve(reqValue, zTable)

    print(f'The area under the graph = {np.round(reqArea * 100, 10)}%')

# Function that processes questions of "Score at least" type
def calculateAndPrintAtLeast(reqValue, zTable):
    lessThanArea = returnAreaUnderCurve(reqValue, zTable)
    reqArea = 1 - lessThanArea

    print(f'The required area = 1 - {lessThanArea} = {reqArea}\nThe area under the graph = {np.round(reqArea * 100, 10)}%')

# Function that processes questions of "Score in between" type
def calculateAndPrintBetween(firstReqValue, secondReqValue, zTable):
    firstReading = returnAreaUnderCurve(firstReqValue, zTable)
    secondReading = returnAreaUnderCurve(secondReqValue, zTable)

    inBetweenReading = abs(firstReading - secondReading)

    print(f'The required area in between {firstReqValue} and {secondReqValue} = {np.round(inBetweenReading * 100, 10)}%')

# Function to display all the sample questions, call the respective function to calculate the cell value and display results
def calculateCases(zTable):
    print("\n\nPROBLEM STATEMENT:\nConsider the situation:\nThe scores - the number of which are determined by user input - on an exam are normally distributed with a mean of 65 and a std of 9 Find the percentage of scores,\n(i). less than 54\n(ii). less than 20 \n(iii). at least 80\n(iv). at least 90\n(v.) between 40 and 50")

    # Case (i). -> % less than 54
    print("\n\nCase (i). -> percentage less than 54")
    calculateAndPrintLessThan(54, zTable)

    # Case (ii). -> % less than 20
    print("\n\nCase (ii). -> percentage less than 20")
    calculateAndPrintLessThan(20, zTable)

    # Case (iii). -> % at least 80 -> 1 - (% less than 80)
    print("\n\nCase (iii). -> % at least 80 -> 1 - (% less than 80)")
    calculateAndPrintAtLeast(80, zTable)

    # Case (iv). -> % at least 90 -> 1 - (% less than 90)
    print("\n\nCase (iv). -> % at least 90 -> 1 - (% less than 90)")
    calculateAndPrintAtLeast(90, zTable)

    # # Case (v). -> % between 40 and 50
    print("\n\nCase (v). -> between 40 and 50")
    calculateAndPrintBetween(40, 50, zTable)

# Main driver function
def main():
    # Read the zTable from the csv file containing details of the zTable in comma separated format & set the first column as the index col
    zTable = pd.read_csv(r"C:\Users\Volt\Code\B.Tech-CSE-Y2\applied-statistics\lab\experiment-5\zTable.txt", index_col=0)

    print("The generated z-Table is:\n", zTable)

    zTable = zTable.astype(str)
    zTable.index = zTable.index.map(str)

    generateNormalDataAndConvertToStdNormal()
    calculateCases(zTable)

# Invoke main()
if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Mon Feb 22 17:08:24 2021

@author: Chi Ding

Conjugate Gradient:
    Ax = b

    A: Hilbert matrix
    x: coefficients
    b: ones vector
"""

import numpy as np

# dimensionality for the system
D = 5,8,12,20

# stopping criterion
threshold = 1e-6

A = []

x = []

for n in D:

    k = 0

    A.append(
                np.column_stack([1/(i+j-1)
                         for i in range(1,n+1)
                         for j in range(1,n+1)]).reshape(n,n)
                )

    b = np.ones(n)

    # initial point
    x.append(np.zeros(n))

    # initial residual/gradient
    r = A[-1] @ x[-1] - b

    p = -r

    while True:
        """
            The conjugate gradient algorithm
        """
        r2 = r @ r
        alpha = r2 / (p @ A[-1] @ p)
        x[-1] = x[-1] + alpha*p
        r_ = r + alpha * A[-1] @ p
        beta = (r_ @ r_)/r2
        p = -r_ + beta * p
        r = r_.copy()
        k += 1

        if (r @ r)**0.5 < threshold:
            break

    print('dimension',n,'steps',k)


import numpy as np

# arange
# sqrt
# exp
# random
# addition
# maximun

# reference link for many others

A = np.arange(15)
print A
A = np.arange(1, 15, 2)
print A

# sqrt
B = np.sqrt(A)
print "B is"
print B

# Exp

C = np.exp(A)
print "C is"
print C

# addition
D = np.add(A, B)
print "D is"
print D

# Maximum

E = np.maximum(A, B)
print "E is"
print E

# addition resources
# scipy.org - all functions associated np.array
# docs.scipy.org