# The Nelson-Siegel-Svensson is a popular extension of the 4-parameter Nelson-Siegel method to 6 parameters. It is an algorithm for interpolatin/extrapolating the yield curve among other uses.
# The Scennson introduces two extra parameters to better fit the variety of shapes of either the instantaneous forward rate or yield curves that are observed in practice.
# A desirable property of the model is that it produces a smooth and well behaved forward rate curve.
# Another desirable property is the intuitive explanation of the parameters. beta0 is the long term interest rate and beta0+beta1 is the instantaneous short-term rate.
# To find the optimal value of the parameters, the Nelder-Mead simplex algorithm is used (Already implemented in the scipy package). The link to the optimization algorithm is
# Gao, F. and Han, L. Implementing the Nelder-Mead simplex algorithm with adaptive parameters. 2012. Computational Optimization and Applications. 51:1, pp. 259-277

from nelsonsiegelsvensson import *
import numpy as np

## Inputs
#   - Observed yield rates (YieldVec)
#   - Maturity of each observed yield (TimeVec)
#   - Initial guess for parameters (beta0, beta1, beta2,beta3, labda0 , and lambda1)
#   - Target maturities (TimeResultVec)

TimeVec = np.array([1,2,5,10,25])
YieldVec = np.array([0.0039, 0.0061, 0.0166, 0.0258, 0.0332])
beta0   = 0.1 # initial guess
beta1   = 0.1 # initial guess
beta2   = 0.1 # initial guess
beta3   = 0.1 # initial guess
lambda0 = 1 # initial guess
lambda1 = 1 # initial guess

TimeResultVec = np.array([1,2,5,10,25,30,31]) # Maturities for yields that we are interested in

## Implementation
OptiParam = NSSMinimize(beta0, beta1, beta2, beta3, lambda0, lambda1, TimeVec, YieldVec) # The Nelder-Mead simplex algorithem is used to find the parameters that result in a curve with the minimum residuals compared to the market data.

# Print the yield curve with optimal parameter to compare with the data provided
print(NelsonSiegelSvansson(TimeResultVec, OptiParam[0], OptiParam[1], OptiParam[2], OptiParam[3], OptiParam[4], OptiParam[5]))


'''
Reference
[1] Dabhade S A ,  Bewoor M S ,  Dabhade S A , et al. Face Recognition using Principle Component Analysis[J]. Core.kmi.open.ac.uk, 2013, ETCSIT(100):45 - 50.

The variable names used in this source code are consistent with this paper.
'''
import numpy as np
import sys

from settings import EIGENIMAGES_CONTRAST
from settings import FACE_COUNT
from settings import IMAGE_COUNT_PER_FACE
from settings import TRAIN_COUNT_PER_FACE
from image_parser import read_all_images
from image_parser import write_an_image
from math_utils import get_dist

def get_omegas_of_training_set(X, M_):
    '''
    m represents the mean image.
    W represents the mean centered images.
    '''
    m = np.array([np.mean(X, axis=1)]).T
    W = X-m
    '''
    C = np.dot(W,W.T), which The size of C is N × N which could be enormous.
    The eigenvalues and eigenvectors of C are obtained by eigen decomposition of C_ (C').
    The first (M − 1) (λi, ei) of C are given by (μi, Wdi) of C_, respectively.
    Wdi needs to be normalized in order to be equal to ei.
    The dimension of C_ is much smaller than that of C, reducing operations greatly.
    '''
    C_ = np.dot(W.T, W)
    eigenvalues_C, eigenvectors_C_ = np.linalg.eig(C_)
    eigenvectors_C = np.dot(W, eigenvectors_C_)
    for i in range(len(eigenvectors_C[0])):
        eigenvectors_C[:, i] /= np.linalg.norm(eigenvectors_C[:, i])
    '''
    {Wdi} corresponding to nonzero {μi} of C_ produce an orthonormal basis for the subspace,
    within which most image data can be represented with a small amount of error.
    {Wdi} are sorted from high to low according to their corresponding {μi}.
    The 90% of the total variance is contained in the first 5% - 10% of the dimensions.
    Only the M_ eigenvectors with the largest eigenvalues are examined.
    '''
    assert M_ < len(eigenvalues_C)
    sorted_eigenvalues_rank = np.argsort(eigenvalues_C)
    '''
    The vectors ei are also images, so called, eigenimages, or eigenfaces.
    '''
    E = []
    max_rank = len(eigenvalues_C)-1
    for i in range(M_):
        rank = sorted_eigenvalues_rank[max_rank-i]
        E.append(eigenvectors_C[:, rank])
    E = np.array(E).T
    '''
    A facial image can be projected onto M_ dimensions by computing omega.
    '''
    omegas_of_training_set = np.dot(E.T, W)
    return m, E, omegas_of_training_set

def recognize_face(m, E, omegas_of_training_set, face):
    W = face-m
    omega_of_face = np.dot(E.T, W)
    assert len(omegas_of_training_set[0]) == TRAIN_COUNT_PER_FACE*FACE_COUNT
    min_dist = get_dist(
        np.array([omegas_of_training_set[:, 0]]).T, omega_of_face)
    face_recognized = 0
    for i in range(FACE_COUNT):
        for j in range(TRAIN_COUNT_PER_FACE):
            if i == 0 and j == 0:
                continue
            dist = get_dist(
                np.array([omegas_of_training_set[:, TRAIN_COUNT_PER_FACE*i+j]]).T, omega_of_face)
            if dist < min_dist:
                min_dist = dist
                face_recognized = i
    return face_recognized

if __name__ == '__main__':
    if len(sys.argv) == 1:
        print('Please specify a series of `M_` via command line parameters.')
        print(f'0 < `M_` < {TRAIN_COUNT_PER_FACE*FACE_COUNT}')
        print('E.g. python pca_recognizer.py 199 (...)')
        sys.exit()

    print('###### Principal Component Analysis (PCA) Face Recognition ######')
    print('The face recognition results of each person are listed in the following format:')
    print('Face Number | Image Number | Face Recognized | Right or Wrong (T/F)\n...\nError Rate')
    training_images, test_images = read_all_images()
    for i in range(1, len(sys.argv)):
        M_ = int(sys.argv[i])
        print('################################################################################')
        print(f'The dimension of omega is set to {M_}.')
        m, E, omegas_of_training_set = get_omegas_of_training_set(
            training_images, M_)
        for j in range(len(E[0])):
            write_an_image(E[:, j]*EIGENIMAGES_CONTRAST, 'eigenimages', j+1)
        print('The eigenimages of training set are as follows:')
        print(E)
        print(f'(Its shape: {len(E)} * {len(E[0])})')
        print('You can view them at file://./output/eigenimages/\n')
        error_count = 0
        for j in range(FACE_COUNT):
            error_count_j = 0
            for k in range(IMAGE_COUNT_PER_FACE-TRAIN_COUNT_PER_FACE):
                face = np.array(
                    [test_images[:, (IMAGE_COUNT_PER_FACE-TRAIN_COUNT_PER_FACE)*j+k]]).T
                face_recognized = recognize_face(
                    m, E, omegas_of_training_set, face)
                t_or_f = 'T'
                if face_recognized != j:
                    t_or_f = 'F'
                    error_count_j += 1
                print(f'{j+1}\t{k+1}\t{face_recognized+1}\t{t_or_f}')
            error_count += error_count_j
            print(f'{error_count_j} / {IMAGE_COUNT_PER_FACE-TRAIN_COUNT_PER_FACE} = {error_count_j/(IMAGE_COUNT_PER_FACE-TRAIN_COUNT_PER_FACE)}\n')
        print('Total Error Rate:')
        print(f'{error_count} / {(IMAGE_COUNT_PER_FACE-TRAIN_COUNT_PER_FACE)*FACE_COUNT} = {error_count/((IMAGE_COUNT_PER_FACE-TRAIN_COUNT_PER_FACE)*FACE_COUNT)}')
        print('################################################################################')


import numpy as np
import sympy as sp
from sympy.utilities.lambdify import lambdify, implemented_function

def Weierstrass_Enneper(f, g, z, imag_unit=1j):
    """
    Compute the Weierstrass Enneper parametrization for given 'Weierstrass data'.
    :param sympy expression f: h'/g, with h the height function
    :param sympy expression g: Gauss map.
    :param sympy variable z: Complex variable.
    :param imag_unit: Representation of imaginary unit, e.g. 1j (numpy) or sympy.I.
    :return: Weierstrass Enneper parametrization as a tuple of sympy expressions.
    """
    psi1 = sp.integrate(f*(1-g**2)/2, z)
    psi2 = imag_unit*sp.integrate(f*(1+g**2)/2, z)
    psi3 = sp.integrate(f*g, z)

    return psi1, psi2, psi3

def Enneper_parametrization(k, z, verbose=True):
    """
    Compute a parametrization of the higher order Enneper surface of order k.
    :param int k: Order of the Enneper surface
    :param sympy variable z: Complex variable.
    :param bool verbose: Whether to output auxilliary information.
    :return: function s parametrizing the higher order Enneper surface of order k
    """
    f = 2
    g = z**k

    psi = Weierstrass_Enneper(f, g, z)
    if verbose:
        print(f"Minimal curve psi_{k} = {psi}")

    psi_func = lambdify(z, psi)
    polar = lambda r, phi: r*np.cos(phi) + 1j*r*np.sin(phi)

    return lambda u, v: map(np.real, psi_func(polar(u, v)))


import math

import numpy as np
import pandas as pd
import reader
from sklearn.linear_model import Ridge
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold

# file path constants
FILE_PATH_TRAIN = "train1b.csv"

# feature transformation functions
functions = [lambda x: float(x), lambda x: math.pow(x, 2), lambda x: math.exp(x), lambda x: math.cos(x), lambda x: float(1)]

# read input data
data = reader.read_csv(FILE_PATH_TRAIN, False)

# drop the first column
data = pd.DataFrame.drop(data, columns='Id')

# splitting the data set into x & y values
# y-values
Y = pd.DataFrame(data, columns=['y'], copy=True)
Y = Y['y'].values

# x-values
X = pd.DataFrame(data, columns=['x1', 'x2', 'x3', 'x4', 'x5'], copy=True)

# new data frames for feature transformations
quad = pd.DataFrame(data, columns=['x6', 'x7', 'x8', 'x9', 'x10'])
exp = pd.DataFrame(data, columns=['x11', 'x12', 'x13', 'x14', 'x15'])
cos = pd.DataFrame(data, columns=['x16', 'x17', 'x18', 'x19', 'x20'])
const = pd.DataFrame(data, columns=['x21'])

# performing the feature transformations
for index, row in X.iterrows():
    # 1. transformation: quadratic
    quad['x6'][index] = functions[1](X['x1'][index])
    quad['x7'][index] = functions[1](X['x2'][index])
    quad['x8'][index] = functions[1](X['x3'][index])
    quad['x9'][index] = functions[1](X['x4'][index])
    quad['x10'][index] = functions[1](X['x5'][index])

    # 2. transformation: exponential
    exp['x11'][index] = functions[2](X['x1'][index])
    exp['x12'][index] = functions[2](X['x2'][index])
    exp['x13'][index] = functions[2](X['x3'][index])
    exp['x14'][index] = functions[2](X['x4'][index])
    exp['x15'][index] = functions[2](X['x5'][index])

    # 3. transformation: cosine
    cos['x16'][index] = functions[3](X['x1'][index])
    cos['x17'][index] = functions[3](X['x2'][index])
    cos['x18'][index] = functions[3](X['x3'][index])
    cos['x19'][index] = functions[3](X['x4'][index])
    cos['x20'][index] = functions[3](X['x5'][index])

    # 4. transformation: constant
    const['x21'][index] = functions[4](1)

# concatenating all data frames
X_transformed = pd.concat([X, quad, exp, cos, const], axis=1)
X_transformed = X_transformed.values

# storing the evaluation scores
eval = []

# storing the coefficients
av_coeffs = []

# alpha parameter values
alphas = [10.0e-4, 10.0e-3, 10.0e-2, 10.0e-1, 10.0e0, 10.0, 20.0, 100.0]

for entry in alphas:
    # creating the 10-fold split
    ten_fold = KFold(n_splits=10, shuffle=False)

    # fold counter
    i = 1

    # create model
    ridge = Ridge(alpha=entry, fit_intercept=True, normalize=False,
                  copy_X=True, max_iter=None, tol=1e-10, solver='auto')

    # errors per alpha value
    errors = []

    # coefficients per alpha value
    coeffs = []

    # training the ridge cv model
    for train_ind, val_ind in ten_fold.split(X_transformed, Y):
        # x & y training set (no separated test set)
        x_train = X_transformed[train_ind, :]
        y_train = Y[train_ind]

        # x & y validation set (no separated test set)
        x_val = X_transformed[val_ind, :]
        y_val = Y[val_ind]

        # train the model
        ridge.fit(x_train, y_train)

        # remember the coefficients
        coeffs.append(ridge.coef_)

        # evaluate with validation set
        y_val_predictions = ridge.predict(x_val)

        # compute root mean square error
        rmse = math.sqrt(mean_squared_error(y_val, y_val_predictions))
        errors.append(rmse)

        # printing alpha, fold number and root mean squared error
        # print("Alpha:\t", str(entry)+"\t", "Fold:", str(i)+"\t", "RMSE:\t", rmse)
        # print("Coefficients:\t", ridge.coef_)

        # increment fold counter
        i += 1

    # compute mean errors for rmse of each alpha
    eval.append(np.mean(errors))

    # compute the averaged coefficients
    av_coeffs.append(np.mean(coeffs, axis=0, dtype=np.float64))

# print out the rmse averages for all lambdas
min_rmse = 10000000
min_rmse_ind = None

# printing a summary
print("Summary of all alpha values tested:")

for ind in range(0, len(alphas)):
    print("Alpha:\t", alphas[ind], "\t\t", "Avg RMSE:\t", eval[ind])

    if eval[ind] < min_rmse:
        min_rmse = eval[ind]
        min_rmse_ind = ind

# preparing to write the coefficients to file
out = {"Avg Coeffs": av_coeffs[min_rmse_ind]}

# output data frame
best_coeffs = pd.DataFrame(data=out)

# printing test output
print()
print("Result Written To File:")
print("Alpha:\t", alphas[min_rmse_ind], "\tRMSE:\t", min_rmse)
print()
print(best_coeffs.head(5))

# # write to csv-file
# best_coeffs.to_csv("task1b_ridge_10-fold-cv_Nr4.csv", sep=',', index=False, header=False)


from math import sin,cos,exp,sqrt,log
from numpy import arange
import sys
import matplotlib.pyplot as plt

# Parâmetros da exibição dos gráficos
plt.rcParams['xtick.labelsize'] = 24
plt.rcParams['ytick.labelsize'] = 24
plt.rcParams['axes.labelsize'] = 28

RC = float(sys.argv[1])

def vinquad(t):
    vin = 1.
    if int(2*t)%2 == 1:
        vin = -1.
    return vin

def fquad(x,t):
    return (vinquad(t) - x)/RC

def passo_rk4(f,x,t,h):            # Calcula um passo no método de RK4
    k1 = h*f(x,t)
    k2 = h*f(x+0.5*k1,t+0.5*h)
    k3 = h*f(x+0.5*k2,t+0.5*h)
    k4 = h*f(x+k3,t+h)
    return (k1+2.0*(k2+k3)+k4)/6.0

def sol_exata(t,a,xa):             # Válida somente para fquad
    n = int((t-a)/0.5)
    x0 = xa
    t0 = a
    sinal = 1.
    vin = sinal
    for i in range(n):
        xf = vin - (vin - x0)*exp(-0.5/RC)
        t0 = t0 + 0.5
        x0 = xf
        sinal = -sinal
        vin = sinal
    xf = vin - (vin - x0)*exp(-(t-t0)/RC)
    return xf

def main():

    a = 0.00         # Início do intervalo
    b = 10.0         # Final do intervalo
    xa = 0.0         # Condição inicial, ou seja, x(a)

    N_exato = 1000    # Número de pontos para o gráfico da solução exata
    h_exato = (b-a)/N_exato
    t_exato = arange(a,b,h_exato)
    x_exato = []

    for t in t_exato:  # Criando as listas para o gráfico da função
        x_exato.append(sol_exata(t,a,xa))

    N = 1000           # Número de passos da integração numérica
    h = (b-a)/N        # Tamanho do passo da integração

    xrk4 = xa
    t_rk4 = arange(a,b,h)
    x_rk4 = []

    for t in t_rk4:   # Realizando a integração numérica
        x_rk4.append(xrk4)
        xrk4 += passo_rk4(fquad,xrk4,t,h)

    plt.figure(figsize=(12,9))
    plt.plot(t_rk4,x_rk4,'r-',t_exato,x_exato,'bo')
    plt.xlabel("$t$")
    plt.ylabel("$V_{out}(t)$")
    plt.tight_layout()
    plt.show()

if __name__ == '__main__':
    main()


# Copyright 2018 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
A simple implementation of PCA (Principle Component Analysis)
using numpy. Written to apply to neural network activations.

"""

import numpy as np

def get_pca(acts, compute_dirns=False):
    """ Takes in neuron activations acts and number of components.
    Returns principle components and associated eigenvalues.

    Args:
            acts: numpy array, shape=(num neurons, num datapoints)
            n_components: integer, number of pca components to reduce
                          to

    """

    assert acts.shape[0] < acts.shape[1], ("input must be number of neurons"
                                           "by datapoints")

    # center activations
    means = np.mean(acts, axis=1, keepdims=True)
    cacts = acts - means

    # compute PCA using SVD
    U, S, V = np.linalg.svd(cacts, full_matrices=False)

    return_dict = {}
    return_dict["eigenvals"]  = S
    return_dict["neuron_coefs"] = U.T
    if compute_dirns:
        return_dict["pca_dirns"] = np.dot(U.T, cacts) + means

    return return_dict


import numpy as np
import matplotlib.pyplot as plt

from scipy.spatial.distance import pdist, squareform
from scipy import exp
from scipy.linalg import eigh
from sklearn.datasets import make_circles
from sklearn.decomposition import PCA
from matplotlib.ticker import FormatStrFormatter

def rbf_kernel_pca(x, gamma, n_comps):
    """Radial-Basis Kernel PCA"""

    # Calc pairwise squared Euclidian dists in mxn dimm dataset
    sq_dists = pdist(x,'sqeuclidean')

    # Convert pairwise squared Eculidian dist into a square matrix
    mat_sq_dists = squareform(sq_dists)

    # Compute the symetric kernel matrix
    k = exp(-gamma * mat_sq_dists)

    # Center the kernel matrix
    n = k.shape[0]
    one_n = np.ones((n, n)) / n
    k = k - one_n.dot(k) - k.dot(one_n) + one_n.dot(k).dot(one_n)

    # Obtain eigenpairs from the centered kernel matrix
    eigvals, eigvecs = eigh(k)

    # projected ds
    x_pc = np.column_stack((eigvecs[:, -i]
                            for i in range(1, n_comps + 1)))

    return x_pc

x, y = make_circles(n_samples=1000, random_state=123, noise=0.1, factor=0.2)

plt.scatter(x[y == 0, 0], x[y == 0, 1],
            color='red', marker='^', alpha=0.5)
plt.scatter(x[y == 1, 0], x[y == 1, 1],
            color='blue', marker='o', alpha=0.5)
plt.show()

scikit_pca = PCA(n_components=2)
x_spca = scikit_pca.fit_transform(x)
fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(7, 3))

ax[0].scatter(x_spca[y == 0, 0], x_spca[y == 0, 1],
              color='red', marker='^', alpha=0.5)
ax[0].scatter(x_spca[y == 1, 0], x_spca[y == 1, 1],
              color='blue', marker='o', alpha=0.5)
ax[1].scatter(x_spca[y == 0, 0], np.zeros((500, 1)) + 0.02,
              color='red', marker='^', alpha=0.5)
ax[1].scatter(x_spca[y == 1, 0], np.zeros((500, 1)) - 0.02,
              color='blue', marker='o', alpha=0.5)

ax[0].set_xlabel('PC1')
ax[0].set_ylabel('PC2')
ax[1].set_ylim([-1, 1])
ax[1].set_yticks([])
ax[1].set_xlabel('PC1')
plt.show()

x_kpca = rbf_kernel_pca(x, gamma=15, n_comps=2)
fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(7, 3))
ax[0].scatter(x_kpca[y == 0, 0], x_kpca[y == 0, 1],
              color='red', marker='^', alpha=0.5)
ax[0].scatter(x_kpca[y == 1, 0], x_kpca[y == 1, 1],
              color='blue', marker='o', alpha=0.5)
ax[1].scatter(x_kpca[y == 0, 0], np.zeros((500, 1)) + 0.02,
              color='red', marker='^', alpha=0.5)
ax[1].scatter(x_kpca[y == 1, 0], np.zeros((500, 1)) - 0.02,
              color='blue', marker='o', alpha=0.5)

ax[0].set_xlabel('PC1')
ax[0].set_ylabel('PC2')
ax[1].set_ylim([-1, 1])
ax[1].set_yticks([])
ax[1].set_xlabel('PC1')
ax[0].xaxis.set_major_formatter(FormatStrFormatter('%0.1f'))
ax[1].xaxis.set_major_formatter(FormatStrFormatter('%0.1f'))
plt.show()


import numpy as np
import matplotlib.pyplot as plt
import time
import scipy.integrate as inte

def system_of_ode(x,U):
    u , v = U
    du = v
    dv = 2*v/(1-x) - u/((1-x)**4)
    return np.array([du , dv])

# f(x)
def exact(x):
    y = np.sin(1/(1-x))
    return y

def main():

    # u(0) , v(0)
    u0 , v0 = np.sin(1) , np.cos(1)
    system_of_ode_0 = [u0 , v0]

    # LSODA method is an implicit method. Note that although it is a faster method, advanced explicit methods such as 'DOP853' are generally more precise.
    method = 'LSODA'

    start_time = time.clock()
    s = inte.solve_ivp(fun=system_of_ode,t_span=(0,0.99),y0=system_of_ode_0,method=method,rtol=2.3e-14,atol=1e-20)
    end_time = round(time.clock() - start_time, 3)

    # u(x) , v(x)
    u , v  = s.y
    x = s.t
    exact_f = exact(x)

    print('Method:', method)
    print('Number of data points:', len(u))
    print('Execution time:', end_time, 'seconds')

    # Difference of u(x) and f(x) ( Difference of numerical solution and exact solution; computed in the origianl c++ program )
    difference = exact_f-u

    # Plot of u(x)
    plt.plot(x, u,'b')
    plt.xlabel(r'$x$',fontsize=16)
    plt.ylabel(r'$u(x)$',fontsize=16)
    plt.tight_layout()
    plt.show()

    # Plot of v(x)
    plt.plot(x, v,'g')
    plt.xlabel(r'$x$',fontsize=16)
    plt.ylabel(r'$v(x)$',fontsize=16)
    plt.tight_layout()
    plt.show()

    # Plot of difference, using symmetric log
    plt.plot(x, difference,'.-r')
    plt.yscale('symlog',linthresh=1e-15)
    plt.xlabel(r'$x$',fontsize=16)
    plt.ylabel(r'$difference$',fontsize=16)
    plt.tight_layout()
    plt.show()

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pyplot as plt

info_list = []
def InverseShift(A, mu, converge_range):

    r, c = A.shape

    if r != c:
        raise Exception("not a square matrix")

    #initialize eigenvectors
    vec_list = []
    lambda_list = []
    diff_list = []

    idx = 0
    vec_init = np.zeros(r)
    vec_init[-1] = 1
    vec_list.append(vec_init)

    #initialize eigenvalues
    lambda_init = vec_init.dot(A.dot(vec_init))
    lambda_list.append(lambda_init)

    diff_init = float("inf")
    diff_list.append(diff_init)

    I = np.identity(r)

    while diff_list[idx] > converge_range:
        mat = A - (mu*I)
        #solve the (A-(mu)I)w = v^(k-1) for w=vec_new
        vec_new = np.linalg.solve(mat, vec_list[idx])
        #normalize the newly found vec_new=w
        vec_new = vec_new / np.linalg.norm(vec_new)
        #add this to the vector list
        vec_list.append(vec_new)
        lambda_new = vec_new.dot(A.dot(vec_new))
        lambda_list.append(lambda_new)
        diff = np.abs(lambda_new - lambda_list[idx])
        diff_list.append(diff)
        idx = idx + 1

    print_log(idx, vec_list, lambda_list, diff_list)

    #plot lambda_list
    x = [i for i in range(idx+1)]
    if len(x) > 20:
        x = x[:21]
        diff_list = diff_list[:21]

    plt.plot(x, diff_list)
    plt.show()
    plt.savefig("IwS_difference_list_plot")
    return vec_list[-1], lambda_list[-1]

def print_log(idx, vec_list, lambda_list, diff_list):
    print("Inverse Iteration with Shift:")
    print('Number of Iterations:', idx)
    cor_eig_vec = vec_list[idx]
    sm_eig_val = round(lambda_list[idx], 4)
    inv_eig_val = np.reciprocal(sm_eig_val)
    print('Inverse Eigenvalue:', inv_eig_val) #
    print('Eigenvalue:', sm_eig_val)
    print('Corresponding Eigenvector:', cor_eig_vec)

#Examples to run on
D = np.array([[2, 2, -1], [-5, 9, -3], [-4, 4, 1]]) # eigenvals of D are 3, 4, and 5
E = np.array([[-4, 1, 1], [0, 3, 1], [-2, 0, 15]]) # eigenvals of E are ~ -3.9095, 3.0243 and 14.8852
InverseShift(E, 10, 0.00001)


import numpy as np
import random
import math
import matplotlib.pyplot as plt

def modelo(x):
    return 2 * x ** 1.345 + 0.75 * random.random()

x0, x1 = 0, 10
n = 100
x = x0 + (x1 - x0) * np.random.rand(n)
x.sort()
y = [modelo(xi) for xi in x]

def best_pow(x, y):

    new_x = [math.log(xi) for xi in x]
    new_y = [math.log(yi) for yi in y]
    sum_x = sum(new_x)
    sum_x2 = sum(xi ** 2 for xi in new_x)
    A = [[len(new_x), sum_x], [sum_x, sum_x2]]
    sum_new_y = sum(new_y)
    sum_new_xy = sum(xi * yi for xi, yi in zip(new_x, new_y))
    B = [sum_new_y, sum_new_xy]
    a0, a1 = np.linalg.solve(A, B)
    a, b = math.exp(a0), a1
    return a, b

a, b = best_pow(x, y)

def bpow(x):
    return a * x ** b

if __name__ == '__main__':

    print(a, b)
    t = np.linspace(min(x),max(x), 100)
    bpowt = [bpow(i) for i in t]
    plt.plot(t, bpowt, color='orange', linewidth=3, zorder=10)
    plt.scatter(x, y, zorder=1)
    plt.savefig('best_pow.png')


#Pell number
import numpy as np
def __pow(x, n, I):
    if n==0:
        return I
    elif n == 1:
        return x
    else:
        y = __pow(x,n//2,I)
        y = y.dot(y)
        if n % 2:
            y = x.dot(y)
        return y

def PellNumber(index):
    F = __pow(np.array(((2,1),(1,0)),dtype=object),index,np.eye(2))
    return F[0][1]

def Binetformula(index=1,roundi=True):
    from sympy import Pow as mmmpow
    from sympy import Integer as mmmint
    from sympy import N as mmmn
    q = 2**(1/2)
    r = (mmmpow(1+q,index)-mmmpow(1-q,index))/(2*q)
    if(roundi):
        return mmmint(r)
    else:
        return mmmn(r)

#Pell–Lucas
def CompanionPellLucasNumber(Pell,roundi=True):
    from sympy import Integer as mmmint
    from sympy import N as mmmn
    r = Pell*(2**(1/2))
    if(roundi):
        return mmmint(r)
    else:
        return mmmn(r)

def CompanionPellNumber(index=1,roundi=True):
    from sympy import Pow as mmmpow
    from sympy import Integer as mmmint
    from sympy import N as mmmn
    q = 2**(1/2)
    r = (mmmpow(1+q,index)-mmmpow(1-q,index))/(2)
    if(roundi):
        return mmmint(r)
    else:
        return mmmn(r)

def PellNumbers(toEnd):
    Pells=[1,2,5]
    i=3
    while i<toEnd:
        num=Pells[i-1]*2+Pells[i-2]
        Pells.append(num)
        i+=1
    return Pells

def doTest(toPrint=False,toProgress=False,start=0,toEnd=1000,algo="s"):
    s = set()
    KK = 10000
    from IPython.display import clear_output
    for i in range(start,toEnd+1):
        if(toProgress and (i<KK or (i>=KK and i%(KK/100)==0))):
            clear_output(wait=True)
            print(i,end="\t")
        if(algo=="s"):
            pell = PellNumber(i)
        elif(algo=="binet"):
            pell = Binetformula(i)
        elif(algo=="companion"):
            pell = CompanionPellNumber(i)
        elif(algo=="pelllucas"):
            pell = CompanionPellLucasNumber(i)
        if(pell):
            s.add(pell)
            if(toPrint and not toProgress):
                print(pell,end=", ")
        if(toProgress and (i<KK or (i>=KK and i%(KK/100)==0))):
            print(s)
    if(not toPrint):
        return s

#print(Binetformula(6))
#print(PellNumber(6))
#doTest(True,False,0,10000)


#Acessar elementos
import numpy as np

x = np.linspace(start = 10, stop = 100, num = 10)
print("x: ", x)
print("Shape: ",x.shape)
print('''
Primeiro numero é: {}
Segundo numero é: {}
Terceiro numero é: {}
Ultimo numero é: {}
'''.format(x[0],x[1],x[2],x[9]))

#Slicing
print("x: ",x)
print("Os cincos primeiros mumeros: ", x[0:5])
print("Os dois primeiros numeros: ", x[:2])
print("Os dois ultimos numeros: ", x[-2: ])

print('')

#Acessando elementos D2
x = x.reshape(2, 5) # reshape(remodelar) de x para 2 linhas e 5 colunas
print("x:\n ", x)
print(f'''
Primeira linha, segunda coluna: {x[0,1]}
Segunda lina, penultima coluna: {x[1,-2]}
Ultima linha, ultima coluna: {x[1,-4]}
Ultima linha, ultima coluna: {x[-1, -1]}
''')

#Slicing
print("x:\n",x)
print(f'''
Primeira linha inteira: {x[0,:]}
Primeira linha, segunda ate a quarta coluna: {x[0,1:4]}
Ultima coluna inteira:\n {x[:,[-1]]}
''')

#Compartilhamento de memoria
a = np.array([1,2,3,4])
print("A:", a)
z = a[0:4].copy()
z[0] = -90
print("Z:",z)
print("A:",a)
 #Se não for usado o metodo copy(), o array principal(a) será alterado.


#!/usr/bin/env python
# coding: utf-8

# # Week 01 Example Statistics in Python 3
# This Notebook introduces basic statistical analysis from week 01 using Python 3.x.
#
# Jupyter notebooks blend Markdown (rich formatted text) with software code and output to ease the learning process.
#

# Version: 01
#
# Author: Chris Kennedy

# In[1]:

import pandas as pd
from scipy import stats

# In[2]:

dfWine = pd.read_excel(r'W1 - Wine Quality.xlsx')

# In[3]:

dfWine.describe()

# ## Practice Question 1
# What is the 99% confidence interval for the average alcohol level of a bottle of wine?

# In[4]:

n = dfWine['alcohol'].count()
print("# Wines: %6.2f" % n)

# In[5]:

avg = dfWine['alcohol'].mean()
print("Average Alcohol: %6.4f" % avg)

# In[6]:

stderr = dfWine['alcohol'].sem()
print("Standard Error: %6.4f" % stderr)

# In[7]:

t_CI = stats.t.ppf(0.995,df = n - 1)
print("T-statistic: %6.4f" % t_CI)

# In[8]:

moe = t_CI * stderr
print("Margin of error: %6.4f" % moe)

# In[9]:

print("Lower: %6.4f" % (avg - moe))
print("Upper: %6.4f" % (avg + moe))

# In[10]:

# Concise using stats package directly:
stats.t.interval(0.99, loc=avg, scale=stderr, df = n-1)

# ## Practice Question 2
# What is the 90% confidence interval around the proportion of white wines that are rated very good quality (7 or higher)?
#

# In[11]:

filteredDF = dfWine[dfWine['type'] == 'white']
n = filteredDF['type'].count()

# In[12]:

qfilteredDF = filteredDF[filteredDF['quality'] >= 7]
nq = qfilteredDF['type'].count()

# In[13]:

proportion  = nq / n
print("Proportion: %5.2f%%" % (proportion*100))

# In[14]:

stderr = (proportion * (1 - proportion) / n)**0.50
print("Standard error: %5.2f%%" % (stderr*100))

# In[15]:

z_CI = stats.norm.ppf(0.95)

# In[16]:

print ("Z for 90%%: %6.4f" % z_CI)

# In[17]:

stats.norm.interval(0.90, loc=proportion*100, scale=stderr*100)

# ## Question 3
# Can you conclude (at the 5% significance level) that the average fixed acid level for all wines is above 7.2?

# In[18]:

def getStatistic(x, H0, stderr):
    return (x - H0) / stderr

# H$_0$: $\mu \le 7.2$
#
# H$_a$: $\mu > 7.2$
#

# In[19]:

H0 = 7.20
avg = dfWine['fixed acidity'].mean()
stderr = dfWine['fixed acidity'].sem()
tStatistic = getStatistic(avg, H0, stderr)
nWines = dfWine['fixed acidity'].count()
print("T-stat: %7.4f" %tStatistic)
dfWine['fixed acidity'].describe()

# Right tail test due to alternative hypothesis.

# In[20]:

prob_value = 1 - stats.t.cdf(tStatistic, df=nWines-1)
print(prob_value)

# In[21]:

stats.norm.interval(0.9, loc=5,scale=2
                )

# ### End of Notebook!


import tensorflow as tf
import numpy as np

def sigmoid(x):
    return 1.0/(1+np.exp(-x))
#
# 计算方式：对输入的logits先通过sigmoid函数计算，再计算它们的交叉熵，但是它对交叉熵的计算方式进行了优化，使得的结果不至于溢出。
# 适用：每个类别相互独立但互不排斥的情况：例如一幅图可以同时包含一条狗和一只大象。
# output不是一个数，而是一个batch中每个样本的loss,所以一般配合tf.reduce_mean(loss)使用。
#计算公式图片 https://upload-images.jianshu.io/upload_images/5877934-fb698517859fea13.png?imageMogr2/auto-orient/strip%7CimageView2/2/w/720/format/webp
# 5个样本三分类问题，且一个样本可以同时拥有多类
y = np.array([[1,0,0],[0,1,0],[0,0,1],[1,1,0],[0,1,0]])
logits = np.array([[12,3,2],[3,10,1],[1,2,5],[4,6.5,1.2],[3,6,1]])
# 按计算公式计算
y_pred = sigmoid(logits)
E1 = -y*np.log(y_pred)-(1-y)*np.log(1-y_pred)
print(E1)     # 按计算公式计算的结果
# 按封装方法计算
sess =tf.Session()
y = np.array(y).astype(np.float64) # labels是float64的数据类型
E2 = sess.run(tf.nn.sigmoid_cross_entropy_with_logits(labels=y,logits=logits))
print(E2)     # 按 tf 封装方法计算

if E1.all() == E2.all():
    print("True")
else:
    print("False")
# 输出的E1，E2结果相同


"""
% -------------------------------------------------------------
%                                          Matlab code
% -------------------------------------------------------------
% grpah partition using the eigenvector corresponding to the second
% smallest  eigenvalue
% grpah partition using the eigenvector corresponding to the second
% smallest  eigenvalue
t=[randn(500,2)+repmat([-2,-2],500,1) ;randn(500,2)+repmat([2,2],500,1)];
scatter(t(:,1),t(:,2))
W=squareform(pdist(t));
A=W<3;      % create adjacency matrix (set connected notes equal to one)
D       = sum(A,1);
L       = diag(D)-A;
Lsym    = diag(D.^-0.5)*L*diag(D.^-0.5);
[u,s,v] = svd(Lsym);

figure; plot(u(:, (end-1)))
F       = u(:, (end-1));
plot(F);title('Second smallest non-zero eigenvalue eigenvector');
scatter(t(F<0,1),t(F<0,2),'bo','filled');hold on
scatter(t(F>0,1),t(F>0,2),'go','filled');
"""
# Pytorch equivalent code
import torch
from torch.autograd import Variable

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.cm as cm

import matplotlib.colors as colors
import matplotlib.cm as cm
import matplotlib as mpl

color_map =  plt.get_cmap('jet')

def distance_matrix(mat):
    d= ((mat.unsqueeze (0)-mat.unsqueeze (1))**2).sum (2)**0.5
    return d

# Generate Clusters
mat = torch.cat([torch.randn(500,2)+torch.Tensor([-2,-3]),   torch.randn(500,2)+torch.Tensor([2,1])])
plt.scatter(mat[:,0].numpy(),mat[:,1].numpy())
plt.show(block=False)
##-------------------------------------------
#         Compute distance matrix and then the Laplacian
##-------------------------------------------
d= distance_matrix(mat);
da=d<2;
plt.figure()
plt.imshow(da.numpy())
plt.show(block=False)

D= ((da.float()).sum(1)).diag()
L = D -da.float()
plt.figure()
plt.title("Laplacian")
plt.imshow(L.numpy())
plt.show(block=False)

Lsym=torch.mm(torch.mm(torch.diag(torch.pow(torch.diag(D),-0.5)),L),torch.diag(torch.pow(torch.diag(D),-0.5)));
plt.figure()
plt.imshow(Lsym.numpy())
plt.title("Symmetric Laplacian")
plt.show(block=False)

[u,s,v]=torch.svd(Lsym)

# plot fiedler vector

plt.figure()
plt.title('Fiedler vector')
plt.plot(u[:,-2].numpy());
plt.show(block=False)
norm  = colors.Normalize(vmin=-1, vmax=1)

scalarMap = cm.ScalarMappable( norm=norm , cmap=color_map)

plt.figure()
plt.title('clusters')
for i in range(len(u[:,-2])):
	if u[i,-2]<0:
		color = scalarMap.to_rgba(-1)
		plt.scatter(mat[i,0],mat[i,1], color=color,marker='o')
	else:
		color = scalarMap.to_rgba(1)
		plt.scatter(mat[i,0],mat[i,1], color=color,marker='*')

plt.show(block=False)

raw_input("Press Enter to exit..")
plt.close('all')


# %%
import numpy as np
import numpy.linalg as la
import time
import scipy.optimize as sciopt
from mrina.solver_l1_norm import sft, project_l1_ball, MinimizeSumOfSquares

# %%
# Projection onto the nuclear norm ball / tall matrices
def project_nuclear_ball(X, t, eps=1E-5):
    # Compute right singular vectors
    s, V = la.eigh(X.conj().T @ X)
    s = np.sqrt(np.abs(s))

    if np.sum(s) <= t:
        return X

    idx = np.where(s > eps)[0]
    V = V[:, idx]
    s = s[idx]

    st = project_l1_ball(s, t)
    return X @ ((V * (st/s) ) @ V.conj().T)

# Singular value thresholding
def svt(X, t, eps=1E-6):
    # Compute right singular vectors
    s, V = la.eigh(X.conj().T @ X)
    s = np.sqrt(np.abs(s))

    idx = np.where(s >= t)[0]
    if idx.size == 0:
        return np.zeros(shape=X.shape, dtype=X.dtype)

    idx = np.where(s > eps)[0]
    V = V[:, idx]
    s = s[idx]

    st = sft(s, t)
    return X @ ((V * (st/s) ) @ V.conj().T)

# Minimize Sum-of-Squares subject to nuclear norm constraints
#   min_X || A(X) - y ||_2^2 s.t. ||X||_* <= t
def MinimizeSumOfSquaresNuclearBall(t, y, A, Xinit=None, L=None,
                                        maxItns=1E4,
                                        dxAbsTol=1E-4, dxRelTol=1E-6,
                                        dpAbsTol=1E-5, dpRelTol=1E-8,
                                        disp=False, printEvery=10,
                                        restart=True):
    # Project initial iterate on the nuclear-ball
    if Xinit is None:
        X = np.zeros(A.inShape, dtype=np.complex)
    else:
        X = Xinit
    X = project_nuclear_ball(X, t)
    # Lipschitz constant
    if L is None:
        L = 2.05 * A.norm() ** 2
    # Variables
    rX = A.eval(X) - y
    fX = la.norm(rX.ravel()) ** 2
    gX = 2.0 * A.adjoint(rX)
    rpX = X - project_nuclear_ball(X - gX, t)
    rpXNrm = la.norm(rpX.ravel())
    # Initialize variables
    XNrm = la.norm(X.ravel())
    dXNrm = np.inf
    dfX = np.inf
    itn = 0
    s = 1.0
    Z = X
    # Optimization loop
    stop = False
    if disp:
        print('[PGD-SoS-NuclearBall]')
        print(' Initial objective:   {:1.6e}'.format(fX))
        print(' Initial residual:    {:1.6e}'.format(rpXNrm))
    while not stop:
        itn = itn + 1
        gZ = 2.0 * A.adjoint(A.eval(Z) - y)
        Xp = project_nuclear_ball(Z - gZ/L, t)
        sp = 0.5 * (1.0 + np.sqrt(1.0 + 4.0 * s ** 2))
        Zp = Xp + ((s - 1.0)/sp) * (Xp - X)

        # Update
        rXp = A.eval(Xp) - y
        fXp = la.norm(rXp.ravel()) ** 2

        dfX = fXp - fX
        dXNrm = la.norm((Xp - X).ravel())

        if restart and dfX > 1E-12:
            Z = X
            s = 1.0
            dfX = np.inf
            dXNrm = np.inf
        else:
            X = Xp
            gX = 2.0 * A.adjoint(rXp)
            rpX = X - project_nuclear_ball(X - gX, t)
            rpXNrm = la.norm(rpX.ravel())

            Z = Zp
            fX = fXp
            s = sp

        XNrm = la.norm(X.ravel())
        dXTol = np.max([ dxAbsTol, dxRelTol * XNrm ])
        rpXTol = np.max([ dpAbsTol, dpRelTol * XNrm ])
        if (dXNrm < dXTol and rpXNrm < rpXTol) or itn > maxItns:
            stop = True
        if disp and (itn == 1 or np.mod(itn, printEvery) == 0 or stop):
            print(' {:04d} | fX: {:1.3E} | dfX: {:+1.3E} | dXNrm: {:1.3E} | rpXNrm: {:5.3E}'.format(itn, fX, dfX, dXNrm, rpXNrm))
    return X, fX

# Minimize nuclear norm regularized Sum-of-Squares
#   min_X || A(X) - y ||_2^2 + t ||X||_*
def MinimizeNNDN(t, y, A, Xinit=None, L=None,
                    maxItns=1E4,
                    dxAbsTol=1E-4, dxRelTol=1E-6,
                    dpAbsTol=1E-5, dpRelTol=1E-8,
                    disp=False, printEvery=10,
                    restart=True):
    # Project initial iterate on the nuclear-ball
    if Xinit is None:
        X = np.zeros(A.inShape, dtype=np.complex)
    else:
        X = Xinit
    # Lipschitz constant
    if L is None:
        L = 2.05 * A.norm() ** 2
    # Variables
    rX = A.eval(X) - y
    fX = la.norm(rX.ravel()) ** 2 + t * la.norm(np.sqrt(la.eigvalsh(X.conj().T @ X)), 1)
    gX = 2.0 * A.adjoint(rX)
    rpX = X - svt(X - gX, t)
    rpXNrm = la.norm(rpX.ravel())
    # Initialize variables
    XNrm = la.norm(X.ravel())
    dXNrm = np.inf
    dfX = np.inf
    itn = 0
    s = 1.0
    Z = X
    # Optimization loop
    stop = False
    if disp:
        print('[PGD-NNDN]')
        print(' Initial objective:   {:1.6e}'.format(fX))
        print(' Initial residual:    {:1.6e}'.format(rpXNrm))
    while not stop:
        itn = itn + 1
        gZ = 2.0 * A.adjoint(A.eval(Z) - y)
        Xp = svt(Z - gZ/L, t/L)
        sp = 0.5 * (1.0 + np.sqrt(1.0 + 4.0 * s ** 2))
        Zp = Xp + ((s - 1.0)/sp) * (Xp - X)

        # Update
        rXp = A.eval(Xp) - y
        fXp = la.norm(rXp.ravel()) ** 2 + t * la.norm(np.sqrt(la.eigvalsh(Xp.conj().T @ Xp)), 1)

        dfX = fXp - fX
        dXNrm = la.norm((Xp - X).ravel())
        if restart and dfX > 1E-12:
            Z = X
            s = 1.0
            dfX = -np.inf
            dXNrm = np.inf
        else:
            X = Xp
            gX = 2.0 * A.adjoint(rXp)
            rpX = X - svt(X - gX, t)
            rpXNrm = la.norm(rpX.ravel())

            Z = Zp
            fX = fXp
            s = sp

        dXTol = np.max([ dxAbsTol, dxRelTol * XNrm ])
        rpXTol = np.max([ dpAbsTol, dpRelTol * XNrm ])
        if (dXNrm < dXTol and rpXNrm < rpXTol) or itn > maxItns:
            stop = True

        if disp and (itn == 1 or np.mod(itn, printEvery) == 0 or stop):
            print(' {:04d} | obj: {:1.3E} | dfx: {:+1.3E} | dxNrm: {:1.3E} | rpxNrm: {:5.3E}'.format(itn, fX, dfX, dXNrm, rpXNrm))
    return X, fX

class RootSolverNuclearNormNoisy():
    def __init__(self, eta, y, A,
                    maxItns=1E4,
                    dxAbsTol=1E-4, dxRelTol=1E-6,
                    dpAbsTol=1E-5, dpRelTol=1E-8,
                    disp=False, printEvery=10,
                    restart=True,
                    method='NNDN'):
        # Problem parameters
        self.eta = eta
        self.y = y
        self.A = A

        self.ANrm = A.norm()
        self.L = 2.05 * self.ANrm ** 2

        # For bracketing
        self._itn = 0
        self._t = [ None, None ]
        self._ft = [ None, None ]
        self._Xinit = [ None, None ]
        if method == 'SoS-NucBall':
            # Lower bound
            self._t[0] = 0.0
            self._ft[0] = la.norm(y.ravel()) - eta
            self._Xinit[0] = np.zeros(A.inShape, dtype=np.complex)
            # Upper bound
            _Xopt, _fopt = MinimizeSumOfSquares(y, A,
                                                maxItns=maxItns,
                                                dxAbsTol=dxAbsTol, dxRelTol=dxRelTol,
                                                gradAbsTol=dpAbsTol, gradRelTol=dpRelTol,
                                                disp=False)
            self._t[1] = la.norm(np.sqrt(la.eigvalsh(_Xopt.conj().T @ _Xopt)), 1)
            self._ft[1] = la.norm((A.eval(_Xopt) - y).ravel(), 2) - self.eta
            self._Xinit[1] = _Xopt
        elif method == 'NNDN':
            # Lower bound
            _Xopt, _fopt = MinimizeSumOfSquares(y, A,
                                                maxItns=maxItns,
                                                dxAbsTol=dxAbsTol, dxRelTol=dxRelTol,
                                                gradAbsTol=dpAbsTol, gradRelTol=dpRelTol,
                                                disp=False)
            self._t[0] = 0.0
            self._ft[0] = -eta
            self._Xinit[0] = _Xopt
            # Upper bound
            _Z = A.adjoint(y)
            self._t[1] = la.norm(np.sqrt(la.eigvalsh(_Z.conj().T @ _Z)), np.inf)
            self._ft[1] = la.norm(y.ravel(), 2) - self.eta
            self._Xinit[1] = np.zeros(A.inShape, dtype=np.complex)
        else:
            raise ValueError('Method can be either SoS-NucBall or NNDN.')
        self.method = method
        # Solver options
        self.restart = restart
        # Solver parameters
        self.maxItns = maxItns
        self.dxAbsTol = dxAbsTol
        self.dxRelTol = dxRelTol
        self.dpAbsTol = dpAbsTol
        self.dpRelTol = dpRelTol
        # Display options
        self.disp = disp
        self.printEvery = printEvery

    def solve(self, t):
        self._itn = self._itn + 1
        if self.disp:
            print(' {:02d} | t* = {:1.4E} in [{:1.4E}, {:1.4E}] | f: [{:+1.4E}, {:+1.4E}]'.format(self._itn, t, self._t[0], self._t[1], self._ft[0], self._ft[1]))

        if np.abs(t - self._t[0]) < np.abs(t - self._t[1]):
            tNear = 0
        else:
            tNear = 1

        theta = np.abs(t - self._t[0]) / (np.abs(t - self._t[0]) + np.abs(t - self._t[1]))
        Xinit = theta * self._Xinit[0] + (1.0 - theta) * self._Xinit[1]

        if self.method == 'SoS-NucBall':
            Xopt, _ = MinimizeSumOfSquaresNuclearBall(t, y, A, Xinit=Xinit, L=self.L,
                                                        maxItns=self.maxItns,
                                                        dxAbsTol=self.dxAbsTol, dxRelTol=self.dxRelTol,
                                                        dpAbsTol=self.dpAbsTol, dpRelTol=self.dpRelTol,
                                                        disp=False, printEvery=self.printEvery,
                                                        restart=self.restart)
        elif self.method == 'NNDN':
            Xopt, _ = MinimizeNNDN(t, y, A, Xinit=Xinit, L=self.L,
                                    maxItns=self.maxItns,
                                    dxAbsTol=self.dxAbsTol, dxRelTol=self.dxRelTol,
                                    dpAbsTol=self.dpAbsTol, dpRelTol=self.dpRelTol,
                                    disp=False, printEvery=self.printEvery,
                                    restart=self.restart)

        if t > self._t[1]:
            self._t[0] = self._t[1]
            self._ft[0] = self._ft[1]
            self._Xinit[0] = self._Xinit[1]

            self._t[1] = t
            self._ft[1] = la.norm((A.eval(Xopt) - y).ravel(), 2) - self.eta
            self._Xinit[1] = Xopt
        elif t < self._t[0]:
            self._t[1] = self._t[0]
            self._ft[1] = self._ft[0]
            self._Xinit[1] = self._Xinit[0]

            self._t[0] = t
            self._ft[0] = la.norm((A.eval(Xopt) - y).ravel(), 2) - self.eta
            self._Xinit[0] = Xopt
        else:
            self._t[tNear] = t
            self._ft[tNear] = la.norm((A.eval(Xopt) - y).ravel(), 2) - self.eta
            self._Xinit[tNear] = Xopt

        return la.norm((A.eval(Xopt) - y).ravel(), 2) - self.eta

def RecoveryNuclearNormNoisy(eta, y, A,
                    maxItns=1E4,
                    dxAbsTol=1E-4, dxRelTol=1E-6,
                    dpAbsTol=1E-5, dpRelTol=1E-8,
                    disp=False, printEvery=100,
                    method='NNDN'):

    # Create solver object
    T = RootSolverNuclearNormNoisy(eta, y, A,
                                    maxItns=maxItns,
                                    dxAbsTol=dxAbsTol, dxRelTol=dxRelTol,
                                    dpAbsTol=dpAbsTol, dpRelTol=dpRelTol,
                                    disp=disp, printEvery=printEvery,
                                    restart=True, method=method)
    if disp:
        print('[RECOVERY: NUCLEAR NORM-NOISY]' )
        print(' Method:          {:s}'.format(method) )
        print(' Bound:           {:1.4E}'.format(eta) )
        print(' Initial values:')
        print('     t           [{:1.4E},   {:1.4E}]'.format(T._t[0], T._t[1]))
        print('     f(t)        [{:+1.4E}, {:+1.4E}]'.format(T._ft[0], T._ft[1]))
    # Root finding via TOMS748
    tStart = time.time()
    tMin = T._t[0]
    tMax = T._t[1]
    topt, rnfo = sciopt.toms748(lambda t: T.solve(t), tMin, tMax, xtol=5E-4, full_output=disp, disp=disp)
    if np.abs(topt - T._t[0]) < np.abs(topt - T._t[1]):
        Xopt = T._Xinit[0]
    else:
        Xopt = T._Xinit[1]
    tStop = time.time() - tStart
    if disp:
        print('  Summary');
        print('      Iterations:          {:d}'.format(rnfo.iterations) )
        print('      Function calls:      {:d}'.format(rnfo.function_calls) )
        print('      Optimal value:       {:5.3E}'.format(topt) )
        print('      Elapsed time:        {:8.4f} seconds'.format(tStop) )
    return Xopt, la.norm(Xopt.ravel(), 1)

# %%
if __name__ == '__main__':
    # Import
    from maps import OperatorFourierLowRank

    print('[TESTING SOLVER ROUTINES]')
    # *** EXPERIMENTS FOR SINGLE IMAGE AND SUBSAMPLING - L1-NORM
    # Set up map
    imShape = (128, 128)
    samplingSet = np.where(np.random.uniform(size=imShape) < 0.75, True, False)
    A = OperatorFourierLowRank(imShape, samplingSet=samplingSet)
    xinit = np.random.normal(size=A.inShape) + 1j * np.random.normal(size=A.inShape)
    y = A.eval(xinit)
    print('[SINGLE IMAGE - RANDOM UNDERSAMPLING - L1-NORM RECOVERY]')
    print('    Input shape:    {:d} x {:d}'.format(A.inShape[0], A.inShape[1]))
    print('    Output shape:   {:d} x {:d}'.format(A.outShape[0], 1))
    print('    Operator norm:  {:1.3e}'.format(A.norm()))
    # Tests
    print('\n[TEST: MinimizeSumOfSquares]')
    xsol, fmin = MinimizeSumOfSquares(y, A, disp=True)
    print('\n[TEST: MinimizeSumOfSquaresNuclearBall]')
    print('  ---> with no restart...')
    xsol, fmin = MinimizeSumOfSquaresNuclearBall(1.0, y, A, disp=True, restart=False, printEvery=10)
    print('  ---> with restart...')
    _xsol, _fmin = MinimizeSumOfSquaresNuclearBall(1.0, y, A, disp=True, restart=True)
    print('  ---> Difference:     {:1.5e}'.format(la.norm((xsol - _xsol).ravel())))
    print('\n[TEST: RecoveryL1Noisy - SoS-NucBall]')
    xsol, fmin = RecoveryNuclearNormNoisy(2.0, y, A, maxItns=1E4, disp=True, printEvery=100, method='SoS-NucBall')
    print('\n[TEST: RecoveryL1Noisy - NNDN]')
    _xsol, _fmin = RecoveryNuclearNormNoisy(2.0, y, A, maxItns=1E4, disp=True, printEvery=100, method='NNDN')
    print('  ---> Difference:')
    print('         Objective:    {:1.5e}'.format(np.abs(fmin - _fmin)))
    print('         Solution:     {:1.5e}'.format(la.norm((xsol - _xsol).ravel())))

# %%


# used for manipulating directory paths
import os

# Scientific and vector computation for python
import numpy as np

def computeCostMulti(X, y, theta):
    """
    Compute cost for linear regression with multiple variables.
    Computes the cost of using theta as the parameter for linear regression to fit the data points in X and y.

    Parameters
    ----------
    X : array_like
        The dataset of shape (m x n+1).

    y : array_like
        A vector of shape (m, ) for the values at a given data point.

    theta : array_like
        The linear regression parameters. A vector of shape (n+1, )

    Returns
    -------
    J : float
        The value of the cost function.

    Instructions
    ------------
    Compute the cost of a particular choice of theta. You should set J to the cost.
    """
    # Initialize some useful values
    m = y.shape[0]  # number of training examples

    # You need to return the following variable correctly
    J = 0

    J = (np.dot((np.transpose(np.dot(X, theta)) - y), (np.dot(X, theta) - y))) / (2 * m)

    return J

if __name__ == '__main__':
    data = np.loadtxt(os.path.join("/home/thelichking/Desktop/ml-coursera-python-assignments/Exercise1/Data",
                                   "ex1data2.txt"), delimiter=',')
    X = data[:, :3]  # the first two rows
    y = data[:, 2]  # the actual 3rd row
    m = y.size

    print(computeCostMulti(X,y,theta=np.array([-1, 2, 3])))


import torch
import numpy as np

def euclidean_distance(x1, x2):
    """Squared distance

    We are ignoring the sqrt here to since it
    does not affect k-means, and we gain a bit
    of speed.

    Note:
        While k-means is unaffected by using the
        squared distance, other clustering
        algorithms, like hierarchical clustering,
        are affected. Instead, you should use the
        regular Euclidean distance.
    """
    return np.sum(np.square(x1 - x2))

class Kmeans:
    """Clustering around k centroids"""

    def __init__(self, k=2, num_iterations=100):
        self.k = k
        self.num_iterations = num_iterations

    def initialize_centroids(self, data):
        """Randomly initialize centroids

        Randomly choose points in our data to be the
        positions of our centroids
        """
        num_samples, _ = data.shape

        idx = np.random.choice(
            num_samples, (self.k,), replace=False
        )

        return data[idx, :]

    def nearest_centroid(self, sample, centroids):
        """Find the nearest centroid for a data point

        Args:
            sample: np.array of shape (1, num_features),
                    a single data point
            centroids: current centroid positions
        """
        nearest = None
        nearest_distance = np.inf

        for idx, centroid in enumerate(centroids):
            distance = euclidean_distance(sample, centroid)
            if distance < nearest_distance:
                nearest = idx
                nearest_distance = distance

        return nearest

    def assign_clusters(self, data, centroids):
        """Assign the data to the nearest centroids"""
        num_samples, _ = data.shape

        clusters = [[] for _ in range(self.k)]

        for idx, sample in enumerate(data):
            nearest_centroid = self.nearest_centroid(sample, centroids)
            clusters[nearest_centroid].append(idx)

        return clusters

    def cluster_assignments(self, data, clusters):
        """Get which cluster each sample is assigned to"""
        num_samples, _ = data.shape

        assignments = np.zeros(num_samples)
        for idx, cluster in enumerate(clusters):
            for sample in cluster:
                assignments[sample] = idx

        return assignments

    def update_centroids(self, data, clusters):
        """Set the centroids to be the mean of the samples in each cluster"""
        _, num_features = data.shape

        centroids = np.zeros((self.k, num_features))

        for idx, cluster in enumerate(clusters):
            centroid = np.mean(data[cluster], axis=0)
            centroids[idx] = centroid

        return centroids

    def predict(self, data):
        """K-means clustering over data"""
        centroids = self.initialize_centroids(data)

        for idx in range(self.num_iterations):
            clusters = self.assign_clusters(data, centroids)
            centroids = self.update_centroids(data, clusters)

        cluster_assignments = self.cluster_assignments(data, clusters)

        return cluster_assignments, centroids

def kmeans_save(data, target, predictions, centroids, path):
    """Save both the data and our predictions"""
    torch.save({
        "data": data,
        "target": target,
        "predictions": predictions,
        "centroids": centroids
    }, path)

if __name__ == "__main__":
    from sklearn import datasets

    data, _ = datasets.make_blobs(n_features=10)
    print(f"blobs: {data.shape}")

    model = Kmeans(k=10)
    assignments, _ = model.predict(data)

    print(assignments.shape)
    print(assignments)


"""
newton_algorithm.py

Returns the minimizer of the function
X0 - initial guess
func - anonimous function
func_1st_der - anonimous function 1st derivative
func_2nd_der - anonimous function 2nd derivative
N_iter_max - max number of iterations
"""

import numpy as np

def newton_algorithm(X0, func, func_1st_der, func_2nd_der, options):

    epsilon = 10e-6
    report = {};
    N_iter_max = options['N_iter_max'];
    tolerance_x = options['tolerance_x'];
    tolerance_y = options['tolerance_y'];
    progress_x = np.zeros((1, N_iter_max + 1));
    progress_y = np.zeros((1, N_iter_max + 1));
    progress_x[:, [0]] = X0;
    progress_y[0, [0]] = func(X0);
    X_old = X0;

    for iter_no in range(1, N_iter_max + 1):
        grad = func_1st_der(X_old);

        if (np.linalg.norm(grad) < epsilon):
            print('norm(grad) < epsilon in %d iterations, exit..' % (iter_no));
            break;

        X = X_old - grad / func_2nd_der(X_old);
        progress_x[:, [iter_no]] = X;
        progress_y[0, [iter_no]] = func(X);

        if (np.abs(X - X_old) < tolerance_x * np.abs(X_old)):
            print('Tolerance in X is reached in %d iterations, exit..' % (iter_no));
            break;

        if (np.abs(progress_y[0, [iter_no]] - progress_y[0, [iter_no - 1]]) < tolerance_y * np.abs(progress_y[0, [iter_no - 1]])):
            print('Tolerance in Y is reached in %d iterations, exit..' % (iter_no));
            break;

        X_old = X;

    report = {'N_iter_max' : N_iter_max, 'iter_no' : iter_no, 'X0' : X0, 'X' : X, 'progress_x' : progress_x, 'progress_y' : progress_y};
    return (X, report);


# Fit linear and quadratic surfaces to data
# Based on https://github.com/probml/pmtk3/blob/master/demos/surfaceFitDemo.m

import superimport

import numpy as np
import matplotlib.pyplot as plt
import pyprobml_utils as pml
from scipy.io import loadmat
import requests
from io import BytesIO
from mpl_toolkits.mplot3d import Axes3D

url = 'https://raw.githubusercontent.com/probml/probml-data/main/data/moteData/moteData.mat'
response = requests.get(url)
#rawdata = response.text
rawdata = BytesIO(response.content)
data = loadmat(rawdata)

X = data['X']
y = data['y']

n = len(y)
X_pad = np.column_stack((np.ones(n), X))

for use_quad in (False, True):
  phi = X_pad

  if use_quad:
    phi = np.column_stack((X_pad, X**2))

  fig = plt.figure()
  ax = Axes3D(fig)
  ax.set_zlim(15, 19)
  ax.scatter(X[:,0], X[:,1], y, color='r')

  n = 10
  xrange = np.linspace(min(X[:,0]), max(X[:,0]), n)
  yrange = np.linspace(min(X[:,1]), max(X[:,1]), n)
  xx, yy = np.meshgrid(xrange, yrange)
  flatxx = xx.reshape((n**2, 1))
  flatyy = yy.reshape((n**2, 1))
  w = np.linalg.lstsq(phi, y)[0]

  z = np.column_stack((flatxx, flatyy))
  z = np.column_stack((np.ones(n**2), z))
  if use_quad:
    z = np.column_stack((z, flatxx**2, flatyy**2))

  f = np.dot(z , w)
  ax.plot_surface(xx, yy, f.reshape(n, n),
                  rstride=1, cstride=1, cmap='jet')

  name = 'linregSurfaceLinear.pdf'
  if use_quad:
    name = 'linregSurfaceQuad.pdf'

  pml.savefig(name)
  plt.show()


# Central Limit Theorem
import numpy as np
import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] = (20,10)
from distributionLib import Dist
import os
import seaborn as sns
import warnings
warnings.filterwarnings("ignore")
np.random.seed(404)

N      = 20
p      = 0.8
N5,N25 = 5,25
temp5  = []
temp25 = []

data = Dist.binomialDist(N,p)
data = np.array(data)

xBar = np.mean(data)
var  = np.var(data)
sd   = np.sqrt(var)

plt.subplot(2,2,(1,3))
plt.plot([0,len(data)-1],[xBar,xBar],color='k',linestyle='--',label='mean={:.4f}'.format(xBar))
plt.bar(np.arange(len(data)),data)
plt.title('Random Distribution')
plt.legend()
plt.grid()

ax5 = plt.subplot(2,2,2)
ax25 = plt.subplot(2,2,4)

filenames = []
for ii in range(500):
    ax5.clear()
    ax25.clear()

    ind5  = np.random.randint(0, len(data), size=(N5,1))
    ind25 = np.random.randint(0, len(data), size=(N25,1))
    tempo5, tempo25 = np.mean(data[ind5]),np.mean(data[ind25])

    temp5.append(np.round(tempo5,4))
    temp25.append(np.round(tempo25,4))

    plt.subplot(2,2,2)
    sns.distplot(temp5,label='{} Samples mean={:.4f}'.format(N5,np.mean(temp5)))#, fit='norm', kde=False)
    plt.title('{} samples'.format(N5))
    plt.legend()
    plt.grid()

    plt.subplot(2,2,4)
    sns.distplot(temp25,label='{} Samples mean={:.4f}'.format(N25,np.mean(temp25)))
    plt.legend()
    plt.grid()
    plt.title('{} samples'.format(N25))

    plt.pause(0.0001)
    plt.draw()
    if not ii%10:
        filename = f'{ii}.png'
        filenames.append(filename)
        plt.savefig(filename)
plt.close()
Dist.gif(filenames,'CLF.gif')

print("{} samples mean = {:.4f}".format(N5,np.mean(temp5)))
print("{} samples mean = {:.4f}".format(N25,np.mean(temp25)))


"""
Example usage of functions provided by beam module.

Simply supported beam of total length 1.2m with rectangular
cross-section.

The loads are:
* Constant continuous load q = 50kN/m between coordinates
  a = 0.5m and a+b = 0.8m.
* Two point loads F = -20kN (i.e. in the upwards direction)
  at a = 0.5m and a+b = 0.8m.

Dimensions of the cross-section (height H and width B) are to be determined so that maximum stress is 120MPa. H/B = 2 holds.

Deflection and angle of deflection are to be determined at point
C at the location a+b=0.8m.
"""
import os, sys
import numpy as np
import matplotlib.pyplot as plt
sys.path.insert(1, os.path.join(sys.path[0], '..'))
from beam import Beam, ConstantContinuousLoad, PointLoad

# known quantities
q = 50e3
F = 20e3
a, b, c = .5, .3, .4
C = a + b
E = 2e11
sigD = 120e6

### computation of reactions: ###

# loads - list of known loads
loads = [
    ConstantContinuousLoad(q, a, a+b),
    PointLoad(-F, a),
    PointLoad(-F, a+b)
]
# reactions - list of unknown loads
#  All must have magnitude = 1, otherwise the results of
#  Beam.get_reactions would be wrong.
reactions = [PointLoad(1, 0), PointLoad(1, a+b+c)]

# Initialize the beam object.
# For now we don't need the actual value of Jz since we are
# going to compute the actual dimensions based on the values
# of reaction forces.
beam = Beam(loads, reactions,
            l = a+b+c,
            E = E,
            Jz = 1.,
)

Rs = beam.get_reactions()
print('reactions:\n R_A = {:.2e}N\n R_B = {:.2e}N'.format(Rs[0], Rs[1]))

### computation and plots of bending force and moment ###
n = 12*5+1
x = np.linspace(0, a+b+c, n)

plt.plot(x, [beam.force(xi) for xi in x], '.-', label='force')
Mo = np.array([beam.moment(xi) for xi in x])
plt.plot(x, Mo, '.-', label='moment')

plt.legend(loc='best')
plt.grid()

### actual dimensions of the cross-section ###

# sigma = Mmax/Wo = Mmax/(Jz/(H/2)) <= sigD
# Jz >= H/2*Mmax/sigD
#   [rectangular cross-section]
#   Jz = 1/12*B*H**3 = (H/B=2) =
#      = 1/24*H**4
# H**4 >= 24*H/2*Mmax/sigD
# H >= (12*Mmax/sigD)**(1/3)

Mmax = max(abs(Mo))
print('Mmax = {:.4}N'.format(Mmax))

H = (12*Mmax/sigD)**(1./3)
print('H = {:.4}m'.format(H))
print('B = {:.4}m'.format(.5*H))

Jz = H**4/24
beam.Jz = Jz

### computation and plots of deflection and angle ###

v = np.array([beam.deflection(xi) for xi in x]).T
v_C = beam.deflection(C)
print('v_C = {:.4}m\nphi_C = {:.4}rad'.format(v_C[0], v_C[1]))

plt.figure()
plt.plot(x, v[0], '.-', label='deflection')
plt.plot(x, v[1], '.-', label='angle')
plt.grid()
plt.legend(loc='best')
plt.show()


import numpy as np
import pandas as pd

def preprocess(path):
    dataset = pd.read_csv(path, names=['x1', 'x2', 'x3', 'x4', 'y'], delim_whitespace=True)
    dataset.insert(0, 'x0', 1)
    training_set = dataset.iloc[:, :-1].values
    labels_set = dataset.iloc[:, -1].values
    return training_set, labels_set

def sign(x):
    if x <= 0:
        return -1
    else:
        return 1

def pla(training_set, labels_set):
    update_count = 0
    correct_count = 0
    weights = np.zeros(training_set.shape[1])
    converge = False
    while not converge:
        for i in range(training_set.shape[0]):
            x = training_set[i]
            y = labels_set[i]
            if sign(np.dot(weights, x)) != y:
                weights = weights + y * x
                update_count += 1
                correct_count = 0
            else:
                correct_count += 1
                if correct_count == labels_set.shape[0]:
                    converge = True
                    break
    return weights, update_count

training_set, labels_set = preprocess("train.dat")
weights, count = pla(training_set, labels_set)
print("Number of Updates:", count)

def unison_shuffled_copies(arr1, arr2):
    assert len(arr1) == len(arr2)
    p = np.random.permutation(len(arr1))
    return arr1[p], arr2[p]

def pla_random(training_set, labels_set, times_repeat):
    total = 0
    for _ in range(times_repeat):
        xs, ys = unison_shuffled_copies(training_set, labels_set)
        _, count = pla(xs, ys)
        total += count
    return total / times_repeat

avg = pla_random(training_set, labels_set, 2000)
print(f"Average Number of Updates in 2000 Experiments:", avg)

def error_counter(wt, xs, ys):
    count = 0
    for i in range(len(xs)):
        if sign(np.dot(wt, xs[i])) != ys[i]:
            count += 1
    return count

def pocket_pla(training_set, labels_set, time_repeat):
    weights = np.zeros(training_set.shape[1])
    weights_updated = weights.copy()
    error = error_counter(weights, training_set, labels_set)
    t = 0
    while t < time_repeat:
        i = np.random.randint(0, len(training_set))
        if sign(np.dot(weights, training_set[i])) != labels_set[i]:
            weights += training_set[i] * labels_set[i]
            new_error = error_counter(weights, training_set, labels_set)
            t += 1
            if new_error < error:
                weights_updated = weights.copy()
                error = new_error
    return weights_updated

training_set_new, labels_set_new = preprocess("train18.dat")
weights = pocket_pla(training_set_new, labels_set_new, 50)

testing_set, testing_label_set = preprocess("test.dat")

def pocket_test(weights, testing_set, testing_label_set):
    err_cnt = error_counter(weights, testing_set, testing_label_set)
    err_rate = err_cnt / len(testing_set)
    return err_rate

error_rate = pocket_test(weights, testing_set, testing_label_set)
print(f"The error rate is {error_rate}.")

Machine Learning Foundations/Perceptron Learning Algorithm/pla.py


import sys
import numpy as np
import matplotlib

print("Python", sys.version)
print("Numpy", np.__version__)
print("Matplotlib", matplotlib.__version__)

inputs = [1.0, 2.0, 3.0, 2.5]

weights = [[0.2, 0.8, -0.5, 1.0],
           [0.5, -0.91, 0.26, -0.5],
           [-0.26, -0.27, 0.17, 0.87]]

bias = [2.0,3.0,0.5]

layer_output = []
for n_weight, n_bias in zip(weights, bias) :
    neuron_output = 0
    for neu_input,neu_weight in zip(inputs,n_weight) :
        neuron_output = neuron_output+ neu_input*neu_weight
    neuron_output = neuron_output+n_bias
    layer_output.append (neuron_output)

print("layer_output from Scratch Python", layer_output)

output = np.dot(weights,inputs)+ bias
print("layer_output using Mumpy", output)


import numpy as np
import matplotlib.pyplot as plt
import random
import math

# define objective function

def f(x):
    x1 = x[0]
    x2 = x[1]
    obj = -20*math.exp(-0.2*math.sqrt(0.5*(x1*x1*x2*x2)))-math.exp(0.5*(math.cos(2*math.pi*x1)+math.cos(2*math.pi*x2))) + math.e + 20
    return obj

#İstenilen test fonksiyonu, f isimli fonksiyonda obj olarak kullanılabilir.

#ackley    obj = -20*math.exp(-0.2*math.sqrt(0.5*(x1*x1*x2*x2)))-math.exp(0.5*(math.cos(2*math.pi*x1)+math.cos(2*math.pi*x2))) + math.e + 20
#beale     obj = (1.5 - x[0] + x[0]*x[1])**2 + (2.25 - x[0] + x[0]*x[1]**2)**2 + (2.625 - x[0] + x[0]*x[1]**3)**2
#goldstein obj = ((1 + (x[0] + x[1] + 1) ** 2 * (19 - 14 * x[0] + 3 * (x[0] ** 2) - 14 * x[1] + 6 * x[0] * x[1] + 3 * (x[1] ** 2))) * (30 + (2 * x[0] - 3 * x[1]) ** 2 * (18 - 32 * x[0] + 12 * (x[0] ** 2) + 48 * x[1] - 36 * x[0] * x[1] + 27 * (x[1] ** 2))))
#levi      obj = (math.sin(3*x[0]*math.pi))**2 + ((x[0] - 1)**2)*(1 + math.sin(3*x[1]*math.pi)**2) + ((x[1] - 1)**2)*(1 + math.sin(2*x[1]*math.pi)**2)

# Start location
x_start = [0.8, -0.5]

##################################################
# Simulated Annealing
##################################################
# Number of cycles
n = 100
# Number of trials per cycle
m = 100
# Number of accepted solutions
na = 0.0
# Probability of accepting worse solution at the start
p1 = 0.7
# Probability of accepting worse solution at the end
p50 = 0.001
# Initial temperature
t1 = -1.0/math.log(p1)
# Final temperature
t50 = -1.0/math.log(p50)
# Fractional reduction every cycle
frac = (t50/t1)**(1.0/(n-1.0))
# Initialize x
x = np.zeros((n+1,2))
x[0] = x_start
xi = np.zeros(2)
xi = x_start
na = na + 1.0
# Current best results so far
xc = np.zeros(2)
xc = x[0]
fc = f(xi)
fs = np.zeros(n+1)
fs[0] = fc
# Current temperature
t = t1
# DeltaE Average
DeltaE_avg = 0.0
for i in range(n):
    print('Cycle: ' + str(i) + ' with Temperature: ' + str(t))
    for j in range(m):
        # Generate new trial points
        xi[0] = xc[0] + random.random() - 0.5
        xi[1] = xc[1] + random.random() - 0.5
        # Clip to upper and lower bounds
        xi[0] = max(min(xi[0],1.0),-1.0)
        xi[1] = max(min(xi[1],1.0),-1.0)
        DeltaE = abs(f(xi)-fc)
        if (f(xi)>fc):
            # Initialize DeltaE_avg if a worse solution was found
            #   on the first iteration
            if (i==0 and j==0): DeltaE_avg = DeltaE
            # objective function is worse
            # generate probability of acceptance
            p = math.exp(-DeltaE/(DeltaE_avg * t))
            # determine whether to accept worse point
            if (random.random()<p):
                # accept the worse solution
                accept = True
            else:
                # don't accept the worse solution
                accept = False
        else:
            # objective function is lower, automatically accept
            accept = True
        if (accept==True):
            # update currently accepted solution
            xc[0] = xi[0]
            xc[1] = xi[1]
            fc = f(xc)
            # increment number of accepted solutions
            na = na + 1.0
            # update DeltaE_avg
            DeltaE_avg = (DeltaE_avg * (na-1.0) +  DeltaE) / na
    # Record the best x values at the end of every cycle
    x[i+1][0] = xc[0]
    x[i+1][1] = xc[1]
    fs[i+1] = fc
    # Lower the temperature for next cycle
    t = frac * t

# print solution
print('Best solution: ' + str(xc))
print('Best objective: ' + str(fc))

plt.plot(x[:,0],x[:,1],'y-o')
plt.savefig('contour.png')

fig = plt.figure()
ax1 = fig.add_subplot(211)
ax1.plot(fs,'r.-')
ax1.legend(['Objective'])
ax2 = fig.add_subplot(212)
ax2.plot(x[:,0],'b.-')
ax2.plot(x[:,1],'g--')
ax2.legend(['x1','x2'])

# Save the figure as a PNG
plt.savefig('iterations.png')

plt.show()


import numpy as np
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import style
from IPython.display import display

%config InlineBackend.figure_format = 'retina'

tau = 160.0/100.0
def response(w,z):

    Kp = 1.0
    a = 1- (tau**2)*(w**2)
    b = 2*tau*z*w
    sol = Kp/np.sqrt(a**2+b**2)
    sol1 = np.arctan(b/a)
    return sol
w = np.linspace(0.01,100,100000)

zeta = [0.2,1,2]
plt.figure();
plt.plot(np.log10(w), np.log10(response(w,zeta[0])), 'r', label = r"$\zeta$ = %.1f" %(zeta[0]))
plt.plot(np.log10(w), np.log10(response(w,zeta[1])), 'b', label = r"$\zeta$ = %.1f" %(zeta[1]))
plt.plot(np.log10(w), np.log10(response(w,zeta[2])), 'g', label = r"$\zeta$ = %.1f" %(zeta[2]))
plt.xlabel("Time (hours)", fontsize=12)
#plt.xlabel(r"{log}_{10}(\omega )")
#plt.ylabel(r"{log}_{10}(\frac{AR}{K_P} )")
plt.title(r"$\frac{AR}{K_P}$"+ "for 2nd Order System (" + r"$\tau =$ %.2f)"% (tau))
plt.legend();


import numpy as np

def entropy(x):
    x = x[x > 0]
    return -np.sum(x * np.log(x))

def information_gain(y, y_left, y_right):
    n = y.shape[0]
    n_l = y_left.shape[0]
    if n_l == 0:
        H_l = 0.0
    else:
        H_l = entropy(y_left.mean(axis=0))

    n_r = y_right.shape[0]
    if n_r == 0:
        H_r = 0
    else:
        H_r = entropy(y_right.mean(axis=0))

    H = entropy(y.mean(axis=0))
    return H - n_l / n * H_l - n_r / n * H_r

def predict_node_label(values, num_distinct_classes):
    probabilities = predict_node_probability(values, num_distinct_classes)
    return np.argmax(probabilities)

def predict_node_probability(values, num_distinct_classes):
    return np.bincount(values, minlength=num_distinct_classes) / values.shape[0]


import numpy

'''
compute angle (in degrees) for p0p1p2 corner
Inputs:
    p0,p1,p2 - points in the form of [x,y]
'''

def calc_angle(p0, p1, p2):
    v0 = numpy.array(p0) - numpy.array(p1)
    v1 = numpy.array(p2) - numpy.array(p1)

    angle = numpy.math.atan2(numpy.linalg.det([v0, v1]), numpy.dot(v0, v1))
    return numpy.degrees(angle)


from math import sqrt, pow, pi
from pylab import *
import scipy
from scipy.integrate.odepack import odeint

R0 = 6378137
mu = 3.9860047e14

"""Расчет текущей высоты"""

def h_x_y(x, y):
    return sqrt(pow(x, 2) + pow(y, 2)) - R0

"""Расчет периода обращения"""

def T(R):
    return 2 * pi * sqrt(pow(R, 3) / mu)

"""Правые части"""

def right_part(t, StateVector):
    R = sqrt(StateVector[0] * StateVector[0] + StateVector[1] * StateVector[1])
    R3 = pow(R, 3)

    result = [0, 0, 0, 0]
    result[0] = StateVector[2]
    result[1] = StateVector[3]
    result[2] = -mu / R3 * StateVector[0]
    result[3] = -mu / R3 * StateVector[1]
    return result

"""Интегрированния методом Рунге-Кутта (4-го порядка)"""

def runge_kutta_t_state_vector(t, params, t_step, f):
    k1, k2, k3, k4 = [], [], [], []

    h = t_step

    k1 = f(t, params)

    new_params = []

    for i in range(0, len(params)):
        p = params[i]
        new_params.append(p + h / 2 * k1[i])

    k2 = f(t + h / 2, new_params)

    new_params = []

    for i in range(0, len(params)):
        p = params[i]
        new_params.append(p + h / 2 * k2[i])

    k3 = f(t + h / 2, new_params)

    new_params = []

    for i in range(0, len(params)):
        p = params[i]
        new_params.append(p + h * k3[i])

    k4 = f(t + h, new_params)

    for i in range(0, len(params)):
        p = params[i]
        params[i] = p + h / 6 * (k1[i] + 2 * k2[i] + 2 * k3[i] + k4[i])

    return params

"""Обычный расчет (с выводом вектора состояния в файл output.tr)"""

def regular_calculation(Vx, Vy, t, t_end, t_step, x, y):
    file = open("output.tr", mode="w")
    print('t, x, y, Vx, Vy', file=file)

    while t <= t_end:
        print("%(t).3f %(x)d %(y)d %(Vx).3f %(Vy).3f" % {"t": t, "x": x, "y": y, "Vx": Vx, "Vy": Vy},
              file=file)

        t += t_step

        x, y, Vx, Vy = runge_kutta_t_state_vector(t, [x, y, Vx, Vy], t_step, right_part)
    file.close()

"""Тест производительности (без вывода вектора состояния в файл output.tr)"""

def benchmark_test(Vx, Vy, t, t_end, t_step, x, y):
    while t <= t_end:
        t += t_step
        x, y, Vx, Vy = runge_kutta_t_state_vector(t, [x, y, Vx, Vy], t_step, right_part)

"""Тест на корректность результатов
(с проверкой высоты и проверкой соответствия вектора состояния с момента начала движения
на момент завершения первого витка)"""

def test(Vx, Vy, t, t_end, t_step, x, y, R):
    T_period = T(R)

    parameters = [x, y, Vx, Vy]

    x_data = []
    y_data = []

    while t <= t_end:
        x_data.append(x)
        y_data.append(y)

        t += t_step
        x, y, Vx, Vy = runge_kutta_t_state_vector(t, [x, y, Vx, Vy], t_step, right_part)

        k = round(T_period, 1)
        # print(round(t, 1) % k)
        if round(t, 1) % k == 0:
            print("Виток № ", round(t / T_period))
            print("T = ", t)

            for text, x0, x_period in zip(["x", "y", "Vx", "Vy"], parameters, [x, y, Vx, Vy]):
                x1, x2 = round(x0), round(x_period)
                if x1 != x2:
                    print("Разница в значениях:", text, x1, x2)

        assert (round(h_x_y(x, y), 5) == 1000000)

    plot(x_data, y_data)
    show()

"""Начальные условия и выбор режима расчета (обычный, тест производительности, тест на корректность полученных данных)"""

def main(type, step):
    t_end = 10000  # Конец времени движения
    t_step = float(step)  # Шаг интегрирования
    h0 = 1000000
    R = R0 + h0
    t, x, y, Vx, Vy = 0, 0, R, sqrt(mu / R), 0

    if type == "regular":
        regular_calculation(Vx, Vy, t, t_end, t_step, x, y)
    elif type == "benchmark":
        benchmark_test(Vx, Vy, t, t_end, t_step, x, y)
    elif type == "test":
        test(Vx, Vy, t, t_end, t_step, x, y, R)

def run_calculation_and_measure_time(run_type, step, file):
    timeit_timeit = timeit.timeit("main('" + run_type + "','" + str(step) + "')", setup="from __main__ import main",
                                  number=1)
    print(run_type, "t_step =", step, "\n",
          timeit_timeit,
          file=file)  # Замер затраченного времени (вывод в файл)
    print(run_type, "t_step =", step, "\n",
          timeit_timeit,
          )  # Замер затраченного времени (вывод на экран)

if __name__ == '__main__':
    import timeit

    file = open("results.txt", "w")

    for run_type in ["benchmark", "regular", "test"]:
        if run_type != "test":
            for step in [1, 0.1, 0.01]:
                run_calculation_and_measure_time(run_type, step, file)
        else:
            step = 0.1
            run_calculation_and_measure_time(run_type, step, file)

    file.close()

# odeint()


from scipy.stats import beta
from matplotlib import pyplot as plt
import numpy as np

def samples(a, b, success, trials, num_episodes=100):
    '''

    :param a: the shape param for prior dist
    :param b: the shape param for prior dist
    :param success: num success in the experiments
    :param trials:  num trails conducted
    :param num_episodes:  num samples to draw from this distribution
    :return:
    '''
    dist = beta(a+success, b+trials-success)
    episodes = num_episodes
    nums = [dist.rvs() for _ in range(episodes)]
    return nums

def stats(nums):
    avg = sum(nums)/len(nums)
    var = sum(
        [pow((x - avg), 2) for x in nums]
    )
    print(avg, var)

def plots(data, bin_size=20):
    bins = np.arange(0, bin_size, 1) # fixed bin size
    bins = bins/bin_size # normalize bins
    num_plots = len(data)
    for i, nums in enumerate(data):
        plt.subplot(num_plots, 1, i+1)
        # plot histogram
        plt.hist(nums, bins=bins, alpha=0.5)
        # hist = np.histogram(nums, bin_size)

    plt.show()

'''
The conclusion is better prior requires less trails to converge.
Worse prior requires more trails to converge.
'''

successes = 3
trials =10
# alpha, beta defines the shape of beta dist, success and trials is number of experiments.
a, b = 1, 1  # uniform
num_episodes = 2000 # num samples sampled from distribution in order to draw distribution
bin_size = 100
container = []
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

a, b = 0.5, 0.5  # convex shape prior
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

a, b = 1.1, 30  # 0-0.2 prior
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

a, b = 2, 5  # .0-0.8 prior
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

a, b = 2, 2  # bell shape between 0,1
ret = samples(a, b, successes, trials, num_episodes=num_episodes)
container.append(ret)
stats(ret)

plots(container, bin_size=bin_size)


import numpy as np
import matplotlib.pyplot as plt

# numpy.linspace(start, stop, num=50, endpoint=True, retstep=False, dtype=None)
#
# 在指定的间隔内返回均匀间隔的数字。
#
# 返回num均匀分布的样本，在[start, stop]。
#
# 这个区间的端点可以任意的被排除在外。

X = np.linspace(-np.pi, np.pi, 256, endpoint=True)
print(X)
C, S = np.cos(X), np.sin(X)

plt.plot(X, C)
plt.plot(X, S)

plt.show()


import numpy as np
import numpy as np
import pylab as pl
import scipy.special as ss

def beta(a, b, mew):
    e1 = ss.gamma(a + b)
    e2 = ss.gamma(a)
    e3 = ss.gamma(b)
    e4 = mew ** (a - 1)
    e5 = (1 - mew) ** (b - 1)
    return (e1/(e2*e3)) * e4 * e5

def plot_beta(a, b):
    Ly = []
    Lx = []
    mews = np.mgrid[0:1:100j]
    for mew in mews:
        Lx.append(mew)
        Ly.append(beta(a, b, mew))
    pl.plot(Lx, Ly, label="a=%f, b=%f" %(a,b))
 o
def main():
    plot_beta(0.1, 0.1)
    plot_beta(1, 1)
    plot_beta(2, 3)
    plot_beta(8, 4)
    pl.xlim(0.0, 1.0)
    pl.ylim(0.0, 3.0)
    pl.legend()
    pl.show()

if __name__ == "__main__":
    main()


import numpy as np
import statsmodels.api as sm
import matplotlib.pyplot as plt
from scipy.fftpack import rfft
from scipy.fftpack import fftshift

data_loader = sm.datasets.sunspots.load_pandas()
sunspots = data_loader.data["SUNACTIVITY"].values

t = np.linspace(-2 * np.pi, 2 * np.pi, len(sunspots))
mid = np.ptp(sunspots)/2
sine = mid + mid * np.sin(np.sin(t))

sine_fft = np.abs(fftshift(rfft(sine)))
print "Index of max sine FFT", np.argsort(sine_fft)[-5:]

transformed = np.abs(fftshift(rfft(sunspots)))
print "Indices of max sunspots FFT", np.argsort(transformed)[-5:]

plt.subplot(311)
plt.plot(sunspots, label="Sunspots")
plt.plot(sine, lw=2, label="Sine")
plt.grid(True)
plt.legend()
plt.subplot(312)
plt.plot(transformed, label="Transformed Sunspots")
plt.grid(True)
plt.legend()
plt.subplot(313)
plt.plot(sine_fft, lw=2, label="Transformed Sine")
plt.grid(True)
plt.legend()
plt.show()

import numpy,from scipy,import statsmodels


import numpy as np
import matplotlib.pyplot as plt

n = 1
m = 97
theta = np.zeros((2, 1))

alpha = 0.01
iterations = 1500

x1, y = np.loadtxt("data/OneFeature.txt", delimiter=",", unpack = True)  # Ouput is 1D. Vectors are 2D m x 1 dimension so need to change to 2D using reshape

x1 = np.reshape(x1, (x1.shape[0], 1))  # Makes x1 a vector with each new line different training set
y = np.reshape(y, (y.shape[0], 1))  # Makes y a vector with each new line different training set

x1_mean = np.mean(x1)
x1_std = np.std(x1)
x1_scaled = (x1 - x1_mean) / x1_std
onesVector = np.ones((x1.shape[0], 1))

X = np.concatenate((onesVector,x1_scaled),axis=1)  # Creates matrix X, where each new line is a different training example, where x0 on each line is 1

iterationsX = np.arange(1, iterations + 1, dtype= int)
costVector = np.zeros([iterations, 1])  # Vector of cost for each iteration

for j in range(0, iterations):  # Repeat many times to optimise theta
    sumCost = 0
    sumTheta = 0
    for i in range(0, m):  # Iterate through dataset to find sigma (the cost's derivative for current theta values)
        # Change shape of feature so it's 2D and a vector
        currentFeatures = np.reshape(X[i, :], (X[i, :].shape[0], 1))  # Current features is a vector of x0 followed by x1, ... xn
        h = np.matmul(theta.transpose(), currentFeatures)  # Calculate new hypothesis based on current training set (vectorised h(x) version)

        sumCost += (h - y[i, 0]) ** 2
        sumTheta += (h - y[i, 0]) * currentFeatures  # Add on the cost functions derivative associated with this training set

    cost = (1 / (2 * m)) * sumCost  # Cost for current theta parameters
    costVector[j, 0] = cost  # Add cost for current theta parameters to costVector
    sigma = (1/m) * sumTheta
    theta += - alpha * sigma  # update theta vector to more optimal values

#unscale theta
theta[0, 0] = theta[0,0] - (x1_mean / x1_std) * theta[1, 0]
theta[1, 0] = theta[1, 0] / x1_std

print("Cost function = ", cost)
print("Theta =", theta)

# Plot graph of x against y
plt.plot(x1, y, 'rx')  # need x and y values separate or pyplot autogenerates x values.

# Plot line of best fit
xline = np.linspace(0,20,2000)
yline = theta[0, 0] + xline * theta[1, 0]
plt.plot(xline, yline, '-b')

plt.xlabel('Population of City in 10,000s')
plt.ylabel('Profit in $10,000s')
plt.show()

# Plot graph of cost function against iterations
plt.plot(iterationsX, costVector, '-r')  # need x and y values separate or pyplot autogenerates x values.
plt.xlabel('Iterations')
plt.ylabel('Cost Function')
plt.xlim((1, iterations))
plt.show()

One variable with feature scaling (inefficient).py


import numpy as np

# Use "new-style" classes for easier inheritance
__metaclass__ = type

class transformation:
  def __init__(self):
    self.matrix = np.zeros((4,4), dtype = np.float64)
    self.invmatrix = np.zeros((4,4), dtype = np.float64)

  def getMatrix(self):
    return self.matrix

  def getInvMatrix(self):
    return self.invmatrix

  def transformContraVector(self, v):
    """
    Transforms column vectors (1-times contravariant tensors).

    x'^i = T^i_j * x^j  <=>  x' = Tx

    Matrix index convention is row-column
    """
    # See transformation rule for x^j above
    return np.einsum('ij,j', self.matrix, v)

  def transformCoMatrix(self, m):
    """
    Transform matrices representing bilinear forms (2-times covariant tensors).

    We start with using the transformation T for contravariant vectors,

    g(x',x') = x'^i * g'_i_j * x'^j
             = T^i_k * x^k * g'_i_j * T^j_l * x^l
             = x^k * (T^i_k * g'_i_j * T^j_l) * x^l

    We thus get

    g_k_l = T^i_k * g'_i_j * T^j_l

    Applying the inverse matrices on both sides,

    T^i_k * (T^-1)^k_j = delta^i_j

    where delta is the Kronecker symbol, we get the transformation rule

    g'_i_j = (T^-1)^k_i * g_k_l * (T^-1)^l_j

    The bilinear form thus transforms with the inverse matrix T^-1, hence
    its covariant transformation behaviour. To write this as a matrix product,
    we need to use the matrix transpose (the order of indices is important!),

    (T^-1)^k_i = (T^-1^t)_i^k

    which results in

    g' = T^-1^t g T^-1
    """
    # See transformation rule for g_k_l above
    return np.einsum('ki,kl,lj', self.invmatrix, m, self.invmatrix)

# -----------------------------------------------------------------------

class lorentzRotation(transformation):
  """
  Rotations of contravariant vectors in counter-clockwise direction (for angle > 0)
  in a right-handed coordinate system; the inverse matrix is used for covariant
  vectors
  """
  def __init__(self, axis, angle):
    assert (isinstance(axis, int)), "Axis parameter must be an integer number"
    assert (axis > 0 and axis < 4), "Axis must be in range 1..3"
    assert (isinstance(angle, (int, float))), "Angle parameter must be a number"
    self.axis = axis
    self.angle = angle

    super(lorentzRotation, self).__init__()

    self.matrix[0,0] = 1
    if axis == 1:
      self.matrix[1,1] = 1
      self.matrix[2,2] = self.matrix[3,3] = np.cos(self.angle)
      self.matrix[2,3] = -np.sin(self.angle)
      self.matrix[3,2] = np.sin(self.angle)
    elif axis == 2:
      self.matrix[2,2] = 1
      self.matrix[1,1] = self.matrix[3,3] = np.cos(self.angle)
      self.matrix[1,3] = np.sin(self.angle)
      self.matrix[3,1] = -np.sin(self.angle)
    elif axis == 3:
      self.matrix[3,3] = 1
      self.matrix[1,1] = self.matrix[2,2] = np.cos(self.angle)
      self.matrix[1,2] = -np.sin(self.angle)
      self.matrix[2,1] = np.sin(self.angle)

    # Rotation matrices are orthogonal
    self.invmatrix = np.transpose(self.matrix)

# -----------------------------------------------------------------------

class lorentzBoost(transformation):
  """
  Boosts of contravariant vectors, the inverse matrix is used for covariant
  vectors
  """
  def __init__(self, axis, beta):
    assert (isinstance(axis, int)), "Axis parameter must be an integer number"
    assert (axis > 0 and axis < 4), "Axis must be in range 1..3"
    assert (isinstance(beta, (int, float))), "Angle parameter must be a number"
    assert (np.abs(beta) < 1), "Beta must be > -1 and < 1"
    self.axis = axis
    self.beta = beta
    self.rapidity = np.arctanh(self.beta)

    super(lorentzBoost, self).__init__()

    if axis == 1:
      self.matrix[0,0] = np.cosh(self.rapidity)
      self.matrix[0,1] = -np.sinh(self.rapidity)
      self.matrix[1,0] = -np.sinh(self.rapidity)
      self.matrix[1,1] = np.cosh(self.rapidity)
      self.matrix[2,2] = 1
      self.matrix[3,3] = 1
    elif axis == 2:
      self.matrix[0,0] = np.cosh(self.rapidity)
      self.matrix[0,2] = -np.sinh(self.rapidity)
      self.matrix[1,1] = 1
      self.matrix[2,0] = -np.sinh(self.rapidity)
      self.matrix[2,2] = np.cosh(self.rapidity)
      self.matrix[3,3] = 1
    elif axis == 3:
      self.matrix[0,0] = np.cosh(self.rapidity)
      self.matrix[0,3] = -np.sinh(self.rapidity)
      self.matrix[1,1] = 1
      self.matrix[2,2] = 1
      self.matrix[3,0] = -np.sinh(self.rapidity)
      self.matrix[3,3] = np.cosh(self.rapidity)

    # Off-diagonal elements of inverse matrix change sign
    self.invmatrix = -self.matrix
    self.invmatrix[0,0] *= -1
    self.invmatrix[1,1] *= -1
    self.invmatrix[2,2] *= -1
    self.invmatrix[3,3] *= -1


import numpy as np
import scipy.optimize as opt
import matplotlib.pyplot as plt

from plotData import plot_data
import costFunction as cf
import plotDecisionBoundary as pdb
from sigmoid import sigmoid
from predict import predict
"""
Logistic Regression 进行分类
"""

# ====== 1.读取数据并绘制散点图 ======
dataset = np.loadtxt('ex2data1.txt',delimiter =',')
X = dataset[:,0:2]
Y = dataset[:,2]

plot_data(X,Y)
plt.legend(['Admitted', 'Not admitted'])
plt.xlabel('Exam 1 score')
plt.ylabel('Exam 2 score')
# plt.show()

# ====== 2.计算代价和梯度 ======
(m,n) = X.shape
X = np.c_[np.ones(m),X]	# 添加偏置维度
# 初始theta值和lambda值
init_theta = np.zeros(X.shape[1])
lmd = 1
# 计算代价和梯度
cost,grad = cf.cost_Function_Reg(X,Y,init_theta,lmd)
print('Cost at initial theta (zeros): {:0.3f}'.format(cost))
print('Expected cost (approx): 0.693')
print('Gradient at initial theta (zeros): \n{}'.format(grad))
print('Expected gradients (approx): \n-0.1000\n-12.0092\n-11.2628')

# 使用非0的theta测试函数
test_theta = np.array([-24,0.2,0.2])	#生成的是列向量
cost, grad = cf.cost_Function_Reg(X,Y,test_theta,0)
print('Cost at test theta (zeros): {:0.3f}'.format(cost))
print('Expected cost (approx): 0.218')
print('Gradient at test theta: \n{}'.format(grad))
print('Expected gradients (approx): \n0.043\n2.566\n2.647')

# ======= 3.使用优化函数来优化求解 ========
def cost_func(t):
	return cf.cost_Function_Reg(X,Y,t,0)[0]

def grad_func(t):
	return cf.cost_Function_Reg(X,Y,t,0)[1]
# 使用opt.fmin_bfgs()来获得最优解
theta, cost, *unused = opt.fmin_bfgs(f=cost_func, fprime=grad_func, x0=init_theta, maxiter=400, full_output=True, disp=False)
print('Cost at theta found by fmin: {:0.3f}'.format(cost))
print('Expected cost (approx): 0.203')
print('theta: \n{}'.format(theta))
print('Expected Theta (approx): \n-25.161\n0.206\n0.201')

# 使用计算出的theta来绘制分界线
pdb.plot_Decision_Boundary(X,Y,theta)
plt.xlabel('Exam 1 score')
plt.ylabel('Exam 2 score')
# plt.show()

# ====== 4.预测和计算准确率 ======
prob = sigmoid(np.array([1, 45, 85]).T.dot(theta))
print('For a student with scores 45 and 85, we predict an admission probability of {:0.4f}'.format(prob))
print('Expected value : 0.775 +/- 0.002')

P = predict(X,theta)
print('Train accuracy:{}'.format(np.mean(P == Y)*100))
print('Expected accuracy (approx): 89.0')


from __future__ import annotations
import numpy as np
from numpy.linalg import inv, det, slogdet

pdfLambda = lambda mu,var,x: (1.0 / (var * ((2*np.pi)**0.5))) * np.exp(-0.5*((x - mu) / float(var)) ** 2)

def multivarPdf(X, mu, cov):
    size = len(X)
    if size == len(mu) and (size, size) == cov.shape:
        deter = det(cov)
        if deter == 0:
            raise NameError("The covariance matrix can't be singular")

        norm_const = 1.0 / ((2 * np.pi)**(float(size) / 2) * (deter**(1.0 / 2)))
        x_mu = np.matrix(X - mu)
        invar = inv(cov)
        result = np.exp(-0.5 * (x_mu * invar * x_mu.T))
        return norm_const * result
    else:
        raise NameError("The dimensions of the input don't match")

class UnivariateGaussian:
    """
    Class for univariate Gaussian Distribution Estimator
    """
    def __init__(self, biased_var: bool = False) -> UnivariateGaussian:
        """
        Estimator for univariate Gaussian mean and variance parameters

        Parameters
        ----------
        biased_var : bool, default=False
            Should fitted estimator of variance be a biased or unbiased estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `UnivariateGaussian.fit` function.

        mu_: float
            Estimated expectation initialized as None. To be set in `UnivariateGaussian.fit`
            function.

        var_: float
            Estimated variance initialized as None. To be set in `UnivariateGaussian.fit`
            function.
        """
        self.biased_ = biased_var
        self.fitted_, self.mu_, self.var_ = False, None, None

    def fit(self, X: np.ndarray) -> UnivariateGaussian:
        """
        Estimate Gaussian expectation and variance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Training data

        Returns
        -------
        self : returns an instance of self.

        Notes
        -----
        Sets `self.mu_`, `self.var_` attributes according to calculated estimation (where
        estimator is either biased or unbiased). Then sets `self.fitted_` attribute to `True`
        """
        self.mu_ = np.mean(X)
        if self.biased_:
            self.var_ = np.mean(abs(X-self.mu_))
        else:
            self.var_ = X.var(ddof=1)

        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray) -> np.ndarray:
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, )
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, var_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")

        # denom = (2*np.pi*self.var_)**0.5
        # num = np.exp(-(X-self.mu_)**2/(2*self.var_))
        # pdf = np.exp(-1*(((X-self.mu_)**2)/(2*self.var_)))/((2*np.pi*self.var_)**0.5)
        return np.array(np.array([pdfLambda(self.mu_, self.var_, v) for v in X]))

    @staticmethod
    def log_likelihood(mu: float, sigma: float, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : float
            Expectation of Gaussian
        sigma : float
            Variance of Gaussian
        X : ndarray of shape (n_samples, )
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated
        """
        likelihoodSum = 0
        constant = (-1.0) / (2*sigma)
        for xi in X:
            likelihoodSum += pow(xi-mu, 2)
        # denom = (2*np.pi*var)**0.5
        # num = np.exp(-(X-mu)**2/(2*var))
        # pdf = num/denom
        # logLikelihood = ((1/(2*np.pi*sigma**2)**0.5)**len(X))*np.exp((-1*(1/2*sigma**2)*np.sum((X-mu)**2)))
        # #return sum([np.log(x) for x in pdf])

        return (- len(X) / 2) * np.log(2*np.pi) + np.log(sigma) + constant * likelihoodSum

class MultivariateGaussian:
    """
    Class for multivariate Gaussian Distribution Estimator
    """
    def __init__(self):
        """
        Initialize an instance of multivariate Gaussian estimator

        Attributes
        ----------
        fitted_ : bool
            Initialized as false indicating current estimator instance has not been fitted.
            To be set as True in `MultivariateGaussian.fit` function.

        mu_: ndarray of shape (n_features,)
            Estimated expectation initialized as None. To be set in `MultivariateGaussian.fit`
            function.

        cov_: ndarray of shape (n_features, n_features)
            Estimated covariance initialized as None. To be set in `MultivariateGaussian.fit`
            function.
        """
        self.mu_, self.cov_ = None, None
        self.fitted_ = False

    def fit(self, X: np.ndarray) -> MultivariateGaussian:
        """
        Estimate Gaussian expectation and covariance from given samples

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Training data

        Returns
        -------
        self : returns an instance of self

        Notes
        -----
        Sets `self.mu_`, `self.cov_` attributes according to calculated estimation.
        Then sets `self.fitted_` attribute to `True`
        """
        self.mu_ = np.mean(X, 0)
        self.cov_ = np.cov(X.T)

        self.fitted_ = True
        return self

    def pdf(self, X: np.ndarray):
        """
        Calculate PDF of observations under Gaussian model with fitted estimators

        Parameters
        ----------
        X: ndarray of shape (n_samples, n_features)
            Samples to calculate PDF for

        Returns
        -------
        pdfs: ndarray of shape (n_samples, )
            Calculated values of given samples for PDF function of N(mu_, cov_)

        Raises
        ------
        ValueError: In case function was called prior fitting the model
        """
        if not self.fitted_:
            raise ValueError("Estimator must first be fitted before calling `pdf` function")

        return np.array([multivarPdf(xi, self.mu_, self.cov_) for xi in X])

    @staticmethod
    def log_likelihood(mu: np.ndarray, cov: np.ndarray, X: np.ndarray) -> float:
        """
        Calculate the log-likelihood of the data under a specified Gaussian model

        Parameters
        ----------
        mu : ndarray of shape (n_features,)
            Expectation of Gaussian
        cov : ndarray of shape (n_features, n_features)
            covariance matrix of Gaussian
        X : ndarray of shape (n_samples, n_features)
            Samples to calculate log-likelihood with

        Returns
        -------
        log_likelihood: float
            log-likelihood calculated over all input data and under given parameters of Gaussian
        """
        residuals = X-mu

        calcLogLikelihood = lambda resid, invariant, logDet, dim: -0.5 * (logDet + (resid.T @ invariant @ resid) + (dim * np.log(2 * np.pi)))
        invariantMat = inv(cov)
        logDeter = slogdet(cov)
        d = cov.shape[0]
        logPDF = np.apply_along_axis(lambda xi: calcLogLikelihood(xi, invariantMat, logDeter, d), 1, residuals)
        return logPDF.sum()

if __name__ == "__main__":
    quizX = np.array([1, 5, 2, 3, 8, -4, -2, 5, 1, 10, -10, 4, 5, 2, 7, 1, 1, 3, 2, -1, -3, 1, -4, 1, 2, 1,
          -4, -4, 1, 3, 2, 6, -6, 8, 3, -6, 4, 1, -2, 3, 1, 4, 1, 4, -2, 3, -1, 0, 3, 5, 0, -2])
    univar = UnivariateGaussian().fit(quizX)
    print(UnivariateGaussian.log_likelihood(10,1,quizX))
    pass


import random
import numpy as np
from scipy.stats import multivariate_normal

class LinearRegressionMixturesEM():
    def __init__(self, K, max_iter=100, tol=1e-4):
        self.K = K
        self.max_iter = max_iter
        self.tol = tol

    def fit(self, X, y):
        N = len(X)
        K = self.K

        # initialize parameters
        pi = np.array([1 / K] * K)
        w = np.random.randn(K, 1 + X.shape[1])
        beta = 1 / np.std(y)
        rnk = np.zeros(shape=(N, K))

        for iter in range(self.max_iter):

            # E step calc rnk
            phi = np.hstack((np.ones(shape=(X.shape[0], 1)), X))
            for i in range(N):
                norm_const = 0
                xi = phi[i]
                for j in range(K):
                    wjXi = xi.dot(w[j])
                    norm_const += pi[j] * multivariate_normal(wjXi, np.diag([1 / beta])).pdf(y[i])
                for k in range(K):
                    wkXi = xi.dot(w[k])
                    rnk[i, k] = pi[k] * multivariate_normal(wkXi, np.diag([1 / beta])).pdf(y[i]) / norm_const

            # M step
            nk = np.sum(rnk, axis=0)
            pi = nk / N

            # calc w
            for k in range(K):
                atrka = np.dot(phi.T, np.diag(rnk[:, k])).dot(phi)
                atrky = np.dot(phi.T, np.diag(rnk[:, k])).dot(y)
                w[k] = np.linalg.inv(atrka).dot(atrky).flatten()

            # calc beta
            beta = 0
            for k in range(K):
                beta += phi.dot(w[k]).dot(np.diag(rnk[:, k])).dot(phi.dot(w[k]))
            beta = N * (1 / beta)

            # calc likelihood
            likelihood = 0
            for i in range(N):
                likelihood_i = 0
                xi = phi[i]
                for j in range(K):
                    wjXi = xi.dot(w[j])
                    likelihood_i += pi[j] * multivariate_normal(wjXi, np.diag([1 / beta])).pdf(y[i])
                likelihood += np.log(likelihood_i)

            if iter != 0:
                if abs(prev_likelihood - likelihood) < self.tol:
                    break
            prev_likelihood = likelihood

            if iter == self.max_iter:
                break

        self.w = w
        self.beta = beta
        self.pi = pi
        self.rnk = rnk

    def predict(self, X):
        phi = np.hstack((np.ones(shape=(X.shape[0], 1)), X))
        y = np.zeros(shape=len(X))
        for k in range(self.K):
            y += self.pi[k] * phi.dot(self.w[k].T)

        return y

    def predictive_distribution(self, x):

        w, beta = self.w, self.beta
        phi = np.hstack((np.ones(shape=(1, 1)), x.reshape(1, len(x))))
        f_list = []
        for k in range(self.K):
            f_list.append(multivariate_normal(phi.dot(w[k]), np.diag([1 / beta])).pdf)

        return lambda x: sum([p*f(x) for p, f in zip(self.pi, f_list)])