#!/user/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2020/1/14 22:46
# @Author  : Leslie Wong
# @FileName: naiveLinearRegression.py
# @Software: PyCharm

"""
@title: Three Machine Learning Practices to Implement Linear Regression
@title: Four versions of Machine Learning Practice to Implement Linear Regression with one variable in Numpy (2019.11.15)
@author: LeslieWong & HuangJiaXun
@group: Five
"""
import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression

# Manipulate linear regression using normal regression
# θ = (XTX)-1XTY
def normalEquation(x,y):
	#ax1 = fig.add_subplot(221)
	#ax4 = fig.add_subplot(224)
	constant = np.ones_like(x)
	new_x = np.vstack((x, constant))
	new_x = new_x.transpose()
	new_y = y.reshape((-1, 1))
	theta = (np.linalg.inv(new_x.transpose().dot(new_x))).dot(new_x.transpose()).dot(new_y)
	theta = theta.reshape((1, -1))
	nor_x = np.linspace(0, 10, N)
	nor_y = np.array(theta[0, 0] * nor_x + theta[0, 1])
	# plot the stimulated line generated by normal equation
	ax1.plot(nor_x, nor_y, c='c', marker='o', alpha=0.2, label='Normal Line')
	ax4.plot(nor_x, nor_y, c='c', marker='o', alpha=0.2, label='Normal Line')

# Manipulate linear regression using sklearn library
def sklearn(x, y):
	# ax2 = fig.add_subplot(222)
	# ax4 = fig.add_subplot(224)
	model = LinearRegression(fit_intercept=True)
	model.fit(x[:, np.newaxis], y)
	xfit = np.linspace(0, 10, N)
	yfit = model.predict(xfit[:, np.newaxis])
	ax2.plot(xfit, yfit, c='y', marker='o', alpha=0.2, label='Sklearn Line')
	ax4.plot(xfit, yfit, c='y', marker='o', alpha=0.2, label='Sklearn Line')

# Manipulate linear regression using gradient descent algorithm
# y = mx + b

# Method 1: for loop
# def gradientDescent(x, y):
# 	# ax3 = fig.add_subplot(223)
# 	# ax4 = fig.add_subplot(224)
# 	points = np.vstack((x, y))
# 	learning_rate = 0.01
# 	initial_m = 0
# 	initial_b = 0
# 	num_iterations = 10000
# 	m_current = initial_m
# 	b_current = initial_b
#
# 	for i in range(num_iterations):
# 		b_gradient = 0
# 		m_graident = 0
# 		for j in range(N):
# 			temp_x = points[0, j]
# 			temp_y = points[1, j]
# 			b_gradient += (1 / N) * ((m_current * temp_x + b_current) - temp_y)
# 			m_graident += (1 / N) * ((m_current * temp_x + b_current) - temp_y) * temp_x
# 		new_m = m_current - learning_rate * m_graident
# 		new_b = b_current - learning_rate * b_gradient
# 		m_current = new_m
# 		b_current = new_b
#
# 	m = m_current
# 	b = b_current
# 	gra_x = np.linspace(0, 10, N)
# 	gra_y = np.array(m * gra_x + b)
# 	ax3.plot(gra_x, gra_y, c='g', marker='o', alpha=0.2, label='Gradient Descent Line')
# 	ax4.plot(gra_x, gra_y, c='g', marker='o', alpha=0.2, label='Gradient Descent Line')

# Method 2: matrix multiplication
def gradientDescent(x, y):
	x0 = np.ones_like(x)
	new_x = np.vstack((x0, x))
	x_parameters = new_x.transpose()
	y = np.vstack(y)
	learning_rate = 0.01
	initial_m = 0.
	initial_b = 0.
	num_iterations = 10000
	m_current = initial_m
	b_current = initial_b
	b_gradient = 0
	m_gradient = 0
	theta = np.array([[b_current], [m_current]], dtype='float64')

	for i in range(num_iterations):
		b_gradient = sum((1/N) * ((x_parameters.dot(theta)) - y))
		m_gradient = sum((1/N) * ((x_parameters.dot(theta)) - y).flatten() * (x_parameters[:, 1]).flatten())
		new_b = theta[0, 0] - learning_rate * b_gradient
		new_m = theta[1, 0] - learning_rate * m_gradient
		b_current = new_b
		m_current = new_m
		theta[0, 0] = b_current
		theta[1, 0] = m_current

	m = m_current
	b = b_current
	gra_x = np.linspace(0, 10, N)
	gra_y = np.array(m * gra_x + b)
	ax3.plot(gra_x, gra_y, c='g', marker='o', alpha=0.2, label='Gradient Descent Line')
	ax4.plot(gra_x, gra_y, c='g', marker='o', alpha=0.2, label='Gradient Descent Line')

# Define plot instance
fig = plt.figure(figsize=(18, 8))
# plot for original line and normal line
ax1 = fig.add_subplot(221)
# plot for original line and sklearn line
ax2 = fig.add_subplot(222)
# plot for original line and gradient descent line
ax3 = fig.add_subplot(223)
# plot for original line and all the others
ax4 = fig.add_subplot(224)

# Scatter plot and positive correlation
N = 200
# x = np.array(np.random.rand(1, N) * 10)
x = np.linspace(0, 10, N)
oy = np.array(0.5 * x + 4)
y = np.array(0.5 * x + 4 + np.random.randn(N))

# Plot original line and scatter diagram
ax1.plot(x, oy, c='m', marker='o', alpha=0.2, label='Original Line')
ax2.plot(x, oy, c='m', marker='o', alpha=0.2, label='Original Line')
ax3.plot(x, oy, c='m', marker='o', alpha=0.2, label='Original Line')
ax4.plot(x, oy, c='m', marker='o', alpha=0.2, label='Original Line')
ax1.scatter(x, y, s=30, c='r', marker='*', alpha=1)
ax2.scatter(x, y, s=30, c='r', marker='*', alpha=1)
ax3.scatter(x, y, s=30, c='r', marker='*', alpha=1)
ax4.scatter(x, y, s=30, c='r', marker='*', alpha=1)
normalEquation(x,y)
sklearn(x, y)
gradientDescent(x, y)
# add legend
ax1.legend(loc='upper left')
ax2.legend(loc='upper left')
ax3.legend(loc='upper left')
ax4.legend(loc='upper left')
fig.suptitle('Three Machine Learning Algorithm to Manipulate Linear Regression', va="baseline")
plt.show()
# plt.savefig('linearRegression.png', format='png')


import numpy

def editDistance(s1, s2):
	ln_s1 = len(s1)
	ln_s2 = len(s2)
	# Initializing the matrix
	Matrix = numpy.zeros((ln_s1+1 , ln_s2+1))
	for i in range(ln_s2+1):
		Matrix[0][i] = i
	for i in range(ln_s1+1):
		Matrix[i][0] = i

	# Filling the matrix
	for i in range(1, ln_s1+1):
		for j in range(1, ln_s2+1):
			insertion = Matrix[i][j-1]   + 1
			deletion  = Matrix[i-1][j]   + 1
			mismatch  = Matrix[i-1][j-1] + 1
			match     = Matrix[i-1][j-1]
			if s1[i-1] == s2[j-1]:
				Matrix[i][j] = min(insertion, deletion, match)
			if s1[i-1] != s2[j-1]:
				Matrix[i][j] = min(insertion, deletion, mismatch)

	return (int(Matrix[ln_s1][ln_s2]))

str1 = input()
str2 = input()
print(editDistance(str1, str2))


import time
import cocos.device as cd
from cocos.numerics.numerical_package_selector import select_num_pack
import cocos.numerics as cn
import numpy

def estimate_pi(n: int, gpu: bool = True) -> float:
    np = select_num_pack(gpu)

    x = np.random.rand(n)
    y = np.random.rand(n)

    in_quarter_circle = (x * x + y * y) <= 1.0
    return 4.0 * float(np.mean(in_quarter_circle))

# initialize cocos device - the architecture is selected automatically
# the architecture can be specified explitly by providing one of
#   'cpu', 'cuda', and 'opencl'
cd.init()

# print information regarding the available devices on the machine
cd.info()

# number of draws
n = 100000000
print(f'simulating {n} draws')

# run estimation of Pi on the cpu via numpy
pi_cpu = estimate_pi(n, gpu=False)
print(f'Estimate of Pi on cpu: {pi_cpu}')

# run estimation of Pi on the gpu via Cocos
pi_gpu = estimate_pi(n, gpu=True)
print(f'Estimate of Pi on gpu: {pi_gpu}')

# run a benchmark - repeating the simulation R times on both cpu and gpu
R = 10

# on gpu
tic = time.time()
for r in range(R):
    pi_gpu = float(estimate_pi(n, gpu=True))
    cd.sync()

time_on_gpu = time.time() - tic

print(f'time elapsed on gpu: {time_on_gpu}')

# on cpu
tic = time.time()
for r in range(R):
    pi_cpu = float(estimate_pi(n, gpu=False))

time_on_cpu = time.time() - tic

print(f'time elapsed on cpu: {time_on_cpu}')

# compute and print the speedup factor
print(f'speedup factor on gpu: {time_on_cpu/time_on_gpu}')


import numpy as np
from scipy.spatial.distance import cdist

class KMeans:
    def __init__(
            self,
            k: int,
            metric: str = "euclidean",
            tol: float = 1e-6,
            max_iter: int = 100):
        """
        inputs:
            k: int
                the number of centroids to use in cluster fitting
            metric: str
                the name of the distance metric to use
            tol: float
                the minimum error tolerance from previous error during optimization to quit the model fit
            max_iter: int
                the maximum number of iterations before quitting model fit
        """
        self.k = k
        self.metric = metric
        self.tol = tol
        self.max_iter = max_iter
        np.random.seed()

        if (self.k ==0 ):
            raise ValueError("Cannot provide K = 0")

        if (self.k > 200):
            raise ValueError("Cannot provide K bigger than 200")

    def fit(self, mat: np.ndarray):
        """
        fits the kmeans algorithm onto a provided 2D matrix

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features
        """
        if (mat.shape[0] < self.k):
            raise ValueError("Cannot provide K bigger than observations")

        ## pick a random number to initialize the clusters
        ran = np.random.choice(len(mat), self.k, replace=False,)
        ## This are the first clusters
        centroids = mat[ran, :]

        ### initialize the errors to a large value
        error = np.ones(self.k)* np.inf * -1  ## well use them in _get_error

        while (self.max_iter >= 0):
            distance=(cdist(mat, centroids,metric=self.metric)) ## we calcuate the distance of the matrix points to any of the clusters
            points_classification=[]
            for i in distance:
                points_classification.append(np.argmin(i)) # Get the index with the minimun distance
            points_classification=np.array(points_classification) ## convert to a np array

            centroids = self._get_centroids(mat,points_classification) ## We call the get centroids with our matrix and the label of the points
            ## keep trak of the last error
            prev_error = error
            # get the MSE from the centroids to the points
            error = self._get_error(centroids,mat,points_classification)
            # if the error is less than the tolerance, exit and return
            if (np.all((error - prev_error)< self.tol)):
                self.final_centroids=centroids
                return
            else:
                # continue iterating and count that iteration
                self.max_iter = self.max_iter-1
        # if after all the iterations we still don't get an acceptable error return
        self.final_centroids=centroids
        return

    def predict(self, mat: np.ndarray) -> np.ndarray:
        """
        predicts the cluster labels for a provided 2D matrix

        inputs:
            mat: np.ndarray
                A 2D matrix where the rows are observations and columns are features

        outputs:
            np.ndarray
                a 1D array with the cluster label for each of the observations in `mat`
        """
        distance=(cdist(mat, self.final_centroids,metric=self.metric)) ## we calcuate the distance of the matrix points to any of the clusters
        points_classification=[]

        ## we assign the points to their closer centroid
        for i in distance:
            points_classification.append(np.argmin(i))
        points_classification=np.array(points_classification)
        return(points_classification)

    def _get_error(self,centroids,mat,points_classification) -> float:
        """
        returns the final squared-mean error of the fit model

        outputs:
            float
                the squared-mean error of the fit model
        """
        new_error=[]
        for idx in range(self.k):
            ## get the distance from all the points in the cluster to their centroids
            distance = (cdist(mat[points_classification==idx], centroids))
            distance.tolist()
            # Ge the distance  correspoding to the current cluster
            error_d = np.array([item[idx] for item in distance])
            ## Get the mean error dor the distance abd square it
            new_error.append(np.mean((error_d)**2))
        return(np.array(new_error))

    def _get_centroids(self,mat,points_classification) -> np.ndarray:
        """
        returns the centroid locations of the fit model

        outputs:
            np.ndarray
                a `k x m` 2D matrix representing the cluster centroids of the fit model
        """

        centroids=[]
        for i in range(self.k):
            ## update the new centroids to the center of the current cluster
            centroids_new = mat[points_classification == i].mean(axis=0)
            centroids.append(centroids_new)
        return centroids


# %%

# coding: utf-8

# # Assignment 8
#
# This repository contains a file `data_1.dat` that is actual tabular data from a stress-strain test conducted in a standard test frame (MTS machine).  The columns of data have the following format
#
# |time|Axial Displacement|Axial (engineering) Strain| Axial Force |
# |:-:|:-:|:-:|:-:|
# | (s) | (in) | (in/in) | (lbf) |
#
# Complete the function definition below for the function named `convert_to_true_stress_and_strain()`.  This function should take the name of a file (e.g. `data_1.dat`) as an argument and return a 2-tuple ($\varepsilon_T$, $\sigma_T$), where $\varepsilon_T$ is the *true* strain and $\sigma_T$ is the *true* stress.
#
# Recall that the definition of engineering stress is
#
# $$
# \sigma_E = \frac P A_o
# $$
#
# where $P$ is the axial force and $A_o$ is the original cross-sectional area of the sample.  The cross-sectional area can be computed from the width and thickness dimensions that are stored on the 3rd line of the header of the MTS data file.  Look for `Wo=` and `Thicko=`.

# Your function should parse these values out computationally (not manually) because we expect them to be in every MTS header the same way and woud like the function to work on multiple data files.
#
# The conversion between engineering $\varepsilon_E$ and true strain is
#
# $$
# \varepsilon_T = \ln(1+\varepsilon_E).
# $$
#
# The conversion between engineering and true stress is
#
# $$
# \sigma_T = \sigma_E (1+\varepsilon_E).
# $$
#
# Use Numpy data structures and broadcasted operations to make your code compact and efficient.
#
# ## Please Note
#
# You may write your code directly in the Jupyter Notebook below or convert the notebook to a plain Python file with the command
#
# ```bash
# jupyter nbconvert --to python assignment7.ipynb
# ```
#
# which will create a file `assignment7.py`.  If you commit this file to the repository, only the Python file will be tested (not the Jupyter notebook).  However, if you do not commit this file into the repository, the Jupyter notebook will be converted automatically on Travis before testing.

# %%

import numpy as np
import linecache
import scipy.integrate

class StressStrainConverter():

    def __init__(self, filename):

        self.filename = filename

    def extract_dimensions(self):

        line = linecache.getline(self.filename, 3).split('=')

        self.width = float(line[1].split('"')[0])

        self.thickness = float(line[2].split('"')[0])

        return

    def convert_to_true_stress_and_strain(self):

        self.extract_dimensions()

        eng_strain, force = np.loadtxt(self.filename, skiprows=5, usecols=(2,3)).T

        self.true_strain = np.log(1 + eng_strain)

        self.true_stress = force / self.width / self.thickness * (1 + eng_strain)

        return

class Toughness(StressStrainConverter):

    def compute_toughness_simps(self):

        self.convert_to_true_stress_and_strain()

        return scipy.integrate.simps(self.true_stress, self.true_strain)

    def compute_toughness_trapz(self):

        self.convert_to_true_stress_and_strain()

        return scipy.integrate.trapz(self.true_stress, self.true_strain)


from numpy import *

def loadDataSet():
    dataMat = []
    labelMat = []
    fr = open('testSet.txt')

    for line in fr.readlines():
        lineArr = line.strip().split()
        dataMat.append([1.0, float(lineArr[0]), float(lineArr[1])])
        labelMat.append(int(lineArr[2]))

    return dataMat,labelMat

def sigmoid(inx) :
    return 1.0 / (1 + exp(-inx))

def gradAscent(dataMatIn, classLabels) :
    dataMatrix = mat(dataMatIn)
    labelMat = mat(classLabels).transpose()
    m, n = shape(dataMatrix)

    alpha = 0.001
    maxCycles = 500
    weights = ones((n, 1))

    for k in range(maxCycles) :
        h = sigmoid(dataMatrix * weights)
        error = (labelMat - h)
        weights = weights + alpha * dataMatrix.transpose() * error

    return weights

def plotBestFit(weights) :
    import matplotlib.pyplot as plt

    dataMat, labelMat = loadDataSet()
    dataArr = array(dataMat)
    n = shape(dataArr)[0]
    xcord1 = []
    ycord1 = []

    xcord2 = []
    ycord2 = []

    for i in range(n) :
        if int(labelMat[i]) == 1 :
            xcord1.append(dataArr[i, 1])
            ycord1.append(dataArr[i, 2])
        else :
            xcord2.append(dataArr[i, 1])
            ycord2.append(dataArr[i, 2])

    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.scatter(xcord1, ycord1, s=30, c='red', marker='s')
    ax.scatter(xcord2, ycord2, s=30, c='green')

    x = arange(-3.0, 3.0, 0.1)
    y = (-weights[0] - weights[1] * x) / weights[2]
    ax.plot(x, y)
    plt.xlabel('X1')
    plt.ylabel('X2')
    plt.show()

def stocGradAscent0(dataMatrix, classLabels) :
    m, n = shape(dataMatrix)
    alpha = 0.01
    weights = ones(n)

    for i in range(m) :
        h = sigmoid(sum(dataMatrix[i] * weights))
        error = classLabels[i] - h
        weights = weights + alpha * error * dataMatrix[i]

    return weights

def stocGradAscent1(dataMatrix, classLabels, numIter=150) :
    m, n = shape(dataMatrix)
    weights = ones(n)

    for j in range(numIter) :
        dataIndex = range(m)
        for i in range(m) :
            alpha = 4 / (1.0 + j + i) + 0.01
            randIndex = int(random.uniform(0, len(dataIndex)))
            h = sigmoid(sum(dataMatrix[randIndex] * weights))
            error = classLabels[randIndex] - h
            weights = weights + alpha * error * dataMatrix[randIndex]
            del(dataIndex[randIndex])

    return weights


# -*- coding: utf-8 -*-
'''
	Θ(n^lg7)
'''
import numpy as np
import time

def strassen(a,b):

	if len(a) == 2:
		return np.dot(a,b)

	m = len(a)/2

	a11 = a[:m,:m]
	a12 = a[:m,m:]
	a21 = a[m:,:m]
	a22 = a[m:,m:]

	b11 = b[:m,:m]
	b12 = b[:m,m:]
	b21 = b[m:,:m]
	b22 = b[m:,m:]

	x1 = strassen((a11 + a22), (b11 + b22))
	x2 = strassen((a21 + a22), b11)
	x3 = strassen(a11, (b12 - b22))
	x4 = strassen(a22, (b21 - b11))
	x5 = strassen((a11 + a12), b22)
	x6 = strassen((a21 - a11), (b11 + b12))
	x7 = strassen((a12 - a22), (b21 + b22))

	c11 = x1 + x4 - x5 + x7
	c12 = x3 + x5
	c21 = x2 + x4
	c22 = x1 + x3 - x2 + x6

	c1 = np.hstack((c11,c12))
	c2 = np.hstack((c21,c22))

	return np.vstack((c1,c2))

x = y = np.arange(64).reshape(8, 8)

t = time.time()
print np.dot(x, y)
print time.time() - t

t = time.time()
print strassen(x, y)
print time.time() - t


"""
This script computes the persistence landscape of the  Cantor triangle.

authors: Lee Przybylski, Michael J. Catanzaro
"""

import numpy as np
from ripser import ripser
from persim.landscapes import PersLandscapeApprox
from persim.landscapes import plot_landscape
import matplotlib.pyplot as plt
from persim import plot_diagrams

#%% Initialize the Cantor triangle example
def phi0(x):
    return x / 3

def phi20(x):
    return (x + np.array([2, 0])) / 3

def phi02(x):
    return (x + np.array([0, 2])) / 3

SArray = np.array([[0, 0], [1, 0], [0, 1]])
#%% Chose a scale to compute the ifs up to
# Keep the scale less than 30
scale = 4

for _ in range(scale):
    out0 = phi0(SArray)
    out20 = phi20(SArray)
    out02 = phi02(SArray)
    SArray = np.concatenate((out0, out20, out02), axis=0)

#%% Plot at scale
plt.figure(figsize=(5, 5))
# Color the shape
x = SArray[:, 0]
y = SArray[:, 1]
plt.scatter(SArray[:, 0], SArray[:, 1], linewidth=0.1, Color="blue", zorder=2)
plt.title("$S_3$", size=20)
#%% Compute the persistence
Dgmprox = ripser(SArray, maxdim=1)["dgms"]
plot_diagrams(Dgmprox, show=True, lifetime=False)
print(Dgmprox)
#%% Compute the landscape
Landprox = PersLandscapeApprox(dgms=Dgmprox, hom_deg=0)
ttl = f"PL of Cantor triangle at scale {scale}"
plot_landscape(landscape=Landprox, labels=["", "", ""], title=ttl)


#!/usr/bin/env python3

import numpy as np

def pretty_print(name, to_print):
    print(f'{name}:')
    print(f'{to_print}\n\n')

# Let's first create a sample array to check its functions
int_array = np.arange(0,40)

pretty_print('int_array', int_array)

# We can use mean, min, max and std to immediately extract stats on our datasets
pretty_print('int_array.mean()', int_array.mean())
pretty_print('int_array.min()', int_array.min())
pretty_print('int_array.max())', int_array.max())
pretty_print('int_array.std()', int_array.std())

# Let's reshape this array into a matrix
reshaped_int_matrix = int_array.reshape(5, 8)

# We can apply the very same operations on a numpy matrix
pretty_print('reshaped_int_matrix.mean()', reshaped_int_matrix.mean())
pretty_print('reshaped_int_matrix.min()', reshaped_int_matrix.min())
pretty_print('reshaped_int_matrix.max()', reshaped_int_matrix.max())
pretty_print('reshaped_int_matrix.std()', reshaped_int_matrix.std())

# We then can for example apply the same method to a row only
pretty_print('reshaped_int_matrix[1].mean()', reshaped_int_matrix[1].mean())


# William Nguyen
# email: williamvnguyen2@gmail.com

import numpy as np

class PCA:
    """
    Principal component analysis (PCA) is an unsupervised linear
    technique for dimensionality reduction on a data set. PCA computes
    the principal directions that maximize the variance of the projected data,
    and can compute the principal components by projecting the data onto them.
    This gives us a reduced dimension of the data, while preserving as much of
    the variance as possible.
    - fit(X): fits the model on the data matrix X
    - transform(X, target_d): transforms X to lower dimension target_d
    - fit_transform(X, target_d): Performs fit and transform
    """

    def __init__(self, svd=True):
        """
        Constructor for Principal Component Analysis.
        :param svd: whether to use SVD to compute PCA or not
        """
        self.svd = svd
        self.covariance = None # covariance matrix of data fitted, (d, d)
        self.eigenvectors = None # eigenvectors of the covariance matrix
        self.eigenvalues = None # eigenvalues of the covariance matrix
        self.mean = 0 # mean of data fitted, shape: (d,)
        self.Vh = None # unitary matrix from svd that is transposed, shape: (d, d)
        self.S = None # rectangular diagonal matrix from svd, shape: (d,)
        self.U = None # unitary matrix from svd, shape: (n, n)

    def fit(self, X):
        """
        Fits the model to X by computing the
        SVD of X or by computing the eigenvectors of its
        covariance matrix.
        :param X: the data matrix, shape: (n, d)
        """
        self.mean = np.mean(X, axis=0)
        # demean
        X_demean = X - self.mean
        if self.svd: # note that SVD is way faster at computing PCA!
            # computing our singular value decomposition
            self.U, self.S, self.Vh = np.linalg.svd(X_demean, full_matrices=True)
        else: # non-SVD approach, awfully slow
            # compute covariance matrix
            self.covariance = np.cov(X_demean.T)
            # compute eigenvectors and eigenvalues of covariance
            self.eigenvalues, self.eigenvectors = np.linalg.eig(self.covariance)

    def transform(self, X, target_d):
        """
        Performs PCA dimensionality reduction X to a lower dimension.
        :param X: the data matrix, shape: (n, d)
        :param target_d: the lower dimension in which we reduce to
        :return: a matrix, shape: (n, target_d)
        """
        X_transformed = None
        X_demean = X - self.mean
        if self.svd:
            # find the indices of the singular values in descending order
            # also truncating to get the target dimension target_d
            idx = self.S.argsort()[::-1][:target_d]
            # sort the rows of Vh, then project X onto Vh.T
            X_transformed = X_demean.dot(self.Vh[idx].T)
        else:
            # sort eigenvectors by their eigenvalues in descending order for pc's
            idx = self.eigenvalues.argsort()[::-1]
            self.eigenvalues = self.eigenvalues[idx]
            self.eigenvectors = self.eigenvectors[:,idx]
            # projection
            X_transformed = X_demean.dot(self.eigenvectors[:,:target_d])
        return X_transformed

    def fit_transform(self, X, target_d):
        """
        Performs fit and transform in one routine.
        :param X: the data matrix, shape: (n, d)
        :param target_d: the lower dimension in which we reduce to
        :return: a matrix, shape: (n, target_d)
        """
        X_transformed = None
        if self.svd: # svd is much faster
            self.fit(X)
            # sort the indices by the descending order of the singular values
            idx = self.S.argsort()[::-1][:target_d]
            # Notice that this is not necessarily redundant code!
            # This is a different approach than svd in the transform function!
            S_ = np.diag(self.S[idx])[:target_d,:target_d]
            U_ = self.U[:,idx]
            X_transformed = U_.dot(S_)
        else: # terribly slow approach
            self.fit(X)
            X_transformed = self.transform(X, target_d)
        return X_transformed


def lineIntersect3DPoints(PA,PB):
    import numpy as np
    '''
    Find intersection point of lines in 3D space, in the least squares sense.
    PA :          Nx3-matrix containing starting point of N lines
    PB :          Nx3-matrix containing end point of N lines
    P_Intersect : Best intersection point of the N lines, in least squares sense.
    distances   : Distances from intersection point to the input lines
    From https://uk.mathworks.com/matlabcentral/fileexchange/37192-intersection-point-of-lines-in-3d-space
    '''

    Si = PB-PA      #Describe N lines as vectors
    Si_root = np.sqrt(np.sum(np.square(Si),axis=1))*np.ones(shape=(1,3),dtype=np.uint8)

    ni = np.divide(Si,  Si_root)        #normalise vectors
    nx,ny,nz = ni[:,0],ni[:,1],ni[:,2]

    SXX = np.sum((nx**2)-1)
    SYY = np.sum((ny**2)-1)
    SZZ = np.sum((nz**2)-1)
    SXY = np.sum(nx*ny)
    SXZ = np.sum(nx*nz)
    SYZ = np.sum(ny*nz)
    S = np.array([[SXX,SXY,SXZ],[SXY,SYY,SYZ],[SXZ,SYZ,SZZ]])

    CX  = np.sum(    PA[:,0]*((nx**2)-1)  +  PA[:,1]*(nx*ny)         +   PA[:,2]*(nx*nz)         )
    CY  = np.sum(    PA[:,0]*(nx*ny)      +  PA[:,1]*((ny**2)-1)     +   PA[:,2]*(ny*nz)         )
    CZ  = np.sum(    PA[:,0]*(nx*nz)      +  PA[:,1]*(ny*nz)         +   PA[:,2]*((nz**2)-1)     )
    C   = np.array([[CX],[CY],[CZ]])

    P_intersect = (np.linalg.lstsq(S,C)).H

    N = PA.shape()[0]
    distances = np.zeros(shape=(N,1))
    for i in np.arange(N):
        ui = (  (P_intersect-PA[i,:])*(Si[i,:]).H   ) / (   Si[i,:]*(Si[i,:]).H     )
        distances[i] = np.linalg.norm(P_intersect-PA[i,:]-ui*Si[i,:])

    return P_intersect,distances

def lineIntersectionNormVec3D(ni):
    import numpy as np
    '''
    Find intersection point of lines in 3D space, in the least squares sense.
    ni:          Nx3-matrix containing normalised vectors in x,y,z format
    P_Intersect : Best intersection point of the N lines, in least squares sense.
    From https://uk.mathworks.com/matlabcentral/fileexchange/37192-intersection-point-of-lines-in-3d-space
    '''

    nx,ny,nz = ni[:,0],ni[:,1],ni[:,2]

    SXX = np.sum((nx**2)-1)
    SYY = np.sum((ny**2)-1)
    SZZ = np.sum((nz**2)-1)
    SXY = np.sum(nx*ny)
    SXZ = np.sum(nx*nz)
    SYZ = np.sum(ny*nz)
    S = np.array([[SXX,SXY,SXZ],[SXY,SYY,SYZ],[SXZ,SYZ,SZZ]])

    CX  = np.sum(    nx*((nx**2)-1)  +  ny*(nx*ny)         +   nz*(nx*nz)         )
    CY  = np.sum(    nx*(nx*ny)      +  ny*((ny**2)-1)     +   nz*(ny*nz)         )
    CZ  = np.sum(    nx*(nx*nz)      +  ny*(ny*nz)         +   nz*((nz**2)-1)     )
    C   = np.array([[CX],[CY],[CZ]])

    P_intersect = np.array(np.linalg.lstsq(S,C)[0]).conj().T

    N = ni.shape[0]
    distances = np.zeros(shape=(N,1))
    for i in np.arange(N):
        ui = (  (P_intersect-ni[i,:]) * (ni[i,:]).conj().T   ) / (   ni[i,:]*(ni[i,:]).conj().T     )
        distances[i] = np.linalg.norm(P_intersect-ni[i,:]-ui*ni[i,:])

    return P_intersect,distances

def johannesLSintersect(norms):
    import numpy as np
    """
    P0 and P1 are NxD arrays defining N lines.
    D is the dimension of the space. This function
    returns the least squares intersection of the N
    lines from the system given by eq. 13 in
    http://cal.cs.illinois.edu/~johannes/research/LS_line_intersect.pdf.
    """
    # generate all line direction vectors
    # n = (P1-P0)/np.linalg.norm(P1-P0,axis=1)[:,np.newaxis] # normalized

    # generate the array of all projectors
    projs = np.eye(norms.shape[1]) - norms[:,:,np.newaxis]*norms[:,np.newaxis]  # I - n*n.T
    # see fig. 1

    # generate R matrix and q vector
    R = projs.sum(axis=0)
    q = (projs @ (norms*2)[:,:,np.newaxis]).sum(axis=0)

    # solve the least squares problem for the
    # intersection point p: Rp = q
    p = np.linalg.lstsq(R,q,rcond=None)[0]

    return p


"""
Consider an equilateral triangle in which straight lines are drawn from each vertex to the middle of the opposite side, such as in the size 1 triangle in the sketch below.

Sixteen triangles of either different shape or size or orientation or location can now be observed in that triangle. Using size 1 triangles as building blocks, larger triangles can be formed, such as the size 2 triangle in the above sketch. One-hundred and four triangles of either different shape or size or orientation or location can now be observed in that size 2 triangle.

It can be observed that the size 2 triangle contains 4 size 1 triangle building blocks. A size 3 triangle would contain 9 size 1 triangle building blocks and a size n triangle would thus contain n2 size 1 triangle building blocks.

If we denote T(n) as the number of triangles present in a triangle of size n, then

T(1) = 16
T(2) = 104

Find T(36).

ans: 343047
"""

"""
Solution Method:
1. Create an index matrix that translates x-y coordinates into an index for adjacency matrices
2. Create 6 adjacency matrices for each of the six directions
3. For each ad. matrix, create edges between every pair of vertices in the same subgraph
4. Remove reflexive edges - edges with the same start and end point
5. Take every permutation of 3 ad. matrices and multiply them together
6. Find the trace of each product and sum the result
7. Divide by 6 - this is the solution

This method works because the product of adjacency matrices tells you the number of paths from one vertex to another. We multiply permutations of ad. matrices because this ensures we don't count any paths that double-back on themselves. These ad. matrices don't have reflexive edges because we don't want to count "paths" that stay put at a vertex. However, when we finally take the trace of a ad. matrix product, we're counting the number of length-3 paths that start and stop at the same vertex - i.e. triangles. The answer is divided by six because each triangle was counted 6 times - 3 times starting at each vertex and 2 times more for clockwise and counter-clockwise.
"""

import numpy as np
import scipy.sparse
import sys

# Note: depth is 1-indexed
def main(depth):
	# create index matrix
	print("creating index matrix")
	mat_index, num_vertices = index(depth)

	print("creating adjacency matrices")
	h  = mat_h (mat_index, num_vertices)
	v  = mat_v (mat_index, num_vertices)
	d1 = mat_d1(mat_index, num_vertices)
	d2 = mat_d2(mat_index, num_vertices)
	s1 = mat_s1(mat_index, num_vertices)
	s2 = mat_s2(mat_index, num_vertices)

	print("finding paths")
	print("|" + " " * 120 + "|")
	print("|", end="")
	m = [v, h, d1, d2, s1, s2]
	tot = 0
	for a in m:
		for b in m:
			if b is a: continue
			for c in m:
				if c is a or c is b: continue
				tot += a.dot(b).dot(c).diagonal().sum()
				print("=", end="")
				sys.stdout.flush()
	print("|")
	print(f"T({depth}) = {tot//6}")

# used to convert x-y coordinates into an index into the adjacency matrix
def index(depth):
	# max width = 4*depth - 1 (depth 1-indexed)
	# even rows = r + 1 (r 0-indexed)
	# odd rows  = 4*r - 1
	# height    = 2*depth + 1
	mat = np.full((2*depth+1, 4*depth+1), -1)
	i = 0
	for r in range(2*depth + 1):
		if r % 2 == 0:
			for c in range(0, 2*r + 1, 2):
				mat[r, c] = i
				i += 1
		else:
			for c in range(2*r + 1):
				mat[r, c] = i
				i += 1
	return (mat, i)

# create an adjacency matrix with only horizontal connections
def mat_h(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(2, index.shape[0], 2):
		rep = r
		for c in range(rep):
			c = 2*c

			i = index[r, c]
			j = index[r, c + 2]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only vertical connections
def mat_v(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(0, index.shape[0] - 2, 2):
		rep = r + 1 # number of times to repeat
		for c in range(rep):
			c = 2*c

			i = index[r    , c    ]
			j = index[r + 1, c + 1]
			mat[i, j] = 1
			mat[j, i] = 1

			i = index[r + 2, c + 2]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only diagonal connections (1)
def mat_d1(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(0, index.shape[0] - 2, 2):
		rep = r // 2 + 1
		for c in range(rep):
			c = 4*c

			i = index[r    , c]
			j = index[r + 1, c]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r + 2, c]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only diagonal connections (2)
def mat_d2(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(0, index.shape[0] - 2, 2):
		rep = r // 2 + 1
		for c in range(rep):
			c = 4*c

			i = index[r    , c    ]
			j = index[r + 1, c + 2]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r + 2, c + 4]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only shallow diagonal connections (1)
def mat_s1(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(1, index.shape[0] - 1, 2):
		# do initial half-pattern
		i = index[r + 1, 0]
		j = index[r    , 1]
		mat[i, j] = 1
		mat[j, i] = 1
		i = index[r    , 2]
		mat[i, j] = 1
		mat[j, i] = 1

		rep = (r - 1) // 2 # number of times to repeat a full-pattern

		for c in range(rep):
			c = 4*c + 2

			i = index[r    , c    ]
			j = index[r    , c + 1]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r - 1, c + 2]
			mat[i, j] = 1
			mat[j, i] = 1

			i = index[r + 1, c + 2]
			j = index[r    , c + 3]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r    , c + 4]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# create an adjacency matrix with only shallow diagonal connections (2)
def mat_s2(index, num_vertices):
	# create adjacency matrix
	mat = np.zeros((num_vertices, num_vertices), dtype = np.uint8)
	for r in range(1, index.shape[0] - 1, 2):
		# do initial half-pattern
		i = index[r, 0]
		j = index[r, 1]
		mat[i, j] = 1
		mat[j, i] = 1
		i = index[r + 1, 4]
		mat[i, j] = 1
		mat[j, i] = 1

		rep = (r - 1) // 2 # number of times to repeat a full-pattern

		for c in range(rep):
			c = 4*c

			i = index[r - 1, c    ]
			j = index[r    , c + 3]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r    , c + 4]
			mat[i, j] = 1
			mat[j, i] = 1

			j = index[r    , c + 5]
			mat[i, j] = 1
			mat[j, i] = 1
			i = index[r + 1, c + 8]
			mat[i, j] = 1
			mat[j, i] = 1

	mat = scipy.sparse.coo_matrix(mat, dtype = np.uint32)
	return connect(mat)

# take an initial adjacency matrix and create edges between
# all pairs of vertices that come from the same subgraph -
# do not create reflexive edges
def connect(a):
	while True:
		tmp = normalize(a + a.dot(a))
		if (a != tmp).nnz > 0:
			a = tmp
		else:
			return zero_diagonal(a)

# replace all nonzero entries with ones in a sparse matrix
def normalize(a):
	#return np.ceil(a.power(-0.5))
	nnz_inds = a.nonzero()
	keep = np.where(a.data != 0)[0]
	n_keep = len(keep)
	b = scipy.sparse.coo_matrix(
		(np.ones(n_keep), (nnz_inds[0][keep], nnz_inds[1][keep])),
		shape = a.shape,
		dtype = np.uint32)
	return b

# zero-out the diagonal elements of a sparse matrix
def zero_diagonal(a):
	return a - scipy.sparse.dia_matrix(
		(a.diagonal()[scipy.newaxis, :], [0]),
		shape = a.shape,
		dtype = np.uint32)

#main(1)
#main(2)
main(36)


import numpy as np
import pandas as pd
import tensorflow as tf
from sklearn.metrics import mean_squared_error, mean_absolute_error

def cosine_similarity(a, b):
    """
    calcualte the Cosine_simialrity between two vector
    :param a: array-like vecotr (n*1)
    :param b: array-like vector (n*1)
    :return:  a single value range from 0 to 1, represneting the similarity between a and b
    """
    return a @ b / (np.linalg.norm(a) * np.linalg.norm(b))

def mse(data, pred):
    """

    :param data:
    :param pred:
    :return:
    """
    suffixes = ["_true", "_pred"]
    rating_true_pred = data.merge(pred, on = ['user', 'movie'], how='inner', suffixes = suffixes)

    return np.sqrt(mean_squared_error(rating_true_pred['rating' + suffixes[0]], rating_true_pred['rating' + suffixes[1]]))

def mae(data, pred):
    """

    :param data:
    :param pred:
    :return:
    """
    suffixes = ["_true", "_pred"]
    rating_true_pred = data.merge(pred, on = ['user', 'movie'], how='inner', suffixes = suffixes)

    return np.sqrt(mean_absolute_error(rating_true_pred['rating' + suffixes[0]], rating_true_pred['rating' + suffixes[1]]))

def generate_error_matrix(a, b):
    """

    :param a:
    :param b:
    :return:
    """
    entry_diff = []
    row_index_set, col_index_set = a_nonzero_index = np.nonzero(a)
    for i in range(len(row_index_set)):
        row_index = row_index_set[i]
        col_index = col_index_set[i]

        entry_diff.append(a[row_index, col_index] - b[row_index, col_index])

    return np.array(entry_diff)


import numpy

def weighted_mean_and_std(x, w):
    """Return the mean and standard deviation of the data points x, weighted by
    the weights in w (which do not need to sum to 1)."""
    w = numpy.array(w, dtype=float)
    w /= w.sum()
    x = numpy.asarray(x)
    weighted_mean = (w*x).sum()
    squared_diff = (x - weighted_mean)**2
    weighted_var = (w * squared_diff).sum()
    return weighted_mean, numpy.sqrt(weighted_var)

def weighted_mean(x, w):
    """Return the mean of the data points x, weighted by the weights in w
    (which do not need to sum to 1)."""
    w = numpy.array(w, dtype=float)
    w /= w.sum()
    x = numpy.asarray(x)
    return (w*x).sum()

def _gaussian(x, mu=0, sigma=1):
    return (1/numpy.sqrt(2 * numpy.pi * sigma**2) * numpy.exp(-0.5 * ((numpy.asarray(x)-mu)/sigma)**2))

def gaussian_mean(x, y, p, std=1):
    """Given a set of positions x where values y were observed, calculate
    the gaussian-weighted mean of those values at a set of new positions p,
    where the gaussian has a specied standard deviation.
    """
    return numpy.array([weighted_mean(y, _gaussian(x, mu=pp, sigma=std)) for pp in p])

def savitzky_golay(data, kernel=11, order=4):
    """Apply Savitzky-Golay smoothing to the input data.

    http://en.wikipedia.org/wiki/Savitzky-Golay_filter
    """
    kernel = abs(int(kernel))
    order = abs(int(order))
    if kernel % 2 != 1 or kernel < 1:
        raise TypeError("kernel size must be a positive odd number, was: %d" % kernel)
    if kernel < order + 2:
        raise TypeError("kernel is to small for the polynomals\nshould be > order + 2")
    order_range = range(order+1)
    half_window = (kernel-1) // 2
    m = numpy.linalg.pinv([[k**i for i in order_range] for k in range(-half_window, half_window+1)])[0]
    window_size = len(m)
    half_window = (window_size-1) // 2
    offsets = range(-half_window, half_window+1)
    offset_data = list(zip(offsets, m))
    smooth_data = list()
    data = numpy.concatenate((numpy.ones(half_window)*data[0], data, numpy.ones(half_window)*data[-1]))
    for i in range(half_window, len(data) - half_window):
        value = 0.0
        for offset, weight in offset_data:
            value += weight * data[i + offset]
        smooth_data.append(value)
    return numpy.array(smooth_data)

def lowess(x, y, f=2/3., iters=3, outlier_threshold=6, weights=None, degree=1):
    """Apply LOWESS to fit a nonparametric regression curve to a scatterplot.

    http://en.wikipedia.org/wiki/Local_regression

    Parameters:
        x, y: 1-d arrays containing data points in x and y.
        f: smoothing parameter in range [0, 1]. Lower values = less smoothing.
        iter: number of robustifying iterations (after each of which outliers
            are detected and excluded). Larger numbers = more robustness, but
            slower run-time.
        outlier_threshold: data points that are further from the lowess estimate
            than outlier_threshold * numpy.median(numpy.abs(residuals)) are
            declared outliers.
        degree: degree of locally weighted fit. Generally 1 is fine, though for
            data with local minima/maxima, degree=2 may fit better.

    Returns: array of smoothed y-values for the input x-values.
    """
    x = numpy.asarray(x)
    y = numpy.asarray(y)
    r = max(4, int(numpy.ceil(f*(len(x)-1))))
    # below hogs RAM for large input, without much speed gain.
    # h = [numpy.sort(numpy.abs(x - xv))[r] for xv in x]
    # w = numpy.clip(numpy.abs(numpy.subtract.outer(x, x) / h), 0, 1)
    # w = (1 - w**3)**3
    delta = 1
    max_dists = numpy.empty_like(x)
    if weights is None:
        weights = 1
    for it in range(iters):
        y_est = []
        for i, xv in enumerate(x): # for xv, wv in zip(x, w.T):
            x_dists = numpy.abs(x - xv)
            if it == 0:
                max_dist = numpy.partition(x_dists, r)[r]
                max_dists[i] = max_dist
            else:
                max_dist = max_dists[i]
            wv = numpy.clip(x_dists/max_dist, 0, 1)
            wv = (1 - wv**3)**3
            final_weights = delta * wv * weights
            if degree > 1:
                poly = numpy.poly1d(numpy.polyfit(x, y, degree, w=final_weights))
                y_est.append(poly(xv))
            else: # faster to hard-code weighted linear regression formula
                weighted_x = final_weights * x
                b1 = numpy.dot(final_weights, y)
                b2 = numpy.dot(weighted_x, y)
                A11 = numpy.sum(final_weights)
                A12 = A21 = numpy.sum(weighted_x)
                A22 = numpy.dot(weighted_x, x)
                # solve linear system A*beta = b where A = [[A11, A12], [A21, A22]] and b = [b1, b2]
                determinant = A11 * A22 - A12 * A21
                beta1 = (A22*b1 - A12*b2) / determinant
                beta2 = (A11*b2 - A21*b1) / determinant
                y_est.append(beta1 + beta2 * xv)
        y_est = numpy.array(y_est)
        residuals = y - y_est
        s = numpy.median(numpy.abs(residuals))
        if s > 0:
            delta = numpy.clip(residuals / (outlier_threshold * s), -1, 1)
            delta = (1 - delta**2)**2
    return numpy.array(y_est)

def robust_polyfit(x, y, degree=2, iters=3):
    """Fit a polynomial to scattered data, robust to outliers.

    Parameters:
        x, y: 1-d arrays containing data points in x and y.
        degree: degree of the polynomial to fit.
        iter: number of robustifying iterations (after each of which outliers
            are detected and excluded). Larger numbers = more robustness, but
            slower run-time.

    Returns: polynomial coefficients, array of smoothed y-values for the input x-values.
    """
    x, y = numpy.asarray(x), numpy.asarray(y)
    weights = numpy.ones(len(x), float) # delta in original formulation
    for _ in range(iters):
        cs = numpy.polynomial.polynomial.polyfit(x, y, degree, w=weights)
        y_est = numpy.polynomial.polynomial.polyval(x, cs)
        residuals = y - y_est
        s = numpy.median(numpy.abs(residuals))
        if s > 0:
            weights = (residuals / (6 * s)).clip(-1, 1)
            weights = (1 - weights**2)**2
    return cs, y_est


from __future__ import print_function
import numpy as np

def mse(x, y):
	'''Mean-squared error of two 1D numpy arrays or Python lists'''
	x = np.array(x)
	y = np.array(y)
	return np.mean((x - y) ** 2)

def mae(x, y):
	'''Mean absolute error of two 1D numpy arrays or Python lists'''
	x = np.array(x)
	y = np.array(y)
	return np.mean(np.abs(x - y))

def q(y_true, y_pred):
	'''q value as described in Tropsha, Gramatica, Gombar:
	The Importance of Being Earnest'''
	y_true = np.array(y_true)
	y_pred = np.array(y_pred)
	y_mean = np.mean(y_true)
	return 1 - np.sum((y_true - y_pred) ** 2) / np.sum((y_true - y_mean) ** 2)

def linreg(x, y):
	'''Computes a linear regression through the origin using OLS'''
	x = np.array(x)
	y = np.array(y)
	x = x[:, np.newaxis]
	a, _, _, _ = np.linalg.lstsq(x, y)
	r2 = q(y, (x * a)[:, 0])
	return (r2, a)


from sympy import *
x, t, z, nu = symbols('x t z nu')
init_printing(use_unicode=False)

pprint(diff(sin(x)*exp(x), x))

pprint(integrate(exp(x)*sin(x) + exp(x)*cos(x), x))

pprint(integrate(sin(x**2), (x, -oo, oo)))

pprint(limit(sin(x)/x, x, 0))

y = Function('y')
pprint(dsolve(Eq(y(t).diff(t, t) - y(t), exp(t)), y(t)))

pprint(Matrix([[1, 2], [2, 2]]).eigenvals())

pprint(besselj(nu, z).rewrite(jn))

print(latex(Integral(cos(x)**2, (x, 0, pi))))


import numpy as npm

A = npm.array([[1,2],[1,4]])

B = npm.array([[3],[5]])

AB = npm.concatenate((A,B),axis=1)
AB0 = npm.copy(AB)

tamano = npm.shape(AB)
n = tamano[0]
m = tamano[1]

for i in range(0,n-1,1):
    columna = abs(AB[i:,i])
    dondemax = npm.argmax(columna)

    if (dondemax !=0):
        temporal = npm.copy(AB[i,:])
        AB[i,:] = AB[dondemax+i,:]
        AB[dondemax+i,:] = temporal

AB1 = npm.copy(AB)

for i in range(0,n-1,1):
    pivote = AB[i,i]
    adelante = i + 1
    for k in range(adelante,n,1):
        factor = AB[k,i]/pivote
    AB[k,:] = AB[k,:] - AB[i,:]*factor

AB2 = npm.copy(AB)

ultfila = n-1
ultcolumna = m-1
for i in range(ultfila,0-1,-1):
    pivote = AB[i,i]
    atras = i-1
    for k in range(atras,0-1,-1):
        factor = AB[k,i]/pivote
        AB[k,:] = AB[k,:] - AB[i,:]*factor
    AB[i,:] = AB[i,:]/AB[i,i]

X = npm.copy(AB[:,ultcolumna])
X = npm.transpose([X])

print('Matriz aumentada:')
print(AB0)
print('Pivoteo parcial por filas')
print(AB1)
print('eliminacion hacia adelante')
print(AB2)
print('eliminación hacia atrás')
print(AB)
print('solución de X: ')
print(X)


import numpy as np
import matplotlib.pyplot as plt
import cv2
from os import listdir
from matplotlib import image
from matplotlib.pyplot import imread

train_path = 'Yale-FaceA/trainingset/'
test_path = 'Yale-FaceA/testset/'

###########################Part 1: Loading and creating data matrix ##############################
##################################################################################################

loaded_images = []
for filename in listdir(train_path):
    im = cv2.imread(train_path+filename, 0) #grayscale images
    im = im.flatten('F')                    #flatten the images to a 45045x1 vector for each img
    loaded_images.append(im)

all_data = np.stack(loaded_images).T        #stack the loaded data to form a matrix

#############################Part 2: Preprocess the data #########################################
##################################################################################################

def preprocess(A_pp):

    means = np.zeros((A_pp.shape[0],1))

    for i in range(A_pp.shape[0]):
        means[i] = np.mean(A_pp[i,:], axis=0)  # calculation of the mean of the data

    means_stack = np.tile(means, (1,A_pp.shape[1]))

    Q = A_pp - means_stack                     #centering the data
    A = None
    Q_norms = np.zeros((A_pp.shape[0],1))

    for i in range(A_pp.shape[0]):            #In this code block, the data is normalized by
        x = np.abs(np.amax(Q[i,:]))           # the infinity norm to stabilize the results better
        if x==0:
            Q_norms[i] = 1
        else:
            Q_norms[i] = x

    A = Q/Q_norms                           #data divided by L_infinity norm
    A_means = means
    return A, Q_norms, A_means              #centered data, infinity norm and means returned

A, Q_norms, A_means = preprocess(all_data)

#########################Part 3: Calculate eigenvectors and coefficients #########################
##################################################################################################

#Display the mean face
R = A_means.reshape((231,195),  order='F')
R = np.real(R)
plt.imshow(R, cmap=plt.cm.gray)
plt.show()

def eigen_faces(A_pp):
    A, Q_norms, A_means = preprocess(A_pp) #returns centered data and mu

    w,v = np.linalg.eigh(A.T@A)             #eigendecomposition of A.T@A
    F = A@v                                 #calculate the eigenvectors of the original data cov matrix (A@A.T)

    F = F/np.linalg.norm(F, axis = 0)       #normalize the eigenvectors, these are our eigenfaces
    C = A.T@F                               #coefficients of each image in terms of eigenvector basis

    D = np.diag(w, k=0)                     #eigenvalues

    return C, F, D, Q_norms, A_means

coeff, F, D, Q_norms, A_means = eigen_faces(all_data)

#The above code of preprocess and eigenface is taken from an
#Assignment I did in Intro to ML course and it's my own code.

print(f"Eigenvalues: {D}")
#so the biggest eigenvector is in 134th index

variance_retained = D[:, -15:].sum()/D.sum()
print(f"Variance retained with top 15 eigenvectors: {variance_retained}")

#############################Part 4: Display Eigenfaces #########################################
#################################################################################################

#Here I display only 4 because I reused this code to display and snip all 15 eigenfaces for my report

fig, ax = plt.subplots(2, 2)
plt.rcParams["figure.figsize"] = [15,9]
fig.suptitle('Display top 15 eigenfaces')

ax[0,0].imshow(np.real(F[:,130].reshape((231,195),  order='F')), cmap=plt.cm.gray)
ax[0,0].set(title='1st Eigenface')

ax[0,1].imshow(np.real(F[:,129].reshape((231,195),  order='F')), cmap=plt.cm.gray)
ax[0,1].set(title='2nd Eigenface')

ax[1,0].imshow(np.real(F[:,132].reshape((231,195),  order='F')), cmap=plt.cm.gray)
ax[1,0].set(title='3rd Eigenface')

ax[1,1].imshow(np.real(F[:,131].reshape((231,195),  order='F')), cmap=plt.cm.gray)
ax[1,1].set(title='4th Eigenface')
plt.show()

#####################Part 5: Build an Img recognition system and Test ############################
##################################################################################################

#Read in an image from the test dataset

Img = cv2.imread(test_path+'subject.new.01.jpg', 0)
Img = Img.flatten('F')
Img = Img[:, np.newaxis]

#center the image
Img_centered = Img - A_means

#Take first 15 principal eigenvectors
F_k = F[:, -15:]
F_k.shape

#Get the test image projection on the eigenvectors to get the 15 coefficients
Img_projection = Img_centered.T@F_k

#Get coefficients of 15 eigenvectors of all images in the dataset
coeff = C = A.T@F_k

#We now calculate nearest neighbours based on the vectors of coefficients
diff = coeff - Img_projection
norms = np.linalg.norm(diff, axis=1)

#Take the lowest 3 norms (best 3 matches) as they are least distant
indices = norms.argsort()[:3]

###########################Part 6: Display Results Of Recognition ##################################
####################################################################################################

fig, ax = plt.subplots(2, 2)
plt.rcParams["figure.figsize"] = [15,9]
fig.suptitle('Display test images and closest images')

ax[0,0].imshow(Img.reshape((231,195),  order='F'), cmap=plt.cm.gray)
ax[0,0].set(title='Original Test Img')

ax[0,1].imshow(all_data[:, indices[0]].reshape((231,195),  order='F'), cmap=plt.cm.gray)
ax[0,1].set(title='1st Match')

ax[1,0].imshow(all_data[:, indices[1]].reshape((231,195),  order='F'), cmap=plt.cm.gray)
ax[1,0].set(title='2nd Match')

ax[1,1].imshow(all_data[:, indices[2]].reshape((231,195),  order='F'), cmap=plt.cm.gray)
ax[1,1].set(title='3rd Match')
plt.show()

###########################Part 7: Processing of my images #########################################
######################### Commented because no need to run now #######################################

# for i in range(0,10):
#     fname = f"subject.tanya.{i}.jpg"
#     img = cv2.imread(fname, 1)
#     img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
#     img = cv2.resize(img, (195, 231), interpolation = cv2.INTER_AREA)
#     fname = f"subject.new.{i}.jpg"
#     cv2.imwrite(fname, img)


import numpy as np
from numpy import pi
from scipy.special import gamma
from scipy.stats import norm, multivariate_normal

def volume_ball(d):
    """Returns volume of unit ball (wrt l2 norm) in d-dim euclidian space."""
    return (pi ** (d / 2.)) / gamma(1 + d/2.)

def volume_cube(d):
    """Returns volume of unit ball (wrt sup norm) in d-dim euclidian space."""
    return 2 ** d

dimensions = [1, 2, 3, 5, 7, 10, 15]

vol_ball = [volume_ball(d) for d in dimensions]
vol_cube = [volume_cube(d) for d in dimensions]
vol_ratio = [np.round(x / y, decimals=5) for x, y in zip(vol_ball, vol_cube)]

def probability_cube(d):
    """Returns probability of a d-dim. gaussian falling in a unit ball wrt sup norm."""
    return (norm.cdf(1) - norm.cdf(-1))**d

def upperbound_probability_ball(d):
    """Returns simple upper bound on the probability of a d-dim. gaussian falling in a unit ball wrt l2 norm."""
    return multivariate_normal(np.zeros(d), np.eye(d)).pdf(np.zeros(d)) * volume_ball(d)

upper_prob_ball = [upperbound_probability_ball(d) for d in dimensions]
prob_cube = [probability_cube(d) for d in dimensions]
prob_ratio = [np.round(x/y, decimals=5) for x, y in zip(upper_prob_ball, prob_cube)]

print(f"dimensions: {dimensions}")
print(f"vol-ratio: {vol_ratio}")
print(f"prob-ratio: {prob_ratio}")


from __future__ import division
import numpy as np
import math
import sys

"""This python file contains sample data operations such mse,
correlation, covariance, variance, std etc.
"""
# Calculate label array 'y' entropy
def calculate_entropy(y):
    log2 = lambda x: math.log(x) / math.log(2)
    unique_labels = np.unique(y)
    entropy = 0
    for label in unique_labels:
        count = len(y[y == label])
        p = count / len(y)
        entropy += -p * log2(p)
    return entropy

# Returns the mean squared error (mse) between y_true and y_pred predictions
def mean_squared_error(y_true, y_pred):
    mse = np.mean(np.power(y_true - y_pred, 2))
    return mse

# Return the variance of X data features
def calculate_variance(X):
    mean = np.ones(np.shape(X)) * X.mean(0)
    n_samples = np.shape(X)[0]
    variance = (1 / n_samples) * np.diag((X - mean).T.dot(X - mean))

    return variance

# Calculate the standard deviations (std) of X dataset features
def calculate_std_dev(X):
    std_dev = np.sqrt(calculate_variance(X))
    return std_dev

# Calculates the l2 distance between two vectors
def euclidean_distance(x1, x2):
    distance = 0
    # Squared distance between each coordinate
    for i in range(len(x1)):
        distance += pow((x1[i] - x2[i]), 2)
    return math.sqrt(distance)

# Compare y_true to y_pred and get the accuracy by return
def accuracy_score(y_true, y_pred):
    accuracy = np.sum(y_true == y_pred, axis=0) / len(y_true)
    return accuracy

# Calculate X dataset covariance matrix
def calculate_covariance_matrix(X, Y=None):
    if Y is None:
        Y = X
    n_samples = np.shape(X)[0]
    covariance_matrix = (1 / (n_samples-1)) * (X - X.mean(axis=0)).T.dot(Y - Y.mean(axis=0))

    return np.array(covariance_matrix, dtype=float)

# Calculate X dataset correlation matrix
def calculate_correlation_matrix(X, Y=None):
    if Y is None:
        Y = X
    n_samples = np.shape(X)[0]
    covariance = (1 / n_samples) * (X - X.mean(0)).T.dot(Y - Y.mean(0))
    std_dev_X = np.expand_dims(calculate_std_dev(X), 1)
    std_dev_y = np.expand_dims(calculate_std_dev(Y), 1)
    correlation_matrix = np.divide(covariance, std_dev_X.dot(std_dev_y.T))

    return np.array(correlation_matrix, dtype=float)


from numpy import *
import math
import matplotlib.pyplot as plt
import numpy as np

x = np.linspace(-4,4,10000)
fig, ax = plt.subplots()

# ax.plot(x, np.maximum(np.zeros(len(x)),x),label="ReLU",linewidth=3)

# lrelu=x.copy()
# for i in range(len(x)):
# 	if lrelu[i]<0:
# 		lrelu[i]=lrelu[i]*0.2
# ax.plot(x, lrelu,label="LReLU",linewidth=1)

# ax.plot(x, (1-np.exp(-2*x))/(1+np.exp(-2*x)),label="Tanh")

# new=x.copy()
# for i in range(len(x)):
# 	if new[i]<0:
# 		new[i]=new[i]*exp(new[i])
# ax.plot(x, new,label="New activation function",linewidth=3)

# ax.plot(x, x/(1+exp(-x)),label="Swish")

new=x.copy()
for i in range(len(x)):
	if new[i]<0:
		new[i]=(1+new[i])*exp(new[i])
	else:
		new[i]=1
ax.plot(x, new,label="New activation function",linewidth=1)

def sig(x):
	return 1/(1+exp(-x))
ax.plot(x, sig(x)+x*sig(x)*(1-sig(x)),label="Swish")

# elu=x.copy()
# for i in range(len(x)):
# 	if elu[i]<0:
# 		elu[i]=np.exp(elu[i])-1
# ax.plot(x, elu,label="ELU")

ax.axhline(y=0, color='k',alpha=0.5)
ax.axvline(x=0, color='k',alpha=0.5)

legend = ax.legend(loc='upper left',fontsize=14)
plt.xlabel("x",fontsize=15)
plt.ylabel("y",fontsize=15)

plt.ylim(ymax=1.5)
plt.ylim(ymin=-0.5)
plt.show()


#!/usr/bin/python2.7
# -*- coding:utf-8 -*-

# Author: NetworkRanger
# Date: 2018/12/29 下午8:28

from numpy import *

# PCA算法
def loadDataSet(fileName, delim='\t'):
    fr = open(fileName)
    stringArr = [line.strip().split(delim) for line in fr.readlines()]
    datArr = [map(float, line) for line in stringArr]
    return mat(datArr)

def pca(dataMat, topNfeat=9999999):
    meanVals = mean(dataMat, axis=0)
    meanRemoved = dataMat - meanVals
    # 去平均值
    covMat = cov(meanRemoved, rowvar=0)
    eigVals, eigVects = linalg.eig(mat(covMat))
    eigValInd = argsort(eigVals)
    # 从小到大对N个值排序
    eigValInd = eigValInd[:-(topNfeat+1):-1]
    redEigVects = eigVects[:,eigValInd]
    lowDDataMat = meanRemoved * redEigVects
    # 将数据转换到新空间
    reconMat = (lowDDataMat * redEigVects.T) + meanVals
    return lowDDataMat, reconMat

# 将NaN替换成平均值的函数
def replaceNanWithMean():
    datMat = loadDataSet('secom.data', ' ')
    # 计算所有非NaN的平均值
    numFeat = shape(datMat)[1]
    for i in range(numFeat):
        meanVal = mean(datMat[nonzero(~isnan(datMat[:,i].A))[0],i])
        # 将所有NaN置为平均值
        datMat[nonzero(isnan(datMat[:,i].A))[0],i] = meanVal
    return datMat


import sympy as sy
from sympy import *

m2_1 = sy.Matrix([ [ 1,  0,  0, -1, 0],
                 [-1,  0, -2,  0, 0],
                 [ 0, -1,  0,  0, -3],
                 [ 0,  0,  0, -2, 1],
                 [ 0, -3,  1,  0, 0] ])

print("determinant of M: ", m2_1.det())

a = sy.symbols('a')
m2_2 = sy.Matrix([ [ a,  5,  9],
                 [ a, -1,  4],
                 [ 9, -7,  a] ])

print("det A = ",m2_2.det())
print("a = ", solve(-6*a**2 - 35*a + 261, a))

m2_3 = sy.Matrix([ [-3,  5, -8],
                 [ 8, -5,  1],
                 [ 5,  1,  5] ])
cf = m2_3.cofactor_matrix()
print("cofactor matrix: ",cf)

def minor_matrix(m):
    r , c = m.shape
    new_m = []

    for i in range(r):
        for j in range(c):
            new_m.append(m.minor(i,j))
    return Matrix(r, c, new_m)

print("Minors: ",minor_matrix(m2_3))

m2_4 = sy.Matrix([ [-4,  9, -5],
                   [-9,  7,  5],
                   [ 9,  7, -6] ])

print(m2_4.det())
print(m2_4.inv())

m2_5 = sy.Matrix([ [-5, -5,   4],
                   [ 4,  5,  -5],
                   [ 1,  1,  -1] ])

print(m2_5.inv())

m2_5b = sy.Matrix([ [-2],
                    [ 4],
                    [ 5] ])

print("x is:", m2_5.inv() * m2_5b)

m2_6_A = sy.Matrix([ [ 4, -12, -37],
                     [-2,   7,  20],
                     [ 1,  -3,  -9] ])

m2_6_B = sy.Matrix([ [-4,  12,  25],
                     [ 2,  -7, -15],
                     [ 1,  -3,  -6] ])

m2_6_C = sy.Matrix([ [ 4, -12,  -37,  0,  0,   0],
                     [-2,   7,   20,  0,  0,   0],
                     [ 1,  -3,   -9,  0,  0,   0],
                     [ 0,   0,    0, -4, 12,  25],
                     [ 0,   0,    0,  2, -7, -15],
                     [ 0,   0,    0,  1, -3,  -6] ])

print("m2_6_A inv:", m2_6_A.inv())
print("m2_6_B inv:", m2_6_B.inv())
print("m2_6_C inv:", m2_6_C.inv())

m2_8_A = sy.Matrix([ [ 2, -4,  1],
                     [-1,  1, -1],
                     [ 1, -2,  0] ])

m2_8_B = sy.Matrix([ [3],
                     [0],
                     [3] ])

print("2_8 X is:", m2_8_A.inv() * m2_8_B)

m2_9_A = sy.Matrix([ [ 1, -2, -5],
                     [-4, -2,  4],
                     [ 3,  2, -5] ])

print("A+A.T: ",m2_9_A + m2_9_A.transpose())
print("A-A.T: ",m2_9_A - m2_9_A.transpose())

m2_10_A = sy.Matrix([ [ -4, -5, -8, -9] ])

m2_10_B = sy.Matrix([ [ 5],
                      [-5],
                      [-2],
                      [ 9] ])

print("B*A= ", m2_10_B*m2_10_A)

m2_11_A = sy.Matrix([ [ 6,  4, -3],
                      [-6, -9, -7],
                      [ 2, -6, -4] ])

m2_11_B = sy.Matrix([ [-3, -1, -1],
                      [ 6,  5, -1],
                      [ 4, -3,  0] ])

print("6A = ", 6*m2_11_A)
print("A-7B = ", m2_11_A-(7*m2_11_B))
print("-3A-3B = ", (-3*m2_11_A)-(3*m2_11_B))

m2_12_A = sy.Matrix([ [ 2,  4,  6],
                      [ 2,  4, -2] ])

m2_12_B = sy.Matrix([ [-3,  4,  7],
                      [-4,  1,  3] ])

print(" (-2B - A ) / 3 = ",((-2*m2_12_B)-m2_12_A)/3)

m2_16_A = sy.Matrix([ [-1,  3,  5],
                      [ 1,  8, -6],
                      [-6,  2, -6] ])
print("det(A): ", m2_16_A.det())

m2_18_B = sy.Matrix([ [ 1,  0,  1],
                      [ 2,  2, -1],
                      [ 2, -2, -1] ])
print("det(B^5): ", (m2_18_B**5).det())

a, b, c, d, e, f = sy.symbols('a, b, c, d, e, f')
m2_21_A = sy.Matrix([ [ a,  1,  d],
                      [ b,  1,  e],
                      [ c,  1,  f] ])

m2_21_B = sy.Matrix([ [ a,  1,  d],
                      [ b,  2,  e],
                      [ c,  3,  f] ])
print("det m2_21_A:", m2_21_A.det())
print("det m2_21_B:", m2_21_B.det())

x1, x2, z1, z2 = sy.symbols('x1, x2, z1, z2')
m2_25_A = sy.Matrix([ [ x1, x2],
                      [ z1, z2] ])
m2_25_B = sy.Matrix([ [ 9*x1 + 5*z1, 9*x2 + 5*z2],
                      [ 8*x1 + 4*z1, 8*x2 + 4*z2] ])
print("det(A): ", m2_25_A.det())
print("det(B): ", m2_25_B.det())

m2_27_A = sy.Matrix([ [-3,  7,  0,  1],
                      [ 8, -7,  0,  0],
                      [ 0, -8,  0,  0],
                      [ 3, -8,  6,  4] ])
print("det(A): ", m2_27_A.det())

m2_28_A = sy.Matrix([ [ 0, -4, -7,  0, 0],
                      [ 9,  4, -1,  0, 0],
                      [ 0, -3,  0,  0, 0],
                      [-3, -5, -1, -3, -2],
                      [-5, -7, -1,  0, -2] ])
print("det(A): ", m2_28_A.det())

m2_29_A = sy.Matrix([ [-4, 20,   3],
                      [ 3,  2,  -1],
                      [14,  2, -13] ])
print("det(A): ", m2_29_A.det())


import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
import sklearn.linear_model as skl
#from sklearn.linear_model import LinearRegression, Ridge, Lasso
def R2(y_data, y_model):
    return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
def MSE(y_data,y_model):
    n = np.size(y_model)
    return np.sum((y_data-y_model)**2)/n

# A seed just to ensure that the random numbers are the same for every run.
# Useful for eventual debugging.
np.random.seed(3155)

x = np.random.rand(100)
y = 2.0+5*x*x+0.1*np.random.randn(100)

# number of features p (here degree of polynomial
p = 3
#  The design matrix now as function of a given polynomial
X = np.zeros((len(x),p))
X[:,0] = 1.0
X[:,1] = x
X[:,2] = x*x
# We split the data in test and training data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
scaler = StandardScaler()
scaler.fit(X_train)
X_train_scaled = scaler.transform(X_train)
X_test_scaled = scaler.transform(X_test)

# matrix inversion to find beta
OLSbeta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
print(OLSbeta)
# and then make the prediction
ytildeOLS = X_train @ OLSbeta
print("Training R2 for OLS")
print(R2(y_train,ytildeOLS))
print("Training MSE for OLS")
print(MSE(y_train,ytildeOLS))
ypredictOLS = X_test @ OLSbeta
print("Test R2 for OLS")
print(R2(y_test,ypredictOLS))
print("Test MSE OLS")
print(MSE(y_test,ypredictOLS))

# Repeat now for Ridge regression and various values of the regularization parameter
I = np.eye(p,p)
# Decide which values of lambda to use
nlambdas = 100
MSEPredictLasso = np.zeros(nlambdas)
MSEPredictRidge = np.zeros(nlambdas)
lambdas = np.logspace(-4, 0, nlambdas)
for i in range(nlambdas):
    lmb = lambdas[i]
    # add ridge
    clf_ridge = skl.Ridge(alpha=lmb).fit(X_train, y_train)
    clf_lasso = skl.Lasso(alpha=lmb).fit(X_train, y_train)
    yridge = clf_ridge.predict(X_test)
    ylasso = clf_lasso.predict(X_test)
    MSEPredictLasso[i] = MSE(y_test,ylasso)
    MSEPredictRidge[i] = MSE(y_test,yridge)
#then plot the results
plt.figure()
plt.plot(np.log10(lambdas), MSEPredictRidge, 'r--', label = 'MSE Ridge Test')
plt.plot(np.log10(lambdas), MSEPredictLasso, 'g--', label = 'MSE Lasso Test')
plt.xlabel('log10(lambda)')
plt.ylabel('MSE')
plt.legend()
plt.show()


import numpy as np

from numpy import array, diag, divide, exp, matrix, power, zeros
from numpy.linalg import inv, norm

class logistic_regression_IRLS(object):
    """
    Maximum likelihood approach for logistic regression.
    """
    def __init__(self):
        self.w = None

    def fit(self, X, y, w0=None, erf=None, maxit=None, display=None):
        self.__newton_raphson(X, y, w0, erf, maxit, display)

    def predict(self, X):
        y = X * self.w
        return array((y>0).astype(int).T)[0]

    def __newton_raphson(self, X, y, w0, erf, maxit, display):
        """
        Args:
            X:   nxd matrix.
            y:   nx1 column vector.
            w0:  dx1 column vector.
            erf: error function with two parameters.
                 erf = lambda old,new: ...
        """
        # Pre-process
        X   = matrix(X)
        n,d = X.shape
        y   = matrix(y).reshape(n, 1)
        if w0==None:
            w0 = zeros((d, 1))
        if erf==None:
            erf = lambda old,new: norm(old-new,2)<1e-6
        if maxit==None:
            maxit = 64
        if display==None:
            display = False
        w1 = zeros((d, 1))
        ex = zeros((n, 1))
        # Iterations
        for i in range(maxit):
            ex = exp(X*w0).T
            D  = diag(array(ex/power(1+ex,2))[0])
            w1 = w0 + inv(X.T*D*X)*X.T*(y-(ex/(1+ex)).T)
            if erf(w0, w1):
                if display:
                    print("Convergence @ the %d iteration(s)."%i)
                break
            w0 = w1
        self.w = w1

"""
from sklearn.linear_model import LogisticRegression as Regression

X = np.random.rand(20, 2)-0.5
y = (np.sum(X,1)>0).astype(int)

model_1 = logistic_regression_IRLS()
model_1.fit(X, y)
y_1 = model_1.predict(X)

model_2 = Regression(solver="lbfgs")
model_2.fit(X, y)
y_2 = model_2.predict(X)

dy_1 = norm(y_1-y, 2)
dy_2 = norm(y_2-y, 2)

print(dy_1, dy_2)
"""


import numpy as np
import matplotlib.pyplot as plt
m=10
d_teta1=0.01
teta_1_min=-1
teta_1_max=1

# here we actually have a graph with h_teta=teta1*x
X = np.arange(m)
np.random.seed(42) #use to get the same results every run
rnd = np.random.rand(m)
Y = np.arange(m)*0.5 + rnd # actually we have h = teta1*x where teta1 is actually 0.5

#estimated y -> H
Teta1 = np.arange(teta_1_min,teta_1_max,d_teta1)
J=[]

# mathematically J is (1/2*m)* eT * e
# can i do it without a loop ???
for teta1 in Teta1:
    H = X * teta1
    E = Y - H
    cost = np.dot(E,E)/(2*m)
    J.append(cost)

# compute teta1 for min(J)
min_j_index = np.argmin(J)
teta1 = Teta1[min_j_index]
print("min(J) : {} , tata1 @ min(J) : {}".format(J[min_j_index] , teta1))

h_vec = X * Teta1[min_j_index] # computed linear estimation

# plots
plt.plot(X, Y,'o')
plt.title('data set')
plt.xlabel("X")
plt.ylabel("Y")
plt.show()

plt.plot(Teta1,J)
plt.title('loss function J : sum of (Y-H)^2')
plt.xlabel("teta1")
plt.ylabel("J")
plt.show()

plt.plot(X, h_vec,X, Y,'o')
plt.title('linear estimation H = Teta1 * X.    teta1 is chosen as min(J)')
plt.xlabel("X")
plt.ylabel("H , Y")
plt.show()


import numpy as np
from common.functions import softmax

def cross_entropy_error(y, t):
    """
    y : ソフトマックス関数の出力
        y.shape=(k,)またはy.shape=(N,k)
    t : 正解ラベル(ワンホット表現)
        t.shape=(k,)またはt.shape=(N,k)
    """
    if y.ndim==1:
        t = t.reshape(1,-1)
        y = y.reshape(1,-1)

    batch_size = y.shape[0]
    delta = 1e-7
    return -np.sum(t*np.log(y + delta))/ batch_size

class SoftmaxWithLoss:
    def __init__(self):
        self.loss = None
        self.y = None # ソフトマックス関数の出力を格納するインスタンス変数
        self.t = None # 正解ラベルを格納するインスタンス変数

    def forward(self, x, t):
        self.t = t
        self.y = softmax(x)
        self.loss = cross_entropy_error(self.y, self.t)
        return self.loss

    def backward(self):
        batch_size = self.t.shape[0]
        dx = (self.y-self.t)/batch_size
        return dx

if __name__=="__main__":
    x = np.arange(2*3).reshape(2, 3)
    print("x\n", x)

    t = np.arange(2).reshape(-1, 1)
    print("t\n", t)

    sl = SoftmaxWithLoss()

    loss = sl.forward(x,t)
    print("loss\n", loss)

    dx = sl.backward()
    print("dx\n", dx)


# coding: utf-8

# In[1]:

# Importing numpy and matplotlib
from pylab import *
from scipy.integrate import quad

# In[2]:

rcParams['figure.figsize'] = 12,9
rcParams['font.size'] = 18
rcParams['text.usetex'] = True

# In[3]:

def periodic(a, b):
    interval = b - a
    return lambda f: lambda x: f((x - a) % interval + a)

# In[7]:

def coscos(x):
    return cos(cos(x))
def per_e(x):
    return exp(np.remainder(x,2*pi))

# In[8]:

x = linspace(-2*pi,4*pi,400)
fig1 = figure()
ax = fig1.add_subplot(1,1,1)
ax.set_yscale('log')
grid(True)
plot(x,exp(x))
plot(x,per_e(x))
title("$e^x$ and its periodically extended version")
ylabel(r"$y$ (log)", fontsize = 26)
xlabel(r"$x$ (linear)")
legend([r"$e^x$",r"periodic version of $e^x$"], loc=0)
show()

# In[27]:

fig2 = figure()
grid(True)
plot(x,cos(cos(x)))
plot(x,coscos(x))
title("$\cos(\cos(x))$ and its periodically extended version")
ylabel(r"$y$ (linear)", fontsize = 26)
xlabel(r"$x$ (linear)")
legend([r"$\cos(\cos(x))$",r"periodic version of $\cos(\cos(x))$"], loc=1)
ylim(0.3,1.4)
show()

# In[11]:

def quad_fourier(f,n):
    """
    Find the n even and odd fourier coefficients of f, and the DC
    value, using quad integration.
    Assumes a period from 0 to 2pi.
    """

    # functions to integrate
    u = lambda x,k : f(x)*cos(k*x)
    v = lambda x,k : f(x)*sin(k*x)

    # DC coefficient
    a0 = 1/(2*pi)*quad(f,0,2*pi)[0]

    # find coefficients by integrating
    ret = [a0]
    for k in arange(n)+1:
        ak = 1/pi*quad(u,0,2*pi,args=(k))[0]
        bk = 1/pi*quad(v,0,2*pi,args=(k))[0]
        ret.append(ak)
        ret.append(bk)

    return array(ret)

# In[12]:

ef = quad_fourier(per_e,25)
cf = quad_fourier(coscos,25)

# In[13]:

def plotCoeffs(coeffs, name="Coefficients"):
    """
    Helper function to scatter the given coefficients on a
    semilog and loglog scale
    """

    fig0 = figure()
    ax = fig0.add_subplot(1,1,1)
    ax.set_yscale('log')
    grid(True)
    title("Fourier coefficients of {} (semilog)".format(name))
    ylabel(r"Magnitude of coefficients", fontsize = 26)
    xlabel(r"Index")
    scatter(arange(len(coeffs))+1,abs(coeffs),color='red',s=50)
    show()

    fig1 = figure()
    ax = fig1.add_subplot(1,1,1)
    ax.set_yscale('log')
    ax.set_xscale('log')
    grid(True)
    title("Fourier coefficients of {} (log-log)".format(name))
    ylabel(r"Magnitude of coefficients", fontsize = 26)
    xlabel(r"Index")
    scatter(arange(len(coeffs))+1,abs(coeffs),color='red',s=50)

    return fig0,fig1

# In[14]:

fig3,fig4 = plotCoeffs(ef,r"$e^x$")

# In[15]:

fig5,fig6 = plotCoeffs(cf,"$\cos(\cos(x))$")

# In[16]:

def lstsq_fourier(f,n,steps=400):
    """
    Find the n even and odd fourier coefficients of f, and the DC
    value, using least squares estimation. Defaults to 400 steps.
    Assumes a period from 0 to 2pi.

    Also returns the least squares matrix
    """

    x=linspace(0,2*pi,steps+1)
    x=x[:-1] # drop last term to have a proper periodic integral
    b=f(x) # f has been written to take a vector
    A=zeros((steps,2*n+1)) # allocate space for A
    A[:,0]=1 # col 1 is all ones
    for k in range(1,n+1):
        A[:,2*k-1]=cos(k*x) # cos(kx) column
        A[:,2*k]=sin(k*x) # sin(kx) column

    return lstsq(A,b)[0], A

# In[17]:

s = 400 # number of steps for least square estimation
elc, Aexp = lstsq_fourier(per_e,25,s)
ax = fig3.axes[0]
ax.scatter(arange(len(elc))+1,abs(elc),s=50,color='green')
ax.legend(["Using quad integration", "Using least squares"], loc=3)
fig3

# In[18]:

ax = fig4.axes[0]
ax.scatter(arange(len(elc))+1,abs(elc),color='green',s=50)
ax.legend(["Using quad integration", "Using least squares"],loc=3)
fig4

# In[19]:

clc, Acos = lstsq_fourier(coscos,25,s)
ax = fig5.axes[0]
ax.scatter(arange(len(clc))+1,abs(clc),color='green',s=50)
ax.legend(["Using quad integration", "Using least squares"],loc=1)
fig5

# In[20]:

ax = fig6.axes[0]
ax.scatter(arange(len(clc))+1,abs(clc),color='green',s=50)
ax.legend(["Using quad integration", "Using least squares"],loc=6)
fig6

# In[21]:

print(max(abs(ef-elc)))

# In[22]:

print(max(abs(cf-clc)))

# In[23]:

I = linspace(0,2*pi,s)

# In[24]:

ax = fig1.axes[0]
ax.scatter(I,dot(Aexp,ef),color = 'green')
ax.scatter(I,dot(Aexp,elc),color = 'black',marker='+')
ax.set_ylim(1e-3,1e6)
ax.legend([r"$e^x$",r"periodic version of $e^x$",
           "quad estimated","Least squares estimated"], loc = 4)
fig1

# In[25]:

figure()
grid(True)
xlim(-2,7)
plot(x,per_e(x))
plot(I,dot(Aexp,ef),linewidth=3)
title("$e^x$ and its fourier approximation")
ylabel(r"$y$ (linear)", fontsize = 26)
xlabel(r"$x$ (linear)")
legend([r"Periodic version of $e^x$",r"Fourier estimate of $e^x$"], loc=0)
show()

# In[28]:

ax = fig2.axes[0]
ax.scatter(I,dot(Acos,cf),color = 'green')
ax.scatter(I,dot(Acos,clc),color = 'black')
ax.legend([r"$\cos(\cos(x))$",r"periodic version of $\cos(\cos(x))$"
           ,"quad estimated","Least squares estimated"], loc = 0)
fig2


import numpy as np

#load the library and check its version
import numpy as np
print(np.__version__)

## The data manipulation capabilities of pandas are built on top of the numpy library. In a way, numpy is a dependency of the pandas library.
import pandas as pd

#1) Creating Arrays in numpy
get_zero = np.zeros(10, dtype='int')
print(get_zero)

#creating a 2 row x 4 column matrix
data = np.ones((2,4),dtype=float)
print(data)

#creating a matrix with a predefined value
data = np.full((2,4),4.2)
print(data)

##create an array with a set sequence
data = np.arange(0,20,2)
print(data)

#create an array of even space between the given range of values
data = np.linspace(0,10,4)
print(data)

#create a 3x3 array with mean 0 and standard deviation 1 in a given dimension
data = np.random.normal(0,1,(3,3))
print(data)

#create an identity matrix
data = np.eye(4)
print(data)

#set a random seed
#With the seed reset (every time), the same set of numbers will appear every time
np.random.seed(0)

# 1 d
x1 =np.random.randint(10,size=6)
# 2 D
x2 =np.random.randint(10,size=(3,5))
# 3 D
print("3d")
x3 = np.random.randint(10,size=(3,4,5))
print(x3)
print(x3.size)
print(x3.shape)
print(x3.ndim)

## 2 Array Indexing

x1 = np.array([4, 3, 4, 4, 8, 4])
print(x1[2])
x1[0]

#get the last value
x1[-1]

#in a multidimensional array, we need to specify row and column index
x2= np.array([[3, 7, 5, 5],
      [0, 1, 5, 9],
      [3, 0, 5, 0]])

#1st row and 2nd column value
print(x2[2,2])

#replace value at 0,0 index
x2[0,0] = 12
x2

## 3) Array Slicing
x = np.arange(10)
#from start to 4th position
print(x[:5])
#from 4th position to end
x[4:]

#from 4th to 6th position
x[4:7]

#return elements at even place
print(x[ : : 2])

#return elements from first position step by two
x[1::2]

#reverse the array
x[::-1]

## 4 Array Concatenation

x = np.array([1, 2, 3])
y = np.array([3, 2, 1])
z= np.concatenate([x, y])
print(z)

# for combine a 2D array with 1D  use np.vstack or np.hstack
x = np.array([3,4,5])
grid = np.array([[1,2,3],[17,18,19]])
np.vstack([x,grid])

z = np.array([[9],[9]])
np.hstack([grid,z])

### arrange , reshape it
reshap= np.arange(16).reshape((4,4))
print(reshap)

## function
def greater(a,b):
    print("inside function")
    print(a)
    print(b)
    if a>b :
        return a
    else:
        return b
t = greater(np.array(3),np.array(4))
print("d")
print(t)


#!/usr/bin/env python

"""Vandermonde matrix example

Demonstrates matrix computations using the Vandermonde matrix.
  * http://en.wikipedia.org/wiki/Vandermonde_matrix
"""

from sympy import Matrix, pprint, Rational, sqrt, symbols, Symbol, zeros

def symbol_gen(sym_str):
    """Symbol generator

    Generates sym_str_n where n is the number of times the generator
    has been called.
    """
    n = 0
    while True:
        yield Symbol("%s_%d" % (sym_str, n))
        n += 1

def comb_w_rep(n, k):
    """Combinations with repetition

    Returns the list of k combinations with repetition from n objects.
    """
    if k == 0:
        return [[]]
    combs = [[i] for i in range(n)]
    for i in range(k - 1):
        curr = []
        for p in combs:
            for m in range(p[-1], n):
                curr.append(p + [m])
        combs = curr
    return combs

def vandermonde(order, dim=1, syms='a b c d'):
    """Comptutes a Vandermonde matrix of given order and dimension.

    Define syms to give beginning strings for temporary variables.

    Returns the Matrix, the temporary variables, and the terms for the polynomials
    """
    syms = syms.split()
    if len(syms) < dim:
        new_syms = []
        for i in range(dim - len(syms)):
            new_syms.append(syms[i%len(syms)] + str(i/len(syms)))
        syms.extend(new_syms)
    terms = []
    for i in range(order + 1):
        terms.extend(comb_w_rep(dim, i))
    rank = len(terms)
    V = zeros(rank)
    generators = [symbol_gen(syms[i]) for i in range(dim)]
    all_syms = []
    for i in range(rank):
        row_syms = [g.next() for g in generators]
        all_syms.append(row_syms)
        for j,term in enumerate(terms):
            v_entry = 1
            for k in term:
                v_entry *= row_syms[k]
            V[i*rank + j] = v_entry
    return V, all_syms, terms

def gen_poly(points, order, syms):
    """Generates a polynomial using a Vandermonde system"""
    num_pts = len(points)
    if num_pts == 0:
        raise ValueError("Must provide points")
    dim = len(points[0]) - 1
    if dim > len(syms):
        raise ValueError("Must provide at lease %d symbols for the polynomial" % dim)
    V, tmp_syms, terms = vandermonde(order, dim)
    if num_pts < V.shape[0]:
        raise ValueError(
            "Must provide %d points for order %d, dimension "\
            "%d polynomial, given %d points" % \
            (V.shape[0], order, dim, num_pts))
    elif num_pts > V.shape[0]:
        print "gen_poly given %d points but only requires %d, "\
            "continuing using the first %d points" % \
            (num_pts, V.shape[0], V.shape[0])
        num_pts = V.shape[0]

    subs_dict = {}
    for j in range(dim):
        for i in range(num_pts):
            subs_dict[tmp_syms[i][j]] = points[i][j]
    V_pts = V.subs(subs_dict)
    V_inv = V_pts.inv()

    coeffs = V_inv.multiply(Matrix([points[i][-1] for i in xrange(num_pts)]))

    f = 0
    for j,term in enumerate(terms):
        t = 1
        for k in term:
            t *= syms[k]
        f += coeffs[j]*t
    return f

def main():
    order = 2
    V, tmp_syms, _ = vandermonde(order)
    print "Vandermonde matrix of order 2 in 1 dimension"
    pprint(V)

    print '-'*79
    print "Computing the determinate and comparing to \sum_{0<i<j<=3}(a_j - a_i)"

    det_sum = 1
    for j in range(order + 1):
        for i in range(j):
            det_sum *= (tmp_syms[j][0] - tmp_syms[i][0])

    print """
    det(V) = %(det)s
    \sum   = %(sum)s
           = %(sum_expand)s
    """ % { "det": V.det(),
             "sum": det_sum,
             "sum_expand": det_sum.expand(),
          }

    print '-'*79
    print "Polynomial fitting with a Vandermonde Matrix:"
    x,y,z = symbols('x,y,z')

    points = [(0,3), (1,2), (2,3)]
    print """
    Quadratic function, represented by 3 points:
       points = %(pts)s
       f = %(f)s
    """ % { "pts" : points,
            "f" : gen_poly(points, 2, [x]),
          }

    points = [(0, 1, 1), (1, 0, 0), (1, 1, 0), (Rational(1, 2), 0, 0),
              (0, Rational(1, 2), 0), (Rational(1, 2), Rational(1, 2), 0)]
    print """
    2D Quadratic function, represented by 6 points:
       points = %(pts)s
       f = %(f)s
    """ % { "pts" : points,
            "f" : gen_poly(points, 2, [x, y]),
          }

    points = [(0, 1, 1, 1), (1, 1, 0, 0), (1, 0, 1, 0), (1, 1, 1, 1)]
    print """
    3D linear function, represented by 4 points:
       points = %(pts)s
       f = %(f)s
    """ % { "pts" : points,
            "f" : gen_poly(points, 1, [x, y, z]),
          }

if __name__ == "__main__":
    main()


## Leetcode problem 207: Course schedule.
import numpy as np
import algorith.clr_book.ch22_elemtary_graph.graph as gr

## numCourses=2; prerequisites=[[1,0]]; return: True
## numCourses=2; prerequisites=[[1,0],[0,1]]; return: False
## numCourses=4; prerequisites=[[0,1],[0,2],[1,3],[2,3]]; return: False

## See here: https://walkccc.github.io/CLRS/Chap22/22.3/
def find_cycle_w_dfs_it(g):
    for u in g.vertices:
        if u.color=="white":
            if find_cycle_w_dfs(g,u):
                return True
    return False

def find_cycle_w_dfs(g,u):
    u.color="grey"
    st=[u]
    while len(st)>0:
        u=st[len(st)-1]
        remains=False
        for v in g.adj[u]:
            if v.color=="grey":
                return True
            elif v.color=="white":
                remains=True
                v.color="grey"
                st.append(v)
                break#Exit the for loop
        if not remains:
            st.pop()
            u.color="black"
    return False

def find_cycle_w_dfs_rec(g):
    for ver in g.vertices:
        if ver.color=="white":
            try:
                find_cycles_dfs_visit(g,ver)
            except:
                return True
    return False

def find_cycles_dfs_visit(g,u):
    u.color="grey"
    if u in g.adj.keys():
        for v in g.adj[u]:
            if v.color=="white":
                find_cycles_dfs_visit(g,v)
            elif v.color=="grey":
                raise Exception("cycle detected!")
    u.color="black"

#########################################
## Tests

def tst():
    prerequisites=[[1,0],[2,0],[3,1],[3,2]]
    numCourses=4
    g=gr.Graph().init_from_edge_list(numCourses,prerequisites)
    print(find_cycle_w_dfs_rec(g)==False)
    prerequisites=[[0,1],[1,0]]
    numCourses=2
    g=gr.Graph().init_from_edge_list(numCourses,prerequisites)
    print(find_cycle_w_dfs_rec(g)==True)
    prerequisites=[[1,0],[0,3],[0,2],[3,2],[2,5],[4,5],[5,6],[2,4]]
    numCourses=7
    g=gr.Graph().init_from_edge_list(numCourses,prerequisites)
    print(find_cycle_w_dfs_it(g))
    #find_cycle_w_dfs_rec(g)


import numpy as np
from scipy.optimize import minimize

from utils import computer_cost, sigmoid, load_data, plot_data

epsilon = 1e-5

def cost_function(theta, X, y):
    m = y.size
    h = sigmoid.sigmoid(X.dot(theta))
    J = -1 * (1 / m) * (np.log(h + epsilon).T.dot(y) + np.log(1 - h + epsilon).T.dot(1 - y))
    return J[0]

def gradient(theta, X, y):
    m = y.size
    h = sigmoid.sigmoid(X.dot(theta.reshape(-1, 1)))
    grad = (1 / m) * X.T.dot(h - y)
    return grad.flatten()

def predict(score1, score2, theta):
    return sigmoid.sigmoid(np.array([1, score1, score2]).dot(theta.reshape(3, -1)).sum())

def main():
    X, y, origin_X = load_data.load_and_process('data1.txt')
    initial_theta = np.zeros(X.shape[1])
    res = minimize(cost_function, initial_theta, args=(X, y), method=None, jac=gradient, options={'maxiter': 400})
    print(res)

    theta = res.x

    data = load_data.load('data1.txt')

    pos_values = data[(data[:, 2] == 1)]
    neg_values = data[(data[:, 2] == 0)]

    plot_data.plot(pos_values[:, 0], pos_values[:, 1], 'score1', 'score2',
                   {
                       'fmt': 'bx',
                       'markersize': 5
                   })

    plot_data.plot(neg_values[:, 0], neg_values[:, 1], 'score1', 'score2',
                   {
                       'fmt': 'yo',
                       'markersize': 5,
                       'show': False
                   })

    score1 = np.linspace(25, 100)
    score2 = []
    for item in score1:
        score2.append(((0.5 - theta[0]) - theta[1] * item) / theta[2])
    score2 = np.array(score2)

    plot_data.plot(score1, score2, 'score1', 'score2',
                   {
                       'fmt': 'r-',
                       'label': 'minimize',
                       'show': False
                   })

    theta_from_cal = np.array([-4.81180027, 0.04528064, 0.03819149]) # 0.001 10 0000
    theta_from_cal_2 = np.array([-15.39517866, 0.12825989, 0.12247929]) # 0.001 100 0000
    theta_from_cal_3 = np.array([-22.21628108, 0.18268725, 0.17763448]) # 0.003 100 0000
    score3 = []
    for item in score1:
        score3.append(((0.5 - theta_from_cal_3[0]) - theta_from_cal_3[1] * item) / theta_from_cal_3[2])
    score3 = np.array(score3)
    plot_data.plot(score1, score3, 'score1', 'score2',
                   {
                       'fmt': 'g-',
                       'label': 'gredient-descent',
                       'legend_loc': 'upper right',
                       'show': True
                   })

    print(predict(45, 85, theta_from_cal_3))

if __name__ == '__main__':
    main()