from numpy import linspace
from sympy import symbols

from math import *

# Definição da variável em uso -> f(x): x
x = symbols('x')

def Simpson(f, a, b, e):

    #Um numero muito pequeno
    d = 0.0001

    #Vetor de m pontos cobrindo o intervalo [a,b]
    xd = []
    yd = []

    for i in linspace(a,b,round((b-a)/d) ):
        xd.append(i)
        yd.append((f(i+4*d) - 4*f(i+3*d) + 6*f(i+2*d) - 4*f(i+d) +  f(i) )/(d**4))

    #Valor máximo da derivada segunda dos subintervalos
    Max = max(yd)
    Min = min(yd)
    M = abs(Max) if abs(Max)>abs(Min) else abs(Min)

    #Calculo de m
    m = ( ( ( (b-a)**5 )*M ) / (180*e))**(1/4)

    #O arredondamento deve ser feito para cima
    if m > round(m):
        m = round(m)+1
    else:
        m = round(m)

    #Deve-se tomar o primeiro número par, maior que m
    if m%2 is 1:
        m = m+1

    h = (b-a)/(m)
    s = f(a) + f(b)

    x = a +h

    for i in range(0,m-1,2):
        s = s + 4*f(x)
        x = x + 2*h

    x = a +2*h
    for i in range(1, m-2, 2):
        s = s + 2*f(x)
        x = x + 2*h

    Isr = (h/3)*s

    return Isr

if __name__ == "__main__":
    # INSTRUÇÕES
    print(' #########################################################')
    print(' ###             EXEMPLO DE ENTRADA:                   ###')
    print(' ###       x**2 + 5*x - 4  ou    cos(x)* x**2          ###')
    print(' #########################################################\n')

    # Entrada da função escolhida para o método
    '''
    funcao = input("Entre com uma funcao f(x): ")

    A = float(input("Entre com o ponto A: "))
    B = float(input("Entre com o ponto B: "))
    e = float(input("Entre com o erro e: "))
    '''

    funcao = 'x**2 + 1'
    A = 0
    B = 2
    e = 0.001

    f = lambda x: eval(funcao)

    val = Simpson(f, A, B, e)

    try:
        import matplotlib.pyplot as plt
        matplot = True

    except:
        print("MatplotLib indisponível!!")
        matplot = False

    if matplot:
        _sum = (abs(A)+abs(B))

        x = []
        y = []
        for dot in linspace(A-(1/4)*_sum, B+(1/4)*_sum, _sum*100 ):
            x.append(dot)
            y.append(f(dot))

        x_b = []
        y_b = []
        for i in linspace(A, B, _sum*100 ):
            x_b.append(i)
            y_b.append(f(i))

        plt.grid = True

        plt.ylabel('f(x)')
        plt.xlabel('x')
        plt.title('Método da Integração de 1/3 de Simpson para %s = %10.4f' %(funcao.replace('**','^').replace('*',''), val))

        plt.plot(x, y, '-', label = 'Função f(x)')
        plt.fill_between(x_b, y_b, 0 , facecolor = 'red', alpha = 0.5, label = "Área integrada")

        plt.legend(fancybox = True)
        plt.show()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

# This program is based on the following program.
# https://github.com/oreilly-japan/deep-learning-from-scratch/blob/master/common/functions.py
# Copyright (c) 2016 Koki Saitoh
# The original program is released under the MIT License.
# https://github.com/oreilly-japan/deep-learning-from-scratch/blob/master/LICENSE.md

# 必要なライブラリをインポート
import numpy as np

def sigmoid(x):
    """
    シグモイド関数
    """

    return 1 / (1 + np.exp(-x))

def sigmoid_grad(x):
    """
    シグモイド関数を微分した関数
    """

    return (1.0 - sigmoid(x)) * sigmoid(x)

def softmax(x):
    """
    ソフトマックス関数
    """

    x = x - np.max(x, axis=-1, keepdims=True)
    return np.exp(x) / np.sum(np.exp(x), axis=-1, keepdims=True)

def cross_entropy_error(y, t):
    """
    交差エントロピー誤差を計算する関数
    """

    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)

    if t.size == y.size:
        t = t.argmax(axis=1)

    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size


from src.functions.objectivefn import ObjectiveFn
import math
import numpy as np
import matplotlib.pyplot as plt
from matplotlib import cm

class EasomFn(ObjectiveFn):

    def __init__(self):
        super(EasomFn, self).__init__(2)
        self.domain = (-10, 10)

    def get_minima(self):
        minima_coords = [math.pi for i in range(self.dims)]  # minima val is -1
        minima = tuple(minima_coords)
        return self.evaluate(minima)

    def evaluate(self, params):
        f = 0
        try:
            if self.dims != 2:
                raise Exception('Dimensions more than 2')
            if self.dims != len(params):
                raise Exception('Dimensions not matching params')

            X = params[0]
            Y = params[1]
            f = -1 * math.cos(X) * math.cos(Y) * math.exp(-1 * ((X - math.pi) ** 2 + (Y - math.pi) ** 2))
            return f
        except Exception as error:
            print('Exception in easom_fn.eval: '.repr(error))

    def graph_fn(self):
        X = np.linspace(self.domain[0], self.domain[1], 200)
        Y = np.linspace(self.domain[0], self.domain[1], 200)
        X, Y = np.meshgrid(X, Y)
        Z = -1 * np.cos(X) * np.cos(Y) * np.exp(-1 * ((X - math.pi) ** 2 + (Y - math.pi) ** 2))

        fig = plt.figure()
        ax = fig.gca(projection='3d')
        ax.plot_surface(X, Y, Z, rstride=1, cstride=1, cmap=cm.plasma, linewidth=0, antialiased=False)
        plt.savefig('easom.png')

    def contour_plot(self, save_file_name, points):
        X = np.linspace(self.domain[0], self.domain[1], 200)
        Y = np.linspace(self.domain[0], self.domain[1], 200)
        X, Y = np.meshgrid(X, Y)
        Z = -1 * np.cos(X) * np.cos(Y) * np.exp(-1 * ((X - math.pi) ** 2 + (Y - math.pi) ** 2))

        plt.contour(X, Y, Z)
        for point in points:
            plt.scatter(point[0], point[1], marker='X', color='red')
        plt.savefig(save_file_name, dpi=300, bbox_inches='tight')
        plt.close()

    def is_defined_only_for_2d(self):
        return True

    def name(self):
        return "Easomfn"


import numpy as np

k = 3
X = np.genfromtxt("data-clustering-1.csv", dtype=float, delimiter=',')
X = X.T
max_iter = 100

n_examples = X.shape[0]
n_features = X.shape[1]
n_classes = k

# Initialization
t = 0
u = np.random.rand(k, n_features)
converged = False

indices = np.arange(n_examples)

# Loop
while(not(converged)):
  u_old = u
  distance_to_centroids = np.vstack([np.linalg.norm(X-u[i], axis=1) for i in range(k)])
  closest_cluster = np.argmin(distance_to_centroids, axis=0)
  C = np.array([indices[np.where(closest_cluster == i)] for i in range(k)])
  u = np.vstack([np.mean(X[C[i], :], axis=0)] for i in range(k))
  t = t + 1
  if(t==max_iter): converged = True
  if(np.array_equal(u_old,u)): converged = True


import numpy as np
import os
from math import radians, cos, sin, asin, sqrt
import geopandas as geopd
import scipy.optimize
import itertools

R = 6378

##########################################################
def haversine(p1, p2, unit='km'):
    """Calculate the great circle distance (in kilometers) between two points
    on the earth (specified in decimal degrees). It expects points in the
    format (lon, lat)"""
    lonrad1, latrad1 = np.radians(p1)
    lonrad2, latrad2 = np.radians(p2)

    dlon = lonrad2 - lonrad1
    dlat = latrad2 - latrad1
    a = np.sin(dlat/2)**2 + np.cos(latrad1) * np.cos(latrad2) * np.sin(dlon/2)**2
    c = 2 * np.arcsin(np.sqrt(a))
    if unit in ['m', 'meters']: return c * R * 1000
    else: return c * R

##########################################################
def haversine_all(points, unit='km'):
    """Calculate the great circle distance (in meters) between two points
    on the earth (specified in decimal degrees) between every pair of points
    in @points. It expects points in the format (lon, lat)"""
    n = len(points)
    combs = np.array(list(itertools.combinations(list(range(n)), 2)))
    points = np.radians((points.copy()))

    lon1 = points[combs[:, 0], 0]
    lat1 = points[combs[:, 0], 1]
    lon2 = points[combs[:, 1], 0]
    lat2 = points[combs[:, 1], 1]

    dlon = lon2 - lon1
    dlat = lat2 - lat1

    a = np.sin(dlat/2)**2 + np.cos(lat1) * np.cos(lat2) * np.sin(dlon/2)**2
    c = 2 * np.arcsin(np.sqrt(a))
    if unit in ['m', 'meters']: return combs, c * R * 1000
    else: return combs, c * R

##########################################################
def get_shp_points(shppath):
    """Get points from @shppath and returns list of points, x and y """

    geodf = geopd.read_file(shppath)
    shapefile = geodf.geometry.values[0]
    if shapefile.geom_type == 'Polygon': return shapefile.exterior.xy
    elif shapefile.geom_type == 'MultiPolygon':
        polys = list(shapefile)
        areas = [p.area for p in polys]
        poly = polys[np.argmax(areas)]
        return poly.exterior.xy

##########################################################
def deg2num(lon_deg, lat_deg, zoom, imgsize=256):
    """From the (lat,lon) in degrees, get the (x, y) of the tile,
    as well as the pixel in the image."""
    lat_rad = math.radians(lat_deg)
    n = 2.0 ** zoom
    xtile = ((lon_deg + 180.0) / 360.0 * n)
    ytile = ((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n)
    return (int(xtile), int(ytile),
            int(math.modf(xtile)[0]*imgsize), int(math.modf(ytile)[0]*imgsize))

##########################################################
def num2deg(xtile, ytile, zoom):
    """From the tile x and y, get the (lat,lon) in degrees"""
    n = 2.0 ** zoom
    lon_deg = xtile / n * 360.0 - 180.0
    lat_rad = math.atan(math.sinh(math.pi * (1 - 2 * ytile / n)))
    lat_deg = math.degrees(lat_rad)
    return (lon_deg, lat_deg)

##########################################################
def dist_to_deltalonlat(dist, lonref, latref):
    """Get the deltalon and deltalat based on a reference lat and lon.
    For the same distance, lon varies more than lat."""
    def get_delta_lon_from_d(lon1, lat1=latref, lon2=lonref, lat2=latref):
        return dist - haversine([lon1, lat1], [lon2, lat2])

    def get_delta_lat_from_d(lat1, lon1=lonref, lat2=latref, lon2=lonref):
        return dist - haversine([lon1, lat1], [lon2, lat2])

    lon2 = lonref + 20 # a large enough diff in lon to contain the @dist
    lon2 = scipy.optimize.bisect(get_delta_lon_from_d, lonref, lon2,
            xtol=0.000001, rtol=0.000001)

    lat2 = latref + 20 # a large enough diff in lon to contain the @dist
    lat2 = scipy.optimize.bisect(get_delta_lat_from_d, latref, lat2,
            xtol=0.000001, rtol=0.000001)

    return np.abs(lon2 - lonref), np.abs(lat2 - latref)


import numpy as np

def transform_zero_to_one(features):
    min_value=np.min(features)
    max_value=np.max(features)
    return (features/(max_value-min_value)) -(min_value/(max_value-min_value))


import numpy as np
import matplotlib.pyplot as plt
import sys
import math
import random
import operator

def euclidean(x, x_p):
    return ((x[0] - x_p[0]) ** 2 + (x[1] - x_p[1]) ** 2) ** 0.5

def greatest_euclidean(data, centers):
    maxi = {}
    for x in centers:
        for x_p in data:
            euc = euclidean(x, x_p)
            if x_p not in maxi:
                maxi[x_p] = 0
            maxi[x_p] += euc
    return max(maxi.items(), key=operator.itemgetter(1))[0]

# Uses a greedy approach, selects a data point at random and assigns this as a center for a classification
# it then finds the furthest data point from this and assigns this as a center and places it in the set
# the next center will be the furthest datapoint from all centers until all regions have a center
def gen_centers(M, data):
    centers = []
    N = len(data)
    rand = random.randint(0, N - 1)
    centers.append(data.pop(rand))
    center = (0, 0)
    classifiers = []
    for i in range(M - 1):
        center = greatest_euclidean(data, centers)
        data.remove(center)
        centers.append(center)

    for x in data:
        num = voronoi(x, centers)
        classifiers.append(num)
    return centers, classifiers

# Determine the Voronoi region for the data point. This basically just decides how to classify all the data points
# assigning it to the closest center by euclidean distance
def voronoi(x, centers):
    order = []
    for i in range(len(centers)):
        datapoint = centers[i]
        # Euclidean to x
        order.append((euclidean(x, datapoint), i))
    order.sort()
    g = order[0][1]
    return g

# Generates 10,000 random datapoints with x and y values between 0 and 1
def generate_data():
    data = []
    for x1_ in range(100):
        for x2_ in range(100):
            x1 = np.random.uniform(0, 1)
            x2 = np.random.uniform(0, 1)
            data.append((x1, x2))
    return data

def plot(M):
    data = generate_data()
    centers, classifers = gen_centers(M, data)
    unique=set(classifers)
    print(unique)
    plt.scatter(*zip(*data), c=classifers, cmap='rainbow')
    plt.scatter(*zip(*centers), c='black')
    plt.title('Greedy with {} Regions'.format(M))
    plt.xlabel('x1', color='#1C2833')
    plt.ylabel('x2', color='#1C2833')
    plt.grid()
    plt.show()

if __name__ == "__main__":
    # 10 Clusters for users
    regions = 10
    plot(regions)

    # Assumption: Users will be datapoints, users will create a voronoi region and counselors
    # will be assigned to their closest associated region.

    # Just using greedy. May add in branch and bound.


import numpy as np
import math

from zero2ml.supervised_learning._base import BaseModel
from zero2ml.utils.evaluation_metrics import MeanSquaredError
from zero2ml.utils.data_transformations import Standardize

class LinearRegression(BaseModel):
    """
    Multiple Linear Regression with input features standardization.

    Parameters
    ----------
    learning_rate: scalar
        Learning rate.

    Attributes
    ----------
    X: array_like
        Training features.
    y: array_like
        Training labels.
    samples_num: scalar
        Number of training samples.
    features_num: scalar
        Number of features.
    input_normalization: object
        Object used to standardize data and containing necessary parameters.
    self.W: array_like
        Weights.
    self.b: array_like
        Intercept term.
    self.training_loss: list
        List containing training loss for each iteration of model training.
    """
    def __init__(self, learning_rate=0.01):

        # Specify model type
        self.model_type = "regressor"

        # Training data
        self.X = None

        # Training targets
        self.y = None

        # Number of training samples
        self.samples_num = None

        # Number of features
        self.features_num = None

        # Input features normalization
        self.input_normalization = Standardize()

        # Weights and intercept
        self.W = None
        self.b = None

        # Learning rate
        self.learning_rate = learning_rate

        # Loss and loss function
        self.training_loss = None
        self.__loss_function = MeanSquaredError()

    def fit(self, X, y, iterations=5000):
        """
        Fit the linear regression model with given training feature inputs.

        Parameters
        ----------
        X: array_like
            Training features.
        y: array_like
            Training labels.
        iterations: scalar
            Number of iterations for gradient descent to converge.
        """
        # Save X and y into model object
        self.X = X.copy()
        self.y = y.copy()

        # Apply normalization on features
        X_norm = self.input_normalization(self.X)

        # Calculate number of samples and features
        self.samples_num, self.features_num = self.X.shape

        # Initiate weights and intercept
        self.W = np.zeros(self.features_num)
        self.b = 0

        # Initiate loss
        self.training_loss = []

        # Gradient descent
        for iteration in range(iterations):

            # Make prediction
            pred = np.dot(X_norm, self.W) + self.b

            # Calculate loss and save to training loss list
            self.training_loss.append( self.__loss_function(pred, self.y) )

            # Calculate gradients
            dW = - ( 2 * np.dot(X_norm.T, self.y - pred) ) / self.samples_num
            db = - ( 2 * np.sum(self.y - pred) ) / self.samples_num

            # Update weights and intercept
            self.W -= self.learning_rate * dW
            self.b -= self.learning_rate * db

    def predict(self, X):
        """
        Predict the label(s) with given feature inputs.

        Parameters
        ----------
        X: array_like
            Training features.

        Returns
        -------
        pred: array_like
            Predicted values.
        """
        # Apply Z-score normalization before
        X_norm = (X - self.input_normalization.mean) / self.input_normalization.std

        # Make predictions with the trained data
        pred = np.dot(X_norm, self.W) + self.b

        return pred


# -*- coding: utf-8 -*-
"""
Created on Mon Dec 11 20:02:37 2017

@author: tzave
"""

import numpy as np
import solver

A = np.array([[2., -1., 0., 0.],
              [-1., 2., -1., 0.],
              [0., -1., 2., -1.],
              [0., 0., -1., 2.]])
d = np.array([1., 0., 0., 1.])

taus = np.arange(.1, 2., .1)
for tau in taus:
    result = solver.IterativeMethod(A, d, .5e-6, tau, 1000)
    x = result['x']
    iterations = result['iterations']
    print('tau=', tau, ': ', iterations, ', x=', x)
    delta = np.dot(A, x) - d
    print('Norm (Ax-d) =', np.sqrt(np.dot(delta, delta)))


import numpy as np
import numpy.linalg as la
from scipy import optimize  # type: ignore

from . import geom3d
from ._util import distance_point_point

def centroid_fit(points, weights=None):
    """Calculates the weighted average of a set of points
    This minimizes the sum of the squared distances between the points
    and the centroid.
    """
    if points.ndim == 1:
        return points
    return np.average(points, axis=0, weights=weights)

def _check_input(points, weights) -> None:
    """Check the input data of the fit functionality"""
    points = np.asarray(points)
    if weights is not None:
        weights = np.asarray(weights)

    if points.ndim != 2 or points.shape[1] != 3:
        raise ValueError(
            f"Input data has the wrong shape, expects points to be of shape ('n', 3), got {points.shape}"
        )
    if weights is not None and (weights.ndim != 1 or len(weights) != len(points)):
        raise ValueError(
            "Shape of weights does not match points, weights should be a 1 dimensional array of len(points)"
        )

def line_fit(points, weights=None) -> geom3d.Line:
    """Fits a line through a set points"""
    _check_input(points, weights)
    centroid = centroid_fit(points, weights)
    weights = 1.0 if weights is None else weights
    centered_points = points - centroid
    u, s, v = np.linalg.svd(
        np.matmul(weights * centered_points.transpose(), centered_points)
    )
    return geom3d.Line(anchor_point=centroid, direction=v[0])

def plane_fit(points, weights=None) -> geom3d.Plane:
    """Fits a plane through a set of points"""
    _check_input(points, weights)
    centroid = centroid_fit(points, weights)
    weights = 1.0 if weights is None else weights
    centered_points = points - centroid
    u, s, v = np.linalg.svd(
        np.matmul(weights * centered_points.transpose(), centered_points)
    )
    return geom3d.Plane(anchor_point=centroid, normal=v[2])

# TODO add weights
def fast_sphere_fit(points) -> geom3d.Sphere:
    """A fast algebraic circle fit, that uses a modified error function that
    is more sensitive to outliers
    """
    _check_input(points, None)
    A = np.append(points * 2, np.ones((points.shape[0], 1)), axis=1)
    f = np.sum(points ** 2, axis=1)
    C, _, _, _ = np.linalg.lstsq(A, f, rcond=None)
    center = C[0:3]
    radius = np.average(distance_point_point(points, center))
    return geom3d.Sphere(center=center, radius=radius)

def sphere_fit(
    points, weights=None, initial_guess: geom3d.Sphere = None
) -> geom3d.Sphere:
    """Fits a circle through a set of points"""
    _check_input(points, weights)
    initial_guess = initial_guess or fast_sphere_fit(points)

    def sphere_fit_residuals(center, points, weights):
        distances = distance_point_point(center, points)
        radius = np.average(distances, weights=weights)
        if weights is None:
            return distances - radius
        return (distances - radius) * np.sqrt(weights)

    results = optimize.least_squares(
        sphere_fit_residuals, x0=initial_guess.center, args=(points, weights)
    )
    if not results.success:
        raise RuntimeError(results.message)

    radius = np.average(distance_point_point(points, results.x), weights=weights)
    return geom3d.Sphere(center=results.x, radius=radius)

def cylinder_fit(points, weights=None, initial_guess: geom3d.Cylinder = None):
    """Fits a cylinder trough a set of points"""
    _check_input(points, weights)
    if initial_guess is None:
        raise NotImplementedError(
            "Cylinder fit currently does support running without an intial guess."
        )

    def cylinder_fit_residuals(anchor_direction, points, weights):
        line = geom3d.Line(anchor_direction[:3], anchor_direction[3:])
        distances = line.distance_to_point(points)
        radius = np.average(distances, weights=weights)
        if weights is None:
            return distances - radius
        return (distances - radius) * np.sqrt(weights)

    x0 = np.concatenate([initial_guess.anchor_point, initial_guess.direction])
    results = optimize.least_squares(
        cylinder_fit_residuals, x0=x0, args=(points, weights), ftol=1e-10
    )
    if not results.success:
        return RuntimeError(results.message)

    line = geom3d.Line(results.x[:3], results.x[3:])
    distances = line.distance_to_point(points)
    radius = np.average(distances, weights=weights)
    return geom3d.Cylinder(results.x[:3], results.x[3:], radius)

def circle3D_fit(points, weights=None, initial_guess: geom3d.Circle3D = None):
    """Fits a circle in three dimensions trough a set of points"""
    _check_input(points, weights)
    if initial_guess is None:
        raise NotImplementedError(
            "Circle3D fit currently does support running without an intial guess."
        )

    def circle_fit_residuals(circle_params, points, sqrt_w):
        circle = geom3d.Circle3D(
            circle_params[:3], circle_params[3:], la.norm(circle_params[3:])
        )
        distances = circle.distance_to_point(points)
        return distances * sqrt_w

    x0 = np.concatenate(
        [initial_guess.center, initial_guess.direction * initial_guess.radius]
    )
    results = optimize.least_squares(
        circle_fit_residuals,
        jac="2-point",
        method="lm",
        x0=x0,
        args=(points, 1.0 if weights is None else np.sqrt(weights)),
    )

    results = optimize.minimize(
        lambda *args: np.sum(circle_fit_residuals(*args) ** 2),
        x0=results.x,
        args=(points, 1.0 if weights is None else np.sqrt(weights)),
    )

    if not results.success:
        return RuntimeError(results.message)

    return geom3d.Circle3D(results.x[:3], results.x[3:], la.norm(results.x[3:]))

def torus_fit(points, weights=None, initial_guess: geom3d.Torus = None) -> geom3d.Torus:
    """Fits a torus trough a set of points"""
    _check_input(points, weights)
    if initial_guess is None:
        raise NotImplementedError(
            "Toru fit currently does support running without an intial guess."
        )

    def torus_fit_residuals(circle_params, points, weights):
        circle = geom3d.Circle3D(
            circle_params[:3], circle_params[3:], la.norm(circle_params[3:])
        )
        distances = circle.distance_to_point(points)
        radius = np.average(distances, weights=weights)
        weights = np.sqrt(weights) if weights is not None else 1.0
        return (distances - radius) * weights

    x0 = np.concatenate(
        [initial_guess.center, initial_guess.direction * initial_guess.major_radius]
    )

    results = optimize.least_squares(
        torus_fit_residuals,
        x0=x0,
        args=(points, weights),
    )

    if not results.success:
        raise RuntimeError(results.message)

    circle3D = geom3d.Circle3D(results.x[:3], results.x[3:], la.norm(results.x[3:]))
    distances = circle3D.distance_to_point(points)
    minor_radius = np.average(distances, weights=weights)
    return geom3d.Torus(
        results.x[:3], results.x[3:], la.norm(results.x[3:]), minor_radius
    )


import numpy
from amuse.units.trigo import cos, sin

def new_rotation_matrix(phi, theta, psi):
    """
    Return the rotation matrix, to rotate positions, around the x-axis (phi), y-axis (theta) and z-axis (psi).
    See wikipedia for reference
    """

    return numpy.array( (
        (cos(theta)*cos(psi), -cos(phi)*sin(psi) + sin(phi)*sin(theta)*cos(psi),  sin(phi)*sin(psi) + cos(phi)*sin(theta)*cos(psi)) ,
        (cos(theta)*sin(psi),  cos(phi)*cos(psi) + sin(phi)*sin(theta)*sin(psi), -sin(phi)*cos(psi) + cos(phi)*sin(theta)*sin(psi)) ,
        (-sin(theta)        ,  sin(phi)*cos(theta)                             ,  cos(phi)*cos(theta))
    ) )

def rotated(positions, phi, theta, psi):
    """
    Return the positions, rotated by phi, theta and psi around the x, y and z axes
    """
#    print phi, theta, psi
    rotation_matrix = new_rotation_matrix(phi, theta, psi)
#    print "RT=", rotation_matrix
    return positions.dot(rotation_matrix.transpose())

def rotate(particles, phi, theta, psi):
    """
    Rotate the positions and the velocities around 0,0,0.
    """
    particles.position = rotated(particles.position,  phi, theta, psi)
    particles.velocity = rotated(particles.velocity,  phi, theta, psi)

def add_spin(particles, omega):
    """
    Add solid-body rotation to the velocity of the particles, relative to the
    center-of-mass position.
    """
    if not omega.is_vector():
        omega = omega * [0.0, 0.0, 1.0]
    particles.velocity += omega.cross(particles.position - particles.center_of_mass())


import numpy as np

from supervised.nb_classifier import NBClassifier

EPSILON = 1E-16  # offset to avoid "divide by zero" errors

class GaussianNB(NBClassifier):

    def _pdf(self, x, mean, std):

        num = np.exp(-((x - mean)**2) / (EPSILON + 2 * std**2))
        den = np.sqrt(2 * np.pi * std**2) + EPSILON

        return num / den

    def _fit_evidence(self, X):

        feature_probas = []

        for feature in X.T:  # iterate through the features instead of the samples

            feature_probas.append(dict(mean=np.mean(feature),
                                       n=len(feature),
                                       std=np.std(feature, ddof=1)))

        return np.array(feature_probas)

    def _fit_likelihood(self, X, y):

        likelihood_ = []

        for c in self.classes_:

            samples = X[y == c]  # only keep samples of class c

            likelihood_.append(self._fit_evidence(samples))

        return np.array(likelihood_)

    def _get_evidence(self, sample):

        evidence = 1.0

        for i, feature in enumerate(sample):

            mean = self.evidence_[i]["mean"]
            std = self.evidence_[i]["std"]

            evidence *= self._pdf(feature, mean, std)

        return evidence

    def _get_likelihood(self, sample, c):

        likelihood = 1.0

        for i, feature in enumerate(sample):

            mean = self.likelihood_[c][i]["mean"]
            std = self.likelihood_[c][i]["std"]

            likelihood *= self._pdf(feature, mean, std)

        return likelihood

    def _update_evidence(self, X):

        for i, feature in enumerate(X.T):   # iterate through the features instead of the samples

            self.evidence_[i] = self._update_mean_std_n(feature, self.evidence_[i])

        return self.evidence_

    def _update_likelihood(self, X, y):

        for c in self.classes_:

            samples = X[y == c]  # only keep samples of class c

            for i, feature in enumerate(samples.T):  # iterate through the features instead of the samples

                self.likelihood_[c][i] = self._update_mean_std_n(feature, self.likelihood_[c][i])

        return self.likelihood_

    def _update_mean_std_n(self, feature, mean_std_n):

        old_m = mean_std_n["mean"]
        old_std = mean_std_n["std"]
        old_n = mean_std_n["n"]

        n = old_n + len(feature)

        m = (old_m * old_n + np.mean(feature) * n) / (old_n + n)

        s = np.sqrt((old_n * (old_std**2 + (old_m - m)**2)
                     + len(feature) * (np.var(feature)
                                       + (np.mean(feature) - m)**2)
                     ) / (old_n + len(feature)))

        return dict(mean=m, std=std, n=n)


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import linregress

def draw_plot():
    # Read data from file
    df = pd.read_csv('epa-sea-level.csv')

    # Create scatter plot
    x = df['Year']
    y = df['CSIRO Adjusted Sea Level']
    plt.scatter(x, y)

    # Create first line of best fit
    reg = linregress(x, y)
    x_low = x.min()
    x_high = 2050
    x_ext = np.linspace(x_low, x_high, 100)
    plt.plot(x_ext, reg.intercept + reg.slope*x_ext, 'r', color='blue', label='Data from 1880')

    # Create second line of best fit
    new_x = df['Year'][df['Year'] >= 2000]
    new_y = df['CSIRO Adjusted Sea Level'][df['Year'] >= 2000]
    new_reg = linregress(new_x, new_y)
    x_new_low = 2000
    x_new_ext = np.linspace(x_new_low, x_high, 100)
    plt.plot(x_new_ext, new_reg.intercept + new_reg.slope*x_new_ext, 'r', label='Data from 2000 and on')

    # Add labels and title
    plt.legend()
    plt.xlabel('Year')
    plt.ylabel('Sea Level (inches)')
    plt.title('Rise in Sea Level')

    # Save plot and return data for testing
    plt.savefig('sea_level_plot.png')
    return plt.gca()


# -*- coding: utf-8 -*-
# @Date  : 2020/5/27
# @Author: Luokun
# @Email : olooook@outlook.com

import numpy as np

class SimpleEM:
    """
    Expectation-maximization algorithm(期望最大算法,三硬币模型)
    """

    def __init__(self, prob: list, max_iter=100):
        self.prob, self.max_iter = np.array(prob), max_iter

    def fit(self, X: np.ndarray):
        for _ in range(self.max_iter):
            M = self._expect(X)  # E步
            self._maximize(X, M)  # M步

    def _expect(self, X: np.ndarray):  # E步
        p1, p2, p3 = self.prob
        a = p1 * (p2 ** X) * ((1 - p2) ** (1 - X))
        b = (1 - p1) * (p3 ** X) * ((1 - p3) ** (1 - X))
        return a / (a + b)

    def _maximize(self, X: np.ndarray, M: np.ndarray):  # M步
        self.prob[0] = np.sum(M) / len(X)
        self.prob[1] = np.sum(M * X) / np.sum(M)
        self.prob[2] = np.sum((1 - M) * X) / np.sum(1 - M)

# EM算法与高斯混合模型可参见./gmm.py


from collections import deque
import numpy as np

class FiniteDifference:
    def __init__(self, step_size, max_order=4):
        self.step_size = step_size
        self.max_order = min(max_order, 4)
        self.f_deque = deque()
        self.fd_equations = [
            lambda f, h: (-1*f[1]+1*f[0])/(1*1.0*h**1),
            lambda f, h: (1*f[2]-4*f[1]+3*f[0])/(2*1.0*h**1),
            lambda f, h: (-2*f[3]+9*f[2]-18*f[1]+11*f[0])/(6*1.0*h**1),
            lambda f, h: (3*f[4]-16*f[3]+36*f[2]-48*f[1]+25*f[0])/(12*1.0*h**1)
        ]

    def differentiate(self, value):
        # Append value to function deque
        self.f_deque.appendleft(value)
        len_f_deque = len(self.f_deque)

        if len_f_deque == 1:
            # If this is out first value, return 0.
            return np.zeros(np.shape(value))
        elif len_f_deque > (self.max_order + 1):
            # If we have more items then we need for our derivative.
            # pop the extra one from the deque.
            self.f_deque.pop()

        # Calculate derivative and return.
        return self.fd_equations[len(self.f_deque) - 2](self.f_deque, self.step_size)


""""
Author : Aditya Jain
Contact : https://adityajain.me
"""

import numpy as np

def root_mean_squared_error(y_true,y_pred):
	"""
	Compute root mean square error

	Parameters
	----------
	y_pred : predicted values

	y_true : True values

	Returns
	-------
	rmse

	"""
	loss = np.square(y_pred-y_true)
	cost = np.sqrt( np.mean(loss) )
	return cost

def mean_squared_error(y_true,y_pred):
	"""
	Compute mean square error

	Parameters
	----------
	y_pred : predicted values

	y_true : true values

	Returns
	-------
	mse

	"""
	loss = np.square(y_pred-y_true)
	cost = np.mean(loss)
	return cost

def mean_absolute_error(y_true,y_pred):
	"""
	Compute mean absolute error

	Parameters
	----------
	y_pred : predicted values

	y_true : true values

	Returns
	-------
	mae

	"""
	loss = abs(y_pred-y_true)
	cost = np.mean(loss)
	return cost

def r2_score(y_true,y_pred):
	"""
	Compute Coefficient of Determinance, r2 score

	r2 score = ESS/TSS = 1 - RSS/TSS = 1 - (y_true - y_pred)^2 / (y_true - y_true.mean())

	Parameters
	----------
	y_pred : predicted values

	y_true : true values

	Returns
	-------
	r2_score
	"""
	return 1-(np.sum((( y_true - y_pred)**2))/np.sum((y_true-np.mean(y_true))**2))


from neurons import Neurons
import numpy as np

# inherit from base class Layer
class Layer(Neurons):

    # input_size = number of input neurons
    # output_size = number of output neurons
    def __init__(self, in_neurons=0, out_neurons=0):
        self.weights = np.random.rand(in_neurons, out_neurons) - 0.5
        self.bias = np.random.rand(1, out_neurons) - 0.5

    # returns output for a given input
    def forward_propagation(self, input_data):
        self.input = input_data
        self.output = np.dot(self.input, self.weights) + self.bias
        return self.output

    # computes dE/dW, dE/dB for a given output_error = dE/dY. Returns input_error = dE/dX
    def backward_propagation(self, output_error, learning_rate):
        input_error = np.dot(output_error, self.weights.T)
        weights_error = np.dot(self.input.T, output_error)
        # dBias = output_error

        # update parameters
        self.weights -= learning_rate * weights_error
        self.bias -= learning_rate * output_error
        return input_error

# inherit from base class layer
class ActivationLayer(Layer):

    def __init__(self, activation, activation_prime):
        self.activation = activation
        self.activation_prime = activation_prime

    # returns the activated input
    def forward_propagation(self, input_data):
        self.input = input_data
        self.output = self.activation(self.input)
        return self.output

    # Returns input_error = dE/dX for a given output_error = dE/dY
    # learning_rate is not used because there is no "learnable" parameters.
    def backward_propagation(self, output_error, learning_rate):
        return self.activation_prime(self.input) * output_error
        #raise NotImplementedError


# -*- coding: utf-8 -*-
"""Linear_fourier_filter_exercise.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1KQlihNHOyF1_p6he0IWGrdA23wIfoUvA
"""

import numpy as np
import matplotlib.pyplot as plt
from scipy.misc import ascent, face

# Change this to an image of your choosing!
#image = ascent()  # Boring staircase
image = face().mean(axis=2) # Cute little friend face

image = (image - image.mean()) / image.std()
n_x, n_y = image.shape

plt.imshow(image, cmap='Greys_r')

from numpy.fft import fft2, ifft2, fftshift

image_ft = fftshift(fft2(image))
amp = image_ft.real
phase = np.arctan2(image_ft.imag, image_ft.real)

X,Y = np.meshgrid( np.linspace(-.5, .5, n_y), np.linspace(-.5,.5, n_x) )
k = (X**2 + Y**2)**.5

# Change this filter
filt = np.ones_like(image)

# Example filters
#filt = np.exp(-100*k)

#filt = k < .01

#filt = k > .05

#filt = np.sin(100*k**2)

#x_sh,y_sh = 100,100
#filt = np.exp( -1j *2*np.pi * ((x_sh * X) + (y_sh * Y)))

image_filtered = ifft2( fftshift(image_ft * filt) ).real

plt.imshow(image_filtered, cmap='Greys_r')
plt.show()


import numpy as np
import math
from scipy.special import erfc, erfcinv

def phi(input):
    """Phi function.

    :param input:
      Float (scalar or array) value.
    :returns:
      phi(input).
    """
    return 0.5 * erfc(-input/np.sqrt(2))

def invphi(input):
    """Inverse phi function.

    :param input:
      Float (scalar or array) value.
    :returns:
      invphi(input).
    """
    return -1 * np.sqrt(2) * erfcinv(input/0.5)

def calcEmpiricalProbFromValue(G, e, value):
    """Calculate the empirical probability of a given value of loss (fatalities, dollars).

    :param G:
      Input G statistic.
    :param e:
      Expected number of losses.
    :param value:
      Number of losses for which corresponding probability should be calculated.
    :returns:
      Probability that value will not be exceeded.
    """
    e = e + 0.00001
    value = value + 0.00001
    p = phi((math.log(value) - math.log(e))/G)
    return p

def calcEmpiricalValueFromProb(G, e, p):
    """ Calculate the loss value given an input probability.

    :param G:
      Input G statistic.
    :param e:
      Expected number of losses.
    :param p:
      Input probability for which corresponding loss value should be calculated.
    :returns:
      Number of losses.
    """
    e = e + 0.00001
    value = math.exp(G * invphi(p) + math.log(e))
    return value

def calcEmpiricalProbFromRange(G, e, drange):
    """Calculate the empirical probability of a given loss range.

    :param G:
      Input G statistic.
    :param e:
      Expected number of losses.
    :param drange:
      Two-element sequence, containing range of losses over which probability should be calculated.
    :returns:
      Probability that losses will occur in input range.
    """
    e = e + 0.00001
    if e < 1:
        #e = 0.1
        if G > 1.7:
            G = 1.7613
    if len(drange) == 2:
        if (e-0) < 0.001:
            e = 0.5
        fmin = drange[0] + 0.00001
        fmax = drange[1] + 0.00001
        p1 = phi((math.log(fmax) - math.log(e))/G)
        p2 = phi((math.log(fmin) - math.log(e))/G)
        p = p1-p2
        return p
    else:
        psum = 0
        for i in range(0, len(drange)-1):
            fmax = drange[i+1] + 0.00001
            fmin = drange[i] + 0.00001
            p1 = phi((math.log(fmax) - math.log(e))/G)
            p2 = phi((math.log(fmin) - math.log(e))/G)
            psum += (p1-p2)
        return psum


import numpy as np
import matplotlib.pyplot as plt

class KNearestNeighbor(object):

    def __init__(self, xtrainData, ytrainData, testData):
        self.xTrain = np.reshape(xtrainData, (xtrainData.shape[0], -1))
        self.yTrain = ytrainData
        self.xTest  = np.reshape(testData, (testData.shape[0], -1))
        print(self.xTrain.shape, self.xTest.shape)

    def computerDistanceWith2Loop(self):
        trainImageNum = self.xTrain.shape[0]
        testImageNum = self.xTest.shape[0]
        distance = np.zeros((testImageNum, trainImageNum))

        for i in range(0, testImageNum):
            for j in range (0, trainImageNum):
                distance[i, j] = np.sqrt(np.dot(self.xTest[i] - self.xTrain[j], self.xTest[i] - self.xTrain[j]))

        return distance

    def computerDistanceWith1Loop(self):
        trainImageNum = self.xTrain.shape[0]
        testImageNum  = self.xTest.shape[0]

        distance = np.zeros((testImageNum, trainImageNum))

        for i in range(testImageNum):
            distance[i,:] = np.sqrt(np.sum(np.square(self.xTest[i] - self.xTrain), axis=1))

        return distance

    def computerDistanceNoLoop(self):
        trainImageNum = self.xTrain.shape[0]
        testImageNum  = self.xTest.shape[0]

        distance = np.zeros((testImageNum, trainImageNum))

        distance = np.sqrt(self.getNormMatrix(self.xTest, trainImageNum).T
                       +  self.getNormMatrix(self.xTrain, testImageNum)
                       -  2 * np.dot(self.xTest, self.xTrain.T))

        #M = np.dot(self.xTest, self.xTrain.T)
        #print(M.shape)
        #te = np.square(self.xTest).sum(axis = 1)
        #print(te.shape)
        #tr = np.square(self.xTrain).sum(axis = 1)
        #print(tr.shape)
        #distance = np.sqrt(-2*M+tr+np.matrix(te).T) #tr add to line, te add to row

        #distance += np.sum(self.xTrain ** 2, axis=1).reshape(1, trainImageNum)
        #distance += np.sum(self.xTest ** 2, axis=1).reshape(testImageNum, 1) # reshape for broadcasting
        #distance -= 2 * np.dot(self.xTest, self.xTrain.T)

        return distance

    def getNormMatrix(self, data, lines_num):
        return np.ones((lines_num, 1)) * np.sum(np.square(data), axis = 1)

    def saveDistanceToFile(self, distance, fileName):
        np.savetxt(fileName, distance, fmt='%1.1f')

    def predict_lables(self, distance, k=1):
        '''
        Given a matrix of distance between test points and training points.
        pridect a label for each test point.

        Inputs:
        - distance: A numpy array of shape (num_test, num_train) where distance[i,j]
        gives the distance between the ith test point and the jth training point.

        Returns:
        - y: A numpy array of shape (num_test, ) containing predicted lables for the
        test data, where y[i] is the predicted lable for the test point X[i].
        '''

        numTest = distance.shape[0]
        y_pred=np.zeros(numTest)
        for i in range(numTest):
            # A list of length k storing the lables of the k nearest neighbors to
            # the ith test point

            closest_y=[]
            #########################################################################
            # TODO:                                                                 #
            # Use the distance matrix to find the k nearest neighbors of the ith    #
            # testing point, and use self.y_train to find the labels of these       #
            # neighbors. Store these labels in closest_y.                           #
            # Hint: Look up the function numpy.argsort.                             #
            #########################################################################
            kids = np.argsort(distance[i])
            closest_y = self.yTrain[kids[:k]]

            #########################################################################
            # TODO:                                                                 #
            # Now that you have found the labels of the k nearest neighbors, you    #
            # need to find the most common label in the list closest_y of labels.   #
            # Store this label in y_pred[i]. Break ties by choosing the smaller     #
            # label.                                                                #
            #########################################################################

            count = 0
            label = 0
            for j in closest_y:
                tmp = 0
                for kk in closest_y:
                    tmp += (kk==j)
                if tmp > count:
                    count = tmp
                    label = j
            y_pred[i] = label
        return y_pred

    def crossValidation(self):
        num_folds = 5
        k_choices = [1, 3, 5, 8, 10, 12, 15, 20, 50, 100]
        X_train_folder = []
        y_train_folder = []

        ################################################################################
        # TODO:                                                                        #
        # Split up the training data into folds. After splitting, X_train_folds and    #
        # y_train_folds should each be lists of length num_folds, where                #
        # y_train_folds[i] is the label vector for the points in X_train_folds[i].     #
        # Hint: Look up the numpy array_split function.                                #
        ################################################################################
        X_train_folds = np.array_split(self.xTrain, num_folds)
        y_train_folds = np.array_split(self.yTrain, num_folds)

        ################################################################################
        #                                 END OF YOUR CODE                             #
        ################################################################################

        # A dictionary holding the accuracies for different values of k that we find
        # when running cross-validation. After running cross-validation,
        # k_to_accuracies[k] should be a list of length num_folds giving the different
        # accuracy values that we found when using that value of k.
        k_to_accuracies = {}

        ################################################################################
        # TODO:                                                                        #
        # Perform k-fold cross validation to find the best value of k. For each        #
        # possible value of k, run the k-nearest-neighbor algorithm num_folds times,   #
        # where in each case you use all but one of the folds as training data and the #
        # last fold as a validation set. Store the accuracies for all fold and all     #
        # values of k in the k_to_accuracies dictionary.                               #
        ################################################################################
        for k in k_choices:
            k_to_accuracies[k] = np.zeros(num_folds)
            for i in range(num_folds):
                xTr = np.array(X_train_folds[:i] + X_train_folds[i+1:] )
                yTr = np.array(y_train_folds[:i] + y_train_folds[i+1:] )
                xTe = np.array(X_train_folds[i])
                yTe = np.array(y_train_folds[i])

                xTr = np.reshape(xTr, (xTr.shape[0]*xTr.shape[1], -1))
                yTr = np.reshape(yTr, yTr.shape[0]*yTr.shape[1])

                knnClassifier = KNearestNeighbor(xTr, yTr, xTe)
                distanceNoLoop = knnClassifier.computerDistanceNoLoop()

                yte_pred = knnClassifier.predict_lables(distanceNoLoop, k)

                num_correct = np.sum(yte_pred == yTe)
                accuracy = float(num_correct) / len(yTe)
                k_to_accuracies[k][i] = accuracy
        ################################################################################
        #                                 END OF YOUR CODE                             #
        ################################################################################

        # Print out the computed accuracies
        for k in sorted(k_to_accuracies):
            for accuracy in k_to_accuracies[k]:
                print('k = %d, accuracy = %f' % (k, accuracy))

        return k_to_accuracies


from sklearn import datasets, svm, metrics
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import numpy as np

# The digits dataset
digits = datasets.load_digits()

# Flatten the image to apply classifier
n_samples = len(digits.images)
data = digits.images.reshape((n_samples, -1))

# Create support vector classifier
classifier = svm.SVC(gamma=0.001)

# Split into train and test subsets (50% each)
X_train, X_test, y_train, y_test = train_test_split(
    data, digits.target, test_size=0.5, shuffle=False)

# Learn the digits on the first half of the digits
classifier.fit(X_train, y_train)
n_samples/2

# test on second half of data
n = np.random.randint(int(n_samples/2),n_samples)
plt.imshow(digits.images[n], cmap=plt.cm.gray_r, interpolation='nearest')
print('Predicted: ' + str(classifier.predict(digits.data[n:n+1])[0]))

# Select Option by Number
# 0 = Linear, 1 = Quadratic, 2 = Inner Target
# 3 = Moons, 4 = Concentric Circles, 5 = Distinct Clusters
select_option = 5

# generate data
data_options = ['linear','quadratic','target','moons','circles','blobs']
option = data_options[select_option]
# number of data points
n = 2000
X = np.random.random((n,2))
mixing = 0.0 # add random mixing element to data
xplot = np.linspace(0,1,100)

if option=='linear':
    y = np.array([False if (X[i,0]+X[i,1])>=(1.0+mixing/2-np.random.rand()*mixing) else True for i in range(n)])
    yplot = 1-xplot
elif option=='quadratic':
    y = np.array([False if X[i,0]**2>=X[i,1]+(np.random.rand()-0.5)\
                  *mixing else True for i in range(n)])
    yplot = xplot**2
elif option=='target':
    y = np.array([False if (X[i,0]-0.5)**2+(X[i,1]-0.5)**2<=0.1 +(np.random.rand()-0.5)*0.2*mixing else True for i in range(n)])
    j = False
    yplot = np.empty(100)
    for i,x in enumerate(xplot):
        r = 0.1-(x-0.5)**2
        if r<=0:
            yplot[i] = np.nan
        else:
            j = not j # plot both sides of circle
            yplot[i] = (2*j-1)*np.sqrt(r)+0.5
elif option=='moons':
    X, y = datasets.make_moons(n_samples=n,noise=0.05)
    yplot = xplot*0.0
elif option=='circles':
    X, y = datasets.make_circles(n_samples=n,noise=0.05,factor=0.5)
    yplot = xplot*0.0
elif option=='blobs':
    X, y = datasets.make_blobs(n_samples=n,centers=[[-5,3],[5,-3]],cluster_std=2.0)
    yplot = xplot*0.0

plt.scatter(X[y>0.5,0],X[y>0.5,1],color='blue',marker='^',label='True')
plt.scatter(X[y<0.5,0],X[y<0.5,1],color='red',marker='x',label='False')
if option not in ['moons','circles','blobs']:
    plt.plot(xplot,yplot,'k.',label='Division')
plt.legend()
plt.savefig(str(select_option)+'.png')

# Split into train and test subsets (50% each)
XA, XB, yA, yB = train_test_split(X, y, test_size=0.5, shuffle=False)

# Plot regression results
def assess(P):
    plt.figure()
    plt.scatter(XB[P==1,0],XB[P==1,1],marker='^',color='blue',label='True')
    plt.scatter(XB[P==0,0],XB[P==0,1],marker='x',color='red',label='False')
    plt.scatter(XB[P!=yB,0],XB[P!=yB,1],marker='s',color='orange',alpha=0.5,label='Incorrect')
    if option not in ['moons','circles','blobs']:
        plt.plot(xplot,yplot,'k.',label='Division')
    plt.legend()

# Supervised Classification

# Logistic Regression
from sklearn.linear_model import LogisticRegression
lr = LogisticRegression(solver='lbfgs')
lr.fit(XA,yA)
yP = lr.predict(XB)
assess(yP)

# Naïve Bayes
from sklearn.naive_bayes import GaussianNB
nb = GaussianNB()
nb.fit(XA,yA)
yP = nb.predict(XB)
assess(yP)

# Stochastic Gradient Descent
from sklearn.linear_model import SGDClassifier
sgd = SGDClassifier(loss='modified_huber', shuffle=True,random_state=101)
sgd.fit(XA,yA)
yP = sgd.predict(XB)
assess(yP)

# K-Nearest Neighbors
from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier(n_neighbors=5)
knn.fit(XA,yA)
yP = knn.predict(XB)
assess(yP)

# Decision Tree
from sklearn.tree import DecisionTreeClassifier
dtree = DecisionTreeClassifier(max_depth=10,random_state=101,max_features=None,\
                       min_samples_leaf=5)
dtree.fit(XA,yA)
yP = dtree.predict(XB)
assess(yP)

# Random Forest
from sklearn.ensemble import RandomForestClassifier
rfm = RandomForestClassifier(n_estimators=70,oob_score=True,n_jobs=1,\
                  random_state=101,max_features=None,min_samples_leaf=3)
rfm.fit(XA,yA)
yP = rfm.predict(XB)
assess(yP)

# Support Vector Classifier
from sklearn.svm import SVC
svm = SVC(gamma='scale', C=1.0, random_state=101)
svm.fit(XA,yA)
yP = svm.predict(XB)
assess(yP)

# Neural Network
from sklearn.neural_network import MLPClassifier
clf = MLPClassifier(solver='lbfgs',alpha=1e-5,max_iter=200,\
                    activation='relu',hidden_layer_sizes=(10,30,10),\
                    random_state=1, shuffle=True)
clf.fit(XA,yA)
yP = clf.predict(XB)
assess(yP)

# Unsupervised Classification

# K-Means Clustering
from sklearn.cluster import KMeans
km = KMeans(n_clusters=2)
km.fit(XA)
yP = km.predict(XB)
# Arbitrary labels with unsupervised clustering may need to be reversed
if len(XB[yP!=yB]) > n/4: yP = 1 - yP
assess(yP)

# Gaussian Mixture Model
from sklearn.mixture import GaussianMixture
gmm = GaussianMixture(n_components=2)
gmm.fit(XA)
yP = gmm.predict_proba(XB) # produces probabilities
# Arbitrary labels with unsupervised clustering may need to be reversed
if len(XB[np.round(yP[:,0])!=yB]) > n/4: yP = 1 - yP
assess(np.round(yP[:,0]))

# Spectral Clustering
from sklearn.cluster import SpectralClustering
sc = SpectralClustering(n_clusters=2,eigen_solver='arpack',\
                        affinity='nearest_neighbors')
yP = sc.fit_predict(XB) # No separation between fit and predict calls, need to fit and predict on same dataset
# Arbitrary labels with unsupervised clustering may need to be reversed
if len(XB[yP!=yB]) > n/4: yP = 1 - yP
assess(yP)

plt.show()


import math

import numpy as np

from metagrad.tensor import Tensor

def test_simple_log():
    x = Tensor(10, requires_grad=True)
    z = x.log()

    np.testing.assert_array_almost_equal(z.data, math.log(10))

    z.backward()

    np.testing.assert_array_almost_equal(x.grad.data.tolist(), 0.1)

def test_array_log():
    x = Tensor([1, 2, 3], requires_grad=True)
    z = x.log()

    np.testing.assert_array_almost_equal(z.data, np.log([1, 2, 3]))

    z.backward([1, 1, 1])

    np.testing.assert_array_almost_equal(x.grad.data.tolist(), [1, 0.5, 1 / 3])


# -*- encoding: utf-8 -*-
'''
@File    :   LDA_multi_class.py
@Contact :   guzhouweihu@163.com

@Modify Time      @Author        @Version    @Desciption
------------      -----------    --------    -----------
2020/1/3 15:21   guzhouweihu      1.0         None
'''

import numpy as np
import matplotlib.pyplot as plt

def read_iris():
    from sklearn.datasets import load_iris
    from sklearn import preprocessing
    data_set = load_iris()
    data_x = data_set.data
    label = data_set.target + 1
    #preprocessing.scale(data_x, axis=0, with_mean=True, with_std=True, copy=False)
    return data_x, label

def class_mean(data, label, clusters):
    mean_vec = []
    for c in range(1, clusters + 1):
        mean_vec.append(np.mean(data[label == c, :], axis=0))

    return np.array(mean_vec)

def with_class_SW(data, label, clusters):
    m = data.shape[1]
    SW = np.zeros((m, m))
    mean_vec = class_mean(data, label, clusters)

    for c, mv in zip(range(1, clusters + 1), mean_vec):
        class_sc_mat = np.zeros((m, m))

        SW += (data[label == c] - mv).T @ (data[label == c] - mv)

    return SW

def between_class_SB(data, label, clusters):
    m = data.shape[1]
    all_mean = np.mean(data, axis=0)
    SB = np.zeros((m, m))
    mean_vec = class_mean(data, label, clusters)
    for c, mean_vec in enumerate(mean_vec):
        n = data[label == c + 1, :].shape[0]
        mean_vec = mean_vec[:, np.newaxis]
        all_mean = all_mean.reshape(-1, 1)
        SB += n * ((mean_vec - all_mean) @ (mean_vec - all_mean).T)

    return SB

def LDA(data, label, clusters):
    S_W = with_class_SW(data, label, clusters)
    S_B = between_class_SB(data, label, clusters)
    eig_vals, eig_vecs = np.linalg.eig(np.linalg.inv(S_W) @ S_B)

    for i in range(len(eig_vals)):
        eigvec_sc = eig_vecs[:, i].reshape(4, 1)
        print('\nEigenvector {}: \n{}'.format(i + 1, eigvec_sc.real))
        print('Eigenvalue {:}: {:.2e}'.format(i + 1, eig_vals[i].real))

    eig_pairs = [(np.abs(eig_vals[i]), eig_vecs[:, i])
                 for i in range(len(eig_vals))]
    eig_pairs = sorted(eig_pairs, key=lambda k: k[0], reverse=True)
    W = np.hstack((eig_pairs[0][1].reshape(-1, 1),
                   eig_pairs[1][1].reshape(-1, 1)))
    print('Matrix W:\n', W.real)
    print(data.dot(W))
    return W

def plot_lda():
    data, labels = read_iris()
    W = LDA(data, labels, 3)

    Y = data.dot(W)
    # print Y
    ax = plt.subplot(111)
    for label, marker, color in zip(
            range(1, 4), ('^', 's', 'o'), ('blue', 'red', 'green')):
        plt.scatter(x=Y[:, 0][labels == label],
                    y=Y[:, 1][labels == label],
                    marker=marker,
                    color=color,
                    alpha=0.5,
                    )
    plt.xlabel('LDA1')
    plt.ylabel('LDA2')
    plt.title('LDA: Iris projection onto the first 2 linear discriminants')
    plt.show()

def default_plot_lda():
    Y = sklearnLDA()
    data, labels = read_iris()
    ax = plt.subplot(111)
    for label, marker, color in zip(
            range(1, 4), ('^', 's', 'o'), ('blue', 'red', 'green')):
        plt.scatter(x=Y[:, 0][labels == label],
                    y=Y[:, 1][labels == label],
                    marker=marker,
                    color=color,
                    alpha=0.5,
                    )
    plt.xlabel('LDA1')
    plt.ylabel('LDA2')
    plt.title('LDA:default')

    plt.show()

def sklearnLDA():
    from sklearn import datasets
    from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA

    iris = datasets.load_iris()

    X = iris.data
    y = iris.target
    target_names = iris.target_names

    lda = LDA(n_components=2)
    X_r2 = lda.fit(X, y).transform(X)
    return X_r2

if __name__ == "__main__":
    # lda()
    sklearnLDA()
    plot_lda()
    default_plot_lda()


from numpy import *
from math import sqrt

# Input: expects Nx3 matrix of points
# Returns R,t
# R = 3x3 rotation matrix
# t = 3x1 column vector

def rigid_transform_3D(A, B):
    assert len(A) == len(B)

    N = A.shape[0]; # total points

    centroid_A = mean(A, axis=0)
    centroid_B = mean(B, axis=0)

    # centre the points
    AA = A - tile(centroid_A, (N, 1))
    BB = B - tile(centroid_B, (N, 1))

    # dot is matrix multiplication for array
    H = transpose(AA) * BB

    U, S, Vt = linalg.svd(H)

    R = Vt.T * U.T

    # special reflection case
    if linalg.det(R) < 0:
       print "Reflection detected"
       Vt[2,:] *= -1
       R = Vt.T * U.T

    t = -R*centroid_A.T + centroid_B.T

    print t

    return R, t

# Test with random data

# Random rotation and translation
R = mat(random.rand(3,3))
t = mat(random.rand(3,1))

# make R a proper rotation matrix, force orthonormal
U, S, Vt = linalg.svd(R)
R = U*Vt

# remove reflection
if linalg.det(R) < 0:
   Vt[2,:] *= -1
   R = U*Vt

# number of points
n = 10

A = mat(random.rand(n,3));
B = R*A.T + tile(t, (1, n))
B = B.T;

# recover the transformation
ret_R, ret_t = rigid_transform_3D(A, B)

A2 = (ret_R*A.T) + tile(ret_t, (1, n))
A2 = A2.T

# Find the error
err = A2 - B

err = multiply(err, err)
err = sum(err)
rmse = sqrt(err/n);

print "Points A"
print A
print ""

print "Points B"
print B
print ""

print "Rotation"
print R
print ""

print "Translation"
print t
print ""

print "RMSE:", rmse
print "If RMSE is near zero, the function is correct!"


# TEST: Numeric Integration using Trapezoidal and Simpson methods

import matplotlib.pyplot as plt
import numpy as np
from quad import quad_trapezoid, quad_simpson, quad_romberg

#f = lambda x: 0.2 + 25*x - 200*x**2 + 675*x**3 - 900*x**4 + 400*x**5
#a = 0.0;
#b = 0.8;
#Iexp = 1.640533  # Expected integral value

f = lambda x: np.sin(x)
a = 0.0;
b = np.pi
Iexp = -np.cos(b) + np.cos(a)  # Expected integral value

# n is the number of integration nodes (segments = n-1)
for n in range(2, 11, 2):
    print('-----------------------------------')
    print('n = {}'.format(n))
    # Trapezoid method with relative error
    I = quad_trapezoid(a, b, f, n-1)
    E = abs(I - Iexp) / Iexp
    print("> Trapezoid I = {}, E = {}".format(I, E));
    # Simpson method with relative error
    I = quad_simpson(a, b, f, n-1)
    E = abs(I - Iexp) / Iexp
    print("> Simpson   I = {}, E = {}".format(I, E));
    # Romberg method with relative error
    I = quad_romberg(a, b, f, n-1)
    E = abs(I - Iexp) / Iexp
    print("> Romberg   I = {}, E = {}".format(I, E));

# Plot using the last approximation number of nodes
N = 1000    # Number of points for a smooth function plot
X = np.linspace(a, b, N)
plt.plot(X, f(X))
X = np.linspace(a, b, n)
for x in X:
    plt.plot([x,x], [0, f(x)],
             color='black', linestyle='dashed', linewidth=1)
plt.plot(X, f(X), color='black', linestyle='dashed', linewidth=1)
plt.grid(True)
plt.show()


import numpy
import scipy.spatial as spatial
import pylab

sigma = 10.

def K(Xi, Xj, s):
    return numpy.exp( - ( (numpy.linalg.norm(Xi - Xj) ** 2) / (2 * (s**2)) ) )

# Entrée :
#  - X l'ensemble des points
#  - k le nombre de clusters à former
#  - assign_0 l'assignation courante des points à un cluster
#  - M la matrice des distances entre les points
# Sortie :
#  - assign la nouvelle assignation des points à un cluster
#  - cost la fonction de cout (doit etre décroissante)
def ppcentroide(X, k, assign_0, M):
    global sigma
    n = len(X)
    cost = 0.

    # assign[i] = k ssi le point i est dans le cluster k
    assign = numpy.zeros((n, ), dtype=numpy.int32)

    # Dist_k[k] = inertie interne au cluster k
    # Size_k[k] = taille du cluster k
    Dist_k = numpy.zeros((k, ))
    Size_k = numpy.zeros((k, ), dtype=numpy.int32)

    #    # Calcul des deux quantités définies ci-dessus
    #    for i_k in range(k):
    #        d_tmp = 0
    #        C_size = 0
    #        for j1 in range(n):
    #            if (assign_0[j1] == i_k):
    #                C_size +=1
    #                for j2 in range(n):
    #                    if (assign_0[j2] == i_k):
    #                        d_tmp += M[j1][j2]
    #        Size_k[i_k] = C_size
    #        Dist_k[i_k] = d_tmp/C_size**2

    # Calcul des deux quantités définies ci-dessus
    for i in range(n):
        i_k = assign_0[i]
        Size_k[i_k] += 1
        Dist_k[i_k] += M[i][i]
        for j in range(i+1,n):
            if i_k == assign_0[j]:
                Dist_k[i_k] += 2*M[i][j]
    for i in range(k):
        Dist_k[i] /= Size_k[i]**2

    # Calcul des nouvelles assignations
    for i in range(n):
        # d_min = "distance" de i au cluster auquel il est assigné pour le moment
        d_min = numpy.inf
        # Pour chaque cluster k...
        for i_k in range(k):
            # d_k = "distance" de i au cluster k
            d_k = 0

            d_tmp = 0
            for j in range(n):
                if assign_0[j] == i_k:
                    d_tmp += M[i][j]
            d_k += -2*d_tmp/Size_k[i_k]

            d_k += M[i][i]

            d_k += Dist_k[i_k]

            if d_k <= d_min:
                assign[i] = i_k
                d_min = d_k
        cost += d_min

    return assign,cost

def kmeans(X, k, n_iter):
    n = len(X)
    cost = numpy.inf

    # Pré-calcul des distances entre les clusters
    M = [[K(X[i], X[j], sigma) for j in range(n)] for i in range(n)]

    # Assignation aléatoire des points aux clusters
    assign = [numpy.random.randint(0,k) for i in range(n)]
    for i in range(k):
        assign[i] = i # Pour etre sur d'avoir au moins 1 point par cluster

    for iter in range(n_iter):
        assign,cost = ppcentroide(X, k, assign, M)
        print(iter, cost)
    assign,cost = ppcentroide(X, k, assign, M)
    return assign,cost

### Zone de tests

# Fonctions utiles
def read_data(file_name, X):
    f = open(file_name, 'r')
    data = f.readlines()
    for line in data:
        X.append((map(float, filter(lambda x: x <> '\n', line.split("\t")))))
    f.close()

def write_cluster(file_name, assign, n_cluster, n_iter, s):
    n = len(assign)
    f = open(file_name, 'w')
    f.write("n_cluster = " + str(n_cluster) + "\n")
    f.write("n_iter = " + str(n_iter) + "\n")
    f.write("n_data = " + str(n) + "\n")
    f.write("seed = " + str(s) + "\n")
    for i in range(n):
        f.write(str(assign[i]) + "\n")
    f.close()

# Vrais tests

f = 'cinq_classes1'

X = []
read_data('./data/' + f + '/' + f + '.txt',X)

s = 0
n = len(X)
k = 5
n_iter = 10

numpy.random.seed(s)
assign,cost = kmeans(numpy.array(X), k, n_iter)

pylab.figure()
colors = "rbgkmc"
for i_k in range(k):
    pylab.plot([X[i][0] for i in range(n) if assign[i] == i_k],
               [X[i][1] for i in range(n) if assign[i] == i_k],
               colors[i_k] + ".")
pylab.show()

write_cluster('./data/' + f + '/' + 'kernel' + '.txt', assign, k, n_iter, s)


# https://github.com/zhangqi-chen/Algorithms-Stanford/blob/master/4%20Shortest%20Paths%20Revisited%2C%20NP-Complete%20Problems%20and%20What%20To%20Do%20About%20Them/Week%202/4wk1_a2.py
# 愚直 O(N!)→動的計画法bitDP O(2ˆN * Nˆ2)
import numpy as np
from itertools import combinations

f = open('sample-data/tsp.txt', 'r')
ls = f.readlines()[1:]
graph = [list(map(float, i.split(' '))) for i in ls]

def dist(i, j):
    return np.sqrt((graph[i][0]-graph[j][0])**2+(graph[i][1]-graph[j][1])**2)

N = len(graph)
dic1 = {frozenset([0]): {0: 0}}

for m in range(1, N):
    comb = list(combinations(range(1, N), m))
    dic2 = {frozenset(comb[i]): {list(comb[i])[j]: 0 for j in range(m)} for i in range(len(comb))}
    print(m, len(dic2))
    for s in dic2:
        for j in s:
            ans = []
            if m == 1:
                dic2[s][j] = dist(0, j)
            else:
                sj = set(s)
                sj.remove(j)
                dic2[s][j] = min([dic1[frozenset(sj)][k]+dist(k, j) for k in sj if k != j])
    dic1 = dic2.copy()

tsp = min([dic2[frozenset(comb[0])][j]+dist(0, j) for j in range(1, N)])
print([dic2[frozenset(comb[0])][j]+dist(0, j) for j in range(1, N)])
print(tsp)


#############################################
# University of Idaho                       #
# Carlos Santos                             #
# ECE 351-51                                #
# Lab 10                                    #
# 4/8/2020                                  #
#                                           #
#                                           #
#############################################

import numpy as np
import matplotlib.pyplot as plt
import scipy.signal as sig
import control as con

# RLC Circuit Parameters
R = 1000
L = 27e-3
C = 100e-9

#%% 3.3.1, 3.3.2, 3.3.3
steps = 1e3
omega = np.arange(1e3,1e6+steps,steps)

mag_H = 20*np.log10((omega/(R*C))/(np.sqrt(((1/(L*C))-omega**2)**2 + (omega/(R*C))**2)))
phase_H = ((np.pi/2)-np.arctan((omega/(R*C))/(-omega**2 + 1/(L*C)))) * 180/np.pi # phase is in degrees

for i in range(len(phase_H)):
    if (phase_H[i] > 90):
        phase_H[i] = phase_H[i] - 180

plt.figure(figsize=(10,7))
plt.subplot(2,1,1) #Top Figure
plt.ylabel("Mag")
plt.grid()
plt.semilogx(omega, mag_H)
plt.title("hand solved")
plt.subplot(2,1,2) # Bottom Figure
plt.xlabel("w")
plt.ylabel("Phase")
plt.grid()
plt.semilogx(omega, phase_H)
plt.show()

num_H = [1/(R*C),0]
den_H = [1,1/(R*C),1/(L*C)]
sys = sig.TransferFunction(num_H,den_H)
ang,mag,phase = sig.bode(sys,omega)

plt.figure(figsize=(10,7))
plt.subplot(2,1,1) #Top Figure
plt.ylabel("Mag")
plt.grid()
plt.semilogx(ang, mag)
plt.title("sig.bode()")
plt.subplot(2,1,2) # Bottom  Figure
plt.xlabel("w")
plt.ylabel("Phase")
plt.grid()
plt.semilogx(ang, phase)
plt.show()

sys = con.TransferFunction(num_H,den_H)
_ = con.bode(sys, omega, dB=True,Hz=True, deg=True, Plot=True)

#%% 4.4.1, 4.4.2, 4.4.3, 4.4.4

steps = 1e-9
t = np.arange(0, 0.01+steps, steps)
x = (np.cos(2*np.pi*100*t)+ np.cos(2*np.pi*3024*t) + np.sin(2*np.pi* 50e3*t))

num_Z, den_Z = sig.bilinear(num_H, den_H, 1/steps)
xFiltered = sig.lfilter(num_Z, den_Z, x)

plt.figure(figsize=(10,7))
plt.subplot(2,1,1) #Top Figure
plt.grid()
plt.plot(t,x)
plt.title("Unfiltered Signal")
plt.subplot(2,1,2)
plt.plot(t,xFiltered)
plt.title("Filtered Signal")
plt.grid()
plt.show()


# Plots L2, L1 and Huber losses.

import numpy as np
import matplotlib.pyplot as plt
import os
figdir = os.path.join(os.environ["PYPROBML"], "figures")
def save_fig(fname): plt.savefig(os.path.join(figdir, fname))

delta = 1.5
huber = lambda x: (x**2/2) * (x <= delta) + (delta * abs(x) - delta**2/2) * (x > delta)
l2 = lambda x: abs(x)**2
l1 = abs

funs = [l2, l1, huber]
styles = ['r-', 'b:', 'g-.']
labels = ['l2', '11', 'huber']
x = np.arange(-3, 3, .01)

for i, fun in enumerate(funs):
  plt.plot(x, fun(x), styles[i], label=labels[i])

plt.axis([-3, 3, -0.5, 5])
plt.legend()
save_fig(os.path.join(figdir, 'huberLossPlot.pdf'))
plt.show()


import numpy as np

def init_clusters(n_clusters, n_features):
    return np.random.randint(low=0, high=255, size=(n_clusters, n_features))

def k_means(X, n_clusters, distance_metric):
    n_samples, n_features = X.shape
    classification = np.zeros(n_samples)
    clusters = init_clusters(n_clusters, n_features)
    distance = np.zeros((n_clusters, n_samples))

    while True:
        for i, c in enumerate(clusters):
            distance[i] = distance_metric(X, c)
        new_classification = np.argmin(distance, axis=0)
        if np.sum(new_classification != classification) == 0:
            break
        classification = new_classification
        for i in range(n_clusters):
            mask = classification == i
            total_classified = np.sum(mask)
            is_empty_cluster = total_classified == 0
            if not is_empty_cluster:
                clusters[i] = np.sum(X[mask], axis=0) / total_classified
            else:
                clusters[i] = X[np.argmax(distance[i])]
    return classification, clusters


import numpy as np
from nayzakflow.utils import _onehot

def sigmoid(z):
    return (1/(1+np.exp(-1*z)))

def _diff_sigmoid(z):
    return sigmoid(z)*(1-sigmoid(z))

def tanh(z):
    return np.tanh(z)

def _diff_tanh(z):
    return 1-np.square(tanh(z))

def relu(z):
    return np.maximum(0,z)

def _diff_relu(z):
    a= np.zeros_like(z,dtype='int')
    a[z>0] = 1
    return a

def leaky_relu(z):
    return np.maximum(z,0.1*z)

def _diff_leaky_relu(z):
    a= np.zeros_like(z,dtype='int')+0.1
    a[z>0] = 1
    return a

def identity(z):
    return z

def _diff_identity(z):
    return 1

def softmax(z):
    exp = np.exp(z)
    tot= exp.sum(axis=0)
    t= exp/tot
    return t

def _diff_softmax(z,y):
    yhat_r = softmax(z)
    onehotY = _onehot(y,z.shape[0])
    one_yi = onehotY *-1*(1-yhat_r)
    z=(1-onehotY)*yhat_r
    return one_yi +z

def get_activations():
    return {"identity":identity, "relu":relu, "softmax":softmax,"tanh":tanh, "sigmoid":sigmoid, "leaky_relu":leaky_relu}

def get_activations_diff():
    return {"identity":_diff_identity, "relu":_diff_relu, "softmax":_diff_softmax,"tanh":_diff_tanh, "sigmoid":_diff_sigmoid, "leaky_relu":_diff_leaky_relu}


# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.

import numpy as np
import scipy.linalg as la
from typing import Tuple

def procrustes_error(
        target_matrix: np.ndarray,
        matrix_to_rotate: np.ndarray
) -> Tuple[np.ndarray, np.ndarray]:
    """
    Procrustes rotation rotates a matrix to maximum similarity with a target matrix minimizing sum of squared
    differences. Procrustes rotation is typically used in comparison of ordination results. It is particularly useful
    in comparing alternative solutions in multidimensional scaling.

    For more information:
    https://www.rdocumentation.org/packages/vegan/versions/2.4-2/topics/procrustes

    :param numpy.ndarray target_matrix: A matrix representing an embedding
    :param numpy.ndarray matrix_to_rotate: A matrix representing an embedding which will be rotated
    :return: The error which is the difference between the two matrices and the transformation matrix
    """
    dimension = min(target_matrix.shape[1], matrix_to_rotate.shape[1])
    reduced_target_matrix = target_matrix[:, :dimension]
    reduced_matrix_to_rotate = matrix_to_rotate[:, :dimension]

    tmp = np.dot(reduced_target_matrix.T, reduced_matrix_to_rotate)

    left_singular_vectors, eigenvalues, right_singular_vectors = la.svd(tmp)
    w = left_singular_vectors.dot(right_singular_vectors)
    error = la.norm(reduced_target_matrix.dot(w) - reduced_matrix_to_rotate)

    return error, w


import numpy as np
from scipy.stats import chi2

class MahalanobisDistance:

    """
    Description:
    ------------

    Class that computes the mahalanobis distance in order to detect multidimensional outliers.
    For more info, check:

        https://en.wikipedia.org/wiki/Mahalanobis_distance
        https://blogs.sas.com/content/iml/2012/02/15/what-is-mahalanobis-distance.html

    Parameters:
    -----------

        p_value [optional float]: probability of obtaining test results at least as extreme as the results actually
                         observed, under the assumption that the null hypothesis is correct

                          H0: mu_i = mu_j / i != j
                          H1: mu_i != mu_j

                          * mu_i -> mean of i

                         All the records that have a p-value lower than self.p_value will be detected as outlier

    Important:
    ----------

    This distance only works with 2 or more variables. In case of using 1 variable it will not work and it should be
    fine using the z_score.
    """

    def __init__(self, p_value=0.05):

        self.p_value = p_value

    def compute_mahalanobis_individual(self, vector, means_vector, cov_matrix):

        x_mu = vector - means_vector
        inv_covmat = np.linalg.inv(cov_matrix)

        distance = np.dot(np.dot(x_mu, inv_covmat), x_mu.T)
        p_value = 1 - chi2.cdf(distance, len(vector) - 1)

        if p_value < self.p_value:
            print(f"The record {vector} was detected as an outlier")

    def multidimensional_outliers(self, data, means_vector, cov_matrix):

        for i in range(data.shape[1]):
            self.compute_mahalanobis_individual(data[:, i], means_vector, cov_matrix)


import numpy as np
import matplotlib.pyplot as plt
import matplotlibex as plx

def kern_RBF(x1, x2):
    return np.exp(-20.5*np.sum((x1-x2)**2))

def gramm_matrix(f_kern, X, Xprime=None):
    if Xprime is None:
        Xprime = X
    N = X.shape[0]
    K = np.zeros([N, N])
    K = np.array([[f_kern(xi, xj) for xj in Xprime] for xi in X])
    return K

def gp_sample(mean, cov):
    return np.random.multivariate_normal(mean=mean, cov=cov)

def gp_conditional_pdf(f_kern, Xstar, X, Y, f_mean=None):
    if f_mean is None:
        f_mean = lambda x: 0
    Kxx = gramm_matrix(f_kern, X) + 1.0e-2 * np.diag(np.ones(X.shape[0]))
    Kxstarx = gramm_matrix(f_kern, Xstar, X)
    Ystar_mean = f_mean(Xstar) + Kxstarx.dot(np.linalg.inv(Kxx).dot(Y))
    Ystar_cov = gramm_matrix(f_kern, Xstar) - Kxstarx.dot(np.linalg.inv(Kxx).dot(Kxstarx.T))
    return Ystar_mean, Ystar_cov

if __name__ == "__main__":
    np.random.seed(5)
    N = 100
    D = 1
    X = np.reshape(np.random.uniform(size=N*D), [N, D])
    X = np.sort(X, axis=0)
    K = gramm_matrix(f_kern=kern_RBF, X=X)
    Y = gp_sample(mean=np.zeros(N), cov=K)
    np.random.seed(12)
    istart = [10, 41, 95, 70, 80]
    a = np.array(list(set(range(N)) - set(istart)))
    inds = np.concatenate([istart, np.random.choice(a=a, size=N-len(istart), replace=False)])
    Ninds = [0, 1, 2, 3, 4, 5, 10, N]
    for Nind in Ninds:
        ind = np.sort(inds[:Nind])
        Xind = X[ind]
        Yind = Y[ind]
        Xstar = np.linspace(0, 1.0, N)
        Ystar_mean, Ystar_cov = gp_conditional_pdf(f_kern=kern_RBF, Xstar=Xstar, X=Xind, Y=Yind)
        Ystar_err = np.diag(Ystar_cov) + 0.16
        fig = plt.figure(figsize=(5, 5))
        plt.title("#IP={}".format(Nind))
        plt.plot(X, Y + 0.1*np.random.normal(size=N), "+r")
        plt.plot(Xstar, Ystar_mean)
        plt.plot(Xind, Yind, "ok")
        plt.fill(np.concatenate([Xstar, Xstar[::-1]]),
                np.concatenate([Ystar_mean - Ystar_err, (Ystar_mean + Ystar_err)[::-1]]), alpha=.2)
        plt.ylim([-3, 1.5])
        plt.savefig("./IP({}).pdf".format(Nind))
        plt.close(fig)


import numpy as np
import random
import matplotlib.pyplot as plt
import matplotlib
import pandas
import cv2
import tkinter
from copy import deepcopy
from scipy.io import loadmat
# matplotlib.use('TkAgg')

def isclose(a, b, rtol=1e-05, atol=1e-08):
    return np.abs(a-b) <= (atol + rtol * np.abs(b))

def sigma(z):
	return 1/(1 + np.exp(-z))

def mapFeature(X, p):
    m = len(X)
    X_poly = np.zeros((m, p))
    X_poly[:, 0] = X
    for i in range(1, p):
        X_poly[:, i] = X * X_poly[:, i-1]
    return X_poly

data=loadmat('ex5data1')
x = np.array(data['X'].flatten())
y = np.array(data['y'].flatten())

x_train = deepcopy(x)
y_train = deepcopy(y)

x_val = np.array(data['Xval'].flatten())
y_val = np.array(data['yval'].flatten())
x_test = np.array(data['Xtest'].flatten())
y_test = np.array(data['ytest'].flatten())

m = len(x)

x = mapFeature(x, 8)
x_val = mapFeature(x_val, 8)
x_test = mapFeature(x_test, 8)

x = [[(x[i][j]-x[:, j].mean())/x[:, j].std() for j in range(len(x[i]))] for i in range(len(x))]
x_val = [[(x_val[i][j]-x_val[:, j].mean())/x_val[:, j].std() for j in range(len(x_val[i]))] for i in range(len(x_val))]
x_test = [[(x_test[i][j]-x_test[:, j].mean())/x_test[:, j].std() for j in range(len(x_test[i]))] for i in range(len(x_test))]

x = np.array([np.insert(i, 0, 1) for i in x])
x_val = np.array([np.insert(i, 0, 1) for i in x_val])
x_test = np.array([np.insert(i, 0, 1) for i in x_test])

theta = np.ones(9)
lr = 0.001
alpha = 0.001
def cost_function(X, y, thetas, rl):
    m = len(y)
    a = X.dot(theta.T)
    return np.sum(np.square(a - y))/(2 * m) + rl / (2 * m) * np.sum(np.square(theta[1:]))

def gradient(x, y, theta, rl):
    a = x.dot(theta.T)
    dt = x.T.dot(a - y)/m + theta.dot(rl / m) #rl / m * np.sum(theta[1:])
    dt[0] = x.T.dot(a - y)[0]/m
    return dt

m = len(y)
costs = np.zeros(m)
val_costs = np.zeros(m)
for i in range(m):
    iterations = 5000
    theta = np.zeros(9)

    for j in range(iterations):
        new_theta = gradient(x[0:i+1, :], y[0:i+1], theta, 0)
        theta -= alpha * new_theta
    costs[i] = cost_function(x[0:i+1, :], y[0:i+1], theta, 0)
    val_costs[i] = cost_function(x_val, y_val, theta, 0)

plt.plot(np.arange(1, m+1), costs, label = "Train")
plt.plot(np.arange(1, m+1), val_costs, label = "Val")

plt.title('Learning curve for linear regression')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.show()
print(theta)
rls = [0, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10]
errors = []
val_errors = []
for rl in rls:
    iterations = 10000
    theta = np.zeros(9)
    for j in range(iterations):
        new_theta = gradient(x, y, theta, rl)
        theta -= alpha * new_theta
    errors.append(cost_function(x, y, theta, rl))
    val_errors.append(cost_function(x_val, y_val, theta, rl))

plt.plot(rls, errors, 'r', label = "Train")
plt.plot(rls, val_errors, 'b', label = "Train")
plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', borderaxespad=0.)
plt.title('Selecting lambda')
plt.xlabel('Lambda')
plt.ylabel('Error')
plt.show()
# rl = 100
# m = len(y)
# costs = np.zeros(m)
# val_costs = np.zeros(m)
# for i in range(m):
#     iterations = 5000
#     theta = np.zeros(9)

#     for j in range(iterations):
#         new_theta = gradient(x[0:i+1, :], y[0:i+1], theta, rl)
#         theta -= alpha * new_theta
#     costs[i] = cost_function(x[0:i+1, :], y[0:i+1], theta, rl)
#     val_costs[i] = cost_function(x_val, y_val, theta, rl)

# plt.plot(np.arange(1, m+1), costs, 'r', label = "Train")
# plt.plot(np.arange(1, m+1), val_costs, 'b', label = "Val")
# plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', borderaxespad=0.)
# plt.title('Learning curve for linear regression')
# plt.xlabel('Number of training examples')
# plt.ylabel('Error')
# plt.show()
# print(theta)
# plt.plot(x_train, y_train, 'rx')
# plt.plot(x_train, x.dot(theta.T), 'bo')
# plt.show()


# Example of fruits classification using normalization preprocessing

import numpy as np
import pandas as pd
import seaborn as sn
import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import MinMaxScaler

# read dataset
fruits = pd.read_table('readonly/fruit_data_with_colors.txt')

feature_names_fruits = ['height', 'width', 'mass', 'color_score']
X_fruits = fruits[feature_names_fruits]
y_fruits = fruits['fruit_label']
target_names_fruits = ['apple', 'mandarin', 'orange', 'lemon']

X_fruits_2d = fruits[['height', 'width']]
y_fruits_2d = fruits['fruit_label']

# train test split
X_train, X_test, y_train, y_test = train_test_split(X_fruits, y_fruits, random_state=0)

# scale dataset (normalization)
scaler = MinMaxScaler()
X_train_scaled = scaler.fit_transform(X_train) # find params for the scale based on train set and then apply them
X_test_scaled = scaler.transform(X_test) # apply scaling

# define knn model with 5 nearest neighbours
knn = KNeighborsClassifier(n_neighbors = 5)
knn.fit(X_train_scaled, y_train)

# check train and test accuracy
print('Accuracy of K-NN classifier on training set: {:.2f}'.format(knn.score(X_train_scaled, y_train)))
print('Accuracy of K-NN classifier on test set: {:.2f}'.format(knn.score(X_test_scaled, y_test)))

# do prediction with a new sample
example_fruit = [[5.5, 2.2, 10, 0.70]]
example_fruit_scaled = scaler.transform(example_fruit)
print('Predicted fruit type for ', example_fruit, ' is ', target_names_fruits[knn.predict(example_fruit_scaled)[0]-1])


import numpy as np

def gauss(win, sigma):

    x = np.arange(0, win, 1, float)
    y = x[:,np.newaxis]
    x0 = y0 = win // 2

    g=1/(2*np.pi*sigma**2)*np.exp((((x-x0)**2+(y-y0)**2))/2*sigma**2)
    return g

def gaussx(win, sigma):

    x = np.arange(0, win, 1, float)
    y = x[:,np.newaxis]
    x0 = y0 = win // 2

    gx=(x-x0)/(2*np.pi*sigma**4)*np.exp((((x-x0)**2+(y-y0)**2))/2*sigma**2)
    return gx

def gaussy(win, sigma):

    x = np.arange(0, win, 1, float)
    y = x[:,np.newaxis]
    x0 = y0 = win // 2

    gy=(y-y0)/(2*np.pi*sigma**4)*np.exp((((x-x0)**2+(y-y0)**2))/2*sigma**2)
    return gy