"""
    Mnożenie macierzy przez wektor:
    Mając wektor n-elementowy x oraz macierz A o wymiarach m×n napisać funkcję obliczającą ich iloczyn:
                                    y = Ax

    1) Za pomocą iteracji po elementach (podwójna pętla)
    2) korzystając z faktu, że każdy element wektora yi jest iloczynem skalarnym i-tego rzędu macierzy A oraz wektora x (pojedyncza pętla)
    3) używając funkcji: np.dot lub np.tensordot (bez pętli)
"""

import numpy as np

MESSAGE_REGARDING_DIFFERENT_DIMENSIONS = "The number of columns in the matrix " \
                                         "should be the same as the length of the vector."

def y1(A, x):
    # liczba kolumn(n) w macierzu A
    n = A.shape[1]

    # Jeżeli liczba kolumn w macierzy A nie jest równa długości wektora x
    # wyrzuć wyjątek
    if n != len(x):
        raise ValueError(MESSAGE_REGARDING_DIFFERENT_DIMENSIONS)

    y = []

    # Przechodzimy przez kolejne wiersze macierzy A
    for row in A:

        # Obliczanie każdego kolejnego elementu wektora y,
        # w którym i-ty element to iloczyn skalarny wektora x i i-tego wiersza macierza A
        scalar_product = 0

        for idx in range(n):
            scalar_product += row[idx] * x[idx]

        y.append(scalar_product)

    return y

def y2(A, x):
    # liczba kolumn(n) w macierzu A
    n = A.shape[1]

    # Jeżeli liczba kolumn w macierzy A nie jest równa długości wektora x
    # wyrzuć wyjątek
    if n != len(x):
        raise ValueError(MESSAGE_REGARDING_DIFFERENT_DIMENSIONS)

    y = []

    # Przechodzimy przez kolejne wiersze macierzy A
    for row in A:
        # Obliczanie każdego kolejnego elementu wektora y ( za pomocą funkcji z pakietu numpy (inner)),
        # w którym i-ty element to iloczyn skalarny wektora x i i-tego wiersza macierza A
        scalar_product = np.inner(row, x)
        y.append(scalar_product)

    return y

def y3(A, x):
    # liczba kolumn(n) w macierzu A
    n = A.shape[1]

    # Jeżeli liczba kolumn w macierzy A nie jest równa długości wektora x
    # wyrzuć wyjątek
    if n != len(x):
        raise ValueError(MESSAGE_REGARDING_DIFFERENT_DIMENSIONS)

    return np.dot(A, x)

# Tests
A = np.arange(9).reshape(3, 3)
x = np.arange(3)
np.testing.assert_almost_equal(y1(A, x), np.array([5, 14, 23]))
np.testing.assert_almost_equal(y2(A, x), np.array([5, 14, 23]))
np.testing.assert_almost_equal(y3(A, x), np.array([5, 14, 23]))

try:
    y1(np.arange(9).reshape(3, 3), np.array([1, 1]))
except (BaseException, Exception) as error:
    if type(error) != ValueError or \
            str(error) != MESSAGE_REGARDING_DIFFERENT_DIMENSIONS:
        raise AssertionError("Second dimension of matrix should "
                             "be the same as dimension of vector!.")

try:
    y2(np.arange(9).reshape(3, 3), np.array([1, 1]))
except (BaseException, Exception) as error:
    if type(error) != ValueError or \
            str(error) != MESSAGE_REGARDING_DIFFERENT_DIMENSIONS:
        raise AssertionError("Second dimension of matrix should "
                             "be the same as dimension of vector!.")

try:
    y3(np.arange(9).reshape(3, 3), np.array([1, 1]))
except (BaseException, Exception) as error:
    if type(error) != ValueError or \
            str(error) != MESSAGE_REGARDING_DIFFERENT_DIMENSIONS:
        raise AssertionError("Second dimension of matrix should "
                             "be the same as dimension of vector!.")

print("All tests were successful.")


# -*- coding: utf-8 -*-
"""
Created on Sat Sep 26 22:11:55 2015

@author: Nathan
"""

from __future__ import print_function
import numpy as np

try:
	from numba import autojit
except:
	def autojit(a):
		return a

@autojit
def EulerX(beta, mode='rad', clockwise=False):
	# Return the Euler matrix for rotation about the X axis
	if mode == 'deg':
		beta = np.deg2rad(beta)

	if not clockwise:
		x_rot_mat = np.array([	[1, 	0, 				0			],
								[0, 	np.cos(beta), 	np.sin(beta)],
								[0, 	-np.sin(beta), 	np.cos(beta)]	])
	elif clockwise:
		x_rot_mat = np.array([	[1, 	0, 				0			],
								[0, 	np.cos(beta),	-np.sin(beta)],
								[0, 	np.sin(beta), 	np.cos(beta)]	])

	return x_rot_mat

@autojit
def EulerY(theta, mode='rad', clockwise=False):
	# Return the Euler matrix for rotation about the Y axis
	if mode == 'deg':
		theta = np.deg2rad(theta)

	if not clockwise:
		y_rot_mat = np.array([	[np.cos(theta), 0, 	-np.sin(theta)	],
								[0, 			1,	0				],
								[np.sin(theta), 0, 	np.cos(theta)	]	])
	elif clockwise:
		y_rot_mat = np.array([	[np.cos(theta), 0, 	np.sin(theta)	],
								[0, 			1,	0				],
								[-np.sin(theta), 0, 	np.cos(theta)]	])

	return y_rot_mat

@autojit
def EulerZ(alpha, mode='rad', clockwise=False):
	# Return the Euler matrix for rotation about the Z axis
	if mode == 'deg':
		theta = np.deg2rad(alpha)

	if not clockwise:
		z_rot_mat = np.array([	[np.cos(alpha),		np.sin(alpha),	0],
								[-np.sin(alpha),	np.cos(alpha),	0],
								[0,					0,				1]	])
	elif clockwise:
		z_rot_mat = np.array([	[np.cos(alpha),		-np.sin(alpha),	0],
								[np.sin(alpha),		np.cos(alpha),	0],
								[0,					0,				1]	])

	return z_rot_mat

@autojit
def rotx(points, beta, mode='deg'):
	"""
	Rotate points about the global X axis by the angles beta (roll).  Rotation
	matrices are sourced from http://mathworld.wolfram.com/RotationMatrix.html
	"""

	if mode == 'rad':
		pass
	elif mode == 'deg':
		beta	=	np.deg2rad(beta)
	else:
		print('ERROR: Incorrect angle type specified.  Assuming degrees.')

	# Rotation about X axis
	x_rot_mat = np.array([	[1, 	0, 				0			],
							[0, 	np.cos(beta), 	np.sin(beta)],
							[0, 	-np.sin(beta), 	np.cos(beta)]	])

	# Sequentially rotate input points about X, Y and then Z axes
	if np.size(points) == 3:
		rot_x 	= np.dot(points, x_rot_mat)
	else:
		rot_x 	= np.dot(points, x_rot_mat)

	return rot_x

@autojit
def roty(points, theta, mode='deg'):
	"""
	Rotate points about the global Y axis by the angles theta (yaw).  Rotation
	matrices are sourced from http://mathworld.wolfram.com/RotationMatrix.html
	"""

	if mode == 'rad':
		pass
	elif mode == 'deg':
		theta	=	np.deg2rad(theta)
	else:
		print('ERROR: Incorrect angle type specified.  Assuming degrees.')

	# Rotation about Y axis
	y_rot_mat = np.array([	[np.cos(theta), 0, 	-np.sin(theta)	],
							[0, 			1,	0				],
							[np.sin(theta), 0, 	np.cos(theta)	]	])

	# Sequentially rotate input points about X, Y and then Z axes
	if np.size(points) == 3:
		rot_y 	= np.dot(points, y_rot_mat)
	else:
		rot_y 	= np.dot(points, y_rot_mat)

	return rot_y

@autojit
def rotz(points, alpha, mode='deg'):
	"""
	Rotate points about the global Z axis by the angles theta (pitch).  Rotation
	matrices are sourced from http://mathworld.wolfram.com/RotationMatrix.html
	"""

	if mode == 'rad':
		pass
	elif mode == 'deg':
		alpha	=	np.deg2rad(alpha)
	else:
		print('ERROR: Incorrect angle type specified.  Assuming degrees.')

	# Rotation about Z axis
	z_rot_mat = np.array([	[np.cos(alpha),		np.sin(alpha),	0],
							[-np.sin(alpha),	np.cos(alpha),	0],
							[0,					0,				1]	])

	# Sequentially rotate input points about X, Y and then Z axes
	if np.size(points) == 3:
		rot_z 	= np.dot(points, z_rot_mat)
	else:
		rot_z 	= np.dot(points, z_rot_mat)

	return rot_z

@autojit
def rotxyz(points, beta, theta, alpha, mode='deg', clockwise=False):
	"""
	=== triPy.rotxyz ===
	Rotates points about the global X, Y and Z axes by the angles beta, theta,
	and alpha (roll, yaw, and pitch, respectively) in that order.  Rotation
	matrices are sourced from http://mathworld.wolfram.com/RotationMatrix.html

	=== Inputs ===
	'points'	XYZ coordinates of the points to be rotated
	'beta'		Angle of rotation about the X axis (roll)
	'theta'		Angle of rotation about the Y axis (yaw)
	'alpha'		Angle of rotation about the Z axis (pitch)
	'mode'		Defines whether angles are expressed in radians or degrees
				(degrees are the default)
	'clockwise' Defines direction of rotation about axes
				(may be either True or False - the default is False: anticlockwise)

	=== Usage ===
	import triPy
	rot_x, rot_y, rot_z = rotMatrix(x, y, z, beta, theta, alpha)
	"""

	if mode == 'rad':
		pass
	elif mode == 'deg':
		beta		=	np.deg2rad(beta)
		theta	=	np.deg2rad(theta)
		alpha	=	np.deg2rad(alpha)
	else:
		print('ERROR: Incorrect angle type specified.  Assuming degrees.')

	if clockwise == False:
		# Rotation about X axis
		x_rot_mat = np.array([	[1, 	0, 				0			],
								[0, 	np.cos(beta), 	np.sin(beta)],
								[0, 	-np.sin(beta), 	np.cos(beta)]	])

		# Rotation about Y axis
		y_rot_mat = np.array([	[np.cos(theta), 0, 	-np.sin(theta)	],
								[0, 			1,	0				],
								[np.sin(theta), 0, 	np.cos(theta)	]	])

		# Rotation about Z axis
		z_rot_mat = np.array([	[np.cos(alpha),		np.sin(alpha),	0],
								[-np.sin(alpha),		np.cos(alpha),	0],
								[0,					0,				1]	])

	else:
		# Rotation about X axis
		x_rot_mat = np.array([	[1, 	0, 				0			],
								[0, 	np.cos(beta),	-np.sin(beta)],
								[0, 	np.sin(beta), 	np.cos(beta)]	])

		# Rotation about Y axis
		y_rot_mat = np.array([	[np.cos(theta), 0, 	np.sin(theta)	],
								[0, 			1,	0				],
								[-np.sin(theta), 0, 	np.cos(theta)	]	])

		# Rotation about Z axis
		z_rot_mat = np.array([	[np.cos(alpha),		-np.sin(alpha),	0],
								[np.sin(alpha),	np.cos(alpha),	0],
								[0,					0,				1]	])

	# Sequentially rotate input points about X, Y and then Z axes
	if np.size(points) == 3:
		rot_x 	= np.dot(points, x_rot_mat)
		rot_xy 	= np.dot(rot_x, y_rot_mat)
		rot_xyz = np.dot(rot_xy, z_rot_mat)
	else:
		rot_x 	= np.dot(points, x_rot_mat)
		rot_xy 	= np.dot(rot_x, y_rot_mat)
		rot_xyz = np.dot(rot_xy, z_rot_mat)

	return rot_xyz

@autojit
def rotzyx(points, beta, theta, alpha, mode='deg', clockwise=False):
	"""
	=== triPy.rotxyz ===
	Rotates points about the global X, Y and Z axes by the angles beta, theta,
	and alpha (roll, yaw, and pitch, respectively) in reverse order.  Rotation
	matrices are sourced from http://mathworld.wolfram.com/RotationMatrix.html

	=== Inputs ===
	'points'	XYZ coordinates of the points to be rotated
	'beta'		Angle of rotation about the X axis (roll)
	'theta'		Angle of rotation about the Y axis (pitch)
	'alpha'		Angle of rotation about the Z axis (yaw)
	'mode'		Defines whether angles are expressed in radians or degrees
				(degrees are the default)
	'clockwise' Defines direction of rotation about axes
				(may be either True or False - the default is False: anticlockwise)

	=== Usage ===
	import triPy
	rot_x, rot_y, rot_z = rotMatrix(x, y, z, beta, theta, alpha)
	"""

	if mode == 'rad':
		pass
	elif mode == 'deg':
		beta		=	np.deg2rad(beta)
		theta	=	np.deg2rad(theta)
		alpha	=	np.deg2rad(alpha)
	else:
		print('ERROR: Incorrect angle type specified.  Assuming degrees.')

	if clockwise == False:
		# Rotation about X axis
		x_rot_mat = np.array([	[1, 	0, 				0			],
								[0, 	np.cos(beta), 	np.sin(beta)],
								[0, 	-np.sin(beta), 	np.cos(beta)]	])

		# Rotation about Y axis
		y_rot_mat = np.array([	[np.cos(theta), 0, 	-np.sin(theta)	],
								[0, 			1,	0				],
								[np.sin(theta), 0, 	np.cos(theta)	]	])

		# Rotation about Z axis
		z_rot_mat = np.array([	[np.cos(alpha),		np.sin(alpha),	0],
								[-np.sin(alpha),	np.cos(alpha),	0],
								[0,					0,				1]	])

	else:
		# Rotation about X axis
		x_rot_mat = np.array([	[1, 	0, 				0			],
								[0, 	np.cos(beta),	-np.sin(beta)],
								[0, 	np.sin(beta), 	np.cos(beta)]	])

		# Rotation about Y axis
		y_rot_mat = np.array([	[np.cos(theta), 0, 	np.sin(theta)	],
								[0, 			1,	0				],
								[-np.sin(theta), 0, 	np.cos(theta)	]	])

		# Rotation about Z axis
		z_rot_mat = np.array([	[np.cos(alpha),		-np.sin(alpha),	0],
								[np.sin(alpha),	np.cos(alpha),	0],
								[0,					0,				1]	])

	# Sequentially rotate input points about X, Y and then Z axes
	if np.size(points) == 3:
		rot_z 	= np.dot(points, z_rot_mat)
		rot_zy 	= np.dot(rot_z, y_rot_mat)
		rot_zyx = np.dot(rot_zy, x_rot_mat)
	else:
		rot_z 	= np.dot(points, z_rot_mat)
		rot_zy 	= np.dot(rot_z, y_rot_mat)
		rot_zyx = np.dot(rot_zy, x_rot_mat)

	return rot_zyx

@autojit
def eulerToVector(alpha, theta):
	"""
	Convert Euler angles (pitch and yaw; alpha and theta) to vector components
	in XYZ reference frame.
	"""

	x = np.sin(alpha) * np.cos(theta)
	y = np.sin(alpha) * np.sin(theta)
	z = np.cos(alpha)

	return [x, y, z]

@autojit
def vectorToEuler(x, y, z):
	"""
	Convert XYZ vector components to Euler angles (pitch and yaw; alpha and
	theta)
	"""

	# Trig relations
	r = np.linalg.norm([x, y, z])
	theta = np.arctan(y / x)
	alpha = np.arccos(z / r)

#	1st quad: 0
#	2nd quad: 180-
#	3rd quad: +180
#	4th quad: 360-

	# Identify quadrants in XY plane and adjust returned angle accordingly
	# Rotation about Z axis (yaw, theta)
	if (x > 0) & (y > 0):
		# 1st quadrant
		pass
	elif (x < 0) & (y > 0):
		# 2nd quadrant
		theta = np.pi - np.abs(theta)
	elif (x < 0) & (y < 0):
		# 3rd quadrant
		theta = np.pi + np.abs(theta)
	elif (x > 0) & (y < 0):
		# 4th quadrant
		theta = (2 * np.pi) - np.abs(theta)
	else:
		# Quadrant borders i.e. x = 0 or y = 0
		pass

	# Identify quadrants in XZ plane and adjust returned angle accordingly
	# Rotation about Y axis (pitch, alpha)
	if (x > 0) & (z > 0):
		# 1st quadrant
		pass
	elif (x < 0) & (z > 0):
		# 2nd quadrant
		alpha = np.pi - np.abs(alpha)
	elif (x < 0) & (z < 0):
		# 3rd quadrant
		alpha = np.pi + np.abs(alpha)
	elif (x > 0) & (z < 0):
		# 4th quadrant
		alpha = (2 * np.pi) - np.abs(alpha)
	else:
		# Quadrant borders i.e. x = 0 or y = 0
		pass

	return [alpha, theta]

#@autojit
def localPitchRoll(pitch, yaw, roll, clockwise=False):
	"""
	Calculate pitch and roll of an object in its local vetical plane
	NB: pitch, yaw, and roll denote rotations about the Y, Z, and X axes,
	respectively.
	"""

	# Define initial pointing vectors
	vecX = np.array([1, 0, 0])
	vecZ = np.array([0, 0, 1])

	# Transform vectors using provided Euler angles
	vecX1 = rotxyz(vecX, roll, pitch, yaw, mode='rad', clockwise=clockwise)
	vecZ1 = rotxyz(vecZ, roll, pitch, yaw, mode='rad', clockwise=clockwise)

	# Project vectors onto local YZ plane
	vecX1_proj = np.array([0, vecX1[1], vecX1[2]])
	vecZ_proj = np.array([0, 0, np.linalg.norm(vecX1_proj)])

	# Calculate angle to roll vectors back into XY plane
#	planeRoll = np.arccos(np.dot(vecX1, vecZ_proj) / (np.linalg.norm(vecX1_proj)**2))
	planeRoll = vectorAngle(vecX1_proj, vecZ_proj)

	# Calculate local pitch angle (about local Y axis)
	vecZ2 = rotxyz(vecZ1, planeRoll, 0, 0, mode='rad', clockwise=clockwise)
#	localPitch = np.arccos(np.dot(vecX1, [1, 0, 0]) / (np.linalg.norm(vecX1)))
	localPitch = vectorAngle(vecX1, vecX)

	# Transform normal vector into ZY plane (pitch) to find local roll
	vecZ3 = rotxyz(vecZ2, 0, -localPitch, 0, mode='rad', clockwise=clockwise)
#	localRoll = np.arccos(np.dot(vecZ3, [0, 0, 1]) / (np.linalg.norm(vecZ3)))
	localRoll = vectorAngle(vecZ3, vecZ)

	return localPitch, localRoll, planeRoll

@autojit
def vectorAngle(a, b):
	"""
	Return angle between two vectors
	"""
	return np.arccos(np.dot(a, b) / (np.linalg.norm(a) * np.linalg.norm(b)))


import numpy as np

class LOWESS:

    def __init__(self, sigma=1., frac=1., eps=1e-8):
        self.sigma = sigma
        self.frac = frac
        self.eps = eps
        self.X_ = None
        self.y_ = None

    def _compute_weights(self, x):
        distances = np.linalg.norm(self.X_ - x[:, None], axis=-1)

        # gaussian kernel where sigma define the brandwidth
        weights = np.exp(-(distances ** 2) / self.sigma)

        # take weights of the closest points only
        weights = weights * (distances <= np.quantile(distances, q=self.frac))

        # clip weights close to zero
        weights = np.where(np.abs(weights) >= self.eps, weights, 0.)

        return weights

    def fit(self, X, y):
        self.X_ = X
        self.y_ = y

    def predict(self, X):
        n_samples = X.shape[0]
        y_hat = np.zeros(n_samples)
        for i in range(n_samples):
            y_hat[i] = np.average(
                self.y_,
                weights=self._compute_weights(X[i, :]),
            )

        return y_hat


import matplotlib.pyplot as plt
import numpy as np

def PolynomialFit(x, y):
    # Fit using Vandermonde Matrix, Condition number can explode
    assert len(x) == len(y)
    N = len(x)
    A = np.ones((N,N))
    for i in range(1, N):
        A[:, i] = x
        x = x * x
    return np.linalg.solve(A, y), np.linalg.cond(A)

def PolynomialEval(p, x):
    # Evaluate Polynomial Fit, via Horners Rule
    # p - polynomial coefficients, x - xvalues
    i, v = len(p) - 1, None
    while i > 0:
        if v is None :
            v = p[i]*x
        else:
            v = x*(p[i] + v)
            i = i -1
    v += p[i]
    return v

def CubicFit(x, y):
    # cubic spline with continuity conditions
    d = len(x) -1
    a = np.zeros((4*d,4*d))
    s = np.zeros(4*d)
    r = 0
    for i in range(d):
        b=4*i
        a[r,b] = 1
        s[r] = y[r]
        r += 1
    b = (d-1)*4
    dk = x[d] - x[d-1]
    a[r, b:]= [1, dk, dk**2, dk**3]
    s[r] = y[r]

    r += 1
    for i in range(1,d):
        b = 4*(i-1)
        dx = x[i] - x[i-1]
        a[r,b:b+8] = [1, dx, dx**2, dx**3, -1, 0, 0, 0]
        a[r+1,b:b+8] = [0, 1, 2*dx, 3*dx**2, 0, -1, 0, 0]
        a[r+2,b:b+8] = [0, 0, 2, 6*dx, 0, 0, -2, 0]
        r += 3
    a[r,0:4] = [0, 0, 2, 0]
    b = 4*(d-1)
    a[r+1,b:b+4] = [0, 0, 2, 0]
    return np.linalg.solve(a,s), np.linalg.cond(a)

def CubicEval(x, c, xi):
    # Evaluate the cubic spline
    #
    #    x - original x knot points
    #    c - vector of spline coefficits
    #   xi - x value being evaluated
    def leval(x, c, xi, i):
        dx = xi - x[i-1]
        b = 4*(i-1)
        return c[b] + c[b+1]*dx + c[b+2]*dx**2 + c[b+3]*dx**3

    d = len(x) - 1
    for i in range(1,d):
        if xi < x[i]:
            return leval(x, c, xi, i)
    return leval(x, c, xi, d)


import numpy as np
from random import shuffle

def softmax_loss_naive(W, X, y, reg):
  """
  Softmax loss function, naive implementation (with loops)

  Inputs have dimension D, there are C classes, and we operate on minibatches
  of N examples.

  Inputs:
  - W: A numpy array of shape (D, C) containing weights.
  - X: A numpy array of shape (N, D) containing a minibatch of data.
  - y: A numpy array of shape (N,) containing training labels; y[i] = c means
    that X[i] has label c, where 0 <= c < C.
  - reg: (float) regularization strength

  Returns a tuple of:
  - loss as single float
  - gradient with respect to weights W; an array of same shape as W
  """
  # Initialize the loss and gradient to zero.
  loss = 0.0
  dW = np.zeros_like(W)

  #############################################################################
  # TODO: Compute the softmax loss and its gradient using explicit loops.     #
  # Store the loss in loss and the gradient in dW. If you are not careful     #
  # here, it is easy to run into numeric instability. Don't forget the        #
  # regularization!                                                           #
  #############################################################################
  num_classes = W.shape[1]
  num_train = X.shape[0]
  num_dim = X.shape[1]
  loss = 0.0
  for i in xrange(num_train):
    scores = X[i].dot(W)
    # correct_class_score = scores[y[i]]

    # stable trick
    unnormalized_log_possiblilty = scores - scores.max()
    unnormalized_possiblilty = np.exp(unnormalized_log_possiblilty)
    possibility = unnormalized_possiblilty / unnormalized_possiblilty.sum()
    loss += - np.log(possibility[y[i]])
    possibility[y[i]] -= 1
    dW += X[i].reshape([num_dim, 1]).dot(possibility.reshape([1, num_classes]))

  loss /= num_train
  dW /= num_train
  dW += reg * 2 * W
  #############################################################################
  #                          END OF YOUR CODE                                 #
  #############################################################################

  return loss, dW

def softmax_loss_vectorized(W, X, y, reg):
  """
  Softmax loss function, vectorized version.

  Inputs and outputs are the same as softmax_loss_naive.
  """
  # Initialize the loss and gradient to zero.
  loss = 0.0
  dW = np.zeros_like(W)

  #############################################################################
  # TODO: Compute the softmax loss and its gradient using no explicit loops.  #
  # Store the loss in loss and the gradient in dW. If you are not careful     #
  # here, it is easy to run into numeric instability. Don't forget the        #
  # regularization!                                                           #
  #############################################################################
  num_classes = W.shape[1]
  num_train = X.shape[0]
  loss = 0.0

  scores = X.dot(W)
  # correct_class_score = scores[y[i]]

  # stable trick
  unnormalized_log_possiblilty = scores - scores.max(1).reshape((num_train, 1))
  unnormalized_possiblilty = np.exp(unnormalized_log_possiblilty)
  possibility = unnormalized_possiblilty / unnormalized_possiblilty.sum(1).reshape((num_train, 1))
  loss += - np.log(y.choose(possibility.T)).sum()
  possibility[np.arange(num_train), y] -= 1  # strictly speaking it's no longer possibility after this step
  dscores = possibility
  dW += X.T.dot(dscores)

  loss /= num_train
  dW /= num_train
  dW += reg * 2 * W
  #############################################################################
  #                          END OF YOUR CODE                                 #
  #############################################################################

  return loss, dW


import numpy as np
# Note: please don't import any new package. You should solve this problem using only the package(s) above.
#-------------------------------------------------------------------------

'''
    Problem 1: Logistic Regression (with NumPy) (30 points)
    In this problem, we will implement the logistic regression method for binary classification problems using NumPy.
We will get familiar with gradient computation using the chain rule.
We will use binary cross entropy as the loss function and stochastic gradient descent to train the model parameters.

    A list of all variables being used in this problem is provided at the end of this file.
'''

#--------------------------
def Terms_and_Conditions():
    '''
        By submitting this homework or changing this function, you agree with the following terms:
       (1) Not sharing your code/solution with any student before and after the homework due. For example, sending your code segment to another student, putting your solution online or lending your laptop (if your laptop contains your solution or your Dropbox automatically copied your solution from your desktop computer and your laptop) to another student to work on this homework will violate this term.
       (2) Not using anyone's code in this homework and building your own solution. For example, using some code segments from another student or online resources due to any reason (like too busy recently) will violate this term. Changing other's code as your solution (such as changing the variable names) will also violate this term.
       (3) When discussing with any other students about this homework, only discuss high-level ideas or use pseudo-code. Don't discuss about the solution at the code level. For example, two students discuss about the solution of a function (which needs 5 lines of code to solve) and they then work on the solution "independently", however the code of the two solutions are exactly the same, or only with minor differences (variable names are different). In this case, the two students violate this term.
      All violations of (1),(2) or (3) will be handled in accordance with the WPI Academic Honesty Policy.  For more details, please visit: https://www.wpi.edu/about/policies/academic-integrity/dishonesty
      Note: we may use the Stanford Moss system to check your code for code similarity. https://theory.stanford.edu/~aiken/moss/
      Historical Data: in one year, we ended up finding 25% of the students in that class violating this term in their homework submissions and we handled ALL of these violations according to the WPI Academic Honesty Policy.
    '''
    #*******************************************
    # CHANGE HERE: if you have read and agree with the term above, change "False" to "True".
    Read_and_Agree = True
    #*******************************************
    return Read_and_Agree

#----------------------------------------------------
'''
    (Value Forward Function 1) Given a logistic regression model with parameters w and b, compute the linear logit value on a data sample x.
    ---- Inputs: --------
        * x: the feature vector of a data sample, a float numpy vector of length p.
        * w: the weights parameter of the logistic model, a float numpy vector of length p.
        * b: the bias value of the logistic model, a float scalar.
    ---- Outputs: --------
        * z: the logit value of the instance, a float scalar.
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_z(x, w, b):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    z: float = sum(x * w) + b
    #########################################
    return z
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_compute_z
        --- OR ----
        python3 -m nose -v test1.py:test_compute_z
        --- OR ----
        python -m nose -v test1.py:test_compute_z
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Local Gradient 1.1) Suppose we are given a logistic regression model with parameters w and b. Suppose we have already computed the linear logit z(x) on a training sample x. Please compute the gradient of the linear logit z(x) with w.r.t. the bias b.
    ---- Outputs: --------
        * dz_db: the partial gradient of logit z w.r.t. the bias b, a float scalar. It represents (d_z / d_b).
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_dz_db():
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    dz_db = 1
    #########################################
    return dz_db
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_compute_dz_db
        --- OR ----
        python3 -m nose -v test1.py:test_compute_dz_db
        --- OR ----
        python -m nose -v test1.py:test_compute_dz_db
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Local Gradient 1.2) Suppose we are given a logistic regression model with parameters w and b. Suppose we have already computed the linear logit z(x) on a training sample x. Please compute the partial gradients of the linear logit z(x)  w.r.t. the weights w. Here 'w.r.t.' represents 'with respect to'.
    ---- Inputs: --------
        * x: the feature vector of a data sample, a float numpy vector of length p.
    ---- Outputs: --------
        * dz_dw: the partial gradients of the logit z w.r.t. the weights w, a numpy float matrix of shape (p by 1). It represents (d_z / d_w). The i-th element represents ( d_z / d_w[i]).
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_dz_dw(x):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    dz_dw = x
    #########################################
    return dz_dw
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_compute_dz_dw
        --- OR ----
        python3 -m nose -v test1.py:test_compute_dz_dw
        --- OR ----
        python -m nose -v test1.py:test_compute_dz_dw
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Value Forward Function 2) Suppose we are given a logistic regression model and we have already computed the linear logit z on a data sample x (i.e., z(x)). Please compute the sigmoid activation on the data sample a(x).
    ---- Inputs: --------
        * z: the logit value of the instance, a float scalar.
    ---- Outputs: --------
        * a: the activation value, a float scalar.
    ---- Hints: --------
        * You could use np.exp(x) to compute the exponential of x.
        * When computing exp(-z), you need to be careful about an overflowing case. When the (-z) is a large number (say 1000),  the computer can no longer store the result in a floating-point number. In this case, we may want to avoid computing exp(-z) and assign the final result of a(x) directly..
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_a(z):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    a = 1 / (1 + np.exp(-max(-100, min(z, 100))))
    #########################################
    return a
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_compute_a
        --- OR ----
        python3 -m nose -v test1.py:test_compute_a
        --- OR ----
        python -m nose -v test1.py:test_compute_a
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Local Gradient 2) Suppose we are given a logistic regression model and we have already computed the linear logit z(x) and activation a(x) on a training sample x. Please compute the gradient of the activation (a) w.r.t. the linear logit z.
    ---- Inputs: --------
        * a: the activation value, a float scalar.
    ---- Outputs: --------
        * da_dz: .
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_da_dz(a):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    da_dz = a - (a * a)
    #########################################
    return da_dz
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_compute_da_dz
        --- OR ----
        python3 -m nose -v test1.py:test_compute_da_dz
        --- OR ----
        python -m nose -v test1.py:test_compute_da_dz
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Value Forward Function 2) Suppose we are given a logistic regression model and we have already computed the linear logit z(x) on a training sample x. Suppose the label of the training sample is y. Please compute the loss function of the logistic regression model on the training sample.
    ---- Inputs: --------
        * z: the logit value of the instance, a float scalar.
        * y: the label of a data sample, an integer scalar value. The values can be 0 or 1.
    ---- Outputs: --------
        * L: the cross entropy loss value, a float scalar.
    ---- Hints: --------
        * You could use np.log(x) to compute the natural log of x.
        * You could use np.exp(x) to compute the exponential of x.
        * When computing exp(z), you need to be careful about an overflowing case. When the z is a large number (say 1000),  the computer can no longer store the result in a floating-point number. In this case, we may want to avoid computing exp(z) and assign the final result of L(x) directly. When z is very large (say 1000), 1+exp(z) will be very close to exp(z), so we can simplify the equation "log(1+exp(z))-yz" into   "log (exp(z)) - yz". Here log() and exp() cancel out, so we only have "z-yz" left, which is "z(1-y)".
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_L(z, y):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    L = (z if z >= 1000 else np.log(1 + np.exp(z))) - (y * z)
    #########################################
    return L
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_compute_L
        --- OR ----
        python3 -m nose -v test1.py:test_compute_L
        --- OR ----
        python -m nose -v test1.py:test_compute_L
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Local Gradient 2) Suppose we are given a logistic regression model and we have already computed the logits z(x) on a training sample x. Suppose the label of the training sample is y. Please compute the gradient of the loss function (L) w.r.t. the linear logit (z) .
    ---- Inputs: --------
        * z: the logit value of the instance, a float scalar.
        * y: the label of a data sample, an integer scalar value. The values can be 0 or 1.
    ---- Outputs: --------
        * dL_dz: the partial gradient of the loss function L w.r.t. the linear logit z, a float scalar value. It represents (d_L / d_z).
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_dL_dz(z, y):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    dL_dz = (1 if z >= 1000 else np.exp(z) / (np.exp(z) + 1)) - y
    #########################################
    return dL_dz
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_compute_dL_dz
        --- OR ----
        python3 -m nose -v test1.py:test_compute_dL_dz
        --- OR ----
        python -m nose -v test1.py:test_compute_dL_dz
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Global Gradient 1) Suppose we are given a logistic regression model with parameters (w and b) and we have a training data sample (x,y).  Suppose we have already computed the local gradients on the data sample. Please compute the global gradient of the loss L w.r.t. bias parameter b using chain rule.
    ---- Inputs: --------
        * dL_dz: the partial gradient of the loss function L w.r.t. the linear logit z, a float scalar value. It represents (d_L / d_z).
        * dz_db: the partial gradient of logit z w.r.t. the bias b, a float scalar. It represents (d_z / d_b).
    ---- Outputs: --------
        * dL_db: the partial gradient of the loss function L w.r.t. the bias b, a float scalar.
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_dL_db(dL_dz, dz_db):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    dL_db = dL_dz * dz_db
    #########################################
    return dL_db
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_compute_dL_db
        --- OR ----
        python3 -m nose -v test1.py:test_compute_dL_db
        --- OR ----
        python -m nose -v test1.py:test_compute_dL_db
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Global Gradient 2) Suppose we are given a logistic regression model with parameters (w and b) and we have a training data sample (x,y).  Suppose we have already computed the local gradients on the data sample. Please compute the global gradient of the loss function L w.r.t. the weight parameters w using chain rule.
    ---- Inputs: --------
        * dL_dz: the partial gradient of the loss function L w.r.t. the linear logit z, a float scalar value. It represents (d_L / d_z).
        * dz_dw: the partial gradients of the logit z w.r.t. the weights w, a numpy float matrix of shape (p by 1). It represents (d_z / d_w). The i-th element represents ( d_z / d_w[i]).
    ---- Outputs: --------
        * dL_dw: the partial gradient of the loss function L w.r.t. the weight vector w, a numpy float vector of length p.  The i-th element represents ( d_L / d_w[i]).
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def compute_dL_dw(dL_dz, dz_dw):
    #########################################
    ## INSERT YOUR CODE HERE (2 points)
    dL_dw = dL_dz * dz_dw
    #########################################
    return dL_dw
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_compute_dL_dw
        --- OR ----
        python3 -m nose -v test1.py:test_compute_dL_dw
        --- OR ----
        python -m nose -v test1.py:test_compute_dL_dw
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Back Propagation) Suppose we are given a logistic regression model with parameters (w and b) and we have a training data sample (x) with label (y).  Suppose we have already computed the activation a(x) on the data sample in the forward-pass. Please compute the global gradients of the loss w.r.t. the parameters w and b on the data sample using back propagation.
    ---- Inputs: --------
        * x: the feature vector of a data sample, a float numpy vector of length p.
        * y: the label of a data sample, an integer scalar value. The values can be 0 or 1.
        * z: the logit value of the instance, a float scalar.
    ---- Outputs: --------
        * dL_dw: the partial gradient of the loss function L w.r.t. the weight vector w, a numpy float vector of length p.  The i-th element represents ( d_L / d_w[i]).
        * dL_db: the partial gradient of the loss function L w.r.t. the bias b, a float scalar.
    ---- Hints: --------
        * Step 1: compute all the local gradients using the functions above.
        * Step 2: use the local gradients to build global gradients for the parameters w and b.
        * This problem can be solved using 4 line(s) of code.
'''
#---------------------
def backward(x, y, z):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    dL_dz = compute_dL_dz(z, y)
    dz_dw = compute_dz_dw(x)
    dL_dw = compute_dL_dw(dL_dz, dz_dw)
    dL_db = compute_dL_db(dL_dz, 1)
    #########################################
    return dL_dw, dL_db
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_backward
        --- OR ----
        python3 -m nose -v test1.py:test_backward
        --- OR ----
        python -m nose -v test1.py:test_backward
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Gradient Descent 1) Suppose we are given a logistic regression model with parameters (w and b) and we have a training data sample (x,y).  Suppose we have already computed the gradient of the loss w.r.t. the bias b on the data sample. Please update the bias parameter b using gradient descent.
    ---- Inputs: --------
        * b: the bias value of the logistic model, a float scalar.
        * dL_db: the partial gradient of the loss function L w.r.t. the bias b, a float scalar.
        * alpha: the step-size parameter of gradient descent, a float scalar.
    ---- Outputs: --------
        * b: the bias value of the logistic model, a float scalar.
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def update_b(b, dL_db, alpha=0.001):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    b = b - alpha * dL_db
    #########################################
    return b
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_update_b
        --- OR ----
        python3 -m nose -v test1.py:test_update_b
        --- OR ----
        python -m nose -v test1.py:test_update_b
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Gradient Descent 2) Suppose we are given a logistic regression model with parameters (w and b) and we have a training data sample (x,y).  Suppose we have already computed the partial gradients of the loss w.r.t. the parameters w on the data sample. Please update the weight parameter w using gradient descent.
    ---- Inputs: --------
        * w: the weights parameter of the logistic model, a float numpy vector of length p.
        * dL_dw: the partial gradient of the loss function L w.r.t. the weight vector w, a numpy float vector of length p.  The i-th element represents ( d_L / d_w[i]).
        * alpha: the step-size parameter of gradient descent, a float scalar.
    ---- Outputs: --------
        * w: the weights parameter of the logistic model, a float numpy vector of length p.
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def update_w(w, dL_dw, alpha=0.001):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    w = w - alpha * dL_dw
    #########################################
    return w
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_update_w
        --- OR ----
        python3 -m nose -v test1.py:test_update_w
        --- OR ----
        python -m nose -v test1.py:test_update_w
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Training Logistic Regression) Given a training dataset X (features), Y (labels), train the logistic regression model using stochastic gradient descent: iteratively update the weights w and bias b using the gradients on each random data sample.  We repeat n_epoch passes over all the training instances.
    ---- Inputs: --------
        * X: the feature matrix of training instances, a float numpy matrix of shape (n by p).
        * Y: the labels of training instance, a numpy integer vector of length n. The values can be 0 or 1.
        * alpha: the step-size parameter of gradient descent, a float scalar.
        * n_epoch: the number of passes to go through the training dataset in the training process, an integer scalar.
    ---- Outputs: --------
        * w: the weights parameter of the logistic model, a float numpy vector of length p.
        * b: the bias value of the logistic model, a float scalar.
    ---- Hints: --------
        * Step 1 compute the linear logit.
        * Step 2 Back propagation: compute the gradients of w and b.
        * Step 3 Gradient descent: update the parameters w and b using gradient descent.
        * This problem can be solved using 4 line(s) of code.
'''
#---------------------
def train(X, Y, alpha=0.001, n_epoch=100):
    n,p = X.shape # n: the number training samples, p: the number of input features
    w, b = np.random.randn(p), 0. # initialize w randomly and b as 0
    for _ in range(n_epoch): # iterate through the dataset n_epoch times
        indices = np.random.permutation(n) # shuffle the indices of all samples
        for i in indices: # iterate through each random training sample (x,y)
            x=X[i] # the feature vector of the i-th random sample
            y=Y[i] # the label of the i-th random sample
            #########################################
            ## INSERT YOUR CODE HERE (5 points)
            z = compute_z(x, w, b)
            dL_dw, dL_db = backward(x, y, z)
            b = update_b(b, dL_db, alpha)
            w = update_w(w, dL_dw, alpha)
            #########################################
    return w, b
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_train
        --- OR ----
        python3 -m nose -v test1.py:test_train
        --- OR ----
        python -m nose -v test1.py:test_train
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Using Logistic Regression 1) Given a trained logistic regression model with parameters w and b. Suppose we have a test sample x. Please use the logistic regression model to predict the label of x and the probability of the label of x being positive, i.e. the activation a(x).
    ---- Inputs: --------
        * x: the feature vector of a data sample, a float numpy vector of length p.
        * w: the weights parameter of the logistic model, a float numpy vector of length p.
        * b: the bias value of the logistic model, a float scalar.
    ---- Outputs: --------
        * y: the label of a data sample, an integer scalar value. The values can be 0 or 1.
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def inference(x, w, b):
    #########################################
    ## INSERT YOUR CODE HERE (1 points)
    y = 1 if compute_z(x, w, b) > 0 else 0
    #########################################
    return y
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_inference
        --- OR ----
        python3 -m nose -v test1.py:test_inference
        --- OR ----
        python -m nose -v test1.py:test_inference
        ---------------------------------------------------
    '''

#----------------------------------------------------
'''
    (Using Logistic Regression 2) Given a trained logistic regression model with parameters w and b. Suppose we have a test dataset Xtest (features). For each data sample x in Xtest, use the logistic regression model to predict the label of x.
    ---- Inputs: --------
        * Xtest: .
        * w: the weights parameter of the logistic model, a float numpy vector of length p.
        * b: the bias value of the logistic model, a float scalar.
    ---- Outputs: --------
        * Ytest: the predicted labels of test data samples, an integer numpy array of length ntest. Ytest[i] represents the predicted label on the i-th test sample.
    ---- Hints: --------
        * This problem can be solved using 1 line(s) of code.
'''
#---------------------
def predict(Xtest, w, b):
    n_test = Xtest.shape[0] # number of test samples
    Ytest = np.zeros(n_test) # initialize labels as all zeros
    for i in range(n_test): # iterate through each test sample
        x=Xtest[i] # the feature vector of the i-th data sample
        #########################################
        ## INSERT YOUR CODE HERE (5 points)
        Ytest[i] = inference(x, w, b)
        #########################################
    return Ytest
    #-----------------
    '''
        TEST: Now you can test the correctness of your code above by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py:test_predict
        --- OR ----
        python3 -m nose -v test1.py:test_predict
        --- OR ----
        python -m nose -v test1.py:test_predict
        ---------------------------------------------------
    '''

#--------------------------------------------

'''
    TEST problem 1:
        Now you can test the correctness of all the above functions by typing the following in the terminal:
        ---------------------------------------------------
        nosetests -v test1.py
        --- OR ----
        python3 -m nose -v test1.py
        --- OR ----
        python -m nose -v test1.py
        ---------------------------------------------------

        If your code passed all the tests, you will see the following message in the terminal:
        ----------- Problem 1 (30 points in total)--------------------- ... ok
        * (2 points) compute_z ... ok
        * (1 points) compute_dz_db ... ok
        * (1 points) compute_dz_dw ... ok
        * (2 points) compute_a ... ok
        * (2 points) compute_da_dz ... ok
        * (2 points) compute_L ... ok
        * (2 points) compute_dL_dz ... ok
        * (2 points) compute_dL_db ... ok
        * (2 points) compute_dL_dw ... ok
        * (1 points) backward ... ok
        * (1 points) update_b ... ok
        * (1 points) update_w ... ok
        * (5 points) train ... ok
        * (1 points) inference ... ok
        * (5 points) predict ... ok
        ----------------------------------------------------------------------
        Ran 15 tests in 1.189s

        OK
'''

#--------------------------------------------

#--------------------------------------------
'''
    List of All Variables

* p:  the number of input features.
* x:  the feature vector of a data sample, a float numpy vector of length p.
* y:  the label of a data sample, an integer scalar value. The values can be 0 or 1.
* w:  the weights parameter of the logistic model, a float numpy vector of length p.
* b:  the bias value of the logistic model, a float scalar.
* z:  the logit value of the instance, a float scalar.
* a:  the activation value, a float scalar.
* L:  the cross entropy loss value, a float scalar.
* dL_dz:  the partial gradient of the loss function L w.r.t. the linear logit z, a float scalar value. It represents (d_L / d_z).
* dz_dw:  the partial gradients of the logit z w.r.t. the weights w, a numpy float matrix of shape (p by 1). It represents (d_z / d_w). The i-th element represents ( d_z / d_w[i]).
* dz_db:  the partial gradient of logit z w.r.t. the bias b, a float scalar. It represents (d_z / d_b).
* dL_dw:  the partial gradient of the loss function L w.r.t. the weight vector w, a numpy float vector of length p.  The i-th element represents ( d_L / d_w[i]).
* dL_db:  the partial gradient of the loss function L w.r.t. the bias b, a float scalar.
* alpha:  the step-size parameter of gradient descent, a float scalar.
* n_epoch:  the number of passes to go through the training dataset in the training process, an integer scalar.
* n:  the number of data instance in the training set.
* n_test:  the number of data instance in the test set.
* X:  the feature matrix of training instances, a float numpy matrix of shape (n by p).
* Y:  the labels of training instance, a numpy integer vector of length n. The values can be 0 or 1.
* Ytest:  the predicted labels of test data samples, an integer numpy array of length ntest. Ytest[i] represents the predicted label on the i-th test sample.

'''
#--------------------------------------------


import numpy as np

def polar2cart_2D(r, θ):
    """
    Convert 2D polar coordinates to Cartesian coordinates.

    """

    x = r * np.cos(θ)
    y = r * np.sin(θ)

    return x, y

def order_coplanar_points(points, normal, anticlockwise=True):
    """
    Find the clockwise or anticlockwise ordering of a set of coplanar 3D points.

    Parameters
    ----------
    points : ndarray of shape (3, N)
        The set of coplanar points (three-vector columns) whose ordering is to be found.
    normal : ndarray of shape (3, 1)
        Column three-vector representing the normal of the plane on which all points lie.

    Returns
    -------
    Ordered indices of points according to a clockwise or anticlockwise direction when
    looking in the opposite direction to `normal`.

    """

    # Normalise `normal` to a unit vector:
    normal = normal / np.linalg.norm(normal)

    # Compute the centroid:
    centroid = np.mean(points, axis=1)

    # Get the direction vectors from each point to the centroid
    p = points - centroid[:, np.newaxis]

    # Use the first point as the reference point
    # Find cross product of each point with the reference point
    crs = np.cross(p[:, 0:1], p, axis=0)

    # Find the scalar triple product of the point pairs and the normal vector
    stp = np.einsum('ij,ik->k', normal, crs)

    # Find the dot product of the point pairs
    dot = np.einsum('ij,ik->k', p[:, 0:1], p)

    # Find signed angles from reference point to each other point
    ang = np.arctan2(stp, dot)
    ang_order = np.argsort(ang)

    if not anticlockwise:
        ang_order = ang_order[::-1]

    return ang_order

def circle_points(r, n, centre):
    t = np.linspace(0, 2*np.pi, n)
    x = r * np.cos(t) + centre[0]
    y = r * np.sin(t) + centre[1]

    return np.c_[x, y]

def search_keep_order(A, B):
    sort_idx = A.argsort()
    out = sort_idx[np.searchsorted(A, B, sorter = sort_idx)]
    idx_layer_0 = np.nonzero(B[:,None] == A)[1]

    return idx_layer_0


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import numpy as np

def isFullRank(matrix):
    """Given a numpy (n,m) array prints on screen if it's full rank or not.

    As a plus, it also prints if it's singular.

    Arguments:
        matrix: (n,m) numpy array .

    Returns:
        Nothing

    Raises:
        RuntimeError: when matrix dimension is not 2.
    """

    # '\u2713' = ✓ (unicode tick character)

    print('Analysing matrix: \n', matrix)

    matrixNumDim = np.ndim(matrix)
    if (matrixNumDim != 2):
        raise RuntimeError("I'm not prepared for this. </3")

    matrixShape = np.shape(matrix)
    matrixRange = np.linalg.matrix_rank(matrix)

    print()

    if min(matrixShape) == matrixRange:
        print(u"[\u2713] Matrix is full rank.")
        if matrixShape[0] == matrixShape[1]:
            print(u"[\u2713] This matrix is squeared, so it's nonsingular.")
        else:
            print("[X] This matrix is not squeared, so it's singular.")
    else:
        print("[X] Matrix is NOT full rank.")

    print()

if __name__ == "__main__":
    I = np.eye(4) # Full rank matrix
    isFullRank(I)

    I[-1, -1] = 0 # Make it rank deficient
    isFullRank(I)

    I = np.array([[1,2,3],[4,5,6]]) # Full rank non-squared matrix
    isFullRank(I)


# MIT License
#
# Copyright (c) 2021 Florian
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import numpy as np
from .function import Function

class MeanSquaredErrorLoss(Function):
    """Mean squared error loss function class."""

    def function(self, y, y_hat):
        return np.mean(np.power(y - y_hat, 2))

    def function_derive(self, y, y_hat):
        return 2 * (y_hat - y) / y.size

class BinaryCrossEntropyLoss(Function):
    """Binary cross entropy loss function class."""

    def function(self, y, y_hat):
        return -np.mean(y * np.log(y_hat) + (1 - y) * np.log(1 - y_hat))

    def function_derive(self, y, y_hat):
        return ((1 - y) / (1 - y_hat) - y / y_hat) / np.size(y)


"""
The NumPy module also comes with a number of built-in routines for linear algebra calculations. These can be found in the sub-module linalg.

linalg.det

The linalg.det tool computes the determinant of an array.

print numpy.linalg.det([[1 , 2], [2, 1]])       #Output : -3.0
linalg.eig

The linalg.eig computes the eigenvalues and right eigenvectors of a square array.

vals, vecs = numpy.linalg.eig([[1 , 2], [2, 1]])
print vals                                      #Output : [ 3. -1.]
print vecs                                      #Output : [[ 0.70710678 -0.70710678]
                                                #          [ 0.70710678  0.70710678]]
linalg.inv

The linalg.inv tool computes the (multiplicative) inverse of a matrix.

print numpy.linalg.inv([[1 , 2], [2, 1]])       #Output : [[-0.33333333  0.66666667]
                                                #          [ 0.66666667 -0.33333333]]
Other routines can be found here

Task

You are given a square matrix  with dimensions X. Your task is to find the determinant. Note: Round the answer to 2 places after the decimal.

Input Format

The first line contains the integer .
The next  lines contains the  space separated elements of array .

Output Format

Print the determinant of .

Sample Input

2
1.1 1.1
1.1 1.1
Sample Output

0.0
"""

import numpy as np

np.set_printoptions(legacy="1.13")

N = int(input())
arr1 = np.array([list(map(float, input().split())) for _ in range(N)], float)
print(np.linalg.det(arr1))


# Linear Regression: Inverse Matrix Method
#----------------------------------
#
# This function shows how to use TensorFlow to
# solve linear regression via the matrix inverse.
#
# Given Ax=b, solving for x:
#  x = (t(A) * A)^(-1) * t(A) * b
#  where t(A) is the transpose of A

import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow.python.framework import ops
ops.reset_default_graph()

# Create graph
sess = tf.Session()

# Create the data
x_vals = np.linspace(0, 10, 100)
y_vals = x_vals + np.random.normal(0, 1, 100)

# Create design matrix
x_vals_column = np.transpose(np.matrix(x_vals))
ones_column = np.transpose(np.matrix(np.repeat(1, 100)))
A = np.column_stack((x_vals_column, ones_column))

# Create b matrix
b = np.transpose(np.matrix(y_vals))

# Create tensors
A_tensor = tf.constant(A)
b_tensor = tf.constant(b)

# Matrix inverse solution
tA_A = tf.matmul(tf.transpose(A_tensor), A_tensor)
tA_A_inv = tf.matrix_inverse(tA_A)
product = tf.matmul(tA_A_inv, tf.transpose(A_tensor))
solution = tf.matmul(product, b_tensor)

solution_eval = sess.run(solution)

# Extract coefficients
slope = solution_eval[0][0]
y_intercept = solution_eval[1][0]

print('slope: ' + str(slope))
print('y_intercept: ' + str(y_intercept))

# Get best fit line
best_fit = []
for i in x_vals:
  best_fit.append(slope*i+y_intercept)

# Plot the results
plt.plot(x_vals, y_vals, 'o', label='Data')
plt.plot(x_vals, best_fit, 'r-', label='Best fit line', linewidth=3)
plt.legend(loc='upper left')
plt.show()


# entropy of examples from a classification task with 3 classes
from math import log2
from numpy import asarray
# calculate entropy
def entropy(p):
    return -sum(p[i] * log2(p[i]) for i in range(len(p)))

# class 1
p= asarray([1, 0, 0 ]) + 1e-15
print(entropy(p))
# class 2
p= asarray([0, 1, 0 ]) + 1e-15
print(entropy(p))
# class 3
p= asarray([0, 0, 1]) + 1e-15
print(entropy(p))


import numpy as np
import matplotlib.pyplot as plt

## Metodă numerică de calcul a derivatei unei funcții

def f(x):
    return np.sin(x)

left, right = -np.pi, +np.pi

N = 30

x = np.linspace(left, right, N)
y = f(x)

h = x[1] - x[0]
interior = x[1:-1]

# Calculez derivata și aproximările ei
derivative = np.cos(interior)
progressive_diffs = (y[2:] - y[1:-1]) / h
regressive_diffs = (y[1:-1] - y[0:-2]) / h
central_diffs = (y[2:] - y[0:-2]) / (2 * h)

plt.figure(dpi=150)
plt.suptitle('Aproximarea derivatei')
plt.plot(interior, derivative, label='Derivata exactă')
plt.plot(interior, progressive_diffs, label='Diferențe progresive')
plt.plot(interior, regressive_diffs, label='Diferențe regresive')
plt.plot(interior, central_diffs, label='Diferențe centrale')
plt.legend()
plt.show()

# Afișez erorile de trunchiere
def compute_error(approx):
    return np.abs(approx - derivative)

plt.figure(dpi=150)
plt.suptitle('Eroare de trunchiere')
plt.plot(interior, compute_error(progressive_diffs), label='Diferențe progresive')
plt.plot(interior, compute_error(regressive_diffs), label='Diferențe regresive')
plt.plot(interior, compute_error(central_diffs), label='Diferențe centrale')
plt.legend()
plt.show()


# Multiple Linear Regression

# Importing the libraries
import numpy as np
import pandas as pd
from sklearn.preprocessing import LabelEncoder, OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
import statsmodels.formula.api as sm

# Importing the dataset
dataset = pd.read_csv('../datasets/50_Startups.csv')
X = dataset.iloc[:, :-1].values
y = dataset.iloc[:, -1].values

# Encoding categorical data
labelencoder = LabelEncoder()
X[:, 3] = labelencoder.fit_transform(X[:, 3])
onehotencoder = OneHotEncoder(categorical_features = [3])
X = onehotencoder.fit_transform(X).toarray()

# Avoiding the Dummy Variable Trap
X = X[:, 1:]

# Splitting the dataset into the Training set and Test set
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2, random_state = 0)

# Fitting Multiple Linear Regression to the Training set
regressor = LinearRegression()
regressor.fit(X_train, y_train)

# Predicting the Test set results
y_pred = regressor.predict(X_test)

# Building the optimal model using Backward Elimination
X = np.append(arr = np.ones((50, 1)).astype(int), values = X, axis = 1)

# Backward Elimination
max_SL = 0.05  # maximum significanse level to stay
used_variables = range(6)  # index of the selected

X_opt = X[:, used_variables]
regressor_OLS = sm.OLS(endog = y, exog = X_opt).fit()
actual_max_p_value = np.max(regressor_OLS.pvalues)

while actual_max_p_value > max_SL:
    index_to_remove = np.where(regressor_OLS.pvalues == actual_max_p_value)[0][0]
    used_variables.pop(index_to_remove)  # remove a variavel com maximo p_value
    X_opt = X[:, used_variables]  # o modelo eh gerado novamente
    regressor_OLS = sm.OLS(endog = y, exog = X_opt).fit()
    actual_max_p_value = np.max(regressor_OLS.pvalues)
    print("%s - Max P Value: %.4f" % (str(used_variables), actual_max_p_value))

X_opt = X[:, used_variables]
regressor_OLS = sm.OLS(endog = y, exog = X_opt).fit()
# print(regressor_OLS.summary())

# Forward Elimination
print("Forward Elimination")
max_SL = 0.05  # maximum significanse level to stay
all_variables = range(0,6)
used_variables = [] # index of the selected

# Step2 fit all simple regression models
p_values_list = []
for v in all_variables:
    X_opt = X[:, used_variables + [v]]
    regressor_OLS = sm.OLS(endog = y, exog = X_opt).fit()
    p_values_list.append(regressor_OLS.pvalues[-1])

actual_min_p_value = min(p_values_list)
index_to_add = np.where(p_values_list == actual_min_p_value)[0][0]
# Step3: Keep this variable...
used_variables.append(all_variables.pop(index_to_add))
print("%s - Min P Value: %.4f" % (str(used_variables), actual_min_p_value))

while actual_min_p_value < max_SL:
    if len(all_variables) == 0:
        break
    p_values_list = []
    for v in all_variables:
        X_opt = X[:, used_variables + [v]]  # and fit all possible models
        regressor_OLS = sm.OLS(endog = y, exog = X_opt).fit()
        p_values_list.append(regressor_OLS.pvalues[-1])
    actual_min_p_value = min(p_values_list)
    index_to_add = np.where(p_values_list == actual_min_p_value)[0][0]
    used_variables.append(all_variables.pop(index_to_add))
    print("%s - Min P Value: %.4f" % (str(used_variables), actual_min_p_value))

#TODO: Bidirectional Elimination

Part 2 - Regression/Section 5 - Multiple Linear Regression/multiple_linear_regression.py


'''NAME
       Lagrange
VERSION
        1.0
AUTHOR
        Elizabeth Marquez Gomez <elimqzg@lcg.unam.mx>
DESCRIPTION
        Implementation of lagrange interpolation to construct a polynomial and find a solution for x
CATEGORY
        Numeric methods
USAGE
        Lagrange.py
'''
#IMPORT LIBRARIES
from sympy import *

#DEFINE SYMBOLS THAT WILL BE USE IN EQUATIONS
x = symbols('x')

print('\t\t-- Welcome to Lagrange interpolation method --')

#FUNCTION TO GET x AND y VALUES
def GetValues(axis):
    values = input('Insert '+ axis +' values, please separe them with a coma between each one\n')
    return values.split(',')

#GET x AND y VALUES
correct = False
while not correct:
    xAxis = GetValues('x')
    yAxis = GetValues('y')
    if len(xAxis) != len(yAxis):
        print('The number of elements for x axis is not the same for y axis.\nThe method requires same number of elements for both axis, insert again the values.')
    else:
        correct = True
#SAVE x AND y VALUES IN LISTS
xAxis = list(map(float, xAxis))
yAxis = list(map(float, yAxis))

n = len(xAxis)
sumEq = 0

#ITERATE TO CREATE THE POLYNOMIAL
for i in range(1,n+1):
    numEq = 1
    denominator = 1
    for j in range(1,n+1):
        if i == j:
            continue
        #CREATE EQUATION
        numEq = expand(numEq * (x - xAxis[j-1]))
        denominator = denominator * (xAxis[i-1] - xAxis[j-1])
    sumEq += expand(numEq/denominator) * yAxis[i-1]

print(f'\tA polynomial has been constructed:\t{sumEq}')

solveX = input('Would you like to solve for x? (Y/N) --> ').upper()
if solveX == 'Y':
    xValue = float(input('Insert the x value to evaluate: '))
    #EVALUATE WITH x VALUE
    print(f'\tx=\t{N(sumEq.subs(x, xValue))}')
# ------VARIABLE'S DICTIONARY------
#             +----Main----+
# *NAME*         *TYPE*        *USAGE*
# x              string       Represents the symbol "x" that will be use in equations
# correct        bool         Indicates if values inserted are coherent
# xAxis          list, float  Saves x values
# yAxis          list, float  Saves y values
# n              int          Indicates the number of elements to iterate
# i              int          Controls interations
# numEq          sympify      Saves temporal solutions
# denominator    float        Indicates denominator values
# j              int          Controls intern iteration
# sumEq          sympify      Saves polynomial
# solveX         string       Indicates if has been required a solve for x
# xValue         float        Saves the value for x that will be evaluated
#             +----GetValues----+
# *NAME*         *TYPE*        *USAGE*
# axis           string        Indicates the letter x o y
# values         string        Saves values


"""
Ajay Kc
013213328
EE381
Project 4 Part 1

The problem plots a probability distribution function of S, where S is a Random Variable of sum
of the widths of n books. The value of n is 1,5,10,and 15. For each value of n, the experimental
PDF and normal distribution function is calculated.
"""
import numpy as np
import math
import matplotlib.pyplot as plt
mean = (1+3)/2
standard_deviation = math.sqrt(((3-1)**2)/12)

def getRVMean(n):
    return n*mean

def getRVSd(n):
    return standard_deviation*math.sqrt(n)

def simulateRV(n):

    RV = []
    for i in range(0,10000):
        thickness = np.random.uniform(1,3,n)
        RV.append(thickness.sum())
    return RV

def plotFunctions(RV,n):
    a = 1; b =3 #a = min width; b = max width
    nbooks =n; nbins=30; #Number of books; Number of bins
    edgecolor = 'w';    #Color seperating bars in the bargraph
    #
    #Create bins and histogram
    bins = [float(x) for x in np.linspace(nbooks*a, nbooks*b, nbins+1)]
    h1, bin_edges = np.histogram(RV, bins, density = True)
    #Define points on the horizontal axis
    be1 = bin_edges[0:np.size(bin_edges)-1]
    be2 = bin_edges[1:np.size(bin_edges)]
    b1 = (be1+be2)/2
    barwidth = b1[1]-b1[0] #Width of bars in the bargraph

    figureOne = plt.figure(1)
    xlabel = "Book stack height for n="+str(n)+" books"
    plt.xlabel(xlabel, fontsize = 20)
    plt.ylabel("PDF", fontsize = 20)
    plt.title("PDF of book stack height and comparison with Gaussian",fontsize=20)
    plt.bar(b1,h1,width = barwidth, edgecolor=edgecolor)

    normalDist = []
    for i in range(0,len(b1)):
        normalDist.append((1 / (getRVSd(n) * math.sqrt(2 * math.pi))) * math.exp(
            -((b1[i] - getRVMean(n)) ** 2) / (2 * getRVSd(n) ** 2)))

    plt.plot(b1,normalDist,color='r')
    plt.show()

#For 1 book
RV = simulateRV(1)
plotFunctions(RV,1)

#For 5 books
RV = simulateRV(5)
plotFunctions(RV,5)

#For 10 books
RV = simulateRV(10)
plotFunctions(RV,10)

#For 15 books
RV = simulateRV(15)
plotFunctions(RV,15)


from .__loss_function import LossFunction
import numpy as np

class MClassLogisticLoss(LossFunction):
    """
    This class is responsible for calculating Cross Entropy Loss for multiclass

    - Input is the output of the last layer (not probabilities)
    - Softmax is applied to the input to get the probabilities of each class
    - Loss is -log(probability of the correct class)

     Examples
    ---------
    >>> from nn_recipe.utility import OneHotEncoder
    >>> import  numpy as np
    >>> X = np.array([[5, 6, 7], [5, 6, 7]])
    >>> Y = np.array([["Ahmed"], ["Ali"]])
    >>> encoder = OneHotEncoder(types=["Mohamed", "Ahmed", "Ali"])
    >>> loss(encoder.encode(Y), X)
    np.array([1.40760596, 0.40760596])
    >>> loss.local_grad
    np.array([[ 0.09003057, -0.75527153, 0.66524096], [0.09003057, 0.24472847, -0.33475904]]

    :ivar __softmax_value: Value of the softmax layer applied to the input
    :type __softmax_value: np.ndarray
    """
    ID = 3

    def _compute_loss(self, Y, Y_hat):
        """

        :param Y:
        :param Y_hat:
        :return:
        """
        expZ = np.exp(Y_hat - np.max(Y_hat))
        self.__softmax_value = expZ / np.sum(expZ, axis=1).reshape(-1, 1)
        return -1*np.log(self.__softmax_value[0, np.argmax(Y, axis=1)])

    def _compute_local_grad(self, Y, Y_Hat):
        self.__softmax_value[range(self.__softmax_value.shape[0]), np.argmax(Y, axis=1)] -= 1
        return self.__softmax_value


import numpy as np
import matplotlib.pyplot as plt
"""
Sukhbinder
5 April 2017

Based on:
"""

def _rect_inter_inner(x1,x2):
    n1=x1.shape[0]-1
    n2=x2.shape[0]-1
    X1=np.c_[x1[:-1],x1[1:]]
    X2=np.c_[x2[:-1],x2[1:]]
    S1=np.tile(X1.min(axis=1),(n2,1)).T
    S2=np.tile(X2.max(axis=1),(n1,1))
    S3=np.tile(X1.max(axis=1),(n2,1)).T
    S4=np.tile(X2.min(axis=1),(n1,1))
    return S1,S2,S3,S4

def _rectangle_intersection_(x1,y1,x2,y2):
    S1,S2,S3,S4=_rect_inter_inner(x1,x2)
    S5,S6,S7,S8=_rect_inter_inner(y1,y2)

    C1=np.less_equal(S1,S2)
    C2=np.greater_equal(S3,S4)
    C3=np.less_equal(S5,S6)
    C4=np.greater_equal(S7,S8)

    ii,jj=np.nonzero(C1 & C2 & C3 & C4)
    return ii,jj

def intersection(x1,y1,x2,y2):
    """
INTERSECTIONS Intersections of curves.
   Computes the (x,y) locations where two curves intersect.  The curves
   can be broken with NaNs or have vertical segments.

usage:
x,y=intersection(x1,y1,x2,y2)

    Example:
    a, b = 1, 2
    phi = np.linspace(3, 10, 100)
    x1 = a*phi - b*np.sin(phi)
    y1 = a - b*np.cos(phi)

    x2=phi
    y2=np.sin(phi)+2
    x,y=intersection(x1,y1,x2,y2)

    plt.plot(x1,y1,c='r')
    plt.plot(x2,y2,c='g')
    plt.plot(x,y,'*k')
    plt.show()

    """
    ii,jj=_rectangle_intersection_(x1,y1,x2,y2)
    n=len(ii)

    dxy1=np.diff(np.c_[x1,y1],axis=0)
    dxy2=np.diff(np.c_[x2,y2],axis=0)

    T=np.zeros((4,n))
    AA=np.zeros((4,4,n))
    AA[0:2,2,:]=-1
    AA[2:4,3,:]=-1
    AA[0::2,0,:]=dxy1[ii,:].T
    AA[1::2,1,:]=dxy2[jj,:].T

    BB=np.zeros((4,n))
    BB[0,:]=-x1[ii].ravel()
    BB[1,:]=-x2[jj].ravel()
    BB[2,:]=-y1[ii].ravel()
    BB[3,:]=-y2[jj].ravel()

    for i in range(n):
        try:
            T[:,i]=np.linalg.solve(AA[:,:,i],BB[:,i])
        except:
            T[:,i]=np.NaN

    in_range= (T[0,:] >=0) & (T[1,:] >=0) & (T[0,:] <=1) & (T[1,:] <=1)

    xy0=T[2:,in_range]
    xy0=xy0.T
    return xy0[:,0],xy0[:,1]

if __name__ == '__main__':

    # a piece of a prolate cycloid, and am going to find
    a, b = 1, 2
    phi = np.linspace(3, 10, 100)
    x1 = a*phi - b*np.sin(phi)
    y1 = a - b*np.cos(phi)

    x2=phi
    y2=np.sin(phi)+2
    x,y=intersection(x1,y1,x2,y2)
    plt.plot(x1,y1,c='r')
    plt.plot(x2,y2,c='g')
    plt.plot(x,y,'*k')
    plt.show()


""" Rotation matrices for rotations around x, y, z axes

See: https://en.wikipedia.org/wiki/Rotation_matrix#Basic_rotations
"""

import numpy as np

def x_rotmat(theta):
    """ Rotation matrix for rotation of `theta` radians around x axis

    Parameters
    ----------
    theta : scalar
        Rotation angle in radians

    Returns
    -------
    M : shape (3, 3) array
        Rotation matrix
    """
    cos_t = np.cos(theta)
    sin_t = np.sin(theta)
    return np.array([[1, 0, 0],
                     [0, cos_t, -sin_t],
                     [0, sin_t, cos_t]])

def y_rotmat(theta):
    """ Rotation matrix for rotation of `theta` radians around y axis

    Parameters
    ----------
    theta : scalar
        Rotation angle in radians

    Returns
    -------
    M : shape (3, 3) array
        Rotation matrix
    """
    cos_t = np.cos(theta)
    sin_t = np.sin(theta)
    return np.array([[cos_t, 0, sin_t],
                     [0, 1, 0],
                     [-sin_t, 0, cos_t]])

def z_rotmat(theta):
    """ Rotation matrix for rotation of `theta` radians around z axis

    Parameters
    ----------
    theta : scalar
        Rotation angle in radians

    Returns
    -------
    M : shape (3, 3) array
        Rotation matrix
    """
    cos_t = np.cos(theta)
    sin_t = np.sin(theta)
    return np.array([[cos_t, -sin_t, 0],
                     [sin_t, cos_t, 0],
                     [0, 0, 1]])


"""
If we are presented with the first k terms of a sequence it is impossible to
say with certainty the value of the next term, as there are infinitely many
polynomial functions that can model the sequence.

As an example, let us consider the sequence of cube numbers. This is defined by
the generating function,
    u_n = n3: 1, 8, 27, 64, 125, 216, ...

Suppose we were only given the first two terms of this sequence. Working on the
principle that "simple is best" we should assume a linear relationship and
predict the next term to be 15 (common difference 7). Even if we were presented
with the first three terms, by the same principle of simplicity, a quadratic
relationship should be assumed.

We shall define OP(k, n) to be the nth term of the optimum polynomial
generating function for the first k terms of a sequence. It should be clear
that OP(k, n) will accurately generate the terms of the sequence for n ≤ k, and
potentially the first incorrect term (FIT) will be OP(k, k+1); in which case we
shall call it a bad OP (BOP).

As a basis, if we were only given the first term of sequence, it would be most
sensible to assume constancy; that is, for n ≥ 2, OP(1, n) = u1.

Hence we obtain the following OPs for the cubic sequence:

    OP(1, n) = 1	        1, 1, 1, 1, ...
    OP(2, n) = 7n−6	        1, 8, 15, ...
    OP(3, n) = 6n2−11n+6  	1, 8, 27, 58, ...
    OP(4, n) = n3	        1, 8, 27, 64, 125, ...

Clearly no BOPs exist for k ≥ 4.

By considering the sum of FITs generated by the BOPs (indicated in red above),
we obtain 1 + 15 + 58 = 74.

Consider the following tenth degree polynomial generating function:

un = 1 − n + n2 − n3 + n4 − n5 + n6 − n7 + n8 − n9 + n10

Find the sum of FITs for the BOPs.

Solution comment: Very simple. Just a straight forward implementation
                  of the stuff in the problem statement. Very quick.
"""

import numpy as np
from itertools import count

def OP(f, k):
    n = list(range(1, k + 1))
    u = [f(n_i) for n_i in n]
    g = np.poly1d(np.polyfit(n, u, k - 1))
    return g

def FIT(guess, correct, rtol=1e-5):
    for i, (g, c) in enumerate(zip(guess, correct)):
        if abs((g - c) / g) > rtol:
            return g
    return None

def generator(n):
    return sum((-1) ** p * n ** p for p in range(11))

fit_sum = 0
for k in count(1):
    op = OP(generator, k)
    fit = FIT(*zip(*[(round(op(n)), generator(n)) for n in range(1, k + 2)]))
    if fit is None:
        break
    fit_sum += fit

print('Answer: ', int(fit_sum))


# =============================================================================
# Plots a SIRD model according input parameters
# =============================================================================

import numpy as np
from scipy.integrate import odeint
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()

def deriv(y, t, N, beta, gamma, omega):
    S, I, R, D = y
    dSdt = -beta * S * I / N
    dIdt = beta * S * I / N - gamma * I - omega * I
    dRdt = gamma * I
    dDdt = omega * I
    return dSdt, dIdt, dRdt, dDdt

def SIRD_model_simple(N, I0, Ro, T_r, omega, T):
    """
    Plots a simple SIRD model without interaction between populations.

    Parameters
    ----------
    N : int
        Initial population.
    I0 : int
        Initial infected.
    Ro : float
        Reproduction number.
    T_r : int
        Recovery time.
    omega : float
        Mortality rate.
    T : int
        Simulation time.

    Returns
    -------
    None.

    """

    # Initial number of infected, recovered and deceased individuals
    I0, R0, D0 = I0, 0, 0
    S0 = N - I0 - R0  # The rest are susceptible
    T_c = T_r / Ro  # Typical time betwen contacts

    # Transition rates
    beta = T_c ** (-1)
    gamma = T_r ** (-1)

    t = np.linspace(0, T, T)  # t in units of time

    # Initial conditions
    y0 = S0, I0, R0, D0
    # Integrate the differential equations over t.
    ret = odeint(deriv, y0, t, args=(N, beta, gamma, omega))
    S, I, R, D = ret.T

    fig, ax = plt.subplots(1, 1, figsize=(14, 10))
    plt.plot(t, S, label="Susceptible")
    plt.plot(t, I, label="Infected")
    plt.plot(t, R, label="Recovered")
    plt.plot(t, D, label="Deceased")
    plt.title("SIRD MODEL", fontsize=20)
    ax.set_ylabel("Individuals")
    ax.set_xlabel("Time")
    plt.legend()
    plt.tight_layout()

    return S, I, R, D

if __name__ == '__main__':
    # Runs a simulation ant plot the model
    SIRD_model_simple(47000000, 1, 4, 10, 0.02, 360)


# -*-coding: utf-8 -*-

# Program do znajdowania dodatnich pierwiastków równania
# exp(x) - 3*(x**2) = 0
# metodą Newtona

import numpy as np

def f(u):
    '''Oblicza wartość funkcji z zadania'''
    wartosc = np.exp(u) - 3 * (u**2)
    return wartosc

def pochodnaf(x0, war_h):
    '''Oblicza wartość pochodnej funkcji w zadanym punkcie z dok. O(h**2)'''
    pochf_Oh2 = (f(x0 + war_h) - f(x0 - war_h)) / (war_h * 2)
    return pochf_Oh2

print "Program do znajdowania dodatnich pierwiastków\n"
print "równania exp(x) - 3 * x**2 = 0"
print "metodą Newtona."

# przyjęta wartość punktu końcowego
b = 4.0

h = float(raw_input("\nPodaj wartość h "))

N = int(raw_input("\nPodaj liczbę kroków k "))

#utworzenie wektora argumentów x, wartości f(x) i f'(x)
arg  = []
war  = []
pwar = []
arg.append(b)
war.append(f(b))
pwar.append(pochodnaf(b,h))
#ustal wartość początkową x
x_n = b
#obliczenie
for licz in range(0,2):
    for k in range(0,N):
        #obliczenie wartości x_{n+1} - nowe wartości x
        x_n1 = x_n - ( f(x_n)/pochodnaf(x_n, h) )

        arg.append(x_n1)
        war.append(f(x_n1))
        pwar.append(pochodnaf(x_n1,h))
        #zapamiętaj daną historyczną
        x_n = x_n1

    #utworzenie wektorów kolumnowych
    wek_arg  = np.array([arg])
    wek_war  = np.array([war])
    wek_pwar = np.array([pwar])

    #utworzenie macierzy zwierającej x, f(x) i f'(x)
    macierz = np.concatenate((wek_arg.T, wek_war.T, wek_pwar.T), axis=1)
    print macierz, "\n"

    dlugosc = wek_arg.size
    r = arg[dlugosc-1] / 2
    x_n = r

    #usuń wszystkie wartości z wektorów arg, war, pwar
    for i in range(0, dlugosc):
        arg.pop(0)
        war.pop(0)
        pwar.pop(0)
    #dodaj nową wartości do obliczenia następnego pierwiastka
    arg.append(r)
    war.append(f(r))
    pwar.append(pochodnaf(r, h))


# ===================================================================== #
#                     TRAVEL SALESMAN PROBLEM                           #
# ===================================================================== #

from pyomo.environ import *

'''
El problema del viajante de comercio es un problema de optimización
combinatoria. Al igual que el del Set-Covering, es uno de los 21 problemas
NP-completos de Richard Karp.

En este problema, un viajante debe pasar por todas las ciudades, minimizando
los costes del viaje.
'''

## DEFINICION DEL MODELO
# El modelo será un modelo concreto.
m = ConcreteModel(name = 'Knap-sack problem')

## SETS
# Tenemos un set de ciudades y un alias de este set.
c = m.c = Set(initialize = ['A','B','C','D','E','F'], ordered = True)
cc = m.cc = SetOf(c, ordered = True)

# Tenemos tambien una relación que nos permite decidir que ciudades están
# comunicadas con qué ciudades.
RelDic = {'A': ('B','D','F'),
          'B': ('A','C','D','E'),
          'C': ('B','D','E','F'),
          'D': ('A','B','C','F'),
          'E': ('B','C','F'),
          'F': ('A','C','D','E','F')}

R = m.R = Set(within = c*cc, ordered = True)
for s in RelDic:
    for ss in RelDic[s]:
        R.add((s,ss))

## PARÁMETROS
# Como parámetros, tenemos las distancias entre las ciudades. A las que no existen
# pondremos un cero.
DistDic = {}
DistList = [8,3,4,8,1,5,9,1,7,2,21,3,5,7,3, 9,2,35,4,21,3,35,5]
for n,i in enumerate(R):
    DistDic[i] = DistList[n]

Cf = m.Cf = Param(c,cc, initialize = DistDic, default = 0)

## VARIABLES
# Necesitamos una binaria que nos indique si se va de una ciudad c a una ciudad cc.
y = m.y = Var(c,cc, domain = Binary)

## RESTRICCIONES
# Hay que poner las restricciones

#- De todas las ciudades, debe llegar a una
def Llegada(m,ciudad):
    return sum(y[ciudad,cciudad] for cciudad in cc if (ciudad,cciudad) in R) == 1
R1 = m.r1 = Constraint(c, rule = Llegada)

#- A todas las ciudades, debe llegar a una.
def Ida(m, cciudad):
    return sum(y[ciudad,cciudad] for ciudad in c if (ciudad,cciudad) in R) == 1
R2 = m.r2 = Constraint(cc, rule = Ida)

## OBJETIVO
# El objetivo es minimizar la distancia recorrida.
def fobj(m):
    return sum(Cf[ciudad,cciudad]*y[ciudad,cciudad] for ciudad in c for cciudad in cc)
OBJ = m.obj = Objective(rule = fobj, sense = minimize)

## VERBATIM DE RESOLUCIÓN
from pyomo.opt import SolverFactory
opt = SolverFactory('glpk')
results = opt.solve(m)
results.write()

# 02 -Case Studies/# 07 - Travel salesman problem/TravelSales.py


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Feb 10 21:46:41 2021

@author: abda
"""

import numpy as np
import scipy.ndimage
import cv2

def get_gauss_feat_im(im, sigma=1, normalize=True, norm_fac = None):
    """Gauss derivative feaures for every image pixel.
    Arguments:
        image: a 2D image, shape (r,c).
        sigma: standard deviation for Gaussian derivatives.
        normalize: flag indicating normalization of features.
    Returns:
        imfeat: a 3D array of size (r,c,15) with a 15-dimentional feature
            vector for every image pixel.
    Author: vand@dtu.dk, 2020
    """

    s = np.ceil(sigma*4)
    x = np.arange(-s,s+1).reshape((-1,1));

    g = np.exp(-x**2/(2*sigma**2));
    g /= np.sum(g);
    dg = -x/(sigma**2)*g;
    ddg = -1/(sigma**2)*g - x/(sigma**2)*dg;
    dddg = -2/(sigma**2)*dg - x/(sigma**2)*ddg;
    ddddg = -2/(sigma**2)*ddg - 1/(sigma**2)*ddg - x/(sigma**2)*dddg;

    r,c = im.shape
    imfeat = np.zeros((r,c,15))
    imfeat[:,:,0] = cv2.filter2D(cv2.filter2D(im,-1,g),-1,g.T)
    imfeat[:,:,1] = cv2.filter2D(cv2.filter2D(im,-1,dg),-1,g.T)
    imfeat[:,:,2] = cv2.filter2D(cv2.filter2D(im,-1,g),-1,dg.T)
    imfeat[:,:,3] = cv2.filter2D(cv2.filter2D(im,-1,ddg),-1,g.T)
    imfeat[:,:,4] = cv2.filter2D(cv2.filter2D(im,-1,dg),-1,dg.T)
    imfeat[:,:,5] = cv2.filter2D(cv2.filter2D(im,-1,g),-1,ddg.T)
    imfeat[:,:,6] = cv2.filter2D(cv2.filter2D(im,-1,dddg),-1,g.T)
    imfeat[:,:,7] = cv2.filter2D(cv2.filter2D(im,-1,ddg),-1,dg.T)
    imfeat[:,:,8] = cv2.filter2D(cv2.filter2D(im,-1,dg),-1,ddg.T)
    imfeat[:,:,9] = cv2.filter2D(cv2.filter2D(im,-1,g),-1,dddg.T)
    imfeat[:,:,10] = cv2.filter2D(cv2.filter2D(im,-1,ddddg),-1,g.T)
    imfeat[:,:,11] = cv2.filter2D(cv2.filter2D(im,-1,dddg),-1,dg.T)
    imfeat[:,:,12] = cv2.filter2D(cv2.filter2D(im,-1,ddg),-1,ddg.T)
    imfeat[:,:,13] = cv2.filter2D(cv2.filter2D(im,-1,dg),-1,dddg.T)
    imfeat[:,:,14] = cv2.filter2D(cv2.filter2D(im,-1,g),-1,ddddg.T)

    # r,c = im.shape
    # imfeat = np.zeros((r,c,15))
    # imfeat[:,:,0] = scipy.ndimage.gaussian_filter(im,sigma,order=0)
    # imfeat[:,:,1] = scipy.ndimage.gaussian_filter(im,sigma,order=[0,1])
    # imfeat[:,:,2] = scipy.ndimage.gaussian_filter(im,sigma,order=[1,0])
    # imfeat[:,:,3] = scipy.ndimage.gaussian_filter(im,sigma,order=[0,2])
    # imfeat[:,:,4] = scipy.ndimage.gaussian_filter(im,sigma,order=[1,1])
    # imfeat[:,:,5] = scipy.ndimage.gaussian_filter(im,sigma,order=[2,0])
    # imfeat[:,:,6] = scipy.ndimage.gaussian_filter(im,sigma,order=[0,3])
    # imfeat[:,:,7] = scipy.ndimage.gaussian_filter(im,sigma,order=[1,2])
    # imfeat[:,:,8] = scipy.ndimage.gaussian_filter(im,sigma,order=[2,1])
    # imfeat[:,:,9] = scipy.ndimage.gaussian_filter(im,sigma,order=[3,0])
    # imfeat[:,:,10] = scipy.ndimage.gaussian_filter(im,sigma,order=[0,4])
    # imfeat[:,:,11] = scipy.ndimage.gaussian_filter(im,sigma,order=[1,3])
    # imfeat[:,:,12] = scipy.ndimage.gaussian_filter(im,sigma,order=[2,2])
    # imfeat[:,:,13] = scipy.ndimage.gaussian_filter(im,sigma,order=[3,1])
    # imfeat[:,:,14] = scipy.ndimage.gaussian_filter(im,sigma,order=[4,0])

    if normalize:
        if norm_fac is None:
            norm_fac = np.zeros((15,2))
            norm_fac[:,0] = np.mean(imfeat,axis=(0,1))
            norm_fac[:,1] = np.std(imfeat,axis=(0,1))
        imfeat -= norm_fac[:,0]
        imfeat /= norm_fac[:,1]

    return imfeat, norm_fac

def get_gauss_feat_multi(im, sigma = [1,2,4], normalize = True, norm_fac = None):
    '''Multi-scale Gauss derivative feaures for every image pixel.
    Arguments:
        image: a 2D image, shape (r,c).
        sigma: list of standard deviations for Gaussian derivatives.
        normalize: flag indicating normalization of features.
    Returns:
        imfeat: a 3D array of size (r,c,15*n_scale) with a 15*n_scale-dimentional feature
            n_scale is length of sigma
            vector for every image pixel.
    Author: abda@dtu.dk, 2021

    '''
    imfeats = []
    if norm_fac is None:
        norm_fac = [None]*len(sigma)
    for i in range(0,len(sigma)):
        feat, norm_fac[i] = get_gauss_feat_im(im, sigma[i], normalize, norm_fac[i])
        imfeats.append(feat)
    imfeats = np.asarray(imfeats).transpose(1,2,3,0)

    r,c,nf,ns = imfeats.shape
    # imfeats = np.asarray(imfeats).transpose(1,0,2)
    return imfeats.reshape((r,c,nf*ns)), norm_fac


import numpy as np

import pandas as pd

from matplotlib import rcParams
rcParams['font.family'] = 'monospace'
rcParams['font.sans-serif'] = ['Tahoma']
import matplotlib.pyplot as plt

from matplotlib.ticker import (MultipleLocator, AutoMinorLocator)

# smoothing factor and number of data points
N = 15
ALPHA = 2/(N+1)

NUM_X = N * 4

# simple moving average
sma = [0 if i > N else 1/N for i in range(NUM_X)]

# weighted moving average
wma =  [0 if i > N else (N - i)/(N * (N + 1) / 2) for i in range(NUM_X)]

# exponential moving average alpha=2/(N+1)
ema = [ALPHA*(1-ALPHA)**i for i in range(NUM_X)]

# store the values in a data frame
pd.DataFrame({'sma': sma, 'wma': wma, 'ema': ema}).plot(kind='bar', figsize=(16,9))

plt.xticks(np.arange(0, NUM_X, 5), rotation=0)
plt.yticks(fontsize=10)

plt.legend(labels=[
    'SMA',
    'WMA',
    'EMA, α=2/(N+1)'
    ], fontsize=12)

# title and labels
plt.title(f'Weights of Moving Average(N={N})', fontsize=14)
plt.ylabel('Weights', fontsize=12)
plt.xlabel('n-th Most Recent Smple', fontsize=12)
plt.tight_layout()

plt.savefig('images/weight.svg', format='svg')
#plt.show()


#!/usr/bin/env python
#
#       Author:    J. Benjamin Cook
#       E-mail:    jbenjamincook@gmail.com
#
#       File Name: linregPolyVsDegree.py
#       Description:
#           Linear Regression with Polynomial Basis of different degrees
#           based on code code by Romain Thibaux
#           (Lecture 2 from http://www.cs.berkeley.edu/~asimma/294-fall06/)

from utils import preprocessor_create
from utils import poly_data_make
from SupervisedModels.linearRegression import linreg_fit
from SupervisedModels.linearRegression import linreg_fit_bayes
from SupervisedModels.linearRegression import linreg_predict
import numpy as np
import pylab as pl

N = 21
xtrain, ytrain, xtest, _, ytest, _ = poly_data_make(sampling='thibaux', n=N)

degs = np.arange(1, 22)
Nm = len(degs)

# Plot error vs degree
mseTrain = np.zeros(Nm)
mseTest = np.zeros(Nm)
for m in xrange(len(degs)):
    deg = degs[m]
    pp = preprocessor_create(rescale_X=True, poly=deg, add_ones=True)
    model = linreg_fit(xtrain, ytrain, preproc=pp)
    ypredTrain = linreg_predict(model, xtrain)
    ypredTest = linreg_predict(model, xtest)
    mseTrain[m] = np.mean(np.square(ytrain - ypredTrain))
    mseTest[m] = np.mean(np.square(ytest - ypredTest))

ndx = degs <= 16
fig = pl.figure()
pl.plot(degs[ndx], mseTrain[ndx], lw=3)
pl.plot(degs[ndx], mseTest[ndx], lw=3)
pl.xlabel('degree')
pl.ylabel('mse')
leg = pl.legend(('train', 'test'), loc='upper left')
leg.draw_frame(False)
pl.savefig('linregPolyVsDegreeUcurve.png')
pl.show()

degs = [1, 2, 10, 14, 20]
mseTrain = np.zeros(len(degs))
mseTest = np.zeros(len(degs))

for m, deg in enumerate(degs):
    pp = preprocessor_create(rescale_X=True, poly=deg, add_ones=True)
    model = linreg_fit(xtrain, ytrain, preproc=pp)
    ypredTrain = linreg_predict(model, xtrain)
    ypredTest = linreg_predict(model, xtest)
    mseTrain[m] = np.mean(np.square(ytrain - ypredTrain))
    mseTest[m] = np.mean(np.square(ytest - ypredTest))

    pl.figure(m)
    pl.plot(xtrain, ytrain, 'o')
    pl.plot(xtest, ypredTest, lw=3)
    pl.title("degree %d" % deg)
    pl.savefig('polyfitDemo%d.png' % deg)
    pl.xlim([-1, 21])
    pl.ylim([-10, 15])
    pl.show()


import numpy as np
import math

class ID3:
    def __init__(self):
        pass

    def _calc_shannon_entropy(self, dataset, labels):
        '''
        Calculate shannon entropy form data set and labels
        Parameter
            dataset: numpy.array, data set
            labels: list,tuple,numpy.array samples' labels
        Returns
            shannon entropy: float
        '''
        if dataset.shape[0] != len(labels):
            print("dataset length doesn't equal with labels")
            raise Exception

        # 统计数据集D中的K个类及每个类的的数量|Ck|
        if not isinstance(labels, np.ndarray):
            labels_array = np.array(labels)
        else:
            labels_array = labels

        unique_labels = np.unique(labels)
        labels_counts = np.array([0] * len(unique_labels))
        for i, label in enumerate(unique_labels):
            labels_counts[i] = labels_array[labels_array == label].size

        # 计算每个类的比率|Ck|/|D|及训练集的经验熵H(D)
        sample_num = len(dataset)
        prob = labels_counts / sample_num
        shannon_entropy = - (prob * np.log(prob)).sum()

        return shannon_entropy

    def _split_dateset(self, dataset, feature_index, value):
        match_sample_index = np.where(dataset[:, feature_index] == value)
        match_samples = dataset[match_sample_index]
        subdataset = np.delete(match_samples, feature_index, 1)
        return subdataset

    def _split_labels(self, dataset, labels, feature_index, vlaue):
        match_sample_index = np.where(dataset[:, feature_index] == vlaue)
        subdataset_labels = labels[match_sample_index]
        return subdataset_labels

    def _calc_conditional_entropy(self, dataset, labels, feature_index):
        '''
        calculate conditional entropy H(D|A), D is dataset, A is the feature
        Returns
            conditional entropy H(D|A): float
        '''
        feature_values = dataset[:, feature_index]
        unique_feature_values = np.unique(feature_values)

        new_conditional_entropy = 0.0
        for value in unique_feature_values:
            subdataset = self._split_dateset(dataset, feature_index, value)
            subdataset_labels = self._split_labels(dataset, labels, feature_index, value)
            probability = subdataset.shape[0] / dataset.shape[0]
            new_shannon_entropy = self._calc_shannon_entropy(subdataset, subdataset_labels)
            new_conditional_entropy += probability * new_shannon_entropy

        return new_conditional_entropy

    def _calc_entropy_gain(self, conditional_entropy, shannon_entropy):
        return shannon_entropy - conditional_entropy

    def _choose_best_feature(self, dataset, labels):
        '''
        Choose out base feature from current dataset
        Parameters
            dataset: numpy.array

        Returns
            best feature index: int
        '''
        feature_nums = dataset.shape[1]
        shannon_entropy = self._calc_shannon_entropy(dataset, labels)

        best_gain = 0.0
        best_feature = 0
        for i in range(feature_nums):
            conditional_entropy = self._calc_conditional_entropy(dataset, labels, i)
            entropy_gain = self._calc_entropy_gain(conditional_entropy, shannon_entropy)

            if entropy_gain > best_gain:
                best_gain = entropy_gain
                best_feature = i

        return best_feature

    def _find_max_class(self, labels):
        '''
        Find out max class in the labels

        Parameters
            labels: numpy.array

        Returns:
            max count label
        '''
        unique_labels = np.unique(labels)

        max_class = unique_labels[0]
        max_count = 0
        for label in unique_labels:
            label_num = labels[labels == label].size
            if label_num > max_count:
                max_class = label
                max_count = label_num

        return max_class

    def _create_tree(self, dataset, labels):
        if np.unique(labels).size == 1:
            return labels[0]

        if dataset.shape[1] == 1:
            return self._find_max_class(labels)

        best_feature_index = self._choose_best_feature(dataset, labels)
        tree = {best_feature_index: {}}

        feature_values = dataset[:, best_feature_index]
        unique_feature_values = np.unique(feature_values)

        for feature_value in unique_feature_values:
            sub_dataset = self._split_dateset(dataset,
                                              best_feature_index,
                                              feature_value)
            sub_dataset_label = self._split_labels(dataset,
                                                   labels,
                                                   best_feature_index,
                                                   feature_value)
            tree[best_feature_index][feature_value] = self._create_tree(sub_dataset,
                                                                        sub_dataset_label)

        return tree

    def fit(self, dataset, labels):
        self.tree_ = self._create_tree(dataset, labels)
        return self

    def _classify(self, tree, dataset):
        feature_index = list(tree)[0]
        sub_tree = self.tree_[feature_index]
        labels = self._classify(sub_tree, dataset)
        return labels

    def predict(self, dataset):
        return self._classify(self.tree_, dataset)