import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures

# to encode dummies
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder

# -------------------------

# load data
# ------------
dataset = pd.read_csv('../data/Position_Salaries.csv')
# https://stackoverflow.com/questions/509211/understanding-slice-notation/509295#509295
X = dataset.iloc[:, 1:-1].values
y = dataset.iloc[:, -1].values

# => no split here, we will use entire dataset

# run the linear regression model
# ------------
lin_reg = LinearRegression()
lin_reg.fit(X, y)

# run the polynomial regression model
# ------------
poly_reg = PolynomialFeatures(degree = 4)
X_poly = poly_reg.fit_transform(X)
lin_reg_2 = LinearRegression()
lin_reg_2.fit(X_poly, y)

# visualize regression results
# ------------
# -- plot linear reg
plt.scatter(X, y, color = 'red')
plt.plot(X, lin_reg.predict(X), color = 'blue')
plt.title('Truth or Bluff (Linear Regression)')
plt.xlabel('Position Level')
plt.ylabel('Salary')
plt.show()
# -- plot polyn reg
plt.scatter(X, y, color = 'red')
plt.plot(X, lin_reg_2.predict(X_poly), color = 'blue')
plt.title('Truth or Bluff (Polynomial Regression)')
plt.xlabel('Position level')
plt.ylabel('Salary')
plt.show()

# run a future prediction vs test set
# ------------
# y_pred = regressor.predict(X_test)
# np.set_printoptions(precision=2)
# print(np.concatenate((y_pred.reshape(len(y_pred),1), y_test.reshape(len(y_test),1)),1))


import numpy as np
from utility import eta

class ErrorFunction:
    """
    Functions for the "data term" to evaluate model error.
    """

    @staticmethod
    def squared_error(prediction, target):
        """Computes the squared error between the predicted output vector and the target vector

        Args:
            prediction (np.ndarray): multidimensional array of shape (n,m) of predictions for the n examples
            target (np.ndarray): multidimensional array of shape (n,m) of true value for the n examples

        Returns:
            np.float64: squared error loss
        """
        return (np.dot(target - prediction, target - prediction))/2     # /2 usefull for the derivation

    @staticmethod
    def squared_error_der(prediction, target):
        """Computes the derivative of the squared error

        Args:
            prediction (np.ndarray): multidimensional array of shape (n,m) of predictions for the n examples
            target (np.ndarray): multidimensional array of shape (n,m) of true value for the n examples

        Returns:
            np.ndarray: derivative of the squared error
        """
        return (prediction - target)

    @staticmethod
    def euclidean_error(prediction, target):
        """Computes the euclidian error between the predicted output vector and the target vector

        Args:
            prediction (np.ndarray): multidimensional array of shape (n,m) of predictions for the n examples
            target (np.ndarray): multidimensional array of shape (n,m) of true value for the n examples

        Returns:
            np.float64: euclidian error loss
        """
        return np.linalg.norm(prediction - target) # L2 norm for vectors

    @staticmethod
    def euclidian_error_der(prediction, target):
        """Computes the derivative of the euclidian error

        Args:
            prediction (np.ndarray): multidimensional array of shape (n,m) of predictions for the n examples
            target (np.ndarray): multidimensional array of shape (n,m) of true value for the n examples

        Returns:
            np.ndarray: derivative of the euclidian error
        """
        return (prediction - target) / ErrorFunction.euclidean_loss(prediction, target)

    @staticmethod
    def binary_cross_entropy_error(prediction, target):
        """Computes the binary cross-entropy/log loss between the predicted output vector and the target vector

        Args:
            prediction (np.ndarray): multidimensional array of shape (n,m) of predictions for the n examples
            target (np.ndarray): multidimensional array of shape (n,m) of true value for the n examples

        Returns:
            np.float64: binary cross-entropy loss
        """
        return np.sum(np.where(target == 0, - np.log(eta(1. - prediction)), - np.log(eta(prediction))))

    @staticmethod
    def binary_cross_entropy_error_der(prediction, target):
        """Computes the derivative of the binary cross-entropy

        Args:
            prediction (np.ndarray): multidimensional array of shape (n,m) of predictions for the n examples
            target (np.ndarray): multidimensional array of shape (n,m) of true value for the n examples

        Returns:
            np.ndarray: derivative of the binary cross-entropy
        """
        return np.where(target == 0, 1 / eta(1. - prediction), - 1. / eta(prediction))

    @staticmethod
    def init_error_function(name):
        if name == "squared_error":
            return ErrorFunction.squared_error, ErrorFunction.squared_error_der, name
        elif name == "euclidian_error":
            return ErrorFunction.euclidean_error, ErrorFunction.euclidian_error_der, name
        elif name == "binary_cross_entropy":
            return ErrorFunction.binary_cross_entropy_error, ErrorFunction.binary_cross_entropy_error_der, name
        else:
            raise NameError(name+ " is not recognized! Check for correct names and possible error functions in init_error_function!")


'''
Joe Walter

difficulty: 10%
run time:   0:01
answer:     471745499

	***

788 Dominating Numbers

A dominating number is a positive integer that has more than half of its digits equal.

For example, 2022 is a dominating number because three of its four digits are equal to 2. But 2022 is not a dominating number.

Let D(N) be how many dominating numbers are less than 10^N. For example, D(4)=603 and D(10)=21893256.

Find D(2022). Give your answer modulo 1000000007.

	***

Observations

The number of numbers of length n where at least one digit d is repeated m>n/2 times is 9*choose(n,m)*9^(n-m).

Proof:

For an n-digit number, there are choose(n,m) ways to "place" m repetitions of d.

Case 1. The number starts with d (e.g., ddXdXXddd). Then there are 9 choices for d (1-9) and 9^(n-m) choices for all X's (because each X != d).

Case 2. The number does not start with d (e.g., XXdddddX). Then there are 10 choices for d (0-9). If d is positive, then there are 8 choices the frst X and 9 for the others (because the first X cannot be 0 or d, the others can't be d). If d is 0, then there are 9 choices for each X. 9*8*9^(n-m-1) + 1*9^(n-m) = 9*9^(n-m)

When m > n/2, all digits other than d must have repetitions less than m. This means that no numbers are double-counted. ☐

This is the basis for the main loop:

choose(n+1,m)*9^(n+1-m) = 9*choose(n,m)*9^(n-m) + choose(n,m-1)*9^(n-(m-1))

(This is essentially saying that tacking on an X increases the amount by 9x (9 choices for X), while tacking on a d increases the amount by 1x)

Proof:

choose(n+1,m) = choose(n,m) + choose(n,m-1)		(a common identity, and since m>n/2>0)
'''

import numpy as np

def D(N, M = 1000000007):
	# row = total number of digits
	# col = number of digits that are equal
	a = np.zeros((N,N+1), dtype=np.uint64)
	a[0][:2] = [9,1]
	for i in range(1,N):
		a[i] = 	np.mod(
					9*a[i-1] + np.roll(a[i-1],1), # np.roll used to shift right
				M)
	for i in range(N):
		a[i][:(i+1)//2+1] = 0
	return 9*int(a.sum()) % M

assert D(4) == 603
assert D(10) == 21893256

print(D(2022))


import numpy as np
import matplotlib.pyplot as plt
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Ridge

from sklearn.model_selection import cross_val_score
from sklearn.metrics import mean_squared_error
import pandas as pd

MEASUREMENTS_CSV = "firebase-measurements.csv"

np.set_printoptions(suppress=True)
df = pd.read_csv(MEASUREMENTS_CSV)

print("Original Dataframe")
print(df.head())
x = df[['distance', 'pixel_width']]
y = df[['diameter']]

linear_regression = LinearRegression(normalize=True)
# linear_regression = Ridge(normalize=True)

poly = PolynomialFeatures(2, interaction_only = False)
xPoly = poly.fit_transform(x)

# linear_regression.fit(x,y)

linear_regression.fit(xPoly,y)
y_pred = linear_regression.predict(xPoly)

results = np.concatenate((y_pred, y.to_numpy(), y_pred - y.to_numpy()), axis = 1)
print("Results Dataframe")
print(results[:5])
print("Coefficients = ", linear_regression.coef_[0])
print("Intercept = ", linear_regression.intercept_[0])
print("Feature Names: ", poly.get_feature_names())
print("MSE for model is: ", mean_squared_error(y_pred, y.to_numpy()))

distance = 1.4443359375
pixel_width = 72.0
ground_truth = 22.28169059753418
# x0 = -0.179362
# x1 = -0.2876224
# x0sq = 0.000187
# x0x1 = 0.0026005
# x1sq = 0.00046542
# b = 36.186

x0 = linear_regression.coef_[0][1]
x1 = linear_regression.coef_[0][2]
x0sq = linear_regression.coef_[0][3]
x0x1 = linear_regression.coef_[0][4]
x1sq = linear_regression.coef_[0][5]
b = linear_regression.intercept_[0]

print("C# Variables")
print("var x0 = %.6f\nvar x1 = %.6f,\nvar x0sq = %.6f,\nvar x0x1 = %.6f,\nvar x1sq = %.6f,\nvar b = %.6f" % (x0, x1, x0sq, x0x1, x1sq, b))
diameter = distance * x0 + pixel_width * x1 + distance**2 * x0sq + distance * pixel_width * x0x1 + pixel_width**2 * x1sq + b

print(f"With a distance of {distance}cm and pixel width of {pixel_width}px, the diameter is calcualted to be {diameter}cm")
print(f"Ground truth was {ground_truth}")


import numpy as np

def logit(x):
    return np.log(x) - np.log(1 - x)

def ilogit(x):
    return 1. / (1. + np.exp(-x))

def logpdf(value, mu, var):
    return -0.5 * (
            np.log(2 * np.pi) + np.log(var) + (value - mu) ** 2 / var
    )


import numpy as np

def cross_entropy_error(y, t):
  delta = 1e-7
  return -np.sum(t * np.log(y + delta))

t = [0, 0, 1, 0, 0, 0, 0, 0, 0, 0]
y = [0.1, 0.05, 0.6, 0.0, 0.05, 0.1, 0.0, 0.1, 0.0, 0.0]
print(cross_entropy_error(np.array(y), np.array(t)))

y = [0.1, 0.05, 0.1, 0.0, 0.05, 0.1, 0.0, 0.6, 0.0, 0.0]
print(cross_entropy_error(np.array(y), np.array(t)))


#  Exercise 7 | Principle Component Analysis and K-Means Clustering

from matplotlib import use
use('TkAgg')
import numpy as np
import scipy.io
import scipy.misc
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

from featureNormalize import featureNormalize
from pca import pca
from projectData import projectData
from recoverData import recoverData
from kMeansInitCentroids import kMeansInitCentroids
from runkMeans import runkMeans
from plotDataPoints import plotDataPoints
from displayData import displayData
from matplotlib.pyplot import show

#  ================== Part 1: Load Example Dataset  ===================

print('Visualizing example dataset for PCA.')
# The following command loads the dataset. You should now have the
# variable X in your environment
data = scipy.io.loadmat('ex7data1.mat')
X = data['X']

# Visualize the example dataset
"""plt.figure()
plt.scatter(X[:, 0], X[:, 1], marker='o', color='b', facecolors='none', lw=1.0)
plt.axis([0.5, 6.5, 2, 8])
plt.axis('equal')
show()"""

input('Program paused. Press Enter to continue...')

#  =============== Part 2: Principal Component Analysis ===============

print('Running PCA on example dataset.')

#  Before running PCA, it is important to first normalize X
X_norm, mu, sigma = featureNormalize(X)

# Run PCA
U, S, V = pca(X_norm)

# Compute mu, the mean of the each feature

# Draw the eigenvectors centered at mean of data. These lines show the
# directions of maximum variations in the dataset.
mu2 = mu + 1.5 * S.dot(U.T)
"""plt.figure()
plt.scatter(X[:, 0], X[:, 1], marker='o', color='b', facecolors='none', lw=1.0)
plt.axis([0.5, 6.5, 2, 8])
plt.axis('equal')
plt.plot([mu[0], mu2[0, 0]], [mu[1], mu2[0, 1]], '-k', lw=2)
plt.plot([mu[0], mu2[1, 0]], [mu[1], mu2[1, 1]], '-k', lw=2)
show()"""

print('Top eigenvector: ')
print(' U(:,1) = %f %f ' % (U[0, 0], U[1, 0]))
print('(you should expect to see -0.707107 -0.707107)')

input('Program paused. Press Enter to continue...')

#  =================== Part 3: Dimension Reduction ===================

print('Dimension reduction on example dataset.')

# Plot the normalized dataset (returned from pca)
plt.figure()
plt.scatter(X_norm[:, 0], X_norm[:, 1], marker='o',
            color='b', facecolors='none', lw=1.0)
plt.axis([-4, 3, -4, 3])  # axis square
plt.axis('equal')

# Project the data onto K = 1 dimension
K = 1
Z = projectData(X_norm, U, K)
print('Projection of the first example: %f' % Z[0])
print('(this value should be about 1.481274)')

X_rec = recoverData(Z, U, K)
print('Approximation of the first example: %f %f' % (X_rec[0, 0], X_rec[0, 1]))
print('(this value should be about  -1.047419 -1.047419)')

#  Draw lines connecting the projected points to the original points
plt.scatter(X_rec[:, 0], X_rec[:, 1], marker='o',
            color='r', facecolor='none', lw=1.0)
for i in range(len(X_norm)):
    plt.plot([X_norm[i, 0], X_rec[i, 0]], [X_norm[i, 1], X_rec[i, 1]], '--k')
show()

input('Program paused. Press Enter to continue...')

#  =============== Part 4: Loading and Visualizing Face Data =============

print('Loading face dataset.')

# Load Face dataset
data = scipy.io.loadmat('ex7faces.mat')
X = data['X']

# Display the first 100 faces in the dataset
plt.figure()
displayData(X[0:100, :])

input('Program paused. Press Enter to continue...')

#  =========== Part 5: PCA on Face Data: Eigenfaces  ===================

print('Running PCA on face dataset.\n(this might take a minute or two ...)\n\n')

# Before running PCA, it is important to first normalize X by subtracting
# the mean value from each feature
X_norm, mu, sigma = featureNormalize(X)

#  Run PCA
U, S, V = pca(X_norm)

#  Visualize the top 36 eigenvectors found
plt.figure()
displayData(U[:, 1:36].T)

input('Program paused. Press Enter to continue...')

#  ============= Part 6: Dimension Reduction for Faces =================
print('Dimension reduction for face dataset.')

K = 100
Z = projectData(X_norm, U, K)

print('The projected data Z has a size of: ')
print('%d %d' % Z.shape)

input('Program paused. Press Enter to continue...')

#  ==== Part 7: Visualization of Faces after PCA Dimension Reduction ====

print('Visualizing the projected (reduced dimension) faces.')

K = 100
X_rec = recoverData(Z, U, K)

# Display normalized data
plt.figure()
plt.subplot(1, 2, 1)
displayData(X_norm[:100,:])
plt.title('Original faces')
plt.axis('equal')

# Display reconstructed data from only k eigenfaces
plt.subplot(1, 2, 2)
displayData(X_rec[:100, :])
plt.title('Recovered faces')
plt.axis('equal')
show()
input('Program paused. Press Enter to continue...')

#  === Part 8 -  PCA for Visualization ===
A = scipy.misc.imread('bird_small.png')

# If imread does not work for you, you can try instead
#   load ('bird_small.mat')

A = A / 255.0
img_size = A.shape
X = A.reshape(img_size[0] * img_size[1], 3)
K = 16
max_iters = 10
initial_centroids = kMeansInitCentroids(X, K)
centroids, idx = runkMeans(X, initial_centroids, max_iters)

# Sample 1000 random indexes (since working with all the data is
# too expensive. If you have a fast computer, you may increase this.
sel = np.floor(np.random.random(1000) * len(X))
sel = sel.astype(int)

# Setup Color Palette

# Visualize the data and centroid memberships in 3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
Xs = np.array([X[s] for s in sel])
xs = Xs[:, 0]
ys = Xs[:, 1]
zs = Xs[:, 2]
cmap = plt.get_cmap("jet")
idxn = idx[sel]
idxn = idxn.astype('float') / max(idxn.astype('float'))
colors = cmap(idxn)

ax = Axes3D(fig)
ax.scatter3D(xs, ys, zs=zs, edgecolors=colors,
             marker='o', facecolors='none', lw=0.4, s=10)

plt.title('Pixel dataset plotted in 3D. Color shows centroid memberships')
show()
input('Program paused. Press Enter to continue...')

#  === Part 8 -  PCA for Visualization ===
#  Use PCA to project this cloud to 2D for visualization

# Subtract the mean to use PCA
X_norm, mu, sigma = featureNormalize(X)

# PCA and project the data to 2D
U, S, V = pca(X_norm)
Z = projectData(X_norm, U, 2)

# Plot in 2D
plt.figure()
zs = np.array([Z[s] for s in sel])

plotDataPoints(zs, idxn)
plt.title('Pixel dataset plotted in 2D, using PCA for dimensionality reduction')
show()

input('Program paused. Press Enter to continue...')


import time
import numpy as np

def get_sequence(seed):
	"""
	return a list of the chain generated by the Collatz sequence starting at seed
	"""
	assert seed >= 1
	if seed == 1:
		return [1,4,2,1]
	S = [seed]
	while seed != 1:
		if seed%2:
			seed = 3*seed + 1
		else:
			seed /= 2
		S.append(seed)
	return S

def find_longest_chain(N):
	"""
	Find the longest collatz sequence chain generated by a number less than N
	"""
	maxlength = 4
	seed = 1
	for i in range(1,N,1):
		j = len(get_sequence(i))
		if j > maxlength:
			seed = i
			maxlength = j
	return (seed, maxlength)

def find_longest_chain_alt(N):
	"""
	One line version of previous routine. Runs in roughly the same amount of time. Disadvantage is having to create entire list of sequence lengths and then take the max.
	"""
	return np.argmax([len(get_sequence(i)) for i in range(1,N,1)])+1

def find_longest_chain_alt2(N):
	"""
	alternate approach using dictionary objects {seed: chain length}. Much more efficient since it doesn't need to proceed further once the sequence reaches a term which is a previously computed seed
	"""
	import operator
	r = {}
	r[1] = 1
	r[2] = 2
	for i in range(1,N):
		num = i
		chain = 0
		while num not in r:
			if num % 2 == 0:
				num /= 2
			else:
				num = 3*num + 1
			chain += 1
		r[i] = chain + r[num]
	return (max(r.items(), key=operator.itemgetter(1)))

def test1():
	S = get_sequence(837799)
	print(S)
	print(len(S))

def test2():
	start = time.time()
	print(find_longest_chain_alt2(10**6))
	end = time.time()
	print(end-start)

	start = time.time()
	print(find_longest_chain(10**6))
	end = time.time()
	print(end-start)

	start = time.time()
	print(find_longest_chain_alt(10**6))
	end = time.time()
	print(end-start)

def main():
	start = time.time()
	print(find_longest_chain(10**6))
	end = time.time()
	print(end-start)

if __name__ == "__main__":
	#test1()
	#test2()
	main()


import numpy as np

np.seterr(divide='ignore', invalid='ignore')

def marginal_entropy(x,bins):
    px = np.histogram(x,bins)[0]/float(np.sum(np.histogram(x,bins)[0]))
    return np.nansum(px*np.log(1/px))

def joint_entropy(x,y,bins):
    pxy = np.histogram2d(x,y,bins)[0]/float(np.sum(np.histogram2d(x,y,bins)[0]))
    return np.nansum(pxy*np.log(1/pxy))

def mutual_information(x,y,bins):
    hx = marginal_entropy(x,bins)
    hy = marginal_entropy(y,bins)
    hxy = joint_entropy(x,y,bins)
    return hx+hy-hxy

def conditional_entropy(x,y,bins):
    hx = marginal_entropy(x,bins)
    hy = marginal_entropy(y,bins)
    hxy = joint_entropy(x,y,bins)
    return {'H(X|Y)': hxy-hy,
            'H(Y|X)': hxy-hx}


import numpy as np
from sklearn import preprocessing
import matplotlib.pyplot as plt

# Load the dataset of orb features from file.
orb_features = np.genfromtxt(open("orb_features.txt"))
orb_features = orb_features.T

orb_size = len(orb_features[:, 0])
# TODO: Compute covariance.
cov = np.cov(orb_features)

# TODO: Compute eigenvectors and eigenvalues.
eig_val_cov, eig_vec_cov = np.linalg.eig(cov)

# Sort eigenvectors and corresponding eigenvalues in descending order.
eig_pairs = [(np.abs(eig_val_cov[i]), eig_vec_cov[:, i]) for i in range(len(eig_val_cov))]
eig_pairs.sort(key=lambda x: x[0], reverse=True)

# TODO: Compute 5 dimensional feature vector based on largest eigenvalues and normalize the output.
matrix_w = np.hstack((eig_pairs[0][1].reshape(orb_size, 1), eig_pairs[1][1].reshape(orb_size, 1),
                      eig_pairs[2][1].reshape(orb_size, 1), eig_pairs[3][1].reshape(orb_size, 1),
                      eig_pairs[4][1].reshape(orb_size, 1))).T

pca_features = np.real(matrix_w.dot(orb_features))

pca_features = pca_features

# Normalize pca features.
pca_features = preprocessing.scale(pca_features.T)

# 2D plot of first 2 principal components.
plt.scatter(pca_features[:, 0], pca_features[:, 1], marker = 'o')
plt.xlabel('principal component 1')
plt.ylabel('principal component 2')
plt.title('PCA result')
plt.show()

# Save results.
np.savetxt('results_pca.txt', pca_features)


# Copyright (c) Dietmar Wolz.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory.

# This problem consists of minimizing the weight of a tension/compression spring subject to constraints on
# shear stress, surge frequency and minimum deflection
# The design variables are:

# - x1: the mean coil diameter
# - x2: the wire diameter
# - x3: the number of active coils

# which are subject to four inequality constraints.
# fcmaes retry used with a penalty for constraint violations 'weight_penalty'
# works as well as scipy minimize.

# This example is taken from https://www.sciencedirect.com/science/article/abs/pii/S0096300306015098

import math
import time
import numpy as np
from scipy.optimize import Bounds, minimize
from fcmaes.optimizer import dtime, random_x, logger
from fcmaes import retry, advretry

bounds = Bounds([0.01, 0.01, 0.01], [20, 20, 20])

def _feasible(x):
    x = np.array(x)
    return np.maximum(np.minimum(x, bounds.ub), bounds.lb)

def constraint_ineq(x):
    return [ x[1]**3 * x[2] / (71785 * x[0]**4) - 1,
             1 - (4*x[1]**2 - x[0]*x[1]) / (12566*(x[1]*x[0]**3 - x[0]**4)) - 1/(5108*x[0]**2),
             140.45*x[0]/(x[1]**2 * x[2]) - 1,
             1 - (x[1] + x[0]) / 1.5]

def penalty(x):
    return - np.sum(np.minimum(constraint_ineq(x), 0))

def weight(x):
    return (x[2] + 2) * x[1]*x[0]**2

def weight_penalty(x):
    try:
        val = weight(x)
        return val + 100000*penalty(x)
    except ZeroDivisionError:
        return 1E99

def print_result(ret, best, t0, i):
    x = _feasible(ret.x) # make sure result is _feasible
    w = weight(x)
    val = weight_penalty(x) # add penalty for ineq constraint violation
    if val < best:
        pen = penalty(x) # check ineq constraint
        best = min(val, best)
        print("{0}: time = {1:.1f} best = {2:.8f} f(xmin) = {3:.8f} ineq = {4:.5f}"
              .format(i+1, dtime(t0), best, w, pen))
    return best

def test_minimize_SLSQP(fun, num):
    ineq_cons = {'type': 'ineq', 'fun' : constraint_ineq}

    best = math.inf
    t0 = time.perf_counter();
    for i in range(num):
        guess = random_x(bounds.lb, bounds.ub)
        ret = minimize(fun, x0 = guess, bounds = bounds,
                       method='SLSQP', constraints=[ineq_cons])
        if ret.success:
            best = print_result(ret, best, t0, i)

if __name__ == '__main__':

    # try as alternative
    # test_minimize_SLSQP(weight, 10000)

    t0 = time.perf_counter();
    ret = advretry.minimize(weight_penalty, bounds, logger = logger(), num_retries = 320)
    #ret = retry.minimize(weight_penalty, bounds, logger = logger(), num_retries=32)
    print_result(ret, 10000, t0, 0)


"""
@file:          linearModel.py
@description:   answers to problems in exercises 6.6
                (Linear Algebra and its applications, David C. Lay 3ed)
@author:        Hiu-kong Dan
@date:          June 3, 2021
"""

from sympy import Matrix
import numpy as np
from numpy import linalg
from matplotlib import pyplot as plt
import math

def delimeter():
    print("-"*20)

def printAnswerMsg(title, res):
    """
    @description:
        print answers
    @param:
        title: string
            the question title: string
        res: tuple
            tuple returned by getLeastSquare
    @return:
        none
    """
    print(title, ": ")
    print("b: ", res[0])
    print("e: ", res[1])
    delimeter()

def getLeastSquare(X, y):
    """
    @description:
        calculate the least square solution of Xb = y
    @param:
        X: design matrix
        y: observation vector
    @return:
        (b, e)
        b: parameter vector
        e: error vector
    """
    XTX = np.matmul(X.T, X)
    XTX_INV = np.linalg.inv(XTX)
    XTY = np.matmul(X.T, y)
    b = np.matmul(XTX_INV, XTY)
    XB = np.matmul(X, b)
    e = y - XB
    return (b, e)

def problem7():
    X = np.array([[1,1],
                  [2,4],
                  [3,9],
                  [4,16],
                  [5,25]])
    y = np.array([1.8,2.7,3.4,3.8,3.9])
    res = getLeastSquare(X, y)
    printAnswerMsg("problem7", res)

def problem8():
    X = np.array([[4, 4**2, 4**3],
                  [6, 6**2, 6**3],
                  [8, 8**2, 8**3],
                  [10, 10**2, 10**3],
                  [12, 12**2, 12**3],
                  [14, 14**2, 14**3],
                  [16, 16**2, 16**3],
                  [18, 18**2, 18**3]])
    y = np.array([1.58,2.08,2.5,2.8,3.1,3.4,3.8,4.32])
    res = getLeastSquare(X, y)
    b = res[0]

    func = lambda x: b[0]*x + b[1]*x**2 + b[2]*x**3;
    plt.plot([x for x in range(100)], [func(x) for x in range(100)])
    plt.show()

def problem10():
    X = np.array([[math.e**-.2, math.e**-.7],
                  [math.e**-.22, math.e**(-.07*11)],
                  [math.e**(-.02*12), math.e**(-.07*12)],
                  [math.e**(-.02*14), math.e**(-.07*14)],
                  [math.e**(-.02*15), math.e**(-.07*15)]])
    y = np.array([21.34, 20.68,20.05,18.87,18.30])

    res = getLeastSquare(X, y)

    printAnswerMsg("problem10", res)

def problem11():
    X = np.array([[1, 3.00*math.cos(.88)],
                  [1, 2.30*math.cos(1.10)],
                  [1, 1.65*math.cos(1.42)],
                  [1, 1.25*math.cos(1.77)],
                  [1, 1.01*math.cos(2.14)]])
    y = np.array([3.00, 2.30, 1.65, 1.25, 1.01])

    res = getLeastSquare(X, y)

    printAnswerMsg("problem11", res)

def problem12():
    X = np.array([[1, 1, 1, 1, 1],
                  [3.78, 4.11, 4.41, 4.73, 4.88]]).T
    y = np.array([91.0, 98, 103, 110, 112])

    res = getLeastSquare(X, y)

    func = lambda w: res[0][0] + res[0][1] * math.log(w)

    print("problem 12:", func(100))
    delimeter()

def problem13():
    time = [x for x in range(13)]
    data = [0, 8.8, 29.9, 62.0, 104.7, 159.1, 222.0, 294.5, 380.4, 471.1, 571.7, 686.8, 809.2]
    X = np.array([[1 for x in range(len(time))],
                  [x for x in time],
                  [x*x for x in time],
                  [x*x*x for x in time]]).T

    y = np.array(data)

    res = getLeastSquare(X, y)

    print("b")

    b = res[0]
    func = lambda t: b[1] + 2 * b[2] * t + 3 * b[3] * t * t

    print("velocity of the plane at time 4.5 is ", func(4.5))

if __name__ == "__main__":
    problem13()


import numpy as np

# In[]:
T = np.array([
    [0.9, 0.1],
    [0.5, 0.5]
])

# declaring initial distribution
v = np.array(
    [1, 0]
)

for i in [1,2,3,4,5]:
    T_i = np.linalg.matrix_power(T, i)
    print('Transition Prob after time k=',i,'\n',T_i)
    vti = np.dot(v, T_i)
    print('Prob of being in a specific state after', i, 'iterations = \n', vti)
# the transition probability converges to array([
#           [0.83333333, 0.16666667],
#           [0.83333333, 0.16666667]
#     ])
# Thus the asymptotic belief is that after infinite steps, the state will be
# system will remain in s0 with a prob of 83.3% and move to s1 16.7% of the time
# Upon moving to s1, it will stay in s1 16.7% of the time, and move to s0
# with 83.3% probability
# this is ind. of the initial state as we can see with the initial state vector.
# thus this Markov process has converged.

# In[]:
# there are 12 states, thus the transition matrix should be a 12*12 matrix giving
# the transition prob. from each state to every other state
# Now there are 4 actions also, thus the transition matrix should have this also since
# the prob. of moving to other state is a function of T(s,s',a).
# Thus the transition prob will be a 12*12*4 matrix, where each T[:,:,0] is the transition
# matrix for action 0, and so on.
T = np.load('T.npy')
NUM_ACTIONS = 4

# the agent starts from position (1,1) which is the bottom-most left corner
v = np.array([[
    0.0, 0.0, 0.0, 0.0,
    0.0, 0.0, 0.0, 0.0,
    1.0, 0.0, 0.0, 0.0
]])
# Utility vector : this utility vector came from the value iteration algorithm
u = np.array([[
    0.812, 0.868, 0.918, 1.0,
    0.762, 0.0, 0.660, -1.0,
    0.705, 0.655, 0.611, 0.388
]])
reward = -0.04
gamma = 1

# In[]:

def return_state_utility(v, T, u, reward, gamma):
    """Return the state utility. Computes the best action to be taken in each state
    and returns the state utility if that action is taken. Note that there is no policy
    defined here. # TODO : Why is there no policy defined here??
    @param v the state vector
    @param T transition matrix
    @param u utility vector
    @param reward for that state
    @param gamma discount factor
    @return the utility of the state
    """
    action_array = np.zeros(NUM_ACTIONS)
    for action in range(NUM_ACTIONS):
        #     This bellman equation is basically calculating the utilities, and whichever neighbor
        #     utility is highest, its picking that up and assigning the utility of the current state
        action_array[action] = np.sum(np.multiply(u, np.dot(v, T[:, :, action])))

    # With max oper, its just picking up the action with the highest utility
    return reward + gamma * np.max(action_array)

# In[]:

utility_11 = return_state_utility(v, T, u, reward, gamma)
print('Utility of state (1,1) = ', utility_11)

# In[]:
# The Value iteration algorithm to compute the utility vector

tot_states = 12

# Discount Factor
gamma = 0.999
iteration = 0

#Stopping criteria small value
epsilon = 0.001

# List containing the data for each iteation
graph_list = list()

#Reward vector, note the +1 and -1 for the charging station and the stairs
r = np.array([-0.04, -0.04, -0.04, +1.0,
    -0.04, 0.0, -0.04, -1.0, -0.04, -0.04, -0.04, -0.04
])

# Initial Utility vectors
u1 = np.array([0.0, 0.0, 0.0, 0.0,
    0.0, 0.0, 0.0,  0.0, 0.0, 0.0, 0.0,  0.0
])

# In[]:

# The Value Iteration Algorithm
# You only need to define the initial transition probabilities wrt each action
# and wrt each state to state, and the rewards for each state.
# The utilities and everything else will be learnt
# from here
while True:
    delta = 0
    u = u1.copy()
    iteration += 1
    graph_list.append(u)

    for state in range(tot_states):
        reward = r[state]
        state_vector = np.zeros((1, tot_states))
        state_vector[0, state] = 1
        u1[state] = return_state_utility(state_vector, T, u, reward, gamma)
        delta = max(delta, np.abs(u1[state] - u[state]))

    if delta < epsilon * (1-gamma) / gamma:
        print("=================== FINAL RESULT ==================")
        print("Iterations: " + str(iteration))
        print("Delta: " + str(delta))
        print("Gamma: " + str(gamma))
        print("Epsilon: " + str(epsilon))
        print("===================================================")
        print('Utility Values')
        print(u[0:4])
        print(u[4:8])
        print(u[8:12])
        print("===================================================")
        break

# In[]:
# Plotting the utility value convergence of each state
import matplotlib.pyplot as plt
for state in range(tot_states):
    utility_vals_history = [el[state] for el in graph_list]
    plt.plot(utility_vals_history, label='state'+str(state))
plt.legend()
plt.show()

# In[]:
# With the value iteration algorithm we have a way to estimate the utility
# of each state. What we still miss is a way to estimate an optimal policy.
import numpy as np
def return_policy_evaluation(policy, u, r, T, gamma):
    """Return the policy utility, i.e. wrt the action which the given policy chose, what is the utility
    of each of the states, (not just one state as as done in return_state_utility)

    @param policy policy vector
    @param u utility vector
    @param r reward vector
    @param T transition matrix
    @param gamma discount factor
    @return the utility vector u
    """
    for state in range(tot_states):
        if not np.isnan(policy[state]):
            v = np.zeros((1, tot_states))
            v[0, state] = 1
            action = int(policy[state])
            # Since we have a policy and the policy associate to each state an action,
            # we can get rid of the max operator and use a simplified version of the
            # Bellman equation without the max operator
            u[state] = r[state] + gamma * np.sum(np.multiply(u, np.dot(v, T[:, :, action])))
    return u

def return_expected_action(u, T, v):
    """Return the expected action.

    It returns an action based on the expected utility of doing a in state s,
    according to T and u. This action is the one that maximize the expected utility.
    @param u utility vector
    @param T transition matrix
    @param v starting vector
    @return expected action (int)
    """
    actions_array = np.zeros(NUM_ACTIONS)
    for action in range(NUM_ACTIONS):
        # Expected utility of doing action a in state s, according to T and u.
        actions_array[action] = np.sum(np.multiply(u, np.dot(v, T[:, :, action])))
    return np.argmax(actions_array)

def print_policy(p, shape):
    """Printing utility.

    Print the policy actions using symbols:
    ^, v, <, > up, down, left, right
    * terminal states
    # obstacles
    """
    counter = 0
    policy_string = ""
    for row in range(shape[0]):
        for col in range(shape[1]):
            if(p[counter] == -1): policy_string += " *  "
            elif(p[counter] == 0): policy_string += " ^  "
            elif(p[counter] == 1): policy_string += " <  "
            elif(p[counter] == 2): policy_string += " v  "
            elif(p[counter] == 3): policy_string += " >  "
            elif(np.isnan(p[counter])): policy_string += " #  "
            counter += 1
        policy_string += '\n'
    print(policy_string)

# In[]:

# The next blocks are the implementation of the policy iteration algorithm
gamma = 0.999
epsilon = 0.0001
iteration = 0
tot_states = 12
T = np.load("T.npy")

# Generate the first policy randomly
# NaN=Nothing, -1=Terminal, 0=Up, 1=Left, 2=Down, 3=Right
p = np.random.randint(0, 4, size=(tot_states)).astype(np.float32)
p[5] = np.NaN
# 3 and 7 are the terminal states, no action can be taken there.
# 3 is the charging station, and 7 is the staircase
p[3] = p[7] = -1

# Utility vectors
u = np.array([0.0, 0.0, 0.0,  0.0,
              0.0, 0.0, 0.0,  0.0,
              0.0, 0.0, 0.0,  0.0])
# Reward vector
r = np.array([-0.04, -0.04, -0.04,  +1.0,
              -0.04,   0.0, -0.04,  -1.0,
              -0.04, -0.04, -0.04, -0.04])

# In[]:

while True:
    iteration += 1
    u1 = np.copy(u)
    # 1- Policy evaluation
    u = return_policy_evaluation(p, u, r, T, gamma)
    delta = np.max(np.abs(u - u1))
    if delta < epsilon * (1-gamma) / gamma:
        print(delta)
        break

    # 2- Policy improvement
    for s in range(tot_states):
        if not np.isnan(p[s]) and not p[s] == -1:
            # p[s] != -1 check to confirm the state is not terminal
            v = np.zeros((1, tot_states))
            v[0, s] = 1
            a = return_expected_action(u, T, v)
            if a != p[s]:
                p[s] = a
print_policy(p, shape=(3, 4))
print("=================== FINAL RESULT ==================")
print("Iterations: " + str(iteration))
print("Delta: " + str(delta))
print("Gamma: " + str(gamma))
print("Epsilon: " + str(epsilon))
print("===================================================")
print(u[0:4])
print(u[4:8])
print(u[8:12])
print("===================================================")
print_policy(p, shape=(3, 4))
print("===================================================")

# Policy iteration and value iteration, which is best?
# If you have many actions or you start from a fair policy then choose policy iteration.
# If you have few actions and the transition is acyclic then chose value iteration.
# If you want the best from the two world then give a look to the modified policy iteration algorithm.


import numpy as np
from math import sqrt, sin, cos, exp, pi
import random
import matplotlib.pyplot as plt
from matplotlib import cm, colors
from mpl_toolkits.mplot3d import Axes3D
from sensors import save_points

random.seed(1)

class Point:
    """
    Representation of a point in spherical coordinates.
    """
    def __init__(self, r, phi, th):
        self.r = r
        self.phi = phi
        self.th = th

def create_random_points(n, r):
    """
    Randomly distribute n points on sphere.
    :param n: number of points
    :param r: radium of sphere
    :return: list of Points
    """
    return [Point(r, random.uniform(0,1)*2*pi, random.uniform(0,1)*pi) for _ in range(n)]

def euclidean(p1, p2):
    """
    Calculate euclidean distance in spherical coordinates.
    :param p1: First Point(r, phi, th)
    :param p2: Second Point(r, phi, th)
    :return: euclidean distance calculated in spherical coordinates
    """
    return sqrt(p1.r**2 + p2.r**2 - 2*p1.r*p2.r*(sin(p1.th)*sin(p2.th)*cos(p1.phi-p2.phi)+cos(p1.th)*cos(p2.th)))

def energy(points):
    """
    Calculates Electrostatic-Energy of all points on sphere.
    :param points: list of points
    :return: Electrostatic-Energy without actual physics constants, unit [m**-1]
    """
    E = 0
    n = len(points)
    for i in range(0, n):
        p1 = points[i]
        for j in range(i+1, n):
            p2 = points[j]
            E += 1/abs(euclidean(p1, p2))
    return E

def sphere_optimisation(points, sigma, T, dT, N0=1000, N_max=10000):
    """
    Redistributes random points on sphere in a way that Electrostatic-Energy
    achieves its minimum, based on simulated annealing in combination with
    Monte-Carlo.
    :param points: Randomly distributed list of points on sphere.
    :param sigma: Variance parameter for Gaussian distribution.
    :param T: Initial Temperature of our system.
    :param dT: Step for decreasing T, T -> T-dT
    :param N0: Maximum number of accepted moves for each T.
    :param N_max: Maximum number of iterations for each T.
    :return: List of evenly distributed points on sphere.
    """
    Ec = energy(points)
    print(Ec)
    while T > 0:
        n_iter = 0
        N_accepted = N0
        while N_accepted > 0 and n_iter < N_max:
            n_iter += 1
            point = random.choice(points[1:])  # let one point intact
            dphi = np.random.normal(0, sigma)
            dth = np.random.normal(0, sigma/2)

            point.phi += dphi
            point.th += dth
            En = energy(points)
            dE = En-Ec

            if n_iter % 500 == 0:
                print(T, Ec, dE, N_accepted, n_iter)

            if dE < 0:
                Ec = En
                N_accepted -= 1
                continue
            else:
                r1 = random.uniform(0,1)
                if r1 < exp(-dE/T):
                    Ec = En
                    N_accepted -= 1
                    continue
                else:
                    point.phi -= dphi
                    point.th -= dth
        T -= dT
    return points

def visualize_points(opt_points, r):
    phis = []
    thetas = []
    for p in opt_points:
        phis.append(p.phi)
        thetas.append(p.th)

    coss = np.cos
    sinn = np.sin
    phi, theta = np.mgrid[0.0:pi:100j, 0.0:2.0*pi:100j]
    x = r*sinn(phi)*coss(theta)
    y = r*sinn(phi)*sinn(theta)
    z = r*coss(phi)

    xx = r*coss(phis)*sinn(thetas)
    yy = r*sinn(phis)*sinn(thetas)
    zz = r*coss(thetas)

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')

    ax.plot_surface(
        x, y, z,  rstride=1, cstride=1, color='c', alpha=0.3, linewidth=0)

    ax.scatter(xx,yy,zz,color="k",s=20)

    ax.set_xlim([-r,r])
    ax.set_ylim([-r,r])
    ax.set_zlim([-r,r])
    ax.set_aspect("equal")
    plt.tight_layout()
    plt.show()

def to_coordinates(opt_points, r=1):
    phis = []
    thetas = []
    for p in opt_points:
        phis.append(p.phi)
        thetas.append(p.th)

    coss = np.cos
    sinn = np.sin

    xx = r * coss(phis) * sinn(thetas)
    yy = r * sinn(phis) * sinn(thetas)
    zz = r * coss(thetas)
    points = list(zip(xx, yy, zz))
    return points

R = 1
rpoints = create_random_points(50, R)
opt_points = sphere_optimisation(rpoints, 0.05, 10, 0.1, N_max=1000)
visualize_points(opt_points, R)
save_points("data/generated00.txt", to_coordinates(opt_points))


import scipy
import numpy
import matplotlib.pyplot as plt
import sympy
import math
#puntos a interpolar
A= sympy.Matrix([[-3, -1 , 2, 4],
                [0, 4, 3, 1]])

#array de polinomios
polinomio = sympy.Matrix([[0],
                        [0]])
t = sympy.symbols("t")

def combinatoria(i,n):
    return math.factorial(n) / (math.factorial(i)* math.factorial(n-i))

##### lagrange en x , y
def bernstein(i,n):
        ber = combinatoria(i,n)*((t**i))*((1-t)**(n-i))
        return ber
def Bezier(points):
    for r in range(points):
        polinomio[: , 0 ]  = polinomio[: , 0] + bernstein(r,points-1)*A[:,r]
    return polinomio

poly=Bezier(4)

polysimplex = sympy.expand(poly[0,0])
polysimpley = sympy.expand(poly[1,0])
#print(poly)
print(polysimplex)
print(polysimpley)
fx = sympy.lambdify(t,polysimplex) # funcion
fy = sympy.lambdify(t,polysimpley)
muestras = 100
lin_t = numpy.linspace(0,1,muestras)

pfix = fx(lin_t)
pfiy = fy(lin_t)
#grafica
plt.plot(A[0,:],A[1, :],'o')
plt.plot(pfix,pfiy)
plt.show()


# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for full license information.

"""
Version 1.1
- add MSE
- add Crossentropy for multiple classifier
"""

import numpy as np

from HelperClass2.EnumDef_2_0 import *

class LossFunction_1_1(object):
    def __init__(self, net_type):
        self.net_type = net_type
    # end def

    # fcFunc: feed forward calculation
    def CheckLoss(self, A, Y):
        m = Y.shape[0]
        if self.net_type == NetType.Fitting:
            loss = self.MSE(A, Y, m)
        elif self.net_type == NetType.BinaryClassifier:
            loss = self.CE2(A, Y, m)
        elif self.net_type == NetType.MultipleClassifier:
            loss = self.CE3(A, Y, m)
        #end if
        return loss
    # end def

    def MSE(self, A, Y, count):
        p1 = A - Y
        LOSS = np.multiply(p1, p1)
        loss = LOSS.sum()/count/2
        return loss
    # end def

    # for binary classifier
    def CE2(self, A, Y, count):
        p1 = 1 - Y
        p2 = np.log(1 - A)
        p3 = np.log(A)

        p4 = np.multiply(p1 ,p2)
        p5 = np.multiply(Y, p3)

        LOSS = np.sum(-(p4 + p5))  #binary classification
        loss = LOSS / count
        return loss
    # end def

    # for multiple classifier
    def CE3(self, A, Y, count):
        p1 = np.log(A)
        p2 =  np.multiply(Y, p1)
        LOSS = np.sum(-p2, keepdims=False)
        loss = LOSS / count
        return loss
    # end def


import numpy as np
import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
import pickle
import os
import argparse

RESULTS_FILENAME = 'results.bin'

_F_cache = {}

def F(v):
	if not v in _F_cache:
		_F_cache[v] = (v**4)/4 - 2 * (v**3) - (11/2) * (v**2) - 5 * v

	return _F_cache[v]

_F1_cache = {}

def F1(v):
	if not v in _F1_cache:
		_F1_cache[v] = v**3 - 6 * (v**2) - 11 * v - 5

	return _F1_cache[v]

_F2_cache = {}

def F2(v):
	if not v in _F2_cache:
		_F2_cache[v] = 3 * (v**2) - 12 * v - 11

	return _F2_cache[v]

#unstable interval (obtained theoretically)
unstable_v0 = -0.76887
unstable_v1 = 4.76887

#struct to put together the function and its derivatives
class Function:
	def __init__(self, x):
		self.f = F(x)
		self.first = F1(x)
		self.second = F2(x)

def algorithm(interval_0, interval_1, step, show_progress=False, save=True):
	best_diff = None

	best_v0 = None
	best_v1 = None

	best_m = None
	best_n = None

	v0 = interval_0[0]

	while interval_0[0] <= v0 <= interval_0[1]:
		f0 = Function(v0)
		m0 = f0.first
		n0 = f0.f - m0 * v0

		v1 = interval_1[0]

		if show_progress:
			#show progress as percentage
			print((interval_0[0] - v0)/(interval_0[0] - interval_0[1]) * 100)

		while interval_1[0] <= v1 <= interval_1[1]:
			f1 = Function(v1)
			m1 = f1.first
			n1 = f1.f - m1 * v1

			diff = abs(m0 - m1) + abs(n0 - n1)

			if not best_diff or diff < best_diff:
				best_diff = diff
				best_v0 = v0
				best_v1 = v1
				best_m = (m0 + m1)/2
				best_n = (n0 + n1)/2

			v1 += step

		v0 += step

	print('Best diff: %f' % best_diff)

	if save:
		#save the results to a file
		with open(RESULTS_FILENAME, 'wb') as f:
			pickle.dump(best_v0, f)
			pickle.dump(best_v1, f)
			pickle.dump(best_m, f)
			pickle.dump(best_n, f)

	return best_v0, best_v1, best_m, best_n

def draw(v_l, v_g, m, n, num):
	limits = (-5, v_l, unstable_v0, unstable_v1, v_g, 12)

	num /= len(limits)

	#these arrays hold the value of x, F and y = mx + n for every interval
	x = [None] * (len(limits) - 1)
	y_f = [None] * (len(limits) - 1)
	y_line = [None] * (len(limits) - 1)

	for i in range(len(limits) - 1):
		x[i] = np.linspace(limits[i], limits[i+1], num)
		y_f[i] = [F(v) for v in x[i]]
		y_line[i] = m * x[i] + n

	fig, ax = plt.subplots(1, 1)
	ax.set_xlim([limits[0], limits[-1]])

	#blue interval is drawn on top

	#points until v_l
	ax.plot(x[0], y_line[0], color='black')
	ax.plot(x[0], y_f[0], color='blue')

	#meta-estable points
	ax.plot(x[1], y_f[1], color='darkorange')
	ax.plot(x[1], y_line[1], color='blue')

	#unstable points
	ax.plot(x[2], y_f[2], color='red')
	ax.plot(x[2], y_line[2], color='blue')

	#meta-estable points
	ax.plot(x[3], y_f[3], color='darkorange')
	ax.plot(x[3], y_line[3], color='blue')

	#points after v_g
	ax.plot(x[4], y_line[4], color='black')
	ax.plot(x[4], y_f[4], color='blue')

	#markers
	marker = '.'
	ax.plot(v_l, F(v_l), color='blue', marker=marker)
	ax.plot(v_g, F(v_g), color='blue', marker=marker)
	ax.plot(unstable_v0, F(unstable_v0), color='red', marker=marker)
	ax.plot(unstable_v1, F(unstable_v1), color='red', marker=marker)

	#labels
	ax.set_ylabel('F', rotation=0)
	ax.set_xlabel('v')

	#custom legend
	legend_custom_lines = [Line2D([0], [0], color='blue'), Line2D([0], [0], color='darkorange'), Line2D([0], [0], color='red')]
	ax.legend(legend_custom_lines, ['Equilibrio estable', 'Estados metaestables', 'Estados inestables'])

	plt.savefig('figure.pdf')
	plt.show()

if __name__=='__main__':
	parser = argparse.ArgumentParser(description="Algorithm to obtain v_l and v_g numerically using Maxwell's construction")

	parser.add_argument('--force', help='force the execution of the algorithm and overwrite the previous result', action='store_true')
	parser.add_argument('--accuracy', help='accuracy to which the values are calculated', type=float, default=0.00005)
	parser.add_argument('--show-progress', help='show calculation progress as a percentage', action='store_true')
	parser.add_argument('--first-interval', help='select the interval (v0, v1) in which v_l is contained', type=float, nargs=2, default=[-3.5, -2.5], metavar=('v0', 'v1'))
	parser.add_argument('--second-interval', help='select the interval (v0, v1) in which v_g is contained', type=float, nargs=2, default=[6.5, 7.5], metavar=('v0', 'v1'))
	parser.add_argument('--dont-save', help='dont save the results to a file', action='store_true')

	args = parser.parse_args()

	if not args.force and os.path.isfile(RESULTS_FILENAME):
		#load the results from previous file
		with open(RESULTS_FILENAME, 'rb') as f:
			v_l = pickle.load(f)
			v_g = pickle.load(f)
			m = pickle.load(f)
			n = pickle.load(f)
	else:
		#calculate results and save them to file
		v_l, v_g, m, n = algorithm(args.first_interval, args.second_interval, step=args.accuracy, show_progress=args.show_progress, save=(not args.dont_save))

	print('v_l: %f' % v_l)
	print('v_g: %f' % v_g)
	print('m: %f' % m)
	print('n: %f' % n)

	draw(v_l, v_g, m, n, num=1000)


# 3.02
# Baseball player's BMI
#The MLB also offers to let you analyze their weight data. Again, both are available as regular Python lists: height and weight. height is in inches and weight is in pounds.

#It's now possible to calculate the BMI of each baseball player. Python code to convert height to a numpy array with the correct units is already available in the workspace. Follow the instructions step by step and finish the game!

#Create a numpy array from the weight list with the correct units. Multiply by 0.453592 to go from pounds to kilograms. Store the resulting numpy array as np_weight_kg.
#Use np_height_m and np_weight_kg to calculate the BMI of each player. Use the following equation:
#BMI=weight(kg)height(m)2
#Save the resulting numpy array as bmi.
#Print out bmi.

# height and weight are available as regular lists

# Import numpy
import numpy as np

# Create array from height with correct units: np_height_m
np_height_m = np.array(height) * 0.0254

# Create array from weight with correct units: np_weight_kg
np_weight_kg = np.array(weight) * 0.453592

# Calculate the BMI: bmi
bmi = np_weight_kg / np_height_m ** 2

# Print out bmi
print(bmi)

Python/Datacamp/0.0_Intro to Python for Data Science/3.02_Baseball-players-bmi.py


import numpy as np

def length_squared(vector):
    """ Return the length squared of a vector. """
    return vector[0]**2 + vector[1]**2

def unit_vector(vector):
    """ Returns the unit vector of the vector.  """
    return vector / np.sqrt(length_squared(vector))

def angle_between(v1, v2):
    """ Returns the angle in radians between vectors 'v1' and 'v2'::

            >>> angle_between((1, 0, 0), (0, 1, 0))
            1.5707963267948966
            >>> angle_between((1, 0, 0), (1, 0, 0))
            0.0
            >>> angle_between((1, 0, 0), (-1, 0, 0))
            3.141592653589793
    """
    try:
        v1_u = unit_vector(v1)
        v2_u = unit_vector(v2)
    except:
        return 0.0

    return np.arccos(np.clip(np.dot(v1_u, v2_u), -1.0, 1.0))


from numpy import *

def run():
	points = genfromtxt("data.csv", delimiter=",")
	initB = 0
	initM = 0
	numberOfIterations = 1000
	[b, m] = gradient_decent_runner(points, initB, initM, numberOfIterations)
	print("y=" + str(m) + "x+"+str(b) )
	print("Error: = " + str(compute_error_for_line_given_points(points, b, m)) )

def compute_error_for_line_given_points(points, b, m):
	accumulatedError = 0
	for i in range(0, len(points)):
		x = points[i, 0]
		y = points[i, 1]
		accumulatedError += (y - (m*x+b)) ** 2
	return accumulatedError/float(len(points))

def gradient_decent_runner(points, bI, mI, numberOfIterations):
	b = bI
	m = mI
	for i in range(0, numberOfIterations):
		b, m = step_gradient(b, m, array(points))
	return [b, m]

def step_gradient(bCurr, mCurr, points):
	bPartial = 0
	mPartial = 0
	N = float(len(points))
	for i in range(0, len(points)):
		x = points[i, 0]
		y = points[i, 1]
		mPartial += -(2/N) * x * (y-(mCurr*x+bCurr))
		bPartial += -(2/N) * (y - (mCurr*x+bCurr))
	bNew = bCurr - bPartial * .0001
	mNew = mCurr - mPartial * .0001
	return [bNew, mNew]

if __name__ == '__main__':
	run()


import numpy as np

def logaddexp(a, b):
    m = np.maximum(a, b)
    return m + np.log(np.exp(a - m) + np.exp(b - m))

def logsumexp(x):
    m = np.max(x, axis=1, keepdims=True)
    return m + np.log(np.sum(np.exp(x - m), axis=1, keepdims=True))


import numpy as np      #Numerical Python : lib used for matrix operations
from math import cos,sin      #Math lib of python for various mathematical functions
import matplotlib.pyplot as plt     #plotting lib of python

x_list = np.linspace(-10,10,10000) #linearly spaced vector
y_list = x_list**(2) #Parabolic Equation

theta = float(input("Enter the value of angle (anticlockwise rotation) : theta = "))
rot_x_list = x_list*cos(theta) + y_list*sin(theta) #just apply formula
rot_y_list = -x_list*sin(theta) + y_list*cos(theta)

a, b = input("Enter Translation Vector : ").split()   #split() method in Python split a string into a list of strings after breaking the given string by the specified separator.
trans_x_list = x_list+float(a)
trans_y_list = y_list+float(b)

a, b = input("Enter Scaling Vector : ").split()   #split() method in Python split a string into a list of strings after breaking the given string by the specified separator.
scaled_x_list = x_list * float(a)
scaled_y_list = y_list * float(b)

ref_x_list = x_list
ref_y_list = y_list*-1

a, b = input("Enter Shearing Parameters : ").split()   #split() method in Python split a string into a list of strings after breaking the given string by the specified separator.
shearx_x_list = x_list+float(a)*y_list
shearx_y_list = y_list

sheary_x_list = x_list
sheary_y_list = y_list+float(b)*x_list

plt.clf()

plt.subplot(3,2,1)
plt.plot(x_list,y_list,"b-",label="Before Rotation")
plt.plot(rot_x_list,rot_y_list,"r--",label="After Rotation")
plt.xlabel("X-Points")
plt.ylabel("Y-Points")
plt.title("(Anti-Clockwise) ROTATION")
plt.legend()
plt.grid()

plt.subplot(3,2,2)
plt.plot(x_list,y_list,"b-",label="Before Translation")
plt.plot(trans_x_list,trans_y_list,"r--",label="After Translation")
plt.xlabel("X-Points")
plt.ylabel("Y-Points")
plt.title("TRANSLATION")
plt.legend()
plt.grid()

plt.subplot(3,2,3)
plt.plot(x_list,y_list,"b-",label="Before Scaling")
plt.plot(scaled_x_list,scaled_y_list,"r--",label="After Scaling")
plt.xlabel("X-Points")
plt.ylabel("Y-Points")
plt.title("(Expanding) SCALING")
plt.legend()
plt.grid()

plt.subplot(3,2,4)
plt.plot(x_list,y_list,"b-",label="Before Reflection")
plt.plot(ref_x_list,ref_y_list,"r--",label="After Reflection")
plt.xlabel("X-Points")
plt.ylabel("Y-Points")
plt.title("REFLECTION about X-axis")
plt.legend()
plt.grid()

plt.subplot(3,2,5)
plt.plot(x_list,y_list,"b-",label="Before Shearing")
plt.plot(shearx_x_list,shearx_y_list,"r--",label="After Shearing")
plt.xlabel("X-Points")
plt.ylabel("Y-Points")
plt.title("SHEARING in X-axis")
plt.legend()
plt.grid()

plt.subplot(3,2,6)
plt.plot(x_list,y_list,"b-",label="Before Shearing")
plt.plot(sheary_x_list,sheary_y_list,"r--",label="After Shearing")
plt.xlabel("X-Points")
plt.ylabel("Y-Points")
plt.title("SHEARING in y-axis")
plt.legend()
plt.grid()

plt.show()


import pandas as pd
import numpy as np

def get_data(dataset_switch = 'linear'):
    """
    try the following switches:
    linear
    exponential
    even_spacing
    pups
    """

    # making our dataset with a few different switches or options
    if dataset_switch == 'linear':
        from sklearn.datasets import make_regression
        X,y = make_regression(
            n_samples=100,
            n_features=1,
            random_state = 11,
            noise = 10
        )
        X = X[:,0]
    elif dataset_switch == 'exponential':
        from sklearn.datasets import make_regression
        X,_ = make_regression(
            n_samples=100,
            n_features=1,
            random_state = 11,
            noise = 10
        )
        X = X[:,0]
        from numpy.random import default_rng
        rng = default_rng()
        vals = rng.standard_normal(100)/2
        y = np.power(3,X) + vals
    elif dataset_switch == 'even_spacing':
        X = np.linspace(-2.0, 3.0, num=100)
        y = 40*X
    elif dataset_switch == 'pups':
        pups_df = pd.read_csv('pupSlice.csv',header=None)
        X = np.arange(1,3024+1)
        y = pups_df[0].values

    import pandas as pd

    df = pd.DataFrame({
        'x':X,
        'y':y
    })

    return df.sort_values(by=['x'])


import numpy as np
import matplotlib.pyplot as plt
from pandas.io.parsers import read_csv
import scipy.optimize as opt
from sklearn.preprocessing import PolynomialFeatures

def load_csv(file_name):
    values = read_csv(file_name, header=None).values

    return values.astype(float)

def gradient(thetas, XX, Y, lamb):
    m = np.shape(XX)[0]

    H = h(thetas, XX)
    grad = (1/len(Y)) * np.dot(XX.T, H-Y)
    grad += (lamb/m) * np.c_[thetas]

    return grad

def cost(thetas, X, Y, lamb):
    m = np.shape(X)[0]
    H = h(thetas, X)

    c = (-1/m) * (np.dot(Y.T, np.log(H)) + np.dot((1-Y).T, np.log(1-H)))
    c += (lamb/(2*m)) * (thetas**2).sum()

    return c

def sigmoid(Z):
    return 1/(1 + np.e**(-Z))

def h(thetas, X):
    return np.c_[sigmoid(np.dot(X, thetas))]

def show_decision_boundary(thetas, X, Y, poly):
    plt.figure()

    x1_min, x1_max = X[:, 0].min(), X[:, 0].max()
    x2_min, x2_max = X[:, 1].min(), X[:, 1].max()
    xx1, xx2 = np.meshgrid(np.linspace(x1_min, x1_max), np.linspace(x2_min, x2_max))

    h = sigmoid(poly.fit_transform(np.c_[xx1.ravel(), xx2.ravel()]).dot(thetas))

    h = h.reshape(xx1.shape)

    positives = np.where(Y == 1)
    negatives = np.where(Y == 0)
    plt.scatter(X[positives, 0], X[positives, 1], marker='+', color='blue')
    plt.scatter(X[negatives, 0], X[negatives, 1], color='red')

    plt.contour(xx1, xx2, h, [0.5], linewidths=1, colors='g')
    plt.savefig("images/regresion_logistic_regularized.png")
    plt.show()
    plt.close()

def evaluate(thetas, X, Y, degree):
    poly = PolynomialFeatures(degree)
    X_poly = poly.fit_transform(X)

    result = h(thetas, X_poly)
    passed_missed = np.logical_and((result >= 0.5), (Y == 0)).sum()
    failed_missed = np.logical_and((result < 0.5), (Y == 1)).sum()

    errors = (passed_missed + failed_missed)

    return (result.shape[0] - errors) / (result.shape[0])

def train(X, Y, degree=2, lamb=1, verbose = True):
    poly = PolynomialFeatures(degree)
    X_poly = poly.fit_transform(X)

    m = np.shape(X_poly)[0]
    n = np.shape(X_poly)[1]

    thetas = np.zeros((n, 1), dtype=float)
    result = opt.fmin_tnc(func=cost, x0=thetas, fprime=gradient, args=(X_poly, Y, lamb), disp = 5 if verbose else 0)
    thetas = result[0]

    return thetas

def main():
    # DATA PREPROCESSING
    datos = load_csv("data/ex2data2.csv")
    X = datos[:, :-1]
    Y = datos[:, -1][np.newaxis].T

    thetas = train(X, Y)

    print("Accuracy: ", evaluate(thetas, X, Y, 2)*100, "%")
    show_decision_boundary(thetas, X, Y, poly)

if __name__ == "__main__":
    main()


import numpy as np

def binary_step(x):
    """
    Calculates the binary step activation function
    """
    return np.where(x > 0, 1, 0)

def leaky_relu(x):
    """
    Calculates the leaky relu activation function
    """
    return np.maximum(0.01 * x, x)

def relu(x):
    """
    Calculates the relu activation function
    """
    return np.maximum(0, x)

def sigmoid(x):
    """
    Calculates the sigmoid activation function
    """
    return 1 / (1 + np.exp(-x))

def softmax(x):
    """
    Calculates the softmax activation function
    """
    return np.exp(x) / np.sum(np.exp(x), axis=0)

def softplus(x):
    """
    Calculates the softplus activation function
    """
    return np.log(1 + np.exp(x))

def tanh(x):
    """
    Calculates the tanh activation function
    """
    return np.tanh(x)


import numpy as np
from math import sin, exp, pi, ceil

def f(x):
    return exp(3*x)*sin(2*x)

def trapezoid( start, end, h, f) :
    points= ceil((end - start) / h)
    x = np.linspace(start, end, points+1)  # N+1 points make N subintervals
    y = f(x)
    trapezoidIntegral = (h / 2) * (y[0]+y[points]+np.sum(2*y[1:-1])) #approximation result
    return trapezoidIntegral


import numpy as np
import matplotlib.pyplot as plt
import math
import random

def vec3(x,y,z):
    return np.array((x,y,z))

def unit_vector(v):
    len = math.sqrt(np.dot(v,v))
    return v / len

# The unit vector v returns a color that is a linear blend of light blue and dark blue depending on the y valye of v
def background_color(v):
    u = 0.5*(1.0 + v[1])
    return u*vec3(0.7, 0.8, 0.9) + (1.0-u)*vec3(0.05, 0.05, 0.2)

# The implicit equation for a sphere with origin at center is
#    f(x,y,z) = x^2 + y^2 + z^2 - R^2 = 0
# In vector form, for 3D points p = (x,y,z) on sphere that is:
#    f(p) = dot(p,p) - R^2 = 0
# When the center is at c = (cx, cy, cz) the implict equation for points on the sphere is:
#    f(p) = dot(p-c,p-c) - R^2 = 0
# For a ray p(t) = o + t*v, it will be on the sphere (so "hit" it) when f(p(t)) = 0
#    dot(o + t*v - c,o + t*v - c) - R^2 = 0
#    dot(o-c,o-c) 2*t*dot(o-c,v) + t^2*dot(v,v) - R^2 = 0
# We can write that as At^2 + Bt + C = 0 and solve the quadratic for t
# If the ray missed the sphere there is no real solution (the discriminant is negative)
# If two solutions, take the smallter that is positive (negative hits are "behind" you)
def hit_sphere(ray_origin, ray_direction, center, radius):
    oc = ray_origin - center
    qb = np.dot(oc,ray_direction)
    qc = np.dot(oc,oc)-radius*radius
    discriminant = qb*qb-qc
    if (discriminant > 0):
        t = (-qb - math.sqrt(discriminant))
        if t > 0.00001:
            return t
        t = (-qb + math.sqrt(discriminant))
        if t > 0.0001:
            return t
    return 1.0e9

# returns (hit_something, t_min (of the closest sphere), i (index of the closest sphere)
def hit_array(ray_origin, ray_direction, centers, radii):
    hit_something = False
    t_min = 1.0e20
    i_min = -1

    for i in range(len(centers)):
        t = hit_sphere(ray_origin, ray_direction, centers[i], radii[i])

        if (t < 1.0e8 and t > 0.001):
            hit_something = True
            t_min = t
            i_min = i

    return (hit_something, t_min, i_min)


import numpy as np
import copy
import scipy

# functions -> CatModel class
def z_func(w, X, b):
    """
    Calculates Z function
    """
    return np.dot(w.T,X) + b

def sigmoid(z):
    """
    Calculates sigmoid of Z function
    """
    return 1/(1+np.exp(-z))

def cost(A, Y, m):
    """
    Calculates total cost
    """
    return -(1/m)*np.sum(np.dot(Y*np.log(A)) + (1-Y)*np.log(1-A))

def propagate(m, w, b, X, Y):
    """
    Forward and backward propagation
    """

    # Forward propagation
    A = sigmoid(z_func(w, X, b))
    total_cost = cost(A, Y, m)

    # Backward propagation
    dz = A - Y
    dw = (1/m)*(np.dot(X.T, dz))
    db = (1/m)*(np.sum(dz))

    grads = {
        "dw": dw,
        "db": db
    }

    return grads, total_cost

def train(m, X, Y, num_iterations = 1000, learning_rate = 0.009):
    """
    Trains model based on iterations and learning rate
    """
    # Init values (weights and bias)
    w = np.zeros((m, 1))
    b = 0.0

    # Iterate
    for i in num_iterations:

        # Propagate
        grads, total_cost = propagate(m, w, b, X, Y)

        # Descent
        w = w - learning_rate*grads["dw"]
        b = b - learning_rate*grads["db"]

        # Print costs
        if i % 100 == 0:
            print("Cost at " + i + " = " + total_cost)

    # Returning values
    params = {
        "w": w,
        "b": b
    }
    return grads, params

def predict(w, b, X):
    """
    Predicts a set of data using the model
    """

    A = sigmoid(z_func(w, X, b))
    predictions = np.zeros((X.shape[0], 1))
    for i in range(A.shape[1]):
        if A[0][i] <= 0.5:
            predictions[0][i] = 0
        else:
            predictions[0][1] = 1

    return predictions


#!/usr/bin/env python

import numpy as np

def compute_area_polygon(x, y):
    """Compute the surface area of a polygon.

    Parameters
    ----------
    x : np.array
        X-coordinates of the polygon's corners.
    y : np.array
        Y-coordinates of the polygon's corners.

    Returns
    -------
    area : type of x and y
        The surface area of the polygon.

    Raises
    ------
    TypeError
        When x and y do not have the same length.

    """
    # Shoelace algorithm, Meister, 1769
    if len(x) != len(y):
        raise TypeError("Arguments x and y must have the same length.")
    if len(x) < 2:
        return 0.0
    else:
        return abs( x[-1]*y[0] + np.dot(x[:-1], y[1:])
                   -x[0]*y[-1] - np.dot(x[1:], y[:-1]))/2

def check_single(x, y, area):
    np.testing.assert_almost_equal(compute_area_polygon(x, y), area)

def check_variants(x, y, area):
    x = np.asarray(x)
    y = np.asarray(y)
    check_single(x, y, area)
    check_single(x[::-1], y[::-1], area)
    check_single(x + 0.3, y - 0.5478, area)
    check_single(-2*x, 0.8*y, 1.6*area)
    xp = np.cos(0.3)*x - np.sin(0.3)*y
    yp = np.sin(0.3)*x + np.cos(0.3)*y
    check_single(xp, yp, area)

def test_compute_area_polygon():
    # Simple geometries
    check_single([0, 0, 1, 1], [0, 1, 1, 0], 1.0)
    check_single([0.0, 0.0, 2.0], [0.0, 1.0, 1.0], 1.0)
    check_single([-0.5, 2.5, 1.0, 0.0], [0.0, 0.0, 0.5, 0.5], 1.0)

    # Corner cases: flat, coinciding points, too short vectors
    check_single([0.0, 2.0, -1.0], [0.0, 2.0, -1.0], 0.0)
    check_single([0.0, 0.0, 2.0, 2.0], [0.0, 1.0, 1.0, 1.0], 1.0)
    check_single([], [], 0.0)
    check_single([1], [2], 0.0)
    check_single([2.0, 1.0], [0.0, 0.0], 0.0)


import numpy as np

def compute_mse(real_ratings, predictions):
    """
    Computes the Mean Squared Error given the real and the predicted
    ratings.
    Args:
        real_ratings: The actual ratings
        predictions: Ratings predicted by some model
    Returns:
        mse: The Mean Squared Error value
    """
    error_vector = real_ratings - predictions
    return np.mean(error_vector ** 2)

def compute_rmse(real_ratings, predictions):
    """
    Computes the Root Mean Squared Error given the real and the
    predicted ratings.
    Args:
        real_ratings: The actual ratings
        predictions: Ratings predicted by some model
    Returns:
        rmse: The Root Mean Squared Error value
    """
    rmse = np.sqrt(compute_mse(real_ratings, predictions))
    return rmse


#!/bin/env python
import numpy
##import pylab
from math import *

def Mean (myArray):
    return sum(myArray)/len(myArray)

def Var (myArray):
    mean = Mean(myArray)
    mean2 = Mean(myArray**2)
    N = len(myArray)+0.0
    return N/(N-1) * (mean2 - mean**2)

def NaiveStandardError(myArray):
  # calculate the standard error by calling the standard deviation function
  return sqrt(Var(myArray))/sqrt(len(myArray)+0.0)

def BinData(data, n):
    nbins = len(data)/n
    binned = numpy.zeros(nbins,float)
    # fill in the binned array here
    for i in range(0,nbins):
        binned[i] = numpy.average(data[n*i:(i+1)*n+1])
    return binned

def BinningMethod(data):
    errList = []
    for n in range(1,len(data)/20):
        errList.append(NaiveStandardError(BinData(data,n)))
    return errList

def C(data, i, mean, var):
    delta = data - mean
    N = len(data)
    return numpy.sum(delta[0:N-i]*delta[i:N])/((N-i)*var)

def Kappa(data):
    done = False
    N = len(data)
    i = 1
    mean = Mean (data)
    var  = Var  (data)
    csum = 1.0
    while ((not done) and i<N):
        c = C(data,i,mean,var)
        if (c < 0.0):
            done = True
        else:
            csum = csum + 2.0*c
        i = i+1
    return csum

def StdError(data):
    var = Var(data)
    kappa = Kappa(data)
    Neff = len(data)/kappa
    return sqrt(var/Neff)

def Stats(data):
    if len(data) > 0 :
        return (Mean(data), StdError(data), Kappa(data))


import math
import matplotlib.pyplot as plt
import numpy as np

def gelu(x):
    return 0.5 * x * (1 + math.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * math.pow(x, 3))))
def relu(x):
    return max(x, 0)
def lrelu(x):
    return max(0.01*x, x)

plt.figure(figsize=(15, 5))
plt.subplots_adjust(left=None, bottom=None, right=None, top=None, wspace=.5, hspace=None)

ranges_ = (-10, 3, .25)

plt.subplot(1, 2, 1)
plt.plot([i for i in np.arange(*ranges_)], [relu(i) for i in np.arange(*ranges_)], label='ReLU', marker='.')
plt.plot([i for i in np.arange(*ranges_)], [gelu(i) for i in np.arange(*ranges_)], label='GELU')
plt.hlines(0, -10, 3, colors='gray', linestyles='--', label='0')
plt.title('Figure 7: GELU as an activation function for autoencoders')
plt.ylabel('f(x) for GELU and ReLU')
plt.xlabel('x')
plt.legend()

plt.subplot(1, 2, 2)
plt.plot([i for i in np.arange(*ranges_)], [lrelu(i) for i in np.arange(*ranges_)], label='Leaky ReLU')
plt.hlines(0, -10, 3, colors='gray', linestyles='--', label='0')
plt.ylabel('f(x) for Leaky ReLU')
plt.xlabel('x')
plt.title('Figure 8: LeakyReLU')
plt.legend()
plt.show()


# import packages: numpy, math (you might need pi for gaussian functions)
import numpy as np
import math
import matplotlib.pyplot as plt
from scipy.signal import convolve2d as conv2

"""
Gaussian function taking as argument the standard deviation sigma
The filter should be defined for all integer values x in the range [-3sigma,3sigma]
The function should return the Gaussian values Gx computed at the indexes x
"""
def gauss(sigma):
    x = [y for y in range(int(-3*sigma),int(3*sigma) +1)]
    x = np.array(x)
    Gx = np.exp(-np.power(x, 2.) / (2 * np.power(sigma, 2.))) / np.sqrt(2*sigma*math.pi)
    return Gx, x

"""
Implement a 2D Gaussian filter, leveraging the previous gauss.
Implement the filter from scratch or leverage the convolve2D method (scipy.signal)
Leverage the separability of Gaussian filtering
Input: image, sigma (standard deviation)
Output: smoothed image
"""
def gaussianfilter(img, sigma):

    # leverage gaussian
    gx, x = gauss(sigma)

    # normalize gaussian  vector
    gx *= 1.0 / gx.sum()

    # Reshape and compute convolution
    gx = gx.reshape(1, gx.size)
    img_smooth_X = conv2(img, gx, 'same') # smooth on X axe
    img_smoothed_X_Y = conv2(img_smooth_X, gx.T, 'same') # smooth on y axe

    return img_smoothed_X_Y

"""
Gaussian derivative function taking as argument the standard deviation sigma
The filter should be defined for all integer values x in the range [-3sigma,3sigma]
The function should return the Gaussian derivative values Dx computed at the indexes x
"""
def gaussdx(sigma):

    x = [y for y in range(int(-3 * sigma), int(3 * sigma) + 1)]
    x = np.array(x)
    Dx =  -np.exp(-np.power(x, 2.) / (2 * np.power(sigma, 2.))) / np.sqrt(2 * (sigma**3) * math.pi)
    Dx = x*Dx
    return Dx,x

def gaussderiv(img, sigma):

    Gx, x = gauss(sigma)
    Dx, x = gaussdx(sigma)

    Gx = Gx.reshape(1, Gx.size)
    Dx = Dx.reshape(1, Dx.size)

    img_smoothed = gaussianfilter(img,sigma)

    imgDx = conv2(img_smoothed, Dx, 'same')
    imgDy = conv2(img_smoothed, Dx.T, 'same')

    return imgDx, imgDy


import numpy as np

# Mean Absolute Error
def mae(predict, actual):
    """
    Examples(rounded for precision):
    >>> actual = [1,2,3];predict = [1,4,3]
    >>> np.around(mae(predict,actual),decimals = 2)
    0.67
    >>> actual = [1,1,1];predict = [1,1,1]
    >>> mae(predict,actual)
    0.0
    """
    predict = np.array(predict)
    actual = np.array(actual)

    difference = abs(predict - actual)
    score = difference.mean()

    return score

# Mean Squared Error
def mse(predict, actual):
    """
    Examples(rounded for precision):
    >>> actual = [1,2,3];predict = [1,4,3]
    >>> np.around(mse(predict,actual),decimals = 2)
    1.33
    >>> actual = [1,1,1];predict = [1,1,1]
    >>> mse(predict,actual)
    0.0
    """
    predict = np.array(predict)
    actual = np.array(actual)

    difference = predict - actual
    square_diff = np.square(difference)

    score = square_diff.mean()
    return score

# Root Mean Squared Error
def rmse(predict, actual):
    """
    Examples(rounded for precision):
    >>> actual = [1,2,3];predict = [1,4,3]
    >>> np.around(rmse(predict,actual),decimals = 2)
    1.15
    >>> actual = [1,1,1];predict = [1,1,1]
    >>> rmse(predict,actual)
    0.0
    """
    predict = np.array(predict)
    actual = np.array(actual)

    difference = predict - actual
    square_diff = np.square(difference)
    mean_square_diff = square_diff.mean()
    score = np.sqrt(mean_square_diff)
    return score

# Root Mean Square Logarithmic Error
def rmsle(predict, actual):
    """
    Examples(rounded for precision):
    >>> actual = [10,10,30];predict = [10,2,30]
    >>> np.around(rmsle(predict,actual),decimals = 2)
    0.75
    >>> actual = [1,1,1];predict = [1,1,1]
    >>> rmsle(predict,actual)
    0.0
    """
    predict = np.array(predict)
    actual = np.array(actual)

    log_predict = np.log(predict + 1)
    log_actual = np.log(actual + 1)

    difference = log_predict - log_actual
    square_diff = np.square(difference)
    mean_square_diff = square_diff.mean()

    score = np.sqrt(mean_square_diff)

    return score

# Mean Bias Deviation
def mbd(predict, actual):
    """
    This value is Negative, if the model underpredicts,
    positive, if it overpredicts.
    Example(rounded for precision):
    Here the model overpredicts
    >>> actual = [1,2,3];predict = [2,3,4]
    >>> np.around(mbd(predict,actual),decimals = 2)
    50.0
    Here the model underpredicts
    >>> actual = [1,2,3];predict = [0,1,1]
    >>> np.around(mbd(predict,actual),decimals = 2)
    -66.67
    """
    predict = np.array(predict)
    actual = np.array(actual)

    difference = predict - actual
    numerator = np.sum(difference) / len(predict)
    denumerator = np.sum(actual) / len(predict)
    # print(numerator, denumerator)
    score = float(numerator) / denumerator * 100

    return score


"""
Exercise 6.1.8: Parrot
"""
import argparse
import matplotlib.pyplot as plt
import numpy as np
import sys

from phase_plane.runge_kutta import runge_kutta

def f(v):
    """
    The right-hand side of the Parrot system.

    ..math:: \dot{x} = y, \dot{y} = -x + y (1 - x^2)
    """
    x = v[0]
    y = v[1]
    return np.array([y + y**2, -x + y / 5. - x * y + 6. / 5. * y**2])

if __name__ == "__main__":
    x_min = -10
    x_max = +10
    y_min = -10
    y_max = +10

    # Draw vector field
    x, y = np.meshgrid(np.linspace(x_min, x_max, 35), np.linspace(y_min, y_max, 35))
    u = y + y**2
    v = -x + y / 5. - x * y + 6. / 5. * y**2

    plt.title('Parrot')
    plt.xlabel('x')
    plt.ylabel('y')
    plt.xlim(x_min, x_max)
    plt.ylim(y_min, y_max)
    plt.quiver(x, y, u, v)

    # Draw phase portraits
    x0 = [-3., -3.0]
    t, xy = runge_kutta(f, x0, 0, 100, 0.01)
    x, y = xy.T[0], xy.T[1]
    plt.plot(x, y, c=f"C2", label=f'x0={x0}')

    x0 = [0.5, 0.5]
    t, xy = runge_kutta(f, x0, 0, 10, 0.01)
    x, y = xy.T[0], xy.T[1]
    plt.plot(x, y, c=f"C1", label=f'x0={x0}')

    x0 = [-8.5, -2.5]
    t, xy = runge_kutta(f, x0, 0, 10, 0.01)
    x, y = xy.T[0], xy.T[1]
    plt.plot(x, y, c=f"C5", label=f'x0={x0}')

    x0 = [-9.5, -9.5]
    t, xy = runge_kutta(f, x0, 0, 10, 0.01)
    x, y = xy.T[0], xy.T[1]
    plt.plot(x, y, c=f"C6", label=f'x0={x0}')

    # Plot nullcline
    z = np.linspace(x_min, x_max, 100)
    plt.plot(z, np.zeros(len(z)), c=f"C3", label='Nullcline: $\dot{x} = 0$')

    # For the nullclines associated to y_dot, make sure we only consider
    # x-values in the domain of the function.
    exclude_low = (-11 - np.sqrt(120)) / 5
    exclude_high = (-11 + np.sqrt(120)) / 5

    x1 = np.linspace(x_min, exclude_low, 50)
    x2 = np.linspace(exclude_high, x_max, 50)

    nullcline_y_1 = 5. * ( (x1 - 1. / 5.) - np.sqrt((1. / 5. - x1)**2 + 4. * x1 * 6. / 5.) ) / 12.
    nullcline_y_2 = 5. * ( (x2 - 1. / 5.) - np.sqrt((1. / 5. - x2)**2 + 4. * x2 * 6. / 5.) ) / 12.
    nullcline_y_3 = 5. * ( (x1 - 1. / 5.) + np.sqrt((1. / 5. - x1)**2 + 4. * x1 * 6. / 5.) ) / 12.
    nullcline_y_4 = 5. * ( (x2 - 1. / 5.) + np.sqrt((1. / 5. - x2)**2 + 4. * x2 * 6. / 5.) ) / 12.
    plt.plot(x1, nullcline_y_1, c=f"C4", label='Nullcline: $\dot{y} = 0$')
    plt.plot(x2, nullcline_y_2, c=f"C4")
    plt.plot(x1, nullcline_y_3, c=f"C4")
    plt.plot(x2, nullcline_y_4, c=f"C4")

    # Plot everything
    plt.legend()
    plt.show()


"""
Function: Find Mean, Variance, Standard Variance of X(discrete random variables)
Coding: utf-8
Author: fox2049
Date: 2020/11/10
"""

import numpy as np  # using version-1.19.3 if you are Windows PC
import math

def arr_input():
    array = []
    while True:
        num = input("Input variable or end:")
        if num == "end":
            break
        else:
            array.append(float(num))
            print(array, "-" * 10,len(array))
    return array

def sum_ln(ar):
    sum1 = 0
    for num in ar:
        ln = math.log(num, math.e)
        sum1 += ln
    return sum1

def var(ar):
    sum2 = 0
    for num in ar:
        dis = num - np.mean(ar)
        sum2 += dis * dis
    return sum2/(len(ar) - 1)

if __name__ == '__main__':
    # arr = arr_input()
    arr = [35, 16, 15, 17, 17, 30, 11, 16, 19, 24]
    arr_mean = np.mean(arr)
    arr_var = var(arr)
    arr_std = var(arr) ** 0.5
    print("Mean: %f" % arr_mean)
    print("Variance (unbiased): %f" % arr_var)
    print("Std_Variance (unbiased): %f" % arr_std)


import numpy as np
from math import ceil

def random_connections(population_size, connection_chance, excitatory_ratio = 0.8):
    weights_values = np.random.rand(population_size,population_size)
    np.fill_diagonal(weights_values, 0)
    AdjacencyMatrix = (np.random.rand(population_size, population_size) + connection_chance).astype(np.int)
    excitatory_neurons = np.array([1]* int(excitatory_ratio * population_size) + [-1]* (ceil((1-excitatory_ratio) * population_size))).reshape((population_size, 1))
    weights = AdjacencyMatrix * excitatory_neurons * weights_values
    return weights

def uniform_connections(population_size, connection_chance, excitetory_weight = 0.25, inhibitory_weight = -0.25, excitatory_ratio = 0.8):
    AdjacencyMatrix = (np.random.rand(population_size, population_size) + connection_chance).astype(np.int)
    excitatory_neurons = np.array([excitetory_weight]* int(excitatory_ratio * population_size) + [inhibitory_weight]* (ceil((1-excitatory_ratio) * population_size))).reshape((population_size, 1))
    weights = AdjacencyMatrix * excitatory_neurons
    np.fill_diagonal(weights, 0)
    return weights

def recurrent_layer_wise(layers, recurrent_connection_chance,
    between_connection_chance, inside_connection_chance, excitatory_ratio = 0.8,):
    '''
    recurrent_connection_chance: 'float'
    between_connection_chance: 'float' or 'list of floats'
    inside_connection_chance: 'float' or 'list of floats'
    excitatory_ratio: 'float' or 'list of floats'
    '''
    N_layers = len(layers)
    between_connection_chance = [between_connection_chance] * N_layers if type(between_connection_chance) != list else between_connection_chance
    inside_connection_chance = [inside_connection_chance] * N_layers if type(inside_connection_chance) != list else inside_connection_chance
    excitatory_ratio = [excitatory_ratio] * N_layers if type(excitatory_ratio) != list else excitatory_ratio
    population_size = sum(layers)
    recurrent_weights_values = np.random.rand(population_size,population_size) * (np.random.rand(population_size, population_size) + recurrent_connection_chance).astype(np.int)
    weights =np.tril(recurrent_weights_values,-1)
    index = 0
    excitatory_neurons = []
    for i, (N, between_chance, inside_chance, excitatory) in enumerate(zip(layers, between_connection_chance, inside_connection_chance,excitatory_ratio)):
        # excitatory neurons
        excitatory_neurons += [1]* int(excitatory * N) + [-1]* (ceil((1-excitatory) * N))
        # inside layer:
        weights_values = np.random.rand(N,N) * (np.random.rand(N, N) + inside_chance).astype(np.int)
        np.fill_diagonal(weights_values, 0)
        weights[index:index+N, index:index+N] = weights_values
        #between layers:
        if i < len(layers)-1:
            next_N = layers[i + 1]
            weights_values = np.random.rand(N,next_N) * (np.random.rand(N,next_N) + between_chance).astype(np.int)
            weights[index:index+N, index+N:index+N+next_N] = weights_values
        index+= N
    excitatory_neurons = np.array(excitatory_neurons).reshape(population_size, 1)
    return weights * excitatory_neurons


import numpy as np
import pyailib as pl
import matplotlib.pyplot as plt

Ns, k, b = 100, 6.2, 3.0
x = np.linspace(0, 1, Ns)
y = x * k + b + np.random.randn(Ns)
# x, y = x.to('cuda:1'), y.to('cuda:1')

deg = 1
deg = 2
# deg = (2, 2)

w = pl.polyfit(x, y, deg=deg)
print(w)
w = pl.polyfit(x, y, deg=deg, C=5)
print(w)
yy = pl.polyval(w, x, deg=deg)
print(yy)

print(np.sum(np.abs(y - yy)))

plt.figure()
plt.plot(x, y, 'ob')
plt.plot(x, yy, '-r')
plt.xlabel('x')
plt.ylabel('y')
plt.title('polynomial fitting')
plt.legend(['noised data', 'fitted'])
plt.show()


"""
.. versionadded:: 0.6

Principal component analysis (PCA) is a statistical method how to convert a set of observations with possibly correlated variables into a data-set of linearly uncorrelated variables (principal components). The number of principal components is less or equal than the number of original variables. This transformation is defined in such a way that the first principal component has the largest possible variance.

.. contents::
   :local:
   :depth: 1

See also: :ref:`preprocess-lda`

Usage Explanation
=======================

For reduction of dataset :code:`x` to :code:`n` number of principal components

.. code-block:: python

    new_x = pa.preprocess.PCA(x, n)

If you want to see the ordered eigenvalues of principal components,
you can do it as follows:

.. code-block:: python

    eigenvalues = pa.preprocess.PCA_components(x)

Minimal Working Example
===========================

In this example is generated random numbers (100 samples, with 3 values each).
After the PCA application the reduced data-set is produced
(all samples, but only 2 valueseach)

.. code-block:: python

    import numpy as np
    import padasip as pa

    np.random.seed(100)
    x = np.random.uniform(1, 10, (100, 3))
    new_x = pa.preprocess.PCA(x, 2)

If you do not know, how many principal components you should use,
you can check the eigenvalues of principal components according to
following example

.. code-block:: python

    import numpy as np
    import padasip as pa

    np.random.seed(100)
    x = np.random.uniform(1, 10, (100, 3))
    print pa.preprocess.PCA_components(x)

what prints

>>> [ 8.02948402  7.09335781  5.34116273]

Code Explanation
====================
"""
from __future__ import division
import numpy as np

def PCA_components(x):
    """
    Principal Component Analysis helper to check out eigenvalues of components.

    **Args:**

    * `x` : input matrix (2d array), every row represents new sample

    **Returns:**

    * `components`: sorted array of principal components eigenvalues

    """
    # validate inputs
    try:
        x = np.array(x)
    except:
        raise ValueError('Impossible to convert x to a numpy array.')
    # eigen values and eigen vectors of data covariance matrix
    eigen_values, eigen_vectors = np.linalg.eig(np.cov(x.T))
    # sort eigen vectors according biggest eigen value
    eigen_order = eigen_vectors.T[(-eigen_values).argsort()]
    # form output - order the eigenvalues
    return eigen_values[(-eigen_values).argsort()]

def PCA(x, n=False):
    """
    Principal component analysis function.

    **Args:**

    * `x` : input matrix (2d array), every row represents new sample

    **Kwargs:**

    * `n` : number of features returned (integer) - how many columns
      should the output keep

    **Returns:**

    * `new_x` : matrix with reduced size (lower number of columns)
    """
    # select n if not provided
    if not n:
        n = x.shape[1] - 1
    # validate inputs
    try:
        x = np.array(x)
    except:
        raise ValueError('Impossible to convert x to a numpy array.')
    assert type(n) == int, "Provided n is not an integer."
    assert x.shape[1] > n, "The requested n is bigger than \
        number of features in x."
    # eigen values and eigen vectors of data covariance matrix
    eigen_values, eigen_vectors = np.linalg.eig(np.cov(x.T))
    # sort eigen vectors according biggest eigen value
    eigen_order = eigen_vectors.T[(-eigen_values).argsort()]
    # form output - reduced x matrix
    return eigen_order[:n].dot(x.T).T


"""
    Turn radius projection demo
    How and why to use the Ackermann steering model. https://www.youtube.com/watch?v=i6uBwudwA5o
"""

import matplotlib.pyplot as plt
import math
import numpy as np

CAR_L = 2.634  # Wheel base
CAR_T = 1.733  # Tread
MIN_TURNING_RADIUS = 5.
MAX_STEER = 500

def get_radius(angle, car_l=CAR_L, car_t=CAR_T):
    r = car_l / np.tan(np.deg2rad(angle, dtype=np.float32))
    return r

def get_delta(r, car_l=CAR_L, car_t=CAR_T):
    """
    :param r: Turn radius ( calculated against back center)
    :param car_l: Wheel base
    :param car_t: Tread
    :return: Angles of front center, inner wheel, outer wheel
    """
    delta_i = np.rad2deg(np.arctan(car_l / (r - car_t / 2.)))
    delta = np.rad2deg(np.arctan(car_l / r))
    delta_o = np.rad2deg(np.arctan(car_l / (r + car_t / 2.)))
    return delta, delta_i, delta_o

def get_car_path(r, distance=1., no_points=100, center_x=True, car_l=CAR_L, car_t=CAR_T):
    """
    :param r: Car turn radius ( against back center )
    :param distance: Distance to draw points
    :param no_points: No of points to draw on path
    :param center_x: If center point on the oX axis
    :param car_l: Wheel base
    :param car_t: Tread
    :return: center_points, inner_points, outer_points (on car path)
    """
    r_center = r
    r_inner = r_center - car_t / 2.
    r_outer = r_center + car_t / 2.

    d_inner = r_inner / r_center * distance
    d_outer = r_outer / r_center * distance
    center_points = point_on_circle(r_center, distance=distance, no_points=no_points,
                                    center_x=False)
    inner_points = point_on_circle(r_inner, distance=d_inner, no_points=no_points, center_x=False)
    outer_points = point_on_circle(r_outer, distance=d_outer, no_points=no_points, center_x=False)
    if center_x:
        center_points[:, 0] -= r_center
        inner_points[:, 0] -= r_center
        outer_points[:, 0] -= r_center

    return center_points, inner_points, outer_points

def point_on_circle(r, distance=1., no_points=100, center_x=True):
    """
    Returns a fix number of points on a circle circumference.
    :param r: circle radius
    :param distance: length of circumference to generate points for
    :param no_points: number of points to generate
    :param center: center points on the x axis ( - r)
    :return: np. array of 2D points
    """
    fc = 2 * np.pi * r
    p = distance / fc
    step = 2 * np.pi * p / float(no_points)
    points = np.array([
        (math.cos(step * x) * r, math.sin(step * x) * r) for x in range(0, no_points + 1)
    ])
    if center_x:
        points[:, 0] = points[:, 0] - r
    return points

class TurnRadius:
    def __init__(self, cfg):
        self.car_l = cfg.car_l
        self.car_t = cfg.car_t
        self.min_turning_radius = cfg.min_turning_radius

        self.num_points = num_points = 400
        max_wheel_angle = np.rad2deg(np.arctan(CAR_L / MIN_TURNING_RADIUS))
        self.angles = np.linspace(-max_wheel_angle, max_wheel_angle, num_points)

    def get_car_path(self, steer_factor, distance=20):
        """
        :param steer_factor: [-1, 1] (max left max right)
        :return:
        """
        num_points = self.num_points
        idx = np.clip(int(num_points/2. * steer_factor + num_points/2.), 0, num_points-1)
        r = get_radius(self.angles[idx])
        c, lw, rw = get_car_path(r, distance=distance)
        return c, lw, rw

if __name__ == "__main__":
    # You probably won't need this if you're embedding things in a tkinter plot...
    plt.ion()

    fig = plt.figure()

    car_l, car_t = CAR_L, CAR_T
    r = -10.
    c, lw, rw = get_car_path(r, distance=20)

    plt.plot(c[:, 0], c[:, 1])
    plt.plot(lw[:, 0], lw[:, 1])
    plt.plot(rw[:, 0], rw[:, 1])

    plt.axis('equal')
    plt.show(block=False)

    num = 400
    max_wheel_angle = np.rad2deg(np.arctan(CAR_L / MIN_TURNING_RADIUS))

    angles = np.linspace(-max_wheel_angle, max_wheel_angle, num)

    idx = int(angles.size / 2)
    while True:
        fig.clear()
        fig.canvas.draw()
        fig.canvas.flush_events()

        r = get_radius(angles[idx])
        c, lw, rw = get_car_path(r, distance=20)
        print(c)

        plt.plot(c[:, 0], c[:, 1])
        plt.plot(lw[:, 0], lw[:, 1])
        plt.plot(rw[:, 0], rw[:, 1])

        plt.axis('equal')
        plt.show(block=False)

        q = raw_input("key:\n")
        if q == "q":
            idx -= 1
        elif q == "w":
            idx += 1


import numpy as np
import matplotlib.pyplot as plt
import math

def normal(mu,sigma,x): #normal distribution
    return 1/(math.pi*2)**0.5/sigma*np.exp(-(x-mu)**2/2/sigma**2)

def eval(x):
    return normal(-4,1,x) + normal(4,1,x)

def ref(x):
    return normal(0,2,x)  #normal
    #return np.array([0.2 for i in range(len(x))])  #uniform

N=100000
M=5000
#x = (np.random.rand(N)-0.5)*16
x = np.random.normal(0,2,N)
w_x = eval(x)/ref(x)
w_x = w_x/sum(w_x)
w_xc = np.cumsum(w_x) #accumulate

X=np.array([])
for i in range(M):
    u = np.random.rand()
    X = np.hstack((X,x[w_xc>u][0]))

x = np.linspace(-8,8,500)
plt.plot(x,eval(x)/2)
plt.plot(x,ref(x))
plt.hist(X,bins=100,density=True)
#plt.title('Importance Sampling(Uniform)')
plt.title('Importance Sampling(Normal)')
plt.xlabel('$samples$')
plt.ylabel('$sample prob.$')
#plt.savefig('ImpSampling1.png',dpi=70)
plt.savefig('ImpSampling2.png',dpi=70)
plt.show()


'''
Applying Stochastic Gradient Descent for Linear Regression
'''

import numpy as np
import matplotlib
import matplotlib.pyplot as plt

X = 2 * np.random.rand(100,1)
y = 4 + 3 * X + np.random.randn(100,1)

X_b = np.c_[np.ones((100,1)), X]

eta = .1
m = 100

n_epochs = 50
# Learning schedule hyperparameters
t0, t1 = 5, 50

def learning_schedule(t):
    return t0 / (t + t1)

# Random initialization for theta
theta = np.random.randn(2,1)

for epoch in range(n_epochs):
    for i in range(m):
        random_index = np.random.randint(m)
        xi = X_b[random_index:random_index+1]
        yi = y[random_index:random_index+1]
        gradients = 2 * xi.T.dot(xi.dot(theta) - yi)
        eta = learning_schedule(epoch * m + i)
        theta = theta - eta * gradients
print('Theta: ', theta)

# Using Scikit-Learn
from sklearn.linear_model import SGDRegressor
sgd_reg = SGDRegressor(max_iter=50, penalty=None, eta0=.1)
sgd_reg.fit(X, y.ravel())

print('Scikit-Learn: ', sgd_reg.intercept_, sgd_reg.coef_)


import numpy as np
from numpy.linalg import norm
def lms_regression(X, Y, theta, tol=1E-6):
    converged = False
    alpha = 0.01/len(Y)
    while not converged:
	gradient = 0.
	for xiT, yi in zip(X,Y):
            hi = np.dot(theta, xiT)
            gradient += (hi - yi)*xiT.T
        new_theta = theta - alpha * gradient
	converged = norm(theta-new_theta) < tol * norm(theta)
	theta = new_theta
    return theta

if __name__=="__main__":
    m = 20
    t = np.linspace(0,1,m)
    x = 2 + 2*t
    y = 300 + 100*t

    X = np.array([np.ones(m), x]).T
    Y = y.reshape(m,1)

    theta_0 = np.array([[0.0,0.0]])
    theta =  lms_regression(X, Y, theta_0)
    print theta


#from:  https://stackoverflow.com/questions/50658884/why-this-numba-code-is-6x-slower-than-numpy-code
#runas import numpy as np; x1 = np.array([[1.,2,3]]) ; x2 = np.array([[1.,2,3],[2,3,4], [3,4,5], [4,5,6], [5,6,7]]); euclidean_distance_square(x1, x2)
#pythran export euclidean_distance_square(float64[1,:], float64[:,:])
import numpy as np
def euclidean_distance_square(x1, x2):
    return -2*np.dot(x1, x2.T) + np.sum(np.square(x1), axis=1)[:, np.newaxis] + np.sum(np.square(x2), axis=1)


import numpy as np

def sigmoid(x, deriv=False):
  """Calculates sigmoid function for given input.

  Args:
    x: Numeric input.
    deriv: If true, returns derivative on that point.

  Returns:
    Sigmoid or sigmoid derivative at specific point.
  """
  if deriv:
    return sigmoid(x) * (1 - sigmoid(x))
  return 1 / (1+np.exp(-x))

class MLP():
  """
  A implementation of a simple 2 layers Multilayer Perceptron.
  """
  def __init__(self, input_size, hidden_size, output_size):
    self.Wxh = np.random.randn(input_size+1, hidden_size)
    self.Who = np.random.randn(hidden_size, output_size)

  def forward(self, X):
    """Pass the input through the network and calculates the
    corresponding output for each layer.

    Args:
      X: Input vector with shape (N,x).

    Returns:
      Output vector with shape (L,N,o).
    """
    ones_l0 = np.ones((X.shape[0], 1))

    l0 = np.concatenate((X, ones_l0), axis=1)
    l1 = sigmoid(np.dot(l0, self.Wxh))
    l2 = sigmoid(np.dot(l1, self.Who))

    return l0, l1, l2

  def predict(self, X):
    """Returns the output in the last layer.

    Args:
      X: Input vector with shape (N,x).

    Returns:
      Output vector with shape (N,o).
    """
    l2 = self.forward(X)[2]
    return l2

  def fit(self, X, y, epochs=50, lr=1e-3):
    """Fit the model into given data.

    Args:
      X: Training input vectors with shape (N,x).
      y: Training output classes with shape (N).
      epochs: Number of epochs.
      lr: Learning rate.

    Returns:
      A history of losses in each epoch.
    """
    total_loss = []

    for epoch in range(epochs):
      l0, l1, l2 = self.forward(X)

      l2_error = y - l2
      l2_delta = l2_error * sigmoid(l2, deriv=True)

      l1_error = l2_delta.dot(self.Who.T)
      l1_delta = l1_error * sigmoid(l1, deriv=True)

      self.Who += lr*l1.T.dot(l2_delta)
      self.Wxh += lr*l0.T.dot(l1_delta)

      epoch_loss = np.mean(np.abs(l2_error))

      if (epoch % 1000) == 0:
        total_loss.append(epoch_loss)
        print("Loss: " + str(epoch_loss))

    return total_loss

# Heres a simple example with the XOR gate.
X_train = np.array([
  [0, 0],
  [0, 1],
  [1, 0],
  [1, 1]
]) # Generate the training X data

y_train = np.array([
  [0],
  [1],
  [1],
  [0]
]) # Generate the training y data

model = MLP(2, 3, 1)  # Instanciate the model

loss = model.fit(X_train, y_train, 10000, 1) # Fit the model with given data

X = np.array([
  [0, 0],
  [0, 1],
  [1, 0],
  [1, 1]
]) # Test data

model.predict(X) # Predict test output

Scripts/Miscellaneous/Neural Networks From Scratch/neural_net.py


from numpy import zeros, diag, exp, sqrt

from scipy.sparse import kron

def LinCombShiftedLogNMoments(b, mu, sigma2,a):
    # This function computes the expectation, the standard deviation and the skewness
    # of linear combination of multivariate shifted lognormally distributed
    # random vector
    #  INPUTS

    #  b             : [vector] (n_ x 1)  coefficients of the linear combination
    #  mu            : [vector] (n_ x 1)  location parameter of the lognormal distribution
    #  sigma2        : [matrix] (n_ x n_) dispersion parameter of the shifted lognormal distribution
    #  a             : [vector] (n_ x 1)  shift parameter
    #  OPS
    #  mu_Y      : [scalar] expectation of the linear combination
    #  sd_Y      : [scalar] standard deviation of the linear combination
    #  sk_Y      :[scalar] skewness of the linear combination

    # For details on the exercise, see here .

    ## Code
    n_=len(mu)
    if a is None:
        a=zeros((n_,1))

    #expectation
    mulgn=exp(mu + 1/2*diag(sigma2))
    mu_Y=b.T@mulgn-b.T@a

    #Inizialize covariances and third central moments
    Cov = zeros((n_,n_))
    cent3rd = zeros((n_**3,1))
    i = 0

    for n in range(n_):
        for m in range(n_):
            # Covariances
            Cov[n,m] = exp(mu[m]+mu[n] + 1/2*(sigma2[n,n]+sigma2[m,m]))*(exp(sigma2[m,n])-1)
            noncent2nd_nm = exp(mu[m]+mu[n] + 1/2*(sigma2[n,n]+sigma2[m,m])+sigma2[n,m])
        for l in range(n_):
            i=i+1
            # second non-central moments that enter in the third non-central moments formulas
            noncent2nd_nl = exp(mu[n]+mu[l] + 1/2*(sigma2[n,n]+sigma2[l,l])+sigma2[n,l])
            noncent2nd_ml = exp(mu[m]+mu[l] + 1/2*(sigma2[m,m]+sigma2[l,l])+sigma2[m,l])
            # third non-central moments
            noncent3rd_nml = exp(mu[m]+mu[n]+mu[l] + 1/2*(sigma2[n,n]+sigma2[m,m]+sigma2[l,l])+sigma2[n,l]+sigma2[n,m]+sigma2[l,m])
            cent3rd[i] = noncent3rd_nml**mulgn[l]@mulgn[m]@mulgn[n]-noncent2nd_nm@mulgn[l]-noncent2nd_nl@mulgn[m]-noncent2nd_ml@mulgn[n]

    sd_Y = sqrt(b.T@Cov@b) #standard deviation
    dummy = kron(b,b)@b.T
    vec_h = dummy.flatten()
    sk_Y = (vec_h.T@cent3rd)/(sd_Y**3) #skewness
    return mu_Y, sd_Y, sk_Y