import math
layer_output = [4.8, 1.21, 2.385]

E = math.e

# The final value has to be a probability in a classification problem, the probability does not
# work well with negative values, so we need to way to preserve the information from negative values
# while not dealing with negative values, enter E raised to power x

exp_values = [E**value for value in layer_output]
print(exp_values)

# Now we need to normailise this, normalise is basically the value devided by sum of all other values
# so x/sum(of list of values) this gives us probability

sum_of_exp = sum(exp_values)
prob_values = [element/sum_of_exp for element in exp_values]
print(prob_values)
print("sum of total should be close to one " + str(sum(prob_values)))

# The above with numpy

import numpy as np

# we have a batch of output (for m training set)

layer_output = [[4.8, 1.21, 2.385],
                [4.8, 1.21, 2.385],
                [4.8, 1.21, 2.385]]
exp_values = np.exp(layer_output)
sum_of_exp = np.sum(exp_values, axis=1, keepdims=True)
prob_values = exp_values/sum_of_exp
print(prob_values)


import numpy.linalg as ln
import numpy as np
import random as rd
from featureextractionsom.type_aliases import Vector, Weight_Matrix
from typing import Tuple, List

def distance(vec1: Vector, vec2: Vector) -> float:
	"""
	Return the Euclidean distance between two vectors
	"""
	return ln.norm(np.subtract(vec1, vec2))

def find_closest(node_matrix: Weight_Matrix, input_vector: Vector, *args) -> Tuple[int, int]:
	"""
	Return row and column coordinates of closest vector in array to given vector
	"""
	# initially set closest distance high
	closest_dist = 0
	closest_row = 0
	closest_col = 0

	for i in range(0, node_matrix.shape[0]):
		# u represents rows -> vectorArray[u] is row u.

		for j in range(0, node_matrix.shape[1]):

			# For evaluation: *args are coordinates of vector within matrix, looking for closest node TO
			# that node so must exclude itself.
			if args:
				if i == args[0] and j == args[1]:
					if args[0] == 0 and args[1] == 0:
						closest_dist = distance(input_vector, node_matrix[1][1])
					continue

			# v represents columns --> vectorArray[u][v] is row u column v

			# Each value of vectorArray[u][v] is a *weight vector*.

			d = distance(input_vector, node_matrix[i][j])

			if i == 0 and j == 0:
				# this is the first iteration, so set "closest" to this value for now.
				closest_dist = d
			else:
				# not the first time through the loop, check if this value of distance is lower than current minimum.
				if d < closest_dist:
					# This distance is smaller! Update distance and row/column values.

					closest_dist = d
					closest_row = i
					closest_col = j

	return closest_row, closest_col

def random_list(length: int) -> List[float]:
	"""
	Return a random list of floats (magnitude between 0 and 1) of given length, for creating the node matrix.
	"""
	return [rd.randint(0, 10) / 10 for i in range(length)]

def build_node_matrix(size: int, n: int) -> Weight_Matrix:
	"""
	returns matrix of nodes for given size
	Each node holds a weight vector of length n, initially populated with random values between 0 and 1
	"""
	# set seed so initial matrix is always the same
	rd.seed(42)

	return np.array([[random_list(n) for column in range(size)] for row in range(size)])

def update_weights(input_weight_vector: Vector, matrix_weight_vector: Vector, scale_factor: float) -> Vector:
	"""
	Update weights in matrix_weight_vector according to formula
	"""

	return matrix_weight_vector + scale_factor * (np.subtract(input_weight_vector, matrix_weight_vector))


import sys
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import linalg

# Reading data from csv files
def read_params(x_path,y_path):
  x_data = pd.read_csv(x_path,header=None)
  y_data = pd.read_csv(y_path,header=None)
  x = np.asmatrix(np.array(x_data))
  y = np.asmatrix(np.array(y_data))
  x = normalize(x)
  return (x,y,np.array(x_data),np.array(y_data))

# Normalization function
def normalize(mat):
  # No need for normalization here
  z = 1+np.zeros((mat.shape[0],1),dtype=float)
  z = np.hstack((z,mat))
  return z

# computes the distance of x1 from x parametrized by tow
def exp_function(x1,x,tow):
  diff = x1-x
  z = np.dot(diff,diff.transpose())[0,0]
  rv = np.exp((-1*z)/(2*tow*tow))
  return rv

# Gradient computation for underlying gradient descent
def gradient(x,y,theta,point_x,point_y,weight):
  arr = np.multiply(weight,np.dot(x,theta)-y)
  grad = (np.dot(x.transpose(),arr))/len(x)
  return grad

# Main algorithm for a particular point
def local_weighted(point_x,point_y,x,y,tow):
  # this alpha is for the underlying gradient descent for learning params for a given point
  alpha = 0.03
  old_val = 10
  new_val = 1
  epsilon = 0.0001

  # theta is the vector of parameters for predicting on point_x
  theta = np.asmatrix(np.zeros((x.shape[1],1),dtype=float,order='F'))

  # assigning weight to the m data-points
  weight = np.asmatrix(np.zeros((x.shape[0],1),dtype=float,order='F'))
  for i in range(len(x)):
      weight[i,0] = exp_function(x[i,:],point_x,tow)

  # convergence criteria: change in predicted value<epsilon
  while True:
    grad = gradient(x,y,theta,point_x,point_y,weight)
    theta-=alpha*grad
    new_val = np.dot(point_x,theta)
    if(abs(new_val-old_val)<epsilon):
      break
    old_val = new_val

  return np.dot(point_x,theta)[0,0]

# Algorithm for all points
def algo(x,y,tow):
  output_values = np.asmatrix(np.zeros((x.shape[0],1),dtype=float,order='F'))
  for i in range(len(x)):
    output_values[i,0] = local_weighted(x[i,:],y[i,0],x,y,tow)
  return output_values

# Plotting the graph
def curve_plot(x_data,y_data,value_mat):
  plt.scatter(x_data,y_data,c='r',label='Original')
  new_x,new_y = zip(*sorted(zip(x_data,np.array(value_mat))))
  plt.plot(new_x,new_y,'-',linewidth=2,c='g',label='Predicted')
  plt.xlabel('x')
  plt.ylabel('y')
  plt.title("local_lin_reg")
  plt.legend()
  plt.savefig('local_linear.png',dpi=200)
  plt.show()

# MAIN FUNCTION
def main():
  # Taking parameters from command line
  x_path = sys.argv[1]
  y_path = sys.argv[2]
  tow = float(sys.argv[3])

  # reading the datasets
  (x,y,x_data,y_data) = read_params(x_path,y_path)

  # output values on the given dataset when predicted using locally weighted linear regression
  value_mat = algo(x,y,tow)

  # plotting
  curve_plot(x_data,y_data,value_mat)

if __name__ == "__main__":
	main()


import numpy as np

#ref: https://en.wikipedia.org/wiki/Power_iteration
def Eigs_PowerIter(A, nb_iter = 20):
    # Ideally choose a random vector
    # To decrease the chance that our vector
    # Is orthogonal to the eigenvector
    b_k = np.random.rand(A.shape[1])

    for _ in range(nb_iter):
        # calculate the matrix-by-vector product Ab
        b_k1 = np.dot(A, b_k)

        # calculate the norm
        b_k1_norm = np.linalg.norm(b_k1)

        # re normalize the vector
        b_k = b_k1 / b_k1_norm

    return b_k


#use THEANO_FLAGS=device=cpu or device=cuda[0,1,2,3]
#device is the device you want to use
#cuda (or cuda0) is the gpu
#cuda1 is the second gpu if you have one

# try with the flag floatX=float32
# the same can be obtained by putting this inside the code: theano.config.floatX = 'float32'

# then try with the additional flag profile=True and try to optimize

import theano
from theano import function, config, shared, tensor
import numpy
import matplotlib.pyplot as plt
import time

rng = numpy.random

#Training Data
vlen = 1024 * 1024 * 2
X = numpy.asarray(rng.rand(vlen), config.floatX)
Y = numpy.asarray(rng.rand(vlen), config.floatX)

m_value = rng.randn()
c_value = rng.randn()
m = theano.shared(m_value, name ='m')
c = theano.shared(c_value, name ='c')

x = tensor.vector('x')
y = tensor.vector('y')
num_samples = X.shape[0]

prediction = tensor.dot(x,m)+c
cost = tensor.sum(tensor.pow(prediction - y, 2))/(2 * num_samples)

gradm = tensor.grad(cost, m)
gradc = tensor.grad(cost, c)

learning_rate = 0.01
training_steps = 1000

train = theano.function([x, y], cost, updates = [(m, m - (learning_rate * gradm)),(c, c - (learning_rate * gradc))])
test = theano.function([x], prediction)

t0 = time.time() #start time
for i in range(training_steps):
        costM = train(X,Y)
        if i % 100 == 0:
                print(costM)
t1 = time.time()
print("Looping %d times took %f seconds" % (training_steps, t1 - t0))

print("Slope :")
print(m.get_value())
print("Intercept :")
print(c.get_value())

#a = numpy.linspace(0, 10, 10)
#b = test(a)

if numpy.any([isinstance(m.op, tensor.Elemwise) and
              ('Gpu' not in type(m.op).__name__)
        for m in train.maker.fgraph.toposort()]):
        print('Used the cpu')
else:
        print('Used the gpu')


import numpy as np
from scipy.stats import norm
from numpy.linalg import cholesky
import matplotlib.pyplot as plt

############# problem a #################

def problem_a():
    x = 0.5 * (np.random.normal(-5, 1, 100)) + 0.5 * (np.random.normal(5, 1, 100))
    return x

def generate_samples(s_number=100):
    x_samples = []
    for i in range(s_number):
        u = np.random.uniform(0, 1)
        if u >= 0.5:
            x_samples.append(np.random.normal(5.0, 1))
        else:
            x_samples.append(np.random.normal(-5.0, 1))
    return np.array(x_samples)

#s = generate_samples()
#print(s)
#print(s.shape)

############# problem b #################

def prior(mu1, mu2):
    # Gaussian prior
    return -(mu1**2+mu2**2)/200

def posterior_p(x_samples, mu1, mu2):
    # posterior
    p_x_under_mu = 0.5*np.exp(-0.5*(x_samples - mu1)**2) + 0.5*np.exp(-0.5*(x_samples - mu2)**2)
    p_x_under_mu = np.log(p_x_under_mu)
    p_x_under_mu = np.sum(p_x_under_mu)
    return p_x_under_mu + prior(mu1, mu2)

def accept_rate(x_samples, mu1, mu2, mu1_cap, mu2_cap):
    ar = posterior_p(x_samples,mu1_cap,mu2_cap) - posterior_p(x_samples,mu1,mu2)
    return np.exp(ar)

def MH_sampling(x_samples, sigma):
    #initial_mu1, initial_mu2 = 0.0, 0.0
    #mu1 = initial_mu1
    #mu2 = initial_mu2
    mu1 = 0
    mu2 = 0
    mu1_record = []
    mu2_record = []

    N = 11000 # discard the first 10000 samples and plot the next 1000 samples
    accept = 0

    for n in range(N):
        mu1_cap, mu2_cap = np.random.multivariate_normal([mu1, mu2], [[sigma**2, 0], [0, sigma**2]], 1)[0]
        ar = accept_rate(x_samples, mu1, mu2, mu1_cap, mu2_cap)

        if ar >= 1:
            mu1, mu2 = mu1_cap, mu2_cap
            accept += 1
        else:
            tmp = np.random.uniform(0, 1)
            if tmp <= ar:
                mu1, mu2 = mu1_cap, mu2_cap
                accept += 1
        if n >= 10000:
            mu1_record.append(mu1)
            mu2_record.append(mu2)

    plt.plot(mu1_record, mu2_record, '+')
    plt.title("MH_sigma{}".format(sigma))
    plt.show()

    return np.mean(mu1_record), np.mean(mu2_record), accept/N

############# problem c #################

def Gibbs_sampling(x_samples):
    mu1 = 0
    mu2 = 0
    N = 11000

    mu1_record = []
    mu2_record = []

    for n in range(N):
        prob_record = []
        for i in range(100):
            prob = np.exp(-0.5 * (x_samples[i] - mu1) ** 2) / (np.exp(-0.5*(x_samples[i]-mu2)**2) + np.exp(-0.5*(x_samples[i]-mu1)**2))
            prob_record.append(prob)
        latent = np.random.binomial(1, prob_record, 100)

        st_1 = np.sum((latent == 1))
        st_0 = np.sum((latent == 0))
        mu1 = np.random.normal(100 * np.sum((latent == 1)*x_samples)/(100*st_1+1), np.sqrt(200/(100*st_1+1)))
        mu2 = np.random.normal(100 * np.sum((latent == 0)*x_samples)/(100*st_0+1), np.sqrt(200/(100*st_0+1)))

        if n >= 10000:
            mu1_record.append(mu1)
            mu2_record.append(mu2)

    plt.plot(mu1_record, mu2_record, '+')
    plt.title("Gibbs")
    plt.show()

    return np.mean(mu1_record), np.mean(mu2_record)

if __name__ == "__main__":

    for i in range(6):
        x_samples = generate_samples()
        mu1_MH, mu2_MH, acc_rate = MH_sampling(x_samples, sigma=0.5)
        print("sigma=0.5", "ave_mu1: ", mu1_MH, "-- ave_mu2: ", mu2_MH, "-- accept_rate: ",acc_rate)

        mu1_MH, mu2_MH, acc_rate = MH_sampling(x_samples, sigma=5)
        print("sigma=5", "ave_mu1: ", mu1_MH, "-- ave_mu2: ", mu2_MH, "-- accept_rate: ", acc_rate)

        mu1_G, mu2_G = Gibbs_sampling(x_samples)
        print("ave_mu1_G: ", mu1_G, "-- ave_mu2_G: ", mu2_G)

    #plt.plot(MU1_G, MU2_G, '+')
    #plt.show()


"""Classes and functions for Information Theory concepts

Classes:

HarmonicSeries for calculation of alphabet entropy under Zepf's law

BinaryBayes for modeling binary-input, binary-output channel with Bayes theorem and entropy functions

MiscFuncs contains functions for calculating entropy, kl divergence, typical set, number of permutations, etc.

author: Rebecca Stein

"""

import numpy as np
from math import factorial
import networkx as nx
import matplotlib.pyplot as plt

class HarmonicSeries():
    """First n numbers in harmonic series"""

    def __init__(self, n):

        temp = np.arange(1,n+1)
        f = lambda x: 1. / x
        vf = np.vectorize(f)
        self._series = vf(temp)

    def add(self):
        """Return sum of first n numbers in harmonic series"""

        return np.sum(self._series)

class MiscFuncs():

    def __init__(self):
        pass

    @staticmethod
    def entropy(X):
        """Return entropy of probability vector X"""

        f = lambda x: x * np.log2(x)
        vf = np.vectorize(f)
        return -1 * np.sum(vf(X))

    @staticmethod
    def max_entropy(N):
        """Return entropy of event with N outcomes assuming uniform distribution"""

        return np.log2(N)

    @staticmethod
    def kl_divergence(px,qx):
        """Return KL divergence between p(x) and q(x)"""

        f = lambda p,q: p * np.log2(p.q)
        vf = np.vectorize(f)
        return np.sum(vf(px,qx))

    def zipf(k):
        """Return entropy of alphabet with k symbols following Zipf's law

        Zipf's law: if k letters are ordered according to frequency, the probability of
        each letter equals c / k, where k is the letter's place in the order and c is a
        normalizing constant.
        """

        series = HarmonicSeries(k)
        c = 1 / series.add()
        probs = c * series.series
        return self.entropy(probs)

    @staticmethod
    def typical_set(self,n,H):
        """Return size of typical set given length n and entropy H. Uses AEP (Asymptotic Equapartition Property): |A_e^n| -> 2^nH as n -> infinity.
            A_e^n is set of typical sequences of length n with limit in probability -> 1 - e
        """

        return 2**(n*H)

    def count_perms(self,n,k, with_replace = True):
        """Return number of possible permutations of k elements from set size n."""

        if with_replace:
            return n ** k
        else:
            return (factorial(n) / factorial(n-k))

    def n_choose_k(self,n,k):
        """Return number of combinations of k elements from set size n."""

        return self.count_perms(n,k,with_replace = False) / factorial(k)

if __name__ == "main":
    Funcs = MiscFuncs()


from dataloader import *
import numpy as np
from numpy.linalg import pinv, norm
from scipy.linalg.blas import dgemm

class DiscriminativeClassifier(object):
	def __init__(self,preprocessing="", bias=False):
		self.x_train, self.y_train, self.x_test, self.y_test, self.feature_size = \
			load_data(t=preprocessing, bias=bias)
		self.train_size = len(self.y_train)
		self.test_size = len(self.x_test)

def sigmoid(x):
	return 1.0 / (1.0 + np.exp((-1)*x))

# calc gradient (and/or Hessian matrix)
# TODO: you should implement gradient descent or other optimizer to this
#		so please calculate the gradient (and/or) hessian in this function
#		to help TA undertand your code
#		notice that you are only allow to use numpy here
def calc_grad(W, X, Y):
	# W: weight matrix
	# X: input
	# Y: ground true

	grad, Hess = np.zeros_like(W), None
	for i in xrange(len(X)):
		grad += np.insert(X[i], 0, 1) * np.asscalar(sigmoid(np.insert(X[i], 0, 1).dot(W)) - Y[i])
	return grad, Hess

class LogisticRegression(DiscriminativeClassifier):

	def __init__(self, l2norm=1, preprocessing="z", eta=1e-4, max_epoch=30, l2_on=True):
		"""
		:param l2norm: l2 norm penalty
		:param preprocessing: preprocessing method
		:param eta: learning rate (step size)
		:param max_epoch: how many epochs are you going to train the regression (optional)
		:param l2_on: use l2 or not
		"""
		super(LogisticRegression, self).__init__(preprocessing=preprocessing, bias=True)
		self.weight = np.ones(self.feature_size + 1)
		self.L2norm = l2norm
		self.eta = eta
		self.max_epoch = max_epoch
		self.l2_on = l2_on
		# mask is to filter out the weight and discard the bias
		self.mask = np.ones_like(self.weight)
		self.mask[0] = 0

	def train(self, eps=1e-4):
		epoch = 0
		# Error rate
		ratio = -1

		while True:
			# Count Right & Wrong Number
			self.count  = [0, 0]

			# update weight
			update = calc_grad(self.weight, self.x_train, self.y_train)
			self.weight -= self.eta * update[0]
			if self.l2_on:
				# L2 regularization
				self.weight -= self.eta * self.L2norm * np.insert(self.weight[1:], 0, 0);

			if norm(update[0]) < self.eta * 0.1: # TODO: you should think about some early stopping scheme here
				break

			err = 0
			for i in xrange(len(self.x_train)):
				y_1 = sigmoid(self.weight.dot(np.insert(self.x_train[i], 0, 1)))
				y_0 = 1-y_1
				if (y_0 > y_1): y = 0
				else: y = 1
				if y == self.y_train[i]: self.count[1]  += 1
				else: self.count[0]  += 1
				err -= self.y_train[i]*math.log(y_1+eps) + (1-self.y_train[i])*math.log(y_0+eps)
			if self.l2_on:
				for para in self.weight[1:]:
					err += self.L2norm/2.0*para*para

			epoch += 1
			# print "epoch\t", epoch, "\ttraining loss:", err
			ratio = 100 * self.count[0] / float(len(self.y_train))
			# print "Training Error Ratio: ", ratio, "%"
			# print "-" * 59

			if epoch == self.max_epoch: break
		return (ratio, err)

	def test(self, eps=1e-4):
		# Count Right & Wrong Number
		self.count  = [0, 0]
		# Count Error Type & Number
		self.ecount = [0, 0]
		# Error rate
		ratio = -1

		err = 0
		for i in xrange(len(self.x_test)):
			# P(y = 1) = \sigmoid ( W ^ T * X)
			# P(y = 0) = 1 - P(y = 1)
			y_1 = sigmoid(self.weight.dot(np.insert(self.x_test[i], 0, 1)))
			y_0 = 1-y_1
			# J(\theta) = - \sum{ y_i * \log(h_theta{x_i}) + (1-y_i) * \log(1-h_theta{x_i}) }
			err -= self.y_test[i]*math.log(y_1+eps) + (1-self.y_test[i])*math.log(y_0+eps)
			# Count Error
			if (y_0 > y_1): y = 0
			else: y = 1
			if y == self.y_test[i]:
				self.count[1]  += 1
			else:
				self.count[0]  += 1
				self.ecount[y] += 1

		# Regularization = lambda / 2 * \sum{ W_j ^ 2 }
		if self.l2_on:
			for para in self.weight[1:]:
				err += self.L2norm/2.0*para*para

		print
		# print "-" * 10, "Logistic Regression Classifier", "-" * 18
		# print "Correct Classcification:", self.count[1], ", Wrong Classcification:", self.count[0]
		# print "Spam => Normal:", self.ecount[0], ", Normal => Spam:", self.ecount[1]
		ratio = 100 * self.count[0] / float(len(self.y_test))
		# print "Error Ratio: ", ratio, "%"
		# print "-" * 59
		# print "Loss:", err
		return (ratio, err)

class LinearRegression(DiscriminativeClassifier):

	def __init__(self, l2norm=1, preprocessing="z", eta=1e-4, max_epoch=30, l2_on=True, method="lsq", gamma = 0.5, miniBatch=32):
		"""
		:param l2norm: l2 norm penalty
		:param preprocessing: preprocessing method
		:param eta: learning rate (step size)
		:param max_epoch: how many epochs are you going to train the regression (optional)
		:param l2_on: use l2 or not
		"""
		super(LinearRegression, self).__init__(preprocessing=preprocessing, bias=True)
		self.weight = np.ones(self.feature_size + 1)
		self.L2norm = l2norm
		self.eta = eta
		self.max_epoch = max_epoch
		self.l2_on = l2_on
		self.method = method
		self.gamma = gamma
		self.miniBatch = miniBatch

	def train(self, eps=1e-4):
		if (self.method == "lsq"):
			if self.l2_on:
				self.weight, err = np.linalg.lstsq(np.r_[np.c_[np.ones(len(self.x_train)), self.x_train],\
					np.mat(np.insert(np.ones(self.feature_size) * np.sqrt(self.L2norm), 0, 0))],\
					np.append(self.y_train, 0))[:2]
			else:
				self.weight, err = np.linalg.lstsq(np.c_[np.ones(len(self.x_train)), self.x_train], self.y_train)[:2]
			if err != []:
				err = np.asscalar(err) / len(self.x_train)
			else:
				err = 0
			self.count = [0, 0]
			for i in xrange(len(self.x_train)):
				y_1 = self.weight.dot(np.insert(self.x_train[i], 0, 1))
				y_0 = 1 - y_1
				if (y_0 > y_1): y = 0
				else: y = 1
				if y == self.y_train[i]: self.count[1]  += 1
				else: self.count[0]  += 1
			ratio = 100 * self.count[0] / float(len(self.y_train))
			if self.l2_on:
				for para in self.weight[1:]:
					err += self.L2norm/2.0*para*para
		else:

			epoch = 0

			Recall_training    = []
			Precision_training = []
			Recall_test        = []
			Precision_test     = []

			while True:
				momentum = 0
				for i in xrange(0, len(self.x_train), self.miniBatch):
					length = min(self.miniBatch, len(self.x_train) - i)
					update = np.zeros_like(self.weight)
					for j in xrange(length):
						update += np.insert(self.x_train[i+j], 0, 1) \
									* ( self.weight.dot(np.insert(self.x_train[i+j], 0, 1))\
										- self.y_train[i+j] )
					momentum = momentum * self.gamma + self.eta * update
					if self.l2_on:
						momentum += self.eta * self.L2norm * np.insert(self.weight[1:], 0, 0);

					self.weight -= momentum

				recall = 0
				precision = 0
				for i in xrange(len(self.x_train)):
					y_1 = self.weight.dot(np.insert(self.x_train[i], 0, 1))
					y_0 = 1-y_1
					if (y_0 > y_1): y = 0
					else: y = 1

					if (y == 1 and self.y_train[i] == 1):
						recall += 1
					if (y == self.y_train[i]):
						precision += 1
				precision /= float(len(self.y_train))
				recall /= float(sum(self.y_train))
				Recall_training.append(recall)
				Precision_training.append(precision)

				recall = 0
				precision = 0
				for i in xrange(len(self.x_test)):
					y_1 = self.weight.dot(np.insert(self.x_test[i], 0, 1))
					y_0 = 1-y_1
					if (y_0 > y_1): y = 0
					else: y = 1

					if (y == 1 and self.y_test[i] == 1):
						recall += 1
					if (y == self.y_test[i]):
						precision += 1
				precision /= float(len(self.y_test))
				recall /= float(sum(self.y_test))
				Recall_test.append(recall)
				Precision_test.append(precision)

				epoch += 1
				if epoch == self.max_epoch:
					return (Precision_training, Recall_training, Precision_test, Recall_test)

		return (ratio, err)

	def test(self, eps=1e-4):
		# Count Right & Wrong Number
		self.count  = [0, 0]
		# Count Error Type & Number
		self.ecount = [0, 0]
		# Error rate
		ratio = -1
		if self.l2_on:
			err = norm(np.mat(self.weight) * np.r_[np.c_[np.ones(len(self.x_test)), self.x_test],\
					np.mat(np.insert(np.ones(self.feature_size) * np.sqrt(self.L2norm), 0, 0))].T -\
					np.append(self.y_test, 0)) / len(self.x_test)
		else:
			err = norm(np.mat(self.weight) * np.c_[np.ones(len(self.x_test)), self.x_test].T - self.y_test) / len(self.x_test)
		for i in xrange(len(self.x_test)):
			y_1 = self.weight.dot(np.insert(self.x_test[i], 0, 1))
			y_0 = 1 - y_1
			# Count Error
			if (y_0 > y_1): y = 0
			else: y = 1
			if y == self.y_test[i]:
				self.count[1]  += 1
			else:
				self.count[0]  += 1
				self.ecount[y] += 1

		# Regularization = lambda / 2 * \sum{ W_j ^ 2 }
		if self.l2_on:
			for para in self.weight[1:]:
				err += self.L2norm/2.0*para*para

		ratio = 100 * self.count[0] / float(len(self.y_test))
		return (ratio, err)

class KNNClassifier(DiscriminativeClassifier):

	def __init__(self, preprocessing="", K=4):
		"""

		:param preprocessing: preprocessing method
		:param K: how much neighbours you want
		"""
		super(KNNClassifier, self).__init__(preprocessing=preprocessing)
		self.binary = (preprocessing == "binary")
		self.K = K

	# TODO: implement distance between sample here
	def __calc_distance(self, a, b):
		if self.binary:
			return np.sum(abs(a-b))
		else:
			return norm(a-b)

	def train(self):
		self.count = [0, 0]
		for i in xrange(len(self.x_train)):
			temp = sorted(range(len(self.x_train)), key = lambda x:self.__calc_distance(self.x_train[i], self.x_train[x]))[:self.K]
			y_1 = sum([self.y_train[w] for w in temp])
			y_0 = self.K - y_1
			if (y_0 > y_1): y = 0
			else: y = 1
			if y == self.y_train[i]: self.count[1]  += 1
			else: self.count[0]  += 1
		ratio = 100 * self.count[0] / float(len(self.y_train))
		return ratio

	def test(self):
		self.count = [0, 0]
		for i in xrange(len(self.x_test)):
			temp = sorted(range(len(self.x_train)), key = lambda x:self.__calc_distance(self.x_test[i], self.x_train[x]))[:self.K]
			y_1 = sum([self.y_train[w] for w in temp])
			y_0 = self.K - y_1
			if (y_0 > y_1): y = 0
			else: y = 1
			if y == self.y_test[i]: self.count[1]  += 1
			else: self.count[0]  += 1
		ratio = 100 * self.count[0] / float(len(self.y_test))
		return ratio

if __name__ == '__main__':
	# logistic = LogisticRegression()
	# logistic.train()
	# logistic.test()
	# linear = LinearRegression(method = "sgd", miniBatch = 256)
	# file = open("../output/sgd.out", "w")
	# print >> file, linear.train()
	knn = KNNClassifier()
	print knn.train()
	print knn.test()


import numpy as _np
import pandas as _pd
import scipy as _sp

def crossCorrelation(y1,y2,mode='same'):
	""" cross correlation wrapper for numpy function """
	CC=_np.correlate(y1,y2,mode=mode)
	return CC

def correlationCoefficient(data,fit):
	"""
	Correlation coefficient.
	Compares a fit to data.  Note that this is only valid for a linear fit.

	Reference
	---------
	https://mathworld.wolfram.com/CorrelationCoefficient.html
	"""
	if type(data)==_pd.core.frame.DataFrame or type(data)==_pd.core.frame.Series:
		y=data.values.reshape(-1)
		f=fit.values.reshape(-1)
	elif type(data)==_np.ndarray:
		y=data.reshape(-1)
		f=fit.reshape(-1)
	SSxy=((f-f.mean())*(y-y.mean())).sum()
	SSxx=((f-f.mean())**2).sum()
	SSyy=((y-y.mean())**2).sum()
	rho=SSxy**2/(SSxx*SSyy)
	return rho

def errorPropagationMonteCarlo(	func,
								parameterMeanValues=[],
								parameterStdValues=[],
								N=int(1e4),
								verbose=True):
	"""
	This is the "Monte Carlo error propagation" method.  I sometimes hear it also
	called the "Monte Carlo uncertaintly analysis" method.

	Parameters
	----------
	func : function
		Function to be studied.  The function takes a list of variables as input
	parameterMeanValues : list of floats
		Variables in func.
	parameterStdValues : list of floats
		Standard deviations associated with the variables in func.
	N : int
		Number of iterations to perform the Monte Carlo method.  10,000 is default
	verbose : bool
		Optionally prints the results in addition to returning them

	Returns
	-------
	mean : float
		The mean value of the result.  Should be approximately equal to the actual
		solved value of the function
	uncertainty : float
		The standard deviation of the result.  This is the "uncertainty" or "error"
		produced from this analysis.
	df : pandas.core.frame.DataFrame
		Dataframe containing the intermediate steps.  It's honestly not needed for
		anything other than sanity checking the steps.

	Examples
	--------
	def g(params):
		return 4*sp.pi**2*params[0]/params[1]**2
	mean,uncertainty,df=errorPropagationMonteCarlo( g,	[1,2],	[0.004,0.005])

	References
	----------
	http://www.eg.bucknell.edu/physics/ph310/jupyter/error_propagation_examples.ipynb.pdf

	"""

	# make sure N is an interger
	N=int(N)

	# initialize dataframe
	df = _pd.DataFrame(	_np.zeros((N,len(parameterMeanValues))),
						dtype=float)

	# populate dataframe with monte carlo uncertainties
	for i,(key,val) in enumerate(df.iteritems()):
		df[key]=_sp.random.normal(parameterMeanValues[i],parameterStdValues[i],N)

	# apply the result to each row of the dataframe
	df['result']=df.apply(func,axis=1)

	# calculate mean and standard deviation
	mean=df.result.mean()
	uncertainty=df.result.std()

	# optional output
	if verbose:
		print('Actual solved value: %.12f'%func(parameterMeanValues))
		print('Mean: %.12f'%mean )
		print('Std ("uncertainty"): %.12f'%uncertainty )

	return mean,uncertainty,df

def findNearestNeighbors(X,Y,numberOfNearestPoints=1):
	"""
	Find the nearest neighbors in X to each point in Y

	Example
	-------
	::

		from johnspythonlibraries2.Plot import finalizeSubplot as _finalizeSubplot

		# create data
		x=np.arange(0,10+1)
		y=np.arange(100,110+1)
		X,Y=np.meshgrid(x,y)
		X=X.reshape((-1,1))
		Y=Y.reshape((-1,1))
		A=np.concatenate((X,Y),axis=1)

		# points to investigate
		B=[[5.1,105.1],[8.9,102.55]]

		points,indices,radii=findNearestNeighbors(A,B,numberOfNearestPoints=5)

		for i in range(len(B)):
			fig,ax=plt.subplots()
			ax.plot(X,Y,'.',label='original data')
			ax.plot(B[i][0],B[i][1],'x',label='point of interest')
			ax.plot(points[i][:,0],points[i][:,1],label='nearest neighbors',marker='o',linestyle='', markerfacecolor="None")
			_finalizeSubplot(ax)

	"""

	from sklearn.neighbors import NearestNeighbors

	neigh = NearestNeighbors(n_neighbors=numberOfNearestPoints)
	neigh.fit(X)
	radii,indices=neigh.kneighbors(Y)
	points=X[indices]

	return points, indices, radii

def rms(data):
	"""
	Root mean square function.   Ignores NaNs.

	Parameters
	----------
	data : numpy.array
		Data to be processed

	Return
	------
	: numpy.array
		root mean square of data

	References
	----------
	# http://stackoverflow.com/questions/17197492/root-mean-square-error-in-python
	# http://statweb.stanford.edu/~susan/courses/s60/split/node60.html

	Examples
	--------
	>>> rms([-1,1,-1,1,-1,1])
	1.0
	>>> rms([-1,1,-1,1,-1,1,_np.nan])
	1.0
	>>> rms([-1,1,-1,1,-1,1,_np.inf])
	Traceback (most recent call last):
	...
	Exception: data contains +- inf
	"""
	if type(data)==list:
		data=_np.array(data)
	if True in _np.isinf(data):
		raise Exception('data contains +- inf')
	return _np.sqrt(_np.nanmean((data - 0) ** 2))

def rejectOutliers(data, sigma=2):
	"""
	remove outliers from set of data

	Parameters
	----------
	data : numpy.ndarray
		data array being considered
	sigma : int
		the number of std. devs. about which to reject data.  E.g. sigma=2
		rejects outliers outside of +-2*sigma

	Return
	------
	 : numpy.ndarray
		Same as databut missing entires considered as outliers
	indicesToKeep : numpy.ndarray (of bool)
		Boolean indices associated with entries of data that are kept

	References
	----------
	http://stackoverflow.com/questions/11686720/is-there-a-numpy-builtin-to-reject-outliers-from-a-list

	Examples
	--------
	>>> rejectOutliers([1,1.1,1,1,1,10])
	(array([1. , 1.1, 1. , 1. , 1. ]), array([ True,  True,  True,  True,  True, False]))
	>>> rejectOutliers([1,1.1,1,1,1,10,_np.nan])
	(array([1. , 1.1, 1. , 1. , 1. ]), array([ True,  True,  True,  True,  True, False, False]))
	"""
	if type(data)==list:
		data=_np.array(data)
	if True in _np.isinf(data):
		raise Exception('data contains +- inf')
	indicesToKeep=_np.less(abs(data - _np.nanmean(data)), sigma* _np.nanstd(data))
	return data[indicesToKeep],indicesToKeep

if __name__ == '__main__':
    import doctest
    doctest.testmod()


import numpy as np
import matplotlib.pyplot as plt

print("""
Current mode: Single fixed end, single point load

|                        F
|                        |
|                       \|/
|---------------------------------
| x-->
|<----------a----------->
|<---------------L--------------->
|
|
""")

F = float(input('Input F in N:\n'))
L = float(input('Input L in m:\n'))
a = float(input('Input a in m:\n'))
E = float(input('Input E in GPa:\n'))
I = float(input('Input I in mm^4:\n'))
xQuery = float(input('Input x in m:\n'))

# Convert to SI units
E = E*pow(1000,3)
I = I/pow(1000,4)

X = np.arange(0, L+L/1000, L/1000).tolist()
Y = []
for x in X:
  if x >= 0 and x <= a:
    Y.append(-F*x*x/6/E/I*(3*a-x))
  elif x >= a and x <= L:
    Y.append(-F*a*a/6/E/I*(3*x-a))

x = xQuery
if x >= 0 and x <= a:
  y = -F*x*x/6/E/I*(3*a-x)
elif x >= a and x <= L:
  y = -F*a*a/6/E/I*(3*x-a)

print('Deflection at ',x,' m = ',y,' m')

fig, ax = plt.subplots(nrows=1, ncols=1)
plt.plot(X,Y)
plt.plot(x,y,'rD')
plt.annotate(f"({x},{y})",(x,y))
plt.ylabel('Deflection (m)')
plt.xlabel('x (m)')
fig.savefig('graph.png')


import numpy as np
import math
import time
import copy

def gauss_jordan_partial_pivoting_solve(A, B):
    n = np.shape(A)[0]

    P = np.zeros((n, n))

    for i in range(0, n):
        P[i][i] = 1

    for i in range(0, n):
        best_pivot_row = i

        for j in range(i+1, n):
            if abs(A[j][i]) > abs(A[best_pivot_row][i]):
                best_pivot_row = j

        if A[best_pivot_row][i] == 0 or math.isclose(A[best_pivot_row][i],0.0,abs_tol=1e-9):
            raise ValueError("Unique solution does not exist!")

        if best_pivot_row != i:
            A[[i, best_pivot_row]] = A[[best_pivot_row, i]]
            B[i], B[best_pivot_row] = B[best_pivot_row], B[i]

        for j in range(0, n):
            if i != j:
                multiplier = A[j][i] / A[i][i]
                for k in range(0, n):
                    A[j][k] = A[j][k] - multiplier * A[i][k]
                B[j] = B[j] - multiplier * B[i]

    for i in range(0, n):
        B[i] = B[i] / A[i][i]

    return B

def gauss_jordan_complete_pivoting_solve(A, B):
    n = np.shape(A)[0]

    P = np.zeros((n, n))

    for i in range(0, n):
        P[i][i] = 1

    for i in range(0, n):
        best_pivot_row = i
        best_pivot_column = i

        for j in range(i+1, n):
            for k in range(j, n):
                if abs(A[j][k]) > abs(A[best_pivot_row][best_pivot_column]):
                    best_pivot_row = j
                    best_pivot_column = k

        if A[best_pivot_row][best_pivot_column] == 0 or math.isclose(A[best_pivot_row][best_pivot_column],0.0,abs_tol=1e-9):
            raise ValueError("Unique solution does not exist!")

        if best_pivot_row != i:
            A[[i, best_pivot_row]] = A[[best_pivot_row, i]]
            B[i], B[best_pivot_row] = B[best_pivot_row], B[i]

        if best_pivot_column != i:
            P[[i, best_pivot_column]] = P[[best_pivot_column, i]]
            A[:, [i, best_pivot_column]] = A[:, [best_pivot_column, i]]

        for j in range(0, n):
            if i != j:
                multiplier = A[j][i] / A[i][i]
                for k in range(0, n):
                    A[j][k] = A[j][k] - multiplier * A[i][k]
                B[j] = B[j] - multiplier * B[i]

    inverse_P = np.linalg.inv(P)

    A = np.dot(A, inverse_P)
    A = np.dot(inverse_P, A)
    B = np.dot(inverse_P, B)

    for i in range(0, n):
        B[i] = B[i] / A[i][i]

    return B

# simple LU
def LU_decomposition_solve(A, B):
    n = np.shape(A)[0]

    for k in range(0, n):
        for i in range(k+1, n):
            multiplier = A[i][k] / A[k][k]
            A[i][k+1:n] = A[i][k+1:n] - multiplier * A[k][k+1:n]
            A[i][k] = multiplier

    X = [0] * n
    Y = [0] * n

    # L solve, forward substitution
    for i in range(0, n):
        partial_row_sum = 0
        j = 0
        while j < i:
            partial_row_sum = partial_row_sum + A[i][j] * Y[j]
            j += 1
        Y[i] = B[i] - partial_row_sum

    # U solve, backward substitution
    for i in range(n - 1, -1, -1):
        partial_row_sum = 0
        for j in range(i + 1, n):
            partial_row_sum = partial_row_sum + A[i][j] * X[j]
        X[i] = (Y[i] - partial_row_sum) / A[i][i]

    return X

# LU with partial pivoting
def LU_decomposition_solve_partial_pivoting(A, B):
    n = np.shape(A)[0]

    for k in range(0, n):
        best_pivot_row = k

        for i in range(k+1, n):
            if abs(A[i][k]) > abs(A[best_pivot_row][k]):
                best_pivot_row = i

        if A[best_pivot_row][k] == 0 or math.isclose(A[best_pivot_row][k], 0.0, abs_tol=1e-9):
            raise ValueError("Unique solution does not exist!")

        A[[k, best_pivot_row]] = A[[best_pivot_row, k]]
        B[k], B[best_pivot_row] = B[best_pivot_row], B[k]

        for i in range(k+1, n):
            multiplier = A[i][k] / A[k][k]
            A[i][k+1:n] = A[i][k+1:n] - multiplier * A[k][k+1:n]
            A[i][k] = multiplier

    X = [0] * n
    Y = [0] * n

    # L solve, forward substitution
    for i in range(0, n):
        partial_row_sum = 0
        j = 0
        while j < i:
            partial_row_sum = partial_row_sum + A[i][j] * Y[j]
            j += 1
        Y[i] = B[i] - partial_row_sum

    # U solve, backward substitution
    for i in range(n - 1, -1, -1):
        partial_row_sum = 0
        for j in range(i + 1, n):
            partial_row_sum = partial_row_sum + A[i][j] * X[j]
        X[i] = (Y[i] - partial_row_sum) / A[i][i]

    return X

# LU with partial pivoting and matrix scaling
def LU_decomposition_solve_partial_pivoting_scaling(A, B):
    n = np.shape(A)[0]

    for k in range(0, n):
        max_row_element = 0
        for i in range(0, n):
            if abs(A[k][i]) > max_row_element:
                max_row_element = abs(A[k][i])
        if B[k] > max_row_element:
            max_row_element = abs(B[k])

        if math.isclose(max_row_element, 0.0, abs_tol=1e-9):
            raise ValueError("Unique solution does not exist!")

        for i in range(0, n):
            A[k][i] = A[k][i] / max_row_element

    for k in range(0, n):
        best_pivot_row = k

        for i in range(k+1, n):
            if abs(A[i][k]) > abs(A[best_pivot_row][k]):
                best_pivot_row = i

        if A[best_pivot_row][k] == 0 or math.isclose(A[best_pivot_row][k], 0.0, abs_tol=1e-9):
            raise ValueError("Unique solution does not exist!")

        A[[k, best_pivot_row]] = A[[best_pivot_row, k]]
        B[k], B[best_pivot_row] = B[best_pivot_row], B[k]

        for i in range(k+1, n):
            multiplier = A[i][k] / A[k][k]
            A[i][k+1:n] = A[i][k+1:n] - multiplier * A[k][k+1:n]
            A[i][k] = multiplier

    X = [0] * n
    Y = [0] * n

    # L solve, forward substitution
    for i in range(0, n):
        partial_row_sum = 0
        j = 0
        while j < i:
            partial_row_sum = partial_row_sum + A[i][j] * Y[j]
            j += 1
        Y[i] = B[i] - partial_row_sum

    # U solve, backward substitution
    for i in range(n - 1, -1, -1):
        partial_row_sum = 0
        for j in range(i + 1, n):
            partial_row_sum = partial_row_sum + A[i][j] * X[j]
        X[i] = (Y[i] - partial_row_sum) / A[i][i]

    return X

A = 100 * np.random.rand(500, 500)
A1 = copy.deepcopy(A)
A2 = copy.deepcopy(A)
A3 = copy.deepcopy(A)
A4 = copy.deepcopy(A)
A5 = copy.deepcopy(A)

B = 100 * np.random.rand(500)
B1 = copy.deepcopy(B)
B2 = copy.deepcopy(B)
B3 = copy.deepcopy(B)
B4 = copy.deepcopy(B)
B5 = copy.deepcopy(B)

simple_LU_start_time = time.time()
Xs2 = LU_decomposition_solve(A2, B2)
simple_LU_end_time = time.time()

pivoting_LU_start_time = time.time()
Xs3 = LU_decomposition_solve_partial_pivoting(A3, B3)
pivoting_LU_end_time = time.time()

simple_LU_time = simple_LU_end_time - simple_LU_start_time
pivoting_LU_time = pivoting_LU_end_time - pivoting_LU_start_time

print("Simple LU decomposition: " + str(simple_LU_time) + "s")
print("Partial pivoting LU decomposition: " + str(pivoting_LU_time) + "s")

"""
custom_partial_pivoting_start_time = time.time()
Xs = gauss_jordan_partial_pivoting_solve(A, B)
custom_partial_pivoting_end_time = time.time()

custom_complete_pivoting_start_time = time.time()
Xs1 = gauss_jordan_partial_pivoting_solve(A1, B1)
custom_complete_pivoting_end_time = time.time()

simple_LU_start_time = time.time()
Xs2 = LU_decomposition_solve(A2, B2)
simple_LU_end_time = time.time()

pivoting_LU_start_time = time.time()
Xs3 = LU_decomposition_solve_partial_pivoting(A3, B3)
pivoting_LU_end_time = time.time()

pivoting_scaling_LU_start_time = time.time()
Xs4 = LU_decomposition_solve_partial_pivoting_scaling(A4, B4)
pivoting_scaling_LU_end_time = time.time()

library_start_time = time.time()
Xs5 = np.linalg.solve(A5, B5)
library_end_time = time.time()

custom_partial_pivoting_time = custom_partial_pivoting_end_time - custom_partial_pivoting_start_time
custom_complete_pivoting_time = custom_complete_pivoting_end_time - custom_complete_pivoting_start_time
simple_LU_time = simple_LU_end_time - simple_LU_start_time
pivoting_LU_time = pivoting_LU_end_time - pivoting_LU_start_time
pivoting_scaling_LU_time = pivoting_scaling_LU_end_time - pivoting_scaling_LU_start_time
library_time = library_end_time - library_start_time

print("Partial pivoting Gauss-Jordan: " + str(custom_partial_pivoting_time) + "s")
print("Complete pivoting Gauss-Jordan: " + str(custom_complete_pivoting_time) + "s")
print("Simple LU decomposition: " + str(simple_LU_time) + "s")
print("Partial pivoting LU decomposition: " + str(pivoting_LU_time) + "s")
print("Partial pivoting and matrix scaling LU decomposition: " + str(pivoting_scaling_LU_time) + "s")
print("Library LU decomposition: " + str(library_time) + "s")
"""


import numpy as np
import pylab as pylab

#Evaluate the linear regression
def compute_cost(X, y, theta):
    '''
    Comput cost for linear regression
    '''
    #Number of training samples
    m = y.size

    predictions = X.dot(theta).flatten()
    sqErrors = (predictions - y) ** 2

    J = (1.0 / (2 * m)) * sqErrors.sum()

    return J

def gradient_descent(X, y, theta, alpha, parada):
    '''
    Performs gradient descent to learn theta
    by taking num_items gradient steps with learning
    rate alpha
    '''
    m = y.size

    custo_atual = 9999999999
    i = 0
    J_history = np.zeros(shape=(4000, 1))
    while custo_atual > parada:
        i = i + 1
        predictions = X.dot(theta).flatten()

        errors_x1 = (predictions - y) * X[:, 0]
        errors_x2 = (predictions - y) * X[:, 1]

        theta[0][0] = theta[0][0] - alpha * (1.0 / m) * errors_x1.sum()
        theta[1][0] = theta[1][0] - alpha * (1.0 / m) * errors_x2.sum()

        custo_atual = compute_cost(X, y, theta)
        J_history[i, 0] = custo_atual

    print 'numero de iteracoes', i
    return theta, custo_atual, J_history

#Load the dataset
data = np.genfromtxt('height.csv', delimiter=',')

X = data[:, 0]
y = data[:, 1]

#Some gradient descent settings
alpha = 0.01
parada = 0.001
m = y.size

#Add a column of ones to X (interception data)
it = np.ones(shape=(m, 2))

it[:, 1] = X

#Initialize theta parameters
theta = np.zeros(shape=(2, 1))

pylab.plot( X, y, 'o')
pylab.title('Idade vs Altura')
pylab.xlabel('Idade')
pylab.ylabel('Altura')
pylab.show()

theta, custo, J_history = gradient_descent(it, y, theta, alpha, parada)

print '\nthetas', theta, '\ncusto', custo

def minha_funcao(x, theta):
	return theta[0][0] + theta[1][0]*x

print J_history
pylab.plot(J_history)
pylab.title('Erro reduzindo a cada iteracao')
pylab.xlabel('Iteracao')
pylab.ylabel('Erro')
pylab.show()

pylab.plot(X, minha_funcao(X, theta), X, y, 'o')
pylab.title('Idade vs Altura')
pylab.xlabel('Idade')
pylab.ylabel('Altura')
pylab.show()


import numpy as np
import matplotlib.pyplot as plt
import random
import math
from functools import reduce

SEQUENCE_LENGTH = 1000000

def generate_random_sequence(length):
    return list(map(lambda x: random.random(), range(0, length)))

def generate_uniform_distribution(a, b):
    return list(
        map(
            lambda x: a + (b - a) * x,
            generate_random_sequence(SEQUENCE_LENGTH)
        )
    )

def generate_gauss_distribution(mean, std, n=6):
    return list(
        map(
            lambda x: mean + std * math.sqrt(12 / n) * (sum(generate_random_sequence(n)) - n / 2),
            range(0, SEQUENCE_LENGTH)
        )
    )

def generate_exponential_distribution(lambda_param):
    return list(
        map(
            lambda x: - 1 / lambda_param * math.log(x),
            generate_random_sequence(SEQUENCE_LENGTH)
        )
    )

def generate_gamma_distribution(eta, lambda_param):
    return list(
        map(
            lambda x: -1 / lambda_param * math.log(reduce(lambda y, z: y * z, generate_random_sequence(eta))),
            range(0, SEQUENCE_LENGTH)
        )
    )

#  2 * (b - x) / (b - a)^2
def generate_min_triangle_distribution(a, b):
    return list(
        map(
            lambda x: a + (b - a) * min(generate_random_sequence(2)),
            range(0, SEQUENCE_LENGTH)
        )
    )

#  2 * (x - a) / (b - a)^2
def generate_max_triangle_distribution(a, b):
    return list(
        map(
            lambda x: a + (b - a) * max(generate_random_sequence(2)),
            range(0, SEQUENCE_LENGTH)
        )
    )

def generate_simpson_distribution(a, b):
    return list(
        map(
            lambda x, y: x + y,
            generate_uniform_distribution(a / 2, b / 2),
            generate_uniform_distribution(a / 2, b / 2)
        )
    )

def plot_hist(sequence, num_of_bins):
    weights = np.ones_like(sequence) / float(len(sequence))

    plt.hist(
        sequence,
        bins=np.linspace(min(sequence), max(sequence), num_of_bins + 1),
        weights=weights,
        histtype='bar',
        color='#C40100',
        rwidth=0.9
    )
    plt.show()

def show_distribution(sequence):
    plot_hist(sequence, num_of_bins=20)

    print(
        '\nMean: ', np.mean(sequence),
        '\nVariance: ', np.var(sequence),
        '\nStandard deviation: ', np.std(sequence)
    )

def main():
    print('\nUniform distribution: ')
    show_distribution(generate_uniform_distribution(int(input('A: ')), int(input('B: '))))

    print('\nGauss distribution: ')
    show_distribution(generate_gauss_distribution(int(input('Mean: ')), int(input('Std: ')), int(input('N: '))))

    print('\nExponential distribution: ')
    show_distribution(generate_exponential_distribution(int(input('λ: '))))

    print('\nGamma distribution: ')
    show_distribution(generate_gamma_distribution(int(input('η: ')), int(input('λ: '))))

    print('\nMin-Triangle distribution')
    show_distribution(generate_min_triangle_distribution(int(input('A: ')), int(input('B: '))))

    print('\nMax-Triangle distribution')
    show_distribution(generate_max_triangle_distribution(int(input('A: ')), int(input('B: '))))

    print('\nSimpson distribution')
    show_distribution(generate_simpson_distribution(int(input('A: ')), int(input('B: '))))

if __name__ == '__main__':
    main()


import scipy as sp
import cv2

def toHomogenious(X):
  X = sp.hstack((X, sp.ones((X.shape[0], 1))))
  return X

def toEuclidean(X):
  W = X[:,2]
  X = X/W[:, sp.newaxis]
  return X[:, 0:2]

def points2DTransform(points, M):
    points = toHomogenious(points)
    points = M.dot(points.T).T
    points = toEuclidean(points)
    return points

def identityMatrix():
  M = sp.array([
                [1, 0, 0],
                [0, 1, 0],
                [0, 0, 1]
                ])
  return M

def xShearMatrix(image_height, angle):
  lambda_ = sp.tan(angle/180.0*sp.pi)
  delta_width = -lambda_*image_height
  dx = 0
  if lambda_ < 0:
    dx = delta_width;
  M = sp.array([[1, lambda_, dx], [0, 1, 0], [0, 0 ,1]])
  return M

def xFlipMatrix(image_width):
  M = sp.array([
                [-1, 0, image_width],
                [0, 1, 0],
                [0, 0, 1]
                ])
  return M

def yFlipMatrix(image_height):
  M = sp.array([
                [1, 0, 0],
                [0, -1, image_height],
                [0, 0, 1]
                ])
  return M

def rotationMatrix(angle):
  angle = sp.deg2rad(angle)
  M = sp.array([
                [sp.cos(angle), -sp.sin(angle), 0],
                [sp.sin(angle),  sp.cos(angle), 0],
                [0,              0,             1]
                ])
  return M

def translationMatrix(x, y):
  M = sp.array([
                [1, 0, x],
                [0, 1, y],
                [0, 0, 1]
                ])
  return M

def scaleMatrix(x, y):
  M = sp.array([
                [x, 0, 0],
                [0, y, 0],
                [0, 0, 1]
                ])
  return M

def transformedImageBoundaries(w, h, M):
  """minx, miny, maxx, maxy"""
  pts = sp.array([[0, 0], [w, 0], [0, h], [w, h]])
  pts = points2DTransform(pts, M)
  r = sp.array([pts[:, 0].min(), pts[:, 1].min(), pts[:, 0].max(), pts[:, 1].max()])
  return r

def calcFittingParameters(img, M):
  h = img.shape[0]
  w = img.shape[1]
  B = transformedImageBoundaries(w, h, M)
  T = translationMatrix(-B[0], -B[1])
  M = T.dot(M)
  B = sp.int32(B)
  newSize = (B[2] - B[0], B[3] - B[1])
  return sp.float32(M), newSize


import numpy as np
from math import *
import matplotlib.pyplot as plt
from numpy import genfromtxt
from matplotlib.animation import FuncAnimation
from scipy.integrate import quad

Nx = np.array([50, 100, 500])
Time_int = {0 : 'RK_2', 1 : 'EXPLICIT'}

TimeIdx = -1

err_vec1 = np.zeros((len(Nx)*len(Time_int),))
err_vec2 = np.zeros((len(Nx)*len(Time_int),))

# Definition of analitic solution
def u_exact(x,t):

    t_star = 1.

    if (t <  t_star):

        if (x <= -0.5):
            return 0.
        elif  ((x > -0.5) and (x <=  -0.5 + t)):
            return (x + 0.5)/t
        elif ((x > -0.5 + t) and (x <= 0.5 + 0.5*t)):
            return 1.
        else:
            return 0.
    else:

        if (x <= -0.5):
            return 0.
        elif ((x > -0.5) and (x <= 1.5)):
            return x/t
        else:
            return 0.

    return

h = []
for k in range(len(Time_int)):
    for n in range(len(Nx)):
        data = genfromtxt('Results/Caso_3/solution_{:}_GODUNOV_VAN_LEER_TRUE_MUSCL_LINEAR_{:}.csv'.format(Time_int[k], Nx[n]),delimiter=',')
        u = data[:,0:-1]
        time = data[TimeIdx,-1]

        Analytic_sol = np.zeros((u.shape[1],))
        print(time)

        xAxis = np.linspace(-1,1,u.shape[1] + 1)
        meshPoints = 0.5*(xAxis[0:-1]+xAxis[1:])
        h.append(meshPoints[1]-meshPoints[0])
        for i in range(len(meshPoints)):
            Analytic_sol[i] = (1./h[n]) * quad(lambda x: u_exact(x, time), meshPoints[i]-0.5*h[n],meshPoints[i]+0.5*h[n])[0]

        err_vec2[n+k*len(Nx)] = np.linalg.norm(u[TimeIdx,:] - Analytic_sol, 2)/Nx[n]

h = np.array(h)
# Setting the figure
fig = plt.figure(1)
ax = fig.add_subplot(111)
ax.set_title('Linear approximation', fontsize=20)
ax.set_xlabel('h', fontsize=20)
ax.set_ylabel('Error (Norm-2)', fontsize=20)
ax.loglog(h[0:len(Nx)], err_vec2[0:len(Nx)], 'k-')
ax.loglog(h[len(Nx):2*len(Nx)], err_vec2[len(Nx):2*len(Nx)], 'b-')
normalize1 = err_vec2[0]/(h[0]**1.5)
normalize2 = err_vec2[len(Nx)]/(h[0]**0.5)
ax.loglog(h,normalize1*h**1.5, 'r-')
ax.loglog(h,normalize2*h**0.5, 'g-')
plt.legend(['Error RK_2', 'Error  Expl', 'O(h^1.5)','O(h^0.5)'])

plt.show()


import sys
sys.path.insert(0,'..')
from ddekf import *
import numpy as np
import matplotlib.pyplot as plt
np.random.seed(1)

rate = 0.01
max_pop = 100.0
def func(x,t):
    return x + rate*x*(1 -(1/max_pop)*x)

def jacobian_func(x,t):
    return 1 + rate - (2*rate/max_pop)*x

x_0 = np.zeros((1,1))
x_0[0][0] = max_pop/2
P_0 = np.array([[1]])
Q = np.array([[1]])
R = np.array([[3]])
C = np.array([[1]])

Q_root = np.linalg.cholesky(Q).transpose()
R_root = np.linalg.cholesky(R).transpose()
P_0_sqrt = np.linalg.cholesky(P_0).transpose()
Q_root = np.array([[1]])

start_time = 0
finish_time = 10
sensor_count = 1
state_count = 1
measurement_count = 1000
dt_between_measurements = (finish_time - start_time)/measurement_count

x = x_0
x_noise = x_0
ideal_data = []
process_noise_data = []
measurements = []
times = []
for ii in range(0,measurement_count):
    current_time = ii*dt_between_measurements
    times.append(current_time)
    x = func(x,current_time)
    ideal_data.append( x )
    x_noise = func(x_noise,current_time) + Q_root.transpose().dot(np.random.randn(state_count,1))
    process_noise_data.append(x_noise)
    measurements.append(C.dot(x_noise) + R_root.transpose().dot(np.random.randn(sensor_count,1)))

estimates, covariances = ddekf( func, jacobian_func, dt_between_measurements, start_time, state_count, sensor_count, measurement_count, C, Q_root, R_root, P_0_sqrt, x_0, measurements)

measurements_flat = [ x[0][0] for x in measurements ]
estimates_flat = [ x[0][0] for x in estimates ]
process_noise_data_flat = [ x[0][0] for x in process_noise_data ]

line1, = plt.plot(times,measurements_flat,color='blue',label='Measurement')
line2, = plt.plot(times,process_noise_data_flat,color='red',label='Real data')
line3, = plt.plot(times,estimates_flat[1:],color='orange',label='Estimate')
plt.legend(handles=[line1,line2,line3])
plt.ylabel('Population')
plt.xlabel('Time')
plt.plot()
plt.savefig('logistic2.png')
plt.show()
plt.close()

line1, = plt.plot(times[0:99],measurements_flat[0:99],color='blue',label='Measurement')
line2, = plt.plot(times[0:99],process_noise_data_flat[0:99],color='red',label='Real data')
line3, = plt.plot(times[0:99],estimates_flat[1:100],color='orange',label='Estimate')
plt.legend(handles=[line1,line2,line3])
plt.ylabel('Population')
plt.xlabel('Time')
plt.plot()
plt.savefig('logistic1.png')
plt.show()
plt.close()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 30 11:34:35 2020

@author: andrei-mihai
"""

#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
import numpy as np
import cvxpy as cp
from numpy import linalg as la
from scipy import optimize

### Fie problema de programare patratica:
##  min_x  (1/2)x^TQx + q^Tx
##    s.l. a^Tx = b

## Codul rezolva problema de mai sus prin
## Metoda Gradient Proiectat

## Date: Q, q
Q = np.array([[1,1],[1,1.02]])
q = np.array([[2],[2]])
a = np.array(np.random.rand(2,1))
sol = -la.inv(Q)@q
b = a.T@sol - np.array(np.random.rand(1,1))
f = lambda x: 0.5*x.T@Q@x + q.T@x
Proj = lambda x: x - (np.maximum(0,a.T@x - b)/(la.norm(a)**2))*a

#### Rezolvare folosing CVXPY
n = Q.shape[0]
z = cp.Variable(n)
objective = cp.Minimize(0.5*cp.quad_form(z,Q) + q.T@z)
constraints = [a.T@z<=b]
prob = cp.Problem(objective, constraints)
result = prob.solve(solver='CVXOPT')
sol = z.value

k = 0
eps = 10e-4
x00 = np.array(70*np.random.rand(2,1))
x0 = Proj(x00)

### Constanta Lipschitz a gradientului
Lips = np.max(la.eigvals(Q))
alpha = 1/Lips

all_x_i = [x0[0]]
all_y_i = [x0[1]]
all_f_i = [f(x0)]
x_old = x0
x = Proj(x_old - alpha*(Q@x_old + q))
all_x_i.append(x[0])
all_y_i.append(x[1])
all_f_i.append(f(x))
criteriu_stop = la.norm(x - x_old)

while (criteriu_stop > eps**2):
    x_old = x

    ## Pas gradient
    grad = Q@x+q
    y = x - alpha*grad

    ## Pas proiectie
    x = Proj(y)

    #print(a.T@x-b)
    criteriu_stop = la.norm(x - x_old)
    k = k+1

    all_x_i.append(x[0])
    all_y_i.append(x[1])
    all_f_i.append(f(x))

solMGP = x

print(k)
#### Plot ####

x, y = np.mgrid[-10:10:0.1, -10:10:0.1]
x = x.T
y = y.T

fig = plt.figure(1, figsize=(4, 4))
plt.clf()
plt.axes([0, 0, 1, 1])

fcont = 0.5*Q[0,0]*(x**2) + 0.5*Q[1,1]*(y**2) + Q[1,0]*x*y + q[0]*x+ q[1]*y
contours = plt.contour(fcont, extent=[-10, 10, -10, 10],
                    cmap=plt.cm.gnuplot)

#Etichete pentru multimile izonivel

plt.clabel(contours, inline=1,
                fmt='%1.1f', fontsize=10)

plt.plot(all_x_i, all_y_i, 'b-', linewidth=2)
plt.plot(all_x_i, all_y_i, 'k+')
plt.plot(sol[0], sol[1], 'rx', markersize = 14)

fig.savefig('MGP_quad_cs.pdf')

#print("#######################")
#print("")
#print("Solutie gasita de MG: ",'(',solMGP[0],solMGP[1],')')
#print("Solutie optima: ",'(',sol[0],sol[1],')')
#print("Valoare optima: ",f(sol))
#print("")
#print("#######################")

Tehnici de Optimizare (TO)/Seminar/3/metoda_gradient_proiectat.py

import numpy,from numpy,from scipy,import cvxpy


"""
stats functions for mean, median, mode,
variance, standard deviation, and coefficient of variation
"""
import numpy as np

def mean(data):
    """
    function to calculate mean of data
    """
    size = len(data)
    mean = sum(data)/size
    return mean

def median(data):
    """
    function to calculate median of data
    """
    data.sort()
    size = len(data)

    if len(data) % 2 == 1:
        median = data[int(size/2)]
    else:
        median = (data[int(size/2 - 1)] +
                    data[int(size/2)]) / 2
    return median

def mode(data):
    """
    function to get mode of data, only returns the first it encounters if
    there are more than 1 mode
    """
    mode = max(data, key=data.count)
    return mode

def variance(data, sample=True):
    """
    function to calculate variance of data

    by default, it calculates sample variance, change sample=False if you
    want to calculate population.
    """
    size = len(data)
    distance_squared = list(map(lambda x: (x - sum(data)/size) **
                                2, data))

    if sample is True:
        variance = sum(distance_squared)/(size - 1)
    if sample is False:
        variance = sum(distance_squared)/(size)
    return variance

def stddev(data, sample=True):
    """
    function to calculate standard deviation of data

    by default, it calculates sample standard deviation, change
    sample=False if you want to calculate population.
    """
    size = len(data)
    distance_squared = list(map(lambda x: (x - sum(data)/size) **
                                2, data))

    if sample is True:
        variance = sum(distance_squared)/(size - 1)
        stddev = variance**(1/2)
    if sample is False:
        variance = sum(distance_squared)/(size)
        stddev = variance**(1/2)
    return stddev

def coeffvar(data, sample=True):
    """
    function to calculate the coefficient of variance of data

    by default, it calculates sample coefficient of variance, change
    sample=False if you want to calculate population.
    """
    size = len(data)
    mean = sum(data)/size
    distance_squared = list(map(lambda x: (x - sum(data)/size) **
                                2, data))

    if sample is True:
        variance = sum(distance_squared)/(size - 1)
        stddev = variance**(1/2)
        coeffvar = stddev/mean
    if sample is False:
        variance = sum(distance_squared)/(size)
        stddev = variance**(1/2)
        coeffvar = stddev/mean
    return coeffvar


from typing import List

import numpy as np

def mean_squared_error(y_true: List[float], y_pred: List[float]) -> float:
    assert len(y_true) == len(y_pred)
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)
    return np.mean(np.square(y_true - y_pred))

def f1_score(real_labels: List[int], predicted_labels: List[int]) -> float:
    """
    f1 score: https://en.wikipedia.org/wiki/F1_score
    """
    assert len(real_labels) == len(predicted_labels)

    y_true = np.array(real_labels)
    y_pred = np.array(predicted_labels)
    tp = np.sum(y_true * y_pred)
    if tp == 0:
        return 0.0
    precision = tp / np.sum(y_pred)
    recall = tp / np.sum(y_true)
    return 2 * (precision * recall) / (precision + recall)

def polynomial_features(
        features: List[List[float]], k: int
) -> List[List[float]]:
    poly_features = np.array(features)
    for i in range(2, k + 1):
        new_features = np.apply_along_axis(lambda x : np.power(x, i), 0, np.array(features))
        poly_features = np.column_stack((poly_features, new_features))
    return poly_features

def euclidean_distance(point1: List[float], point2: List[float]) -> float:
    x = np.array(point1)
    y = np.array(point2)
    return np.linalg.norm(x - y)

def inner_product_distance(point1: List[float], point2: List[float]) -> float:
    return np.dot(point1, point2)

def gaussian_kernel_distance(
        point1: List[float], point2: List[float]
) -> float:
    d = - 0.5 * euclidean_distance(point1, point2)
    return -np.exp(d)

class NormalizationScaler:
    def __init__(self):
        pass

    def __call__(self, features: List[List[float]]) -> List[List[float]]:
        """
        normalize the feature vector for each sample . For example,
        if the input features = [[3, 4], [1, -1], [0, 0]],
        the output should be [[0.6, 0.8], [0.707107, -0.707107], [0, 0]]
        """
        def norm(x):
            scale = np.linalg.norm(x)
            return x if scale == 0 else x / scale
        return np.apply_along_axis(lambda x : norm(x), 1, features).tolist()

class MinMaxScaler:
    """
    You should keep some states inside the object.
    You can assume that the parameter of the first __call__
        must be the training set.

    Note:
        1. you may assume the parameters are valid when __call__
            is being called the first time (you can find min and max).

    Example:
        train_features = [[0, 10], [2, 0]]
        test_features = [[20, 1]]

        scaler = MinMaxScaler()
        train_features_scaled = scaler(train_features)
        # now train_features_scaled should be [[0, 1], [1, 0]]

        test_features_sacled = scaler(test_features)
        # now test_features_scaled should be [[10, 0.1]]

        new_scaler = MinMaxScale() # creating a new scaler
        _ = new_scaler([[1, 1], [0, 0]]) # new trainfeatures
        test_features_scaled = new_scaler(test_features)
        # now test_features_scaled should be [[20, 1]]

    """

    def __init__(self):
        self.scale = {}

    def __call__(self, features: List[List[float]]) -> List[List[float]]:
        """
        normalize the feature vector for each sample . For example,
        if the input features = [[2, -1], [-1, 5], [0, 0]],
        the output should be [[1, 0], [0, 1], [0.333333, 0.16667]]
        """
        def min_max(line):
            idx = line[0]
            line = line[1:]
            if idx not in self.scale:
                Min, Max = np.min(line), np.max(line)
                self.scale[idx] = (Min, Max)
            else:
                Min, Max = self.scale[idx]
            return (line - Min) / (Max - Min)
        return np.apply_along_axis(min_max, 0, np.row_stack((np.arange(len(features[0])), features))).tolist()


r"""
Solve Poisson equation on the 4-dimensional (0, 2pi)^4 with periodic bcs

    \nabla^2 u = f,

Use Fourier basis and find u in V^4 such that

    (v, div(grad(u))) = (v, f)    for all v in V^4

where V is the Fourier space span{exp(1jkx)}_{k=-N/2}^{N/2-1} and
V^4 is a 4-dimensional tensorproductspace.

"""
import os
from sympy import symbols, cos, sin
import numpy as np
from shenfun import inner, div, grad, TestFunction, TrialFunction, FunctionSpace, \
    TensorProductSpace, Array, Function, dx, comm

# Use sympy to compute a rhs, given an analytical solution
x, y, z, r = symbols("x,y,z,r", real=True)
ue = cos(4*x) + sin(4*y) + sin(6*z) + cos(6*r)
fe = ue.diff(x, 2) + ue.diff(y, 2) + ue.diff(z, 2) + ue.diff(r, 2)

# Size of discretization
N = (8, 10, 12, 14)

K0 = FunctionSpace(N[0], 'F', dtype='D')
K1 = FunctionSpace(N[1], 'F', dtype='D')
K2 = FunctionSpace(N[2], 'F', dtype='D')
K3 = FunctionSpace(N[3], 'F', dtype='d')
T = TensorProductSpace(comm, (K0, K1, K2, K3))
u = TrialFunction(T)
v = TestFunction(T)

# Get f on quad points
fj = Array(T, buffer=fe)

# Compute right hand side
f_hat = Function(T)
f_hat = inner(v, fj, output_array=f_hat)

# Solve Poisson equation
A = inner(v, div(grad(u)))
f_hat = A.solve(f_hat)

uq = T.backward(f_hat)

uj = Array(T, buffer=ue)
print(np.sqrt(dx((uj-uq)**2)))
assert np.allclose(uj, uq)

if 'pytest' not in os.environ and comm.Get_size() == 1:
    import matplotlib.pyplot as plt
    plt.figure()
    X = T.local_mesh(True) # With broadcasting=True the shape of X is local_shape, even though the number of datapoints are still the same as in 1D
    plt.contourf(X[0][:, :, 0, 0], X[1][:, :, 0, 0], uq[:, :, 0, 0])
    plt.colorbar()

    plt.figure()
    plt.contourf(X[0][:, :, 0, 0], X[1][:, :, 0, 0], uj[:, :, 0, 0])
    plt.colorbar()

    plt.figure()
    plt.contourf(X[0][:, :, 0, 0], X[1][:, :, 0, 0], uq[:, :, 0, 0]-uj[:, :, 0, 0])
    plt.colorbar()
    plt.title('Error')
    plt.show()


import numpy as np

def block_diagonal_matrices(n):
    """
    Generates the block diagonal matrices mentionen in Problem 4.1 using
    the Kronecker product, a composite array made of blocks of the second
    array scaled by the first.
    """
    A1 = np.kron(np.eye(n,dtype=int), np.ones((2, 2)))
    A2 = np.kron(np.eye(n,dtype=int), np.ones((3, 2)))
    return A1, A2

def main():
    A1, A2 = block_diagonal_matrices(3)
    print(A1)
    print(A2)

if __name__ == "__main__":
    main()


# Dimentionality Reduction with PCA

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import plotly.express as px
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler, LabelEncoder
from tabulate import tabulate

# if set, it prints some variables
is_debug = False
# set this to true if you want to plot Principal Components visualization in browser using pyplot
is_plot = False

# label encoder instance
labelencoder = LabelEncoder()

# read the HR data in csv format
df_features = pd.read_csv("./HR_comma_sep_2.csv")
# make a copy of the original data before any manipulation
df_features_original = df_features.copy()

print(tabulate(df_features.head(5), headers='keys'))

# convert sales using ordinal encoder (label encoder)

# Assign integer value for each categorical value (each research_interest) and assign to new column
df_features['sales_int'] = labelencoder.fit_transform(df_features['sales'])
df_features['salary_int'] = labelencoder.fit_transform(df_features['salary'])

df_features = df_features.drop(['sales', 'salary'], axis = 1)
# print(tabulate(df_features.head(10), headers='keys'))

df_x = df_features[["satisfaction_level", "last_evaluation", "number_project", "average_montly_hours", "time_spend_company"\
                      , "Work_accident", "promotion_last_5years", "sales_int", "salary_int"]]
df_y = df_features[["left"]]
# returns numpy array
X = df_x.iloc[:, 0:9].values
y = df_y.iloc[:, 0].values

if is_debug:
    print(df_features.dtypes)
    print(tabulate(df_x.head(10), headers='keys'))
    print(tabulate(df_y.head(10), headers='keys'))

# Step 1:Scale the data set
scaler = StandardScaler()
# train = scaler.fit(X)
X_std = scaler.fit_transform(X)

# Step 2: Instantiate PCA & choose minimum number of components
#         such that it covers 95% variance
pca = PCA(0.95).fit(X_std)

if is_debug:
    print(type(X_std))
    print(X_std)
    print(X_std.shape)
################################
# VISUALIZE cumulative variance
################################
# cumulative explained variance (cumulative sum explained by i + 1)
plt.plot(np.cumsum(pca.explained_variance_ratio_))
# x axis and y axis scale
plt.xlim(0, 8, 1)
plt.ylim([0, 1])
# x axis and y axis labels
plt.xlabel('# components')
plt.ylabel('Cumulative explained variance')
# show the plot
plt.show()
##############################################################################################
# VISUALIZE PCA COMPONENTS AS MATRIX. SET VARIABLE is_plot = TRUE FOR BELOW PLOT TO WORD
##############################################################################################
if is_plot:
    features = ['satisfaction_level', 'last_evaluation', 'number_project', 'average_montly_hours', 'time_spend_company'
        ,'Work_accident', 'promotion_last_5years', 'sales_int', 'salary_int']
    # create a PCA instance without any variance need
    pca = PCA()
    # generate principal components
    components = pca.fit_transform(df_features[features])
    # labels for the plot
    labels = {
        str(i): f"PC {i+1} ({var:.1f}%)"
        for i, var in enumerate(pca.explained_variance_ratio_ * 100)
    }
    # plot scatter matrix
    fig = px.scatter_matrix(
        components,
        labels=labels,
        dimensions=range(2),
        color=df_features["left"]
    )
    # mark diagonal as invisible
    fig.update_traces(diagonal_visible=False)
    fig.show()
##############################
# Observation from plot:
##############################
# -> Only first two principal components are worth to be used as new features for Machine Learning Algorithm
#    as it covers most of the variance as per the plot.


import math
import numpy as np
import matplotlib.pyplot as plt
def f(x):
    return 2 * math.sin(x) + math.cos(-x**2) + 4

def f1(x):
    return 1

def f2(x):
    return x

def f3(x):
    return math.sin(x)

def f4(x):
    return math.cos(x)

def f5(x):
    return x**2 + 1

def f6(x):
    return x**3 - x

def trapz(f, a, b, n):
    h =(b - a) / n
    soma = 0
    for k in range(1, n):
        soma += f(a + k * h)
    return (h / 2) * (f(a) + 2 * soma + f(b))

def best_func(f, a, b, fs, *, intervalos=100):
    n = len(fs)
    A = [[0] * n for _ in range(n)]
    B = []
    for i in range(n):
        for j in range(n):
            if i >= j:
                #integral de de fj * fi em [a, b]
                def f_ji(x):
                    return fs[j](x) * fs[i](x)
                a_ij = trapz(f_ji, a, b, intervalos)
                A[i][j] = a_ij
            else:
                A[i][j] = A[j][i]
        def ff_i(x):
            return f(x) * fs[i](x)
        b_i = trapz(ff_i, a, b, intervalos)
        B.append(b_i)
    coeffs = np.linalg.solve(A, B)
    return coeffs

def g(x):
    soma = 0
    for c_k, f_k in zip(coeffs, fs):
        soma += c_k * f_k(x)
    return soma

if __name__ == '__main__':
    fs = [f1, f2, f3, f4, f5, f6]
    a, b = [-3.447, 3.402]

    coeffs = best_func(f, a, b, fs, intervalos=128)

    for i in range(len(coeffs)):
        print(f'a_{i+1} = {coeffs[i]}')

    t = np.linspace(a, b, 200)
    ft = [f(i) for i in t]
    gt = [g(i) for i in t]

    plt.plot(t, ft, label="grafico de f")
    plt.plot(t, gt, label="grafico de g")
    plt.savefig('best_func.png')


import os.path
import numpy as np
import matplotlib.pyplot as plt
from numpy.linalg import linalg
import scipy
from scipy.linalg import svd

def display(Xrow,fig_name):
    ''' Display a digit by first reshaping it from the row-vector into the image.  '''
    plt.imshow(np.reshape(Xrow,(28,28)))
    plt.savefig(fig_name)
    plt.gray()
    plt.show()

def load_data(digit, num):
    '''
    Loads all of the images into a data-array (for digits 0 through 5).
pyplot
    The training data has 5000 images per digit and the testing data has 200,
    but loading that many images from the disk may take a while.  So, you can
    just use a subset of them, say 200 for training (otherwise it will take a
    long time to complete.

    Note that each image as a 28x28 grayscale image, loaded as an array and
    then reshaped into a single row-vector.

    Use the function display(row-vector) to visualize an image.

    '''
    X = np.zeros((num,784),dtype=np.uint8)   #784=28*28
    print '\nReading digit %d' % digit,
    for i in xrange(num):
        if not i%100: print '.',
        pth = os.path.join('train%d' % digit,'%05d.pgm' % i)
        with open(pth, 'rb') as infile:
            header = infile.readline()
            header2 = infile.readline()
            header3 = infile.readline()
            image = np.fromfile(infile, dtype=np.uint8).reshape(1,784)
        X[i,:] = image
    print '\n'
    return X

X_0=load_data(digit=0,num=5000)
X_1=load_data(digit=1,num=5000)
X_2=load_data(digit=2,num=5000)

X=np.array([X_0,X_1, X_2]).reshape(15000,784)
print(X.shape)

def zeroMean(X):
    X_mean=np.mean(X,axis=0)
    X_std=X-X_mean
    return X_std,X_mean

X_std,X_mean=zeroMean(X)
print(X_mean.shape)
fig_name = 'mean_image.png'
display(X_mean,fig_name)
#display(X_mean)

U, S, Vt = svd(X_std, full_matrices=False)
V = Vt.T
ind = np.argsort(S)[::-1]
U = U[:, ind]
S = S[ind]
V = V[:, ind]
S = np.diag(S)

# plot the first 20 eigenvectors
for i in range(20):
   xrow = V[:,i]
   fig_name = 'eigenvector_'+str(i+1)+'.png'
   display(xrow,fig_name)

# plot the eigenvalues-dimensions-100
x=np.arange(100)
y=S[x,x]
plt.figure(figsize=(8,5))
plt.plot(x,y,label="eigvals",'o')
plt.xlabel('dimension')
plt.ylabel('egivalues')
plt.savefig('eigenvalues_dimensions.png')
plt.show()

import numpy,from numpy,import scipy,from scipy


#!/usr/bin/env python
# coding: utf-8

# # 📝 Exercise M4.04
#
# In the previous notebook, we saw the effect of applying some regularization
# on the coefficient of a linear model.
#
# In this exercise, we will study the advantage of using some regularization
# when dealing with correlated features.
#
# We will first create a regression dataset. This dataset will contain 2,000
# samples and 5 features from which only 2 features will be informative.

# In[1]:

from sklearn.datasets import make_regression

data, target, coef = make_regression(
    n_samples=2_000, n_features=5, n_informative=2, shuffle=False,
    coef=True, random_state=0, noise=30,
)

# When creating the dataset, `make_regression` returns the true coefficient
# used to generate the dataset. Let's plot this information.

# In[2]:

import pandas as pd

feature_names = [f"Features {i}" for i in range(data.shape[1])]
coef = pd.Series(coef, index=feature_names)
coef.plot.barh()
coef

# Create a `LinearRegression` regressor and fit on the entire dataset and
# check the value of the coefficients. Are the coefficients of the linear
# regressor close to the coefficients used to generate the dataset?

# In[3]:

from sklearn.linear_model import LinearRegression

linear_regression = LinearRegression()
linear_regression.fit(data, target)
linear_regression.coef_

# In[4]:

feature_names = [f"Features {i}" for i in range(data.shape[1])]
coef = pd.Series(linear_regression.coef_, index=feature_names)
_ = coef.plot.barh()

# We see that the coefficients are close to the coefficients used to generate
# the dataset. The dispersion is indeed cause by the noise injected during the
# dataset generation.

# Now, create a new dataset that will be the same as `data` with 4 additional
# columns that will repeat twice features 0 and 1. This procedure will create
# perfectly correlated features.

# In[20]:

# Write your code here.
# data['Features 5']=data['Features 0']
# data['Features 6']=data['Features 0']
# data['Features 7']=data['Features 1']
# data['Features 8']=data['Features 1']
new_data = [[dat[0], dat[1], dat[2], dat[3], dat[4], dat[0], dat[0], dat[1], dat[1]] for dat in data]
import numpy as np
new_data = np.array(new_data)
new_data.shape

# Fit again the linear regressor on this new dataset and check the
# coefficients. What do you observe?

# In[23]:

# Write your code here.
from sklearn.linear_model import LinearRegression

linear_regression = LinearRegression()
linear_regression.fit(new_data, target)
print(f' coefficients: {linear_regression.coef_}')
feature_names = [f"Features {i}" for i in range(new_data.shape[1])]
coef = pd.Series(linear_regression.coef_, index=feature_names)
_ = coef.plot.barh()

# Create a ridge regressor and fit on the same dataset. Check the coefficients.
# What do you observe?

# In[33]:

# Write your code here.
from sklearn.linear_model import RidgeCV
model = RidgeCV( alphas=[0.001, 0.1, 1, 10, 1000], store_cv_values=True )
model.fit(new_data, target)
print(model.alpha_)
print(f' coefficients: {model.coef_}')
feature_names = [f"Features {i}" for i in range(new_data.shape[1])]
coef = pd.Series(model.coef_, index=feature_names)
_ = coef.plot.barh()

# Can you find the relationship between the ridge coefficients and the original
# coefficients?

# In[ ]:

# Write your code here.


import os
import scipy.io
import numpy as np
from tqdm import tqdm
import matplotlib.pyplot as plt

data_path = os.path.join("Data-HWK3-2020","Problem-4","SymptomDisease.mat")
data=scipy.io.loadmat(data_path)
#print(data.keys())

W = data['W']
b = data['b']
p = data['p']
s = data['s'] # symptoms

def check_data():
    print(W.shape,b.shape,p.shape,s.shape)
    # (200, 50) (200, 1) (50, 1) (200, 1)

def sigma(x):
    return 1/(1+np.exp(-x))

def p_s1_under_d(W,b,d):
    return sigma(np.dot(W,d)+b.reshape(200))

def joint(d,W,b,s,p):
    p_s1_under_d_x = p_s1_under_d(W,b,d)
    joint = np.prod((p.T**d)*(1-p.T)**(1-d))* \
            np.prod((p_s1_under_d_x**s.T)*(1-p_s1_under_d_x)**(1-s.T))
    return joint

def test_func():
    d = np.zeros(50)
    p_s1_under_d_x = p_s1_under_d(W,b,d)
    print(p_s1_under_d_x)
    print(p_s1_under_d_x.shape) # (200,)
    joint_x = joint(d,W,b,s,p)
    print(joint_x)
    print(joint_x.shape)

def main():

    total_N = 2500
    d = np.zeros(50)
    prob_check_old = p
    d_sampled = np.zeros((total_N, 50))
    a = [0] * total_N
    m = 0

    results_record = []

    for n in tqdm(range(total_N)):
        Prob_check_current = np.zeros(len(d))
        for i in range(len(d)):
            d[i]=0
            d_0= joint(d,W,b,s,p)
            d[i]=1
            d_1= joint(d,W,b,s,p)
            prob1 = d_1/(d_0+d_1)
            Prob_check_current[i]=prob1
            d[i]= np.random.binomial(1,prob1)

        if n >= 2000: # burn-in
            if n % 20 == 0: # sub-sampling
                d_sampled[m] = d
                a[m] = np.sum(np.abs(prob_check_old - Prob_check_current))
                prob_check_old = Prob_check_current
                m = m+1
                #print(Prob_check_current)
                results_record.append(Prob_check_current)

    #plt.plot(a)
    #plt.show()
    with np.printoptions(precision=3):
        print(Prob_check_current)
        print(np.mean(np.array(results_record), axis=0))

if __name__ == '__main__':
    main()

#print(p)


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.10.2
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %% colab={"base_uri": "https://localhost:8080/"} id="PUNO2E6SeURH" outputId="f6cf9813-f481-413c-f991-75022976089f"
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
from tensorflow import keras

Xs = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0], dtype=float)
ys = np.array([1.0, 1.5, 2.0, 2.5, 3.0, 3.5], dtype=float)

tf.keras.backend.clear_session()

model = tf.keras.Sequential()
model.add(keras.layers.Dense(units=1, input_shape=[1]))

# model.compile(optimizer='sgd', loss='mse')
# model.compile(optimizer='RMSprop', loess='mse' )
model.compile(optimizer='adam', loss='mse')

model.summary()

print('ok')

# %% [markdown]
# # Header 1

# %% [markdown]
# ## Header 2

# %% colab={"base_uri": "https://localhost:8080/"} id="5r5mt0chNAv1" outputId="45a03db5-3fef-4e0a-8181-379121662fdf"
history = model.fit(Xs, ys, epochs=1000, verbose=0)

# %% colab={"base_uri": "https://localhost:8080/", "height": 282} id="pbecd6AxKQae" outputId="0dcde4a5-2043-4eb7-869a-632e99fecf79"
# Plot the loss
plt.plot(history.history['loss'], label='loss')

# %% colab={"base_uri": "https://localhost:8080/"} id="K-zVJG0_Lztr" outputId="d59b56a1-1262-4936-ee91-66b781b2a22e"
# Get the slope and bias of the line
w, b = model.layers[0].get_weights()
print('w : ' + str(w))
print('b : ' + str(b))

# %% colab={"base_uri": "https://localhost:8080/"} id="SREFOyWALjWD" outputId="8f8683f9-c106-4b64-e377-9ffce639d514"
print(model.predict([5.0]))

import math
print("This is the end of the file")

# %% [markdown]
# #### Conclusion

1. Introduction to TensorFlow/AZ/Basics/.ipynb_checkpoints/01_house_price_01-checkpoint.py


from typing import Union

from scipy.stats import rv_discrete, nbinom

from probability.distributions.mixins.attributes import PFloatDMixin
from probability.distributions.mixins.calculable_mixin import CalculableMixin
from probability.distributions.mixins.rv_discrete_1d_mixin import \
    RVDiscrete1dMixin
from probability.utils import num_format, is_scalar

class NegativeBinomial(
    RVDiscrete1dMixin,
    PFloatDMixin,
    CalculableMixin,
    object
):
    """
    The negative binomial distribution is a discrete probability distribution
    that models the number of failures k in a sequence of independent and
    identically distributed Bernoulli trials before a specified (non-random)
    number of successes (denoted r) occurs.
    For example, we can define rolling a 6 on a die as a success, and rolling
    any other number as a failure, and ask how many failed rolls will occur
    before we see the third success (r = 3). In such a case, the probability
    distribution of the number of non-6s that appear will be a negative binomial
    distribution.

    https://en.wikipedia.org/wiki/Negative_binomial_distribution
    """
    def __init__(self, r: float, p: float):
        """
        Create a new NegativeBinomial distribution.

        :param r: Number of successes we want.
        :param p: Probability of a failure.
        """
        self._r: float = r
        self._p: float = p
        self._reset_distribution()

    def _reset_distribution(self):
        """
        https://stackoverflow.com/questions/40846992/
        alternative-parametrization-of-the-negative-binomial-in-scipy
        #comment109394209_47406400
        """
        self._distribution: rv_discrete = nbinom(
            self._r, 1 - self._p
        )

    @property
    def lower_bound(self) -> int:
        return 1

    @property
    def upper_bound(self) -> int:
        return int(self.ppf().at(0.99))

    @property
    def r(self) -> float:
        return self._r

    @r.setter
    def r(self, value: float):
        self._r = value
        self._reset_distribution()

    def mode(self) -> int:

        if self._r <= 1:
            return 0
        else:
            return int(self._p * (self._r - 1) / (1 - self._p))

    def __str__(self):

        return (
            f'NegativeBinomial('
            f'r={num_format(self._r, 3)}, '
            f'p={num_format(self._p, 3)})'
        )

    def __repr__(self):

        return f'NegativeBinomial(r={self._r}, p={self._p})'

    def __eq__(self, other: Union['NegativeBinomial', int, float]):

        if is_scalar(other):
            return self.pmf().at(other)
        else:
            return (
                self._r == other._r and
                abs(self._p - other._p) < 1e-10
            )

    def __ne__(
            self, other: Union['NegativeBinomial', int, float]
    ) -> Union[bool, float]:

        if is_scalar(other):
            return 1 - self.pmf().at(other)
        else:
            return not self.__eq__(other)


import numpy as np
import scipy.integrate as integrate

from sdia_python.lab2.box_window import BoxWindow

class BallWindow:
    """BoxWindow class representing a virtual n-dimensional bounded Box
    """

    def __init__(self, center, radius):
        """Initialization of Box's parameters

        Args:
            args (array): this argument represents the bounds of the box
        """

        self.rad = radius
        self.cent = center

    def __str__(self):
        """Description of the Ball

        Returns:
            str: description of the Ball
        """
        return f"BallWindow: {self.cent}, {self.rad}"

    def __contains__(self, args):
        """This method tests if an element (args) is inside the ball

        Args:
            args (array): the element to test

        Returns:
            bool: True if the element is inside the ball , False if not
        """

        return np.all(abs(args - self.cent) < self.rad)

    def dimension(self):
        """This method gives the dimension of the ball
        """
        return len(self.cent)

    def volume(self):
        """This method calculates the volume of the Ball using the formula explained in the following link :https://fr.wikipedia.org/wiki/N-sph%C3%A8re
        """
        n = self.dimension()
        R = self.rad
        if n % 2 == 0:  # formula in case dimension is even
            return (((np.pi) ** (n / 2)) * R ** n) / np.math.factorial(n / 2)
        else:  # formula in case dimension is odd
            odds = [i for i in range(1, n + 1, 2)]
            product = np.product(odds)
            return 2 ** ((n + 1) / 2) * np.pi ** ((n - 1) / 2) * R ** n / product

    def indicator_function(self, args):
        """This method is similar to the method  __contains__  described above

        Args:
            args (array): the element to test

        Returns:
            bool: True if the element is inside the ball , False if not
        """

        return self.__contains__(args)


# coding: utf-8

# # Function haarmatrix

# ## Synopse
#
# Compute the Kernel matrix for the Haar Transform.
# The Haar transform is a particular case of discrete wavelet transform.
#
# - **A = haarmatrix(N)**
#
#   - **A**:output: haar matrix NxN.
#   - **N**:input: matrix size (NxN).

# In[2]:

import numpy as np

def haarmatrix(N):
    n = np.floor(np.log(N)/np.log(2))

    if 2**n != N: raise Exception('error: size '+str(N)+' is not multiple of power of 2')

    z = np.resize(1.*np.arange(N)/N, (len(1.*np.arange(N)), len(1.*np.arange(N)/N)))
    k = np.transpose(np.resize(1.*np.arange(N), (len(1.*np.arange(N)/N), len(1.*np.arange(N)))))

    p  = np.floor(np.log(np.maximum(1,k))/np.log(2))
    q  = k - (2**p) + 1
    z1 = (q-1)   / (2**p)
    z2 = (q-0.5) / (2**p)
    z3 = q       / (2**p)
    A  = (1/np.sqrt(N)) * ((( 2**(p/2.)) * ((z >= z1) & (z < z2))) + ((-2**(p/2.)) * ((z >= z2) & (z < z3))))
    A[0,:] = 1/np.sqrt(N)
    return A

# ## Examples

# In[1]:

testing = (__name__ == "__main__")

if testing:
    get_ipython().system(' jupyter nbconvert --to python haarmatrix.ipynb')
    import numpy as np
    import sys,os
    import matplotlib.image as mpimg
    ea979path = os.path.abspath('../../')
    if ea979path not in sys.path:
        sys.path.append(ea979path)
    import ea979.src as ia

# ### Example 1

# In[2]:

if testing:
    np.set_printoptions(suppress=True, precision=4)
    A = ia.haarmatrix(4)
    print('Visualiza matriz haar 4x4:\n',A)
    B = np.dot(A,np.transpose(A))
    print("\nVisualiza propriedade A*A'= I:\n", B)

# ### Example 2

# In[3]:

if testing:

    A = ia.haarmatrix(128)
    ia.adshow(ia.normalize(A),'Haar matrix 128x128')

# ### Example 3

# In[4]:

if testing:
    f = mpimg.imread('../data/cameraman.tif')

    A = ia.haarmatrix(f.shape[0])
    B = ia.haarmatrix(f.shape[1])
    F = np.dot(np.dot(A, f), np.transpose(B))

    nb = ia.nbshow(2)
    nb.nbshow(f,'Imagem original')
    nb.nbshow(ia.normalize(np.log(abs(F)+1)),'Haar transform')
    nb.nbshow()

# In[ ]:

# In[ ]:


'''
    Based on Bruce E. Hansen 2009: Lecture Notes on Nonparametrics,
    http://www.ssc.wisc.edu/~bhansen/718/NonParametrics1.pdf.
'''
from __future__ import unicode_literals
from __future__ import print_function

import numpy as np
import matplotlib.pyplot as plt

class KernelDensityDerivative(object):

    def __init__(self, data, deriv_order):

        if deriv_order == 0:
            self.kernel = lambda u: np.exp(-u**2/2)
        elif deriv_order == 2:
            self.kernel = lambda u: (u**2-1)*np.exp(-u**2/2)
        else:
            raise ValueError('Not implemented for derivative of order {}'.format(deriv_order))
        self.deriv_order = deriv_order
        self.h = silverman_bandwidth(data, deriv_order)
        self.datah = data/self.h

    def evaluate(self, x):
        xh = np.array(x).reshape(-1)/self.h
        res = np.zeros(len(xh))
        if len(xh) > len(self.datah):  # loop over data
            for data_ in self.datah:
                res += self.kernel(data_-xh)
        else:  # loop over x
            for i, x_ in enumerate(xh):
                res[i] = np.sum(self.kernel(self.datah-x_))
        return res*1./(np.sqrt(2*np.pi)*self.h**(1+self.deriv_order)*len(self.datah))

    def score_samples(self, x):
        return self.evaluate(x)

    def plot(self, ax=None):
        x = self.h*np.linspace(np.min(self.datah)-5, np.max(self.datah)+5, 200)
        y = self.evaluate(x)
        if ax is None:
            fig, ax = plt.subplots()
        ax.plot(x, y)

def silverman_bandwidth(data, deriv_order=0):
    sigmahat = np.std(data, ddof=1)
    return sigmahat*bandwidth_factor(data.shape[0], deriv_order)

def bandwidth_factor(nbr_data_pts, deriv_order=0):
    '''
        Scale factor for one-dimensional plug-in bandwidth selection.
    '''
    if deriv_order == 0:
        return (3.0*nbr_data_pts/4)**(-1.0/5)

    if deriv_order == 2:
        return (7.0*nbr_data_pts/4)**(-1.0/9)

    raise ValueError('Not implemented for derivative of order {}'.format(deriv_order))

if __name__ == '__main__':
    print("bandwidth_factor(1, 0) = {}".format(bandwidth_factor(1, 0)))
    print("bandwidth_factor(1, 2) = {}".format(bandwidth_factor(1, 2)))

    data = np.random.randn(100)
    fig, axs = plt.subplots(1, 2)
    KernelDensityDerivative(data, 0).plot(axs[0])
    KernelDensityDerivative(data, 2).plot(axs[1])
    KernelDensityDerivative(data, 0).plot()
    plt.show()


import numpy as np
from sys import exit

## Metoda Gauss cu pivotare totală

def compute_upper_triangular(A, b, partial_pivot=False, full_pivot=False):
    """Aduce un sistem la forma superior triunghiulară
    folosind Gauss cu/fără pivotare totală/parțială.
    """
    if partial_pivot and full_pivot:
        raise Exception("Se poate folosi doar un tip de pivotare")

    N = A.shape[0]

    # Formez matricea sistemului
    M = np.concatenate((A, b), axis=1)

    indices = np.arange(0, N)

    for k in range(N - 1):
        if partial_pivot:
            # Găsesc indicele elementului de valoare maximă
            index = np.argmax(M[k:, k])

            # Pivotez
            M[[k, index]] = M[[index, k]]
        elif full_pivot:
            submatrix = M[k:, k:N - 1]
            index = np.unravel_index(np.argmax(np.abs(submatrix)), submatrix.shape)

            # Obțin indicii în matricea mare
            index = (index[0] + k, index[1] + k)

            M[[k, index[0]]] = M[[index[0], k]]
            M[:, [k, index[1]]] = M[:, [index[1], k]]

            indices[[k, index[1]]] = indices[[index[1], k]]

        # Selectez coloana
        ratios = M[k + 1:, k]

        # Determin raportul pentru fiecare rând
        ratios = ratios / M[k, k]

        row = M[k, :]

        # Înmulțesc fiecare raport cu primul rând
        difference = np.outer(ratios, row)

        # Actualizez matricea
        M[k + 1:, :] -= difference

    return M, indices

def solve_upper_triangular(U, C, print_steps=False):
    """Rezolvă un sistem în formă superior triunghiulară,
    folosind substituția descendentă.
    """
    N = U.shape[0]

    x = np.zeros(N)

    # Merg de la ultima linie către prima,
    # și rezolv pe rând ecuațiile prin substituție
    for i in range(N - 1, -1, -1):
        coefs = U[i, i + 1:]
        values = x[i + 1:]

        x[i] = (C[i] - coefs @ values) / U[i, i]

        if print_steps:
            print("Pasul", N - i, ":",
                C[i], "-", coefs, "@", values, ")",
                "/", U[i, i])

    return x

def solve_system(A, b, pivot=None):
    "Rezolvă un sistem folosind metoda Gauss."

    determinant = np.linalg.det(A)

    if np.abs(determinant) < 1e-14:
        # Sistemul nu are soluție
        return None

    partial_pivot = False
    full_pivot = False

    if pivot == "full":
        full_pivot = True
    elif pivot == "partial":
        partial_pivot = True

    M, indices = compute_upper_triangular(A, b, partial_pivot=partial_pivot, full_pivot=full_pivot)

    N = A.shape[0]
    U = M[:,:N]
    C = M[:,N]

    x = solve_upper_triangular(U, C)
    x = x[indices]

    return x

def compute_determinant(A):
    N = A.shape[0]
    b = np.zeros((N, 1))
    M, _ = compute_upper_triangular(A, b)

    det = 1
    for i in range(N):
        det *= M[i][i]
    return det

if __name__ == '__main__':
    ## Datele de intrare
    # Matricea coeficienților
    A = np.array([
        [2, -1, -2],
        [4, 2, 0],
        [0, -2, -1]
    ], dtype=float)

    # Matricea termenilor liberi
    b = np.array([
        [-1],
        [6],
        [-3]
    ], dtype=float)

    A = np.array([
        [1, 2, 3],
        [4, 5, 6],
        [7, 8, 10]
    ], dtype=float)
    b = np.array([
        [4],
        [4],
        [7]
    ], dtype=float)

    det = compute_determinant(A)
    print("Determinant:", det)

    x = solve_system(A, b, pivot="full")
    print("Soluția:", x)

    print()
    for k in range(1, 20):
        C = 10.0 ** (-k)
        print("C:", C)
        A = np.array([
            [1, C],
            [1, 1],
        ])
        b = np.array([
            [C],
            [2],
        ])
        x = solve_system(A, b, pivot="full")
        print("Soluția:", x)


from numpy import sum as npsum
from numpy import zeros, diag, eye, sqrt, tile, diagflat
from numpy.linalg import cholesky, solve
from numpy.random import rand

from NormalScenarios import NormalScenarios
from DimRedScenariosNormal import DimRedScenariosNormal

def UnifInsideEllScenarios(mu,sig2,j_,k_=0,method='Riccati',d=None):
    # This function generates j_ Monte carlo scenarios of an elliptical random variable
    # with location vector mu, dispersion matrix sigma2 and radial component
    # with quantile function q_R
    #  INPUTS
    #   mu      : [vector]            (n_ x 1) location vector
    #   sigma2  : [matrix]            (n_ x n_) dispersion matrix
    #   j_      : [scalar]            number of simulations
    #   k_      : [scalar]           (optional) number of factors for dimension
    #                                 reduction (we advise to take k_<<n_)
    # method    : [string]            Riccati (default), CPCA, PCA, LDL-Cholesky, Gram-Schmidt
    # d         : [matrix]           (k_ x n_) full rank constraints matrix for CPCA
    #  OPS
    #   X       : [matrix]           (n_ x j_) matrix of elliptical simulations
    #   R       : [vector]           (1 x j_)  vector of radial component scenarios
    #   Y       : [matrix]           (n_ x j_) matrix of uniform component scenarios

    # For details on the exercise, see here .
    ## Code

    n_ = len(mu) # number of variables

    # Step 1. Radial Scenarios
    R=(rand(1,j_))**(1/n_)

    # Step 2. Correlation
    rho2=diag(diag(sig2)**(-1/2))@sig2@diag(diag(sig2)**(-1/2))

    # Step 3. Normal scenarios
    if k_==0:
       N=NormalScenarios(zeros((n_,1)),rho2,j_,'Riccati',d)[0]
    else:
       [N,beta]=DimRedScenariosNormal(zeros((n_,1)),rho2,k_,j_,'Riccati',d)

    # Step 4. Inverse
    if k_!=0:
       delta2=diag(eye((n_))-beta@beta.T)
       omega2=diag(1/delta2)
       rho2_inv=omega2-omega2@beta/(beta.T@omega2@beta+eye((k_)))@beta.T@omega2
    else:
        rho2_inv=solve(rho2, eye(rho2.shape[0]))

    #Step 5. Cholesky
    rho_inv=cholesky(rho2_inv).T

    #Step 6. Normalizer
    M=sqrt(npsum((rho_inv@N)**2,0))

    #Step 7. Output
    Y=rho_inv@N@diagflat(1/M)
    X=tile(mu, (1,j_))+diagflat(sqrt(diag(sig2)))@N@diagflat(1/M)@diagflat(R)
    return X,R,Y


import numpy as np

################################
# define the class 'simulator' #
################################
'''
this class is used to define the data generating process (Y = Xb + e)

Check this before you run the code:
Plz check if you have 'sci-kit learn', 'numpy', 'matplotlib' and 'tqdm' installed. If not,
    1. run 'pip install scikit-learn numpy matplotlib tqdm' if you use pure Python3
    2. run 'conda install scikit-learn numpy matplotlib tqdm' if you use Anaconda3

Inputs:
    1. sample_size : the total sample size we generate for cv-lars-lasso, cv-cd and solar
    2. n_dim       : the number of total variables in X
    3. n_info      : the number of informative variables

Outputs:
    1. X, Y : the inputs and output of regression
'''

class simul:

    def __init__(self, sample_size, n_dim, n_info):
        # for convinience, we define the common variable (variables we need to use for each of the following functions) in the class as follows (the common variable is defined as self.xxxx)

        self.sample_size = sample_size
        self.n_dim = n_dim
        self.n_info = n_info

    # data-generating process
    def data_gen(self):

        # 1. generating the covariance matrix for X,
        # we add a matrix full of 1/2 with an identity matrix multiplied with 1/2
        a = np.ones((self.n_dim, self.n_dim)) * 0.5
        A = np.eye(self.n_dim) * 0.5

        cov_x = a + A

        # 2. generating the mean of each column in X (which is 0)
        mean_x = np.zeros(self.n_dim)

        # 3. generating X as a multivariate Gaussian
        X = np.random.multivariate_normal(mean_x, cov_x, self.sample_size)

        # 4. generate regression coefficients in DGP as an increasing sequence (2,3,4,5,6 in our paper)
        beta_info = np.arange(2, self.n_info + 2)

        # in DGP, generate regression coefficients of redundant variables as 0
        # concatenate the regression coefficients of informative variables and redundant variables
        beta = np.concatenate(
            (beta_info, np.zeros(self.n_dim - self. n_info)), axis=0)

        # 5. generate the Gaussian random noise
        noise = np.random.normal(0, 1, self.sample_size)

        # transform Gaussian random noise into a column
        # transform regression coefficients in DGP into a row (based on the requirement of np.inner )
        noise.shape = (self.sample_size, 1)
        beta.shape = (1, self.n_dim)

        # 6. generate Y by adding random noise with the inner product of X and beta
        Y = np.inner(X, beta) + noise

        return X, Y

##################################
# test if this module works fine #
##################################

'''
this part is set up to test the functionability of the class above;
you can run all the codes in this file to test if the class works;
when you call the class from this file, the codes (function or class) after " if __name__ == '__main__': " will be ingored
'''

if __name__ == '__main__':

    sample_size = 200
    n_dim = 100
    n_info = 5

    trial = simul(sample_size, n_dim, n_info)
    X, Y = trial.data_gen()

    print(X)
    print(Y)

    # to verfiy if we generate X and Y correctly, we compute and output the OLS coefficients of Y onto all informative variables
    from sklearn.linear_model import LinearRegression

    reg = LinearRegression(); reg.fit(X[:, 0:5], Y)
    print(reg.coef_)