import numpy as np
import linalg_toolbox as linalg_tools

def get_projection_matrix(data, classes, ndim, normalize=False):
    """
    Performs a LDA over the data and return the projection matrix.
    data: the data to be projected. Datapoints are given by rows while features
          are given by columns.
    ndim: the dimension of the lower dimentional subspace the data should be
          projected on.
    normalize: set to True if the data should be centered and whitened before
               computing the projection matrix
    """
    n_datapoints, n_features = data.shape
    data = data.T # to retrieve the usual notation: datapoints are arranged by columns.
    if normalize:
        data = linalg_tools.normalize_data(data) # center and whiten the data

    assert ndim <= n_features, 'We can\'t project onto a higher dimentional space.'
    assert len(classes) == n_datapoints, 'Some points have no associated class, or ' \
                                         'too many classes.'

    # Computing the mean of all classes. That is to say, the mean image of
    # every class.
    mu_c = compute_mu_classes(data, classes)

    # Computing the covariance matrix 'between classes': how much are centers
    # of different classes apart from each other
    Sb = compute_Sb(data, classes, mu_c)

    # Computing the covariance matrix 'within classes': how much points within
    # clusters are separated from each other
    Sw = compute_Sw(data, classes, mu_c)

    # This is the matrix we need to compute the eigenvectors from
    fisher_matrix = np.linalg.inv(Sw) * Sb

    # Getting the eigen vectors (they correspond to the projection matrix we are
    # looking for).
    projection_matrix = linalg_tools.get_largest_eigen_vectors(fisher_matrix, ndim)
    return projection_matrix.T

def project_using_projection_matrix(data_to_transform, projection_matrix):
    """
    Projects given data into lower dimentional subspace using the provided
    projection_matrix.
    WARNING: the data is supposed to be in standard data science notation:
    rows: datapoints/individuals
    columns: variables/features
    """
    projected_data = projection_matrix * data_to_transform.T;
    return projected_data.T # the .T is used to get back to standard data-science
                            # layout: datapoints are arranged by rows

def project(data, classes, ndim, normalize):
    """
    Projects an array of data onto a lower dimentional subspace.
    The procedure here is the 'naive' one (as seen in the lectures).
    data: the data to be projected. Datapoints are given by rows while features
          are given by columns.
    ndim: the dimension of the lower dimentional subspace the data should be
          projected on.
    normalize: set to True if the data should be centered and whitened before
               computing the projection matrix
    """
    projection_matrix = get_projection_matrix(data, classes, ndim, normalize)
    return project_using_projection_matrix(data, projection_matrix)

def get_subdata(data, classes, class_to_select):
    """
    Given a dataset and the associate class for each point, returns
    the points whose class is exactly class_to_select).
    """
    subdata = data[:,classes == class_to_select]
    return subdata

def compute_mu_classes(data, classes):
    """
    Given a dataset and the associate class for each point, returns
    the mean of each of the classes as matrix mu_c where:
    each column is associated to a class
    each row is associated to a coordinate
    """
    n_features,_ = data.shape
    unique_classes = np.unique(classes)
    mu_c = np.matrix(np.zeros([n_features,len(unique_classes)]))
    for class_num,class_name in enumerate(unique_classes):
        subdata = get_subdata(data, classes, class_name)
        mu_subdata = subdata.mean(axis=1)
        mu_c[:,class_num] = mu_subdata

    return mu_c

def compute_Sb(data, classes, mu_c):
    """
    Computes the variance 'between' classes. That is to say, how well are
    each class separated from each other.
    data: the dataset
    mu_c: the mean value of each cluster. One can use compute_mu_classes to get it.
    """
    mu = data.mean(axis=1) # mean of the whole dataset

    n_features,_ = data.shape
    unique_classes = np.unique(classes)
    Sb = np.matrix(np.zeros([n_features, n_features]))
    for class_num,class_name in enumerate(unique_classes):
        subdata = get_subdata(data, classes, class_name)
        _,size_subdata = subdata.shape
        mu_subdata = mu_c[:,class_num]
        diff_mu = mu_subdata - mu
        Sb += size_subdata * (diff_mu * diff_mu.T)

    return Sb

def compute_Sw(data, classes, mu_c):
    """
    Computes the variance 'within' classes. That is to say, how spread are each
    class by themselves (how points are separated from each other within their own class).
    data: the dataset
    mu_c: the mean value of each cluster. One can use compute_mu_classes to get it.
    """
    unique_classes = np.unique(classes)
    n_features,_ = data.shape
    Sw = np.matrix(np.zeros([n_features, n_features]))
    for class_num,class_name in enumerate(unique_classes):
        subdata = get_subdata(data, classes, class_name)
        mu_subdata = mu_c[:,class_num]
        _,size_subdata = subdata.shape
        Sw += np.cov(subdata)

    return Sw


#!/usr/bin/env python3

import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt
import sys
from mpl_toolkits.mplot3d import Axes3D

# Define hyperparameters
TRAINING_EPOCH = 400
LEARNING_RATE = 1
REAL_PARAMS = [1.2, 2.5]
INIT_PARAMS = [[5, 4], [5, 1], [2, 4.5]][2]

''' Select different target function '''
def targetFunction(option, x):
	if option == '1':
		# Use a simple linear function with two parameters
		return lambda w, b: w * x + b
	elif option == '2':
		# Use Tensorflow as a calibrating tool for empirical formula like following
		return lambda w, b: w * x**3 + b * x**2
	elif option == '3':
		# Use the most simplest two parameters and two layers Neural Net, and their local and global minimum
		return lambda w, b: np.sin(b * np.cos(w * x))
	return None

''' Select different training function '''
def trainFunction(option, x):
	if option == '1':
		# Use a simple linear function with two parameters
		return lambda w, b: w * x + b
	elif option == '2':
		# Use Tensorflow as a calibrating tool for empirical formula like following
		return lambda w, b: w * x**3 + b * x**2
	elif option == '3':
		# Use the most simplest two parameters and two layers Neural Net, and their local and global minimum
		return lambda w, b: tf.sin(b * tf.cos(w * x))
	return None

def main(args1, args2):
	''' Create data '''
	# Training data
	x = np.linspace(-1, 1, 200, dtype=np.float32)
	noise = np.random.randn(200) / 10
	y_fun = targetFunction(args1, x)
	y = y_fun(*REAL_PARAMS) + noise

	''' Create TensorFlow model '''
	# Set the learning rate
	global LEARNING_RATE
	LEARNING_RATE = float(args2)

	# Define Weights and biases
	Weights, biases = [tf.Variable(initial_value=val,dtype=tf.float32) for val in INIT_PARAMS]

	# Prediction
	y_pred = trainFunction(args1, x)
	pred = y_pred(Weights, biases)

	# Define the loss function and the optimizer
	loss = tf.reduce_mean(tf.square(y - pred))
	train = tf.train.GradientDescentOptimizer(LEARNING_RATE).minimize(loss)

	''' Start training '''
	Weights_list, biases_list, loss_list = [], [], []
	with tf.Session() as sess:
		# Initialize all variables in TensorFlow
		sess.run(tf.global_variables_initializer())

		# Train 400 times
		for epoch in range(TRAINING_EPOCH):
			w, b, l = sess.run([Weights, biases, loss])

			# Record the changes of parameters
			Weights_list.append(w)
			biases_list.append(b)
			loss_list.append(l)

			# Training
			result, _ = sess.run([pred, train])

	''' Visualization '''
	print('Weight = ', w, 'bias = ', b)

	# Plot the input data and the training result
	plt.figure(1)
	plt.scatter(x, y, c='#74BCFF', s=50, alpha=0.5, label='Train')
	plt.legend(loc='upper left')
	plt.plot(x, result, 'r-', lw=2)
	plt.savefig('input_%s_%s.png' % (args1, args2))

	# Plot loss rate in 3D figure
	fig = plt.figure(2)
	ax = Axes3D(fig)
	weight3D, bias3D = np.meshgrid(np.linspace(-2, 7, 30), np.linspace(-2, 7, 30))      # Parameter space
	loss3D = np.array([np.mean(np.square(y_fun(w, b) - y)) for w, b in zip(weight3D.flatten(), bias3D.flatten())]).reshape(weight3D.shape)
	ax.plot_surface(weight3D, bias3D, loss3D, rstride=1, cstride=1, cmap=plt.get_cmap('rainbow'), alpha=0.5)
	ax.scatter(Weights_list[0], biases_list[0], zs=loss_list[0], s=300, c='r')          # Initial parameter
	ax.set_xlabel('Weight')
	ax.set_ylabel('bias')
	ax.plot(Weights_list, biases_list, zs=loss_list, zdir='z', c='r', lw=3)             # Plot 3D gradient descent
	plt.savefig('%s_%s.png' % (args1, args2))
	plt.show()

''' ENTRY POINT '''
if __name__ == "__main__":
	if len(sys.argv) != 3:
		print('[ERROR] No argument')
		print('[INFO] FORMAT: "python3 main.py 1 [FLOAT]" or "python3 main.py 2 [FLOAT]" or "python3 main.py 3 [FLOAT]"')
		sys.exit()
	else:
		# For option
		if sys.argv[1] == '1':
			print('[INFO] Using linear function as target and training function')
		elif sys.argv[1] == '2':
			print('[INFO] Using non-linear function as target and training function')
		elif sys.argv[1] == '3':
			print('[INFO] Using sin/cos function as target and training function')
		else:
			print('[ERROR] Invalid argument')
			print('[INFO] FORMAT: "python3 main.py 1" or "python3 main.py 2" or "python3 main.py 3"')
			sys.exit()

		# For learning rate
		print('[INFO] Using %s learning rate' % sys.argv[2])

		main(sys.argv[1], sys.argv[2])


import sympy

x = sympy.Symbol('x')
y = sympy.Symbol('y')

print(type(x))
# <class 'sympy.core.symbol.Symbol'>

expr = x**2 + y + 1

print(expr)
# x**2 + y + 1

z = sympy.Symbol('ZZZZ')

expr_z = z**2 + 3 * z

print(expr_z)
# ZZZZ**2 + 3*ZZZZ

print(expr)
# x**2 + y + 1

print(expr.subs(x, 1))
# y + 2

print(expr.subs(x, y))
# y**2 + y + 1

print(expr.subs([(x, 1), (y, 2)]))
# 4

expr = (x + 1)**2

print(expr)
# (x + 1)**2

expr_ex = sympy.expand(expr)

print(expr_ex)
# x**2 + 2*x + 1

expr_factor = sympy.factor(expr_ex)

print(expr_factor)
# (x + 1)**2

print(sympy.factor(x**3 - x**2 - 3 * x + 3))
# (x - 1)*(x**2 - 3)

print(sympy.factor(x * y + x + y + 1))
# (x + 1)*(y + 1)

print(sympy.solve(x**2 - 3 * x + 2))
# [1, 2]

print(sympy.solve(x**2 + x + 1))
# [-1/2 - sqrt(3)*I/2, -1/2 + sqrt(3)*I/2]

expr = x + y**2 - 4

print(sympy.solve(expr, x))
# [-y**2 + 4]

print(sympy.solve(expr, y))
# [-sqrt(-x + 4), sqrt(-x + 4)]

expr1 = 3 * x + 5 * y - 29
expr2 = x + y - 7

print(sympy.solve([expr1, expr2]))
# {x: 3, y: 4}

print(sympy.diff(x**3 + 2 * x**2 + x))
# 3*x**2 + 4*x + 1

expr = x**3 + y**2 - y

print(sympy.diff(expr, x))
# 3*x**2

print(sympy.diff(expr, y))
# 2*y - 1

print(sympy.integrate(3 * x**2 + 4 * x + 1))
# x**3 + 2*x**2 + x

print(sympy.diff(sympy.cos(x)))
# -sin(x)

print(sympy.diff(sympy.exp(x)))
# exp(x)

print(sympy.diff(sympy.log(x)))
# 1/x

print(sympy.integrate(sympy.cos(x)))
# sin(x)

print(sympy.integrate(sympy.exp(x)))
# exp(x)

print(sympy.integrate(sympy.log(x)))
# x*log(x) - x


from sympy import ImmutableMatrix as Matrix
from sympy.vector import CoordSys3D, matrix_to_vector

import numpy as np
import sympy as sy

m3_1_x = Matrix([-5, 5, -2])
m3_1_y = Matrix([ 2, 2, -5])

C = CoordSys3D('C')
x_3_1 = matrix_to_vector(m3_1_x, C)
y_3_1 = matrix_to_vector(m3_1_y, C)

print("x+y = ", x_3_1 + y_3_1)
print("2x = ", 2*x_3_1)
print("x-y = ", x_3_1 - y_3_1)
print("2x+10y = ", 2*x_3_1 + 10*y_3_1)

x_3_2 = np.array([[-2, 2, 1],
                 [-1, 0, 1],
                 [1, -1, 0]])
y_3_2 = ([18, 9, -7])
print("coefficients of linear combo are: ", np.linalg.solve(x_3_2, y_3_2) )

m_3_2 = sy.Matrix([[-2, 2, 1,18],
                  [-1, 0, 1, 9],
                  [1, -1, 0, -7]])

print(m_3_2.rref(pivots=True))

m_3_3 = sy.Matrix([[-1,  1, -1,  13],
                   [ 1,  0,  1,  -8],
                   [ 1, -1,  2, -16]])

print(m_3_3.rref(pivots=True))

m_3_4 = sy.Matrix([[ 5,  5,  0,  -2],
                   [ 1, -1, -2,   3],
                   [-1, -1,  3,  -1]])

print(m_3_4.rref(pivots=True))

m_3_5 = sy.Matrix([[ 3,  1, -5,   26],
                   [ 2,  4,  1,    9]])

print(m_3_5.rref(pivots=True))

m_3_6 = sy.Matrix([[ 0,   0,  1,   -5], #note each column is a vector
                   [ -1,  2, -3,   12],
                   [ -1,  1, -2,   11]])

print(m_3_6.rref(pivots=True))

m_3_7 = sy.Matrix([[ -5,  5, -10,  -30],
                   [ -4,  2,  -6,  -16],
                   [  1,  0,   2,    3]])

print(m_3_7.rref(pivots=True))

m3_8_u = Matrix([ 6, -2,  9]) # this is one vector as a 1 x 3 matrix
m3_8_v = Matrix([ 6,  3,  1])
m3_8_w = Matrix([ 1, -3,  1])

C = CoordSys3D('C')
x_8_u = matrix_to_vector(m3_8_u, C)
x_8_v = matrix_to_vector(m3_8_v, C)
x_8_w = matrix_to_vector(m3_8_w, C)

print("2u+3v-9w = ", 2*x_8_u + 3*x_8_v - 9*x_8_w)

m_3_9 = sy.Matrix([[  -2,   2, -6,  0],
                    [ -1,  -5,  9,  0]])

print(m_3_9.rref(pivots=True))

m_3_10 = sy.Matrix([[  9,  -6,  0,  0],
                    [  9, -10,  2,  0],
                    [ -6,  -8,  6,  0]])

print(m_3_10.rref(pivots=True))

m_3_13_1 = sy.Matrix([[  4,  -2,  8, -10, -5, 0],
                      [ -9,   5, -9,  -4,  5, 0],
                      [ -5,  -1,  7,  -4,  9, 0],
                      [ 10,   9, -8,   1, -7, 0]])

print(m_3_13_1.rref(pivots=True))

m_3_13_2 = sy.Matrix([[  4,   8, 0],
                      [  4,   8, 0],
                      [  2,   4, 0],
                      [  5,   10, 0]])

print(m_3_13_2.rref(pivots=True))

k = sy.symbols('k')
m_3_14 = sy.Matrix([ [ -1,    1,      3, 0],
                     [ -2,   -2,     -5, 0],
                     [ -5,  -15 + k, -7, 0]])

print(m_3_14.rref(pivots=True))

m_4_2 = sy.Matrix([[-2,  2,  3, -9],
                   [-1,  0,  0, -1],
                   [ 1, -1, -1,  5]])

print(m_4_2.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_3 = sy.Matrix([[-1,  5,  1, -5],
                   [-4, -1, -2, -5],
                   [ 1, -4,  2,  2]])

print(m_4_3.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_4 = sy.Matrix([[ 2,  2,  8],
                   [-1, -2, -9],
                   [ 1,  1,  4]])

print(m_4_4.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_5 = sy.Matrix([[-2, -2,  3, -6],
                   [ 1,  0,  0, -5],
                   [ 1,  1, -1,  2]])

print(m_4_5.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_6 = sy.Matrix([[ 1,  1, -3, 0],
                   [ 2, -1, -3, 3],
                   [-4, -5, 13, 18]])

print(m_4_6.rref(pivots=True)) #coefficients of linear combo are in last column

m4_7_x = Matrix([4, -2, 0])
m4_7_y = Matrix([-5, 1, 5])

C = CoordSys3D('C')
x4_7 = matrix_to_vector(m4_7_x, C)
y4_7 = matrix_to_vector(m4_7_y, C)

print("3x = ", 3*x4_7 )
print("x+y = ", x4_7+y4_7)
print("3x+y = ", 3*x4_7 + y4_7)

m_4_8 = sy.Matrix([[ -1, -1, -2],
                   [ -2,  1, -1],
                   [  3,  5,  8]])

print(m_4_8.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_10 = sy.Matrix([[ -16,  16, -5,  -2],
                    [ -9,  -19, -10, -18],
                    [  4,   12, -16,  12],
                    [-11,   -7, -15,  -4]])

print(m_4_10.rref(pivots=True)) #check pivots

m_4_10_w = sy.Matrix([[ -16,  16, -5, -11],
                      [ -9,  -19, -10, -27],
                      [  4,   12, -16,  36],
                      [-11,   -7, -15, -14]])

print(m_4_10_w.rref(pivots=True)) #check pivots

m_4_11 = sy.Matrix([[ -15,  0, 15, 0], #added 0s to check independence
                    [ -10, -8,  2, 0],
                    [  14, 31, 17, 0]])

print(m_4_11.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_12 = sy.Matrix([[   4, -4,  8, 0], #added 0s to check independence
                    [  12,  3, 15, 0],
                    [   9,  6,  9, 0]])

print(m_4_12.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_13 = sy.Matrix([[  1,    k,  0], #added 0s to check independence
                    [  k, 6*k+7, 0]])

print(m_4_13.rref(pivots=True)) #coefficients of linear combo are in last column

m_4_14 = sy.Matrix([  [  4,  -3,  1, -5, 0],
                      [  5,  -1, -3,  2, 0],
                      [ -3,  -4,  5, -3, 0],
                      [ -5,  -2,  2,  1, 0]])

print(m_4_14.rref(pivots=True)) #check pivots

m_4_15 = sy.Matrix([[  -1,  2,    -1, 0], #added 0s to check independence
                    [  -5, -12+k, -3, 0],
                    [  -3, -2,    -1, 0]])

print(m_4_15.rref(pivots=True)) #coefficients of linear combo are in last column