import numpy as np
import matplotlib.pyplot as plt
from itertools import cycle, islice
from sklearn.cluster import KMeans
from sklearn import datasets
def gendata(n_samples):
    x,y = datasets.make_circles(n_samples,factor=0.6,noise=0.03)
    return x,y

#compute the AdjaceincyMatrix
def euclidDistance(x1, x2, sqrt_flag=False):
    res = np.sum((x1-x2)**2)
    if sqrt_flag:
        res = np.sqrt(res)
    return res
def calEuclidDistanceMatrix(X):
    X = np.array(X)
    S = np.zeros((len(X), len(X)))
    for i in range(len(X)):
        for j in range(i+1, len(X)):
            S[i][j] = 1.0 * euclidDistance(X[i], X[j])
            S[j][i] = S[i][j]
    return S
def AdjaceincyMatrix(S,k,sigma=1.0):
    N = len(S)
    W = np.zeros((N,N))

    for i in range(N):
        dist_with_index = zip(S[i], range(N))
        dist_with_index = sorted(dist_with_index, key=lambda x:x[0])
        #Consider xi's k nearest neighbors
        neighbours_id = [dist_with_index[m][1] for m in range(k+1)]

        for j in neighbours_id:
            # xj is ones of the neighbours of xi's
            W[i][j] = np.exp(-S[i][j]/2/sigma/sigma)
            W[j][i] = W[i][j]

    return W

def LaplacecianMatrix(adjaceincyMatrix):
    #compute the D Matrix:D=diag(d1,...,dn)
    # di=w[i][1]+w[i][2]+...
    D = np.diag(np.sum(adjaceincyMatrix,axis=1))

    #compute the Laplacian Matrix:L=D-W
    LaplacecianMatrix = D - adjaceincyMatrix

    #normalize the LaplacecianMatrix
    A = np.sum(adjaceincyMatrix,axis=1)
    nor_lapmatrix = np.dot(np.dot(np.diag(1.0 / (A ** (0.5))),LaplacecianMatrix),np.diag(1.0 / (A ** (0.5))))
    return nor_lapmatrix

def spKmeans(V):
    sp_kmeans = KMeans(n_clusters=2).fit(V)
    return sp_kmeans.labels_

def plot(X, sp_y, km_y):
    colors = np.array(list(islice(cycle(['#377eb8', '#ff7f00', '#4daf4a',
                                                '#f781bf', '#a65628', '#984ea3',
                                                '#999999', '#e41a1c', '#dede00']),
                                        int(max(km_y) + 1))))
    plt.subplot(121)
    plt.scatter(X[:,0], X[:,1], s=10, color=colors[sp_y])
    plt.title("Spectral Clustering")
    plt.subplot(122)
    plt.scatter(X[:,0], X[:,1], s=10, color=colors[km_y])
    plt.title("Kmeans Clustering")
    plt.show()

np.random.seed(123)
data, label = gendata(n_samples=1000)
Similarity = calEuclidDistanceMatrix(data)
Adjacent = AdjaceincyMatrix(Similarity, k=5)
Laplacian = LaplacecianMatrix(Adjacent)

x, Z = np.linalg.eig(Laplacian)
x = zip(x, range(len(x)))
x = sorted(x, key=lambda x:x[0])
V = np.vstack([Z[:,i] for (v, i) in x[:1000]]).T
#If the procedure runs for too long, you can properly adjust the scale of n_sample and the value of k

#SpectralClustering with kmeans
sp_kmeans = KMeans(n_clusters=2).fit(V)
#kmeans
kmeans = KMeans(n_clusters=2).fit(data)
plot(data, sp_kmeans.labels_, kmeans.labels_)

#Reference Songdark's SpectralClustering


import numpy as np
import matplotlib.pyplot as plt
from scipy import optimize

# Lecture 11 2-user water allocation example
# First approach: scipy.optimize.minimize
# does not return duals

# objective function (x is decision variable vector)
def f(x):
  return -1*(30*x[0] - 4*x[0]**2 + 10*x[1] - 2*x[1]**2)

# if the function returns > zero, constraint satisfied
def f_constraint(x):
    return -(x[0] + x[1] - 5)

constraint = {'type': 'ineq', 'fun': f_constraint}

sol = optimize.minimize(f, x0 = [0,5], constraints=[constraint])
print(sol)

# Second approach: cvxpy
# this import is easy but also could be confusing
# because it overwrites common functions (sum, mean, etc) with cvxpy functions

from cvxpy import *

x1 = Variable()
x2 = Variable()
Q = 5 # units of water

obj = Maximize(30*x1 - 4*x1**2 + 10*x2 - 2*x2**2)

constraints = [x1 + x2 <= Q, x1 >= 0, x2 >= 0] # magic?

prob = Problem(obj, constraints)
prob.solve()

print('Objective = %f' % obj.value)
print('X1 = %f' % x1.value)
print('X2 = %f' % x2.value)

for c in constraints:
  print('Dual (%s) = %f' % (c, c.dual_value))


#!/usr/bin/env python
# coding: utf-8

# In[ ]:

import numpy as np
import matplotlib.pyplot as plt
import math

def fact(n):
    if n==0:
        return 1
    else:
        return n*fact(n-1)

#Taylor Series for Log function

x = np.arange(-0.5, 5, 0.00001)

y1 = np.log(x)

k = int(input('Enter the degree upto which taylor series will be taken = '))

y2=0
for i in range(1,k+1):
    y2 +=  ((-1)**(i-1))*((x-1)**(i) / (fact(i)))

plt.plot(x, y1)
plt.plot(x, y2)
plt.grid()
plt.ylabel('log(x)')
plt.xlabel('x')
plt.legend(['log', 'taylor'])
plt.ylim(-3,4)
plt.show()

Taylor Series Visualization for Log Function.py


import numpy as np

def SMAPE(y_true, y_pred):
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)
    score = np.abs(y_pred - y_true) / ((np.abs(y_true) + np.abs(y_pred)) / 2)
    score = np.where(np.isnan(score), 0, score)
    return score

def MAPE(y_true, y_pred):
    y_true = np.array(y_true)
    y_pred = np.array(y_pred)

    idx = y_true > 0
    y_true = y_true[idx]
    y_pred = y_pred[idx]

    score = np.abs(y_pred - y_true) / np.abs(y_true)
    return np.mean(score)

def _get_score_metric(y_true, y_pred, metric='mape'):
    '''
    :param y_true: array-like of shape (n_samples,) or (n_samples, n_outputs). Ground truth (correct) target values.
    :param y_pred: array-like of shape (n_samples,) or (n_samples, n_outputs). Estimated target values.
    :param metric: str, one of ['mae', 'mape', 'mse', 'rmse', 'msle', 'rmsle', 'smape'], default = 'mape'.
    :return:
    '''
    y_true = np.array(y_true) if type(y_true) == list else y_true
    y_pred = np.array(y_pred) if type(y_true) == list else y_pred

    if metric == 'mae':
        return np.mean(np.abs(y_true - y_pred))
    if metric == 'mape':
        return MAPE(y_true, y_pred)
    elif metric == 'mse':
        return np.mean((y_true - y_pred) ** 2)
    elif metric == 'rmse':
        return np.mean((y_true - y_pred) ** 2) ** 0.5
    elif metric == 'msle':
        return np.mean((np.log1p(y_true) - np.log1p(y_pred)) ** 2)
    elif metric == 'rmsle':
        return np.mean((np.log1p(y_true) - np.log1p(y_pred)) ** 2) ** 0.5
    elif metric == 'smape':
        return np.mean(SMAPE(y_true, y_pred))
    return (y_true == y_pred).sum()


import numpy as np
from random_generator import xor_shift_64

# Generate random numbers as required.
a = (2.5-1.1)*xor_shift_64()+1.1
b = (2-0.5)*xor_shift_64()+0.5
c = (4-1.5)*xor_shift_64()+1.5

# Since there's no theta or phi dependence, the 3D spherical
# intergral can be simplied to a 1D integral multiplied with
# 4*pi*x**2. Cancel the constant N_sat on both sides and we have
# an integration equal to 1. A*intergral(n_1d) = 1.
# A is then just 1/result.

n_1d = lambda x:4*np.pi*x**2*(x/b)**(a-3)*np.e**(-(x/b)**c)

# Since the lower bound is 0, the function becomes improper,
# and thus taking the midpoint rule as the integration method.

# Midpoint rule.
def midpoint(f, a, b, N):
    """
    f     ... function to be integrated
    [a,b] ... integration interval
    N     ... number steps(bigger the better but slower)

    This method uses rectangles to approximate the area under the curve.
    Rectangle width is determined by the interval of integration.
    The interval of integration is then divided into n smaller intervals of equal lengths as step sizes increase,
    and n rectangles would used to approximate the integral; each smaller rectangle has the width of the smaller interval.
    The rectangle height is the function value at the midpoint of its base.
    """

    h = float(b-a)/N
    output = 0
    for i in range(N):
        output += f((a+h/2.0)+i*h)
    output *= h
    return output

# Calculate A.
A = 1/midpoint(n_1d,0,5,100)

# Write out the density profile of the satellite galaxies.
N_sat = 100
def n(x):
    return A*N_sat*(x/b)**(a-3)*np.e**(-(x/b)**c)

# Save as a text file.
np.savetxt('integrated_result.txt', [a,b,c,A])


"""
Task

Jordan has $100 to buy some comic books. He really likes the Star Wars books
which cost $12 each, but he could also buy the Marvels books which cost $5 each.
If he has to buy at least 12 books, what is the maximum number of the Star Wars
books that he can buy?
"""

"""
Let s be the number of the Star Wars books and m be the number of the Marvels books

maximize: s
subject to:
    s * 12 + m * 5 <= 100
    s + m >= 12
    s >= 0
    m >= 0
"""

"""
Matrix version

maximize: transpose(c) * x
subject to:
    Ax <= b
    x >= 0

x = [s, m]
c = [1, 0]

# if the first case <= and the second >=
A = [[12, 5], [-1, -1]]
b = [100, -12]
"""

from scipy.optimize import linprog
import pulp

def linear_programming():
    """
    NOTE: this is a linear programming solution, not a linear _integer_
    programming so results are floats!
    """
    print('\n---Solving linear program---')

    # NOTE linprog minimizes a linear objective function!
    # https://docs.scipy.org/doc/scipy-0.18.1/reference/generated/scipy.optimize.linprog.html
    c = [-1, 0]
    A = [[12, 5], [-1, -1]]
    b = [100, -12]

    # x >= 0
    s_bounds = (0, None)
    m_bounds = (0, None)

    result = linprog(
        c, A_ub=A, b_ub=b, bounds=(s_bounds, m_bounds), options={"disp": True}
    )

    print('The number of the Star Wars books:', result['x'][0])

def linear_integer_programming():
    print('\n---Solving linear integer program---')

    prob = pulp.LpProblem('books', pulp.LpMaximize)

    # variables
    s = pulp.LpVariable('s', 0, cat='Integer')
    m = pulp.LpVariable('m', 0, cat='Integer')

    # objective
    prob += s

    # constraints

    prob += (s * 12 + m * 5 <= 100)
    prob += (s + m >= 12)

    prob.solve()

    # for v in prob.variables():
        # print('{}={}'.format(v.name, v.varValue))

    print('The number of the Star Wars books:', pulp.value(prob.objective))

if __name__ == '__main__':

    linear_programming()

    linear_integer_programming()


"""
Straight line fit -- linear regression
======================================

This is probably the most widespread problems in statistics: estimating the
slope and ordinate of a linear relationship y = ax+b given some data (x,y).

The standard least-square (SLS) solution to this problem assumes that the input
data x is exact, and that errors only affect output measurements y. In many
instances, this assumption does not hold and using the same SLS method yields
biased parameters: the slope is underestimated and the ordinate overestimated.

Here, both input and output data are assumed to be corrupted by errors of zero
mean and variances of sigma_x and sigma_y respectively. Under these assumptions,
the most general statistical distribution (maximum entropy) is the normal
distribution. In the following, the parameter distribution is sampled by
marginalizing x from

.. math::
    p(a,b,x \mid \tilde{x}, \tilde{y}) = p(\tilde{y} \mid a, b, x) p(\tilde{x} \mid x) p(x) p(a,b),

where p(x) stands for the prior for the true input and p(a,b) the prior for the
regression parameters.
"""
from pymc import stochastic, observed, deterministic, uniform_like, runiform, rnormal, Sampler, Normal, Uniform
from numpy import inf, log, cos,array
import pylab

# ------------------------------------------------------------------------------
# Synthetic values
# Replace by real data
# ------------------------------------------------------------------------------
slope = 1.5
intercept = 4
N = 30
true_x = runiform(0,50, N)
true_y = slope*true_x + intercept
data_y = rnormal(true_y, 2)
data_x = rnormal(true_x, 2)

# ------------------------------------------------------------------------------
# Calibration of straight line parameters from data
# ------------------------------------------------------------------------------

@stochastic
def theta(value=array([2.,5.])):
    """Slope and intercept parameters for a straight line.
    The likelihood corresponds to the prior probability of the parameters."""
    slope, intercept = value
    prob_intercept = uniform_like(intercept, -10, 10)
    prob_slope = log(1./cos(slope)**2)
    return prob_intercept+prob_slope

init_x = data_x.clip(min=0, max=50)

# Inferred true inputs.
x = Uniform('x', lower=0, upper=50, value=init_x)

@deterministic
def modelled_y(x=x, theta=theta):
    """Return y computed from the straight line model, given the
    inferred true inputs and the model paramters."""
    slope, intercept = theta
    return slope*x + intercept

"""
Input error model.

    Define the probability of measuring x knowing the true value.
"""
measured_input = Normal('measured_input', mu=x, tau=2, value=data_x, observed=True)

"""
Output error model.
    Define the probability of measuring x knowing the true value.
    In this case, the true value is assumed to be given by the model, but
    structural errors could be integrated to the analysis as well.
"""
y = Normal('y', mu=modelled_y, tau=2, value=data_y, observed=True)


"""Permutation testing experiments."""
import typing as t

import numpy as np

def _shuffle(pop_a: np.ndarray,
             pop_b: np.ndarray) -> t.Tuple[np.ndarray, np.ndarray]:
    """Shuffle (not in-place) arrays ``pop_a`` and ``pop_b`` values."""
    aux = np.random.permutation(np.concatenate((pop_a, pop_b)))
    return aux[:pop_a.size], aux[pop_a.size:]

def perm_test(pop_a: np.ndarray,
              pop_b: np.ndarray,
              stat_test: t.Callable[[np.ndarray, np.ndarray], t.
                                    Union[float, int, np.number]],
              perm_num: int = 1000,
              random_seed: t.Optional[int] = None) -> float:
    """Tests whether the ``stat_test`` values of ``pop_a`` and ``pop_b`` match.

    The test works as following:
    H_0 (null hypothesis): the ``stat_test`` value of ``pop_a`` and ``pop_b`` are equal.
    H_1 (alt hypothesis): the ``stat_test`` of ``pop_a`` and ``pop_b`` are different.

    Arguments
    ---------
    stat_test : :obj:`Callable`
        Statistical test. Must be a function that receives two numpy
        arrays and return some numeric value.

    perm_num : :obj:`int`, optional
        Number of permutations.

    random_seed : :obj:`int`, optional
        If given, set the random seed before the first permutation.

    Returns
    -------
    :obj:`float`
        p-value of the permutation test.
    """
    if random_seed is not None:
        np.random.seed(random_seed)

    if pop_a.ndim != 1:
        pop_a = pop_a.ravel()

    if pop_b.ndim != 1:
        pop_b = pop_b.ravel()

    truediff = stat_test(pop_a, pop_b)

    stat_test_vals = np.zeros(perm_num)
    for ind in np.arange(perm_num):
        sh_a, sh_b = _shuffle(pop_a, pop_b)
        stat_test_vals[ind] = stat_test(sh_a, sh_b)

    return (np.sum(truediff <= stat_test_vals) + 1) / (perm_num + 1)

def stratified_perm_test(pop_a_1: np.ndarray,
                         pop_b_1: np.ndarray,
                         pop_a_2: np.ndarray,
                         pop_b_2: np.ndarray,
                         stat_test: t.Callable[[np.ndarray, np.ndarray], t.
                                               Union[float, int, np.number]],
                         perm_num: int = 1000,
                         random_seed: t.Optional[int] = None) -> float:
    """Tests whether the stratified ``test_values`` values of ``pop_a`` and ``pop_b`` match.

    The stratification is made between groups ``1`` and ``2``.

    Arguments
    ---------
    stat_test : :obj:`Callable`
        Statistical test. Must be a function that receives two numpy
        arrays and return some numeric value.

    perm_num : :obj:`int`, optional
        Number of permutations.

    random_seed : :obj:`int`, optional
        If given, set the random seed before the first permutation.

    Returns
    -------
    :obj:`float`
        p-value of the permutation test.
    """
    if random_seed is not None:
        np.random.seed(random_seed)

    if pop_a_1.ndim != 1:
        pop_a_1 = pop_a_1.ravel()

    if pop_b_1.ndim != 1:
        pop_b_1 = pop_b_1.ravel()

    if pop_a_2.ndim != 2:
        pop_a_2 = pop_a_2.ravel()

    if pop_b_2.ndim != 2:
        pop_b_2 = pop_b_2.ravel()

    truediff = stat_test(
        np.concatenate((pop_a_1, pop_a_2)), np.concatenate((pop_b_1, pop_b_2)))

    stat_test_vals = np.zeros(perm_num)
    for ind in np.arange(perm_num):
        sh_a_1, sh_b_1 = _shuffle(pop_a_1, pop_b_1)
        sh_a_2, sh_b_2 = _shuffle(pop_a_2, pop_b_2)
        stat_test_vals[ind] = stat_test(
            np.concatenate((sh_a_1, sh_a_2)), np.concatenate((sh_b_1, sh_b_2)))

    return (np.sum(truediff <= stat_test_vals) + 1) / (perm_num + 1)

def _experiment_01():
    """Permutation testing experiment 01."""

    def mean_test(pop_a: np.ndarray, pop_b: np.ndarray) -> float:
        return np.abs(pop_a.mean() - pop_b.mean())

    random_seed = 16
    np.random.seed(random_seed)

    pop_a = np.random.normal(size=500)
    pop_b = np.random.normal(size=500) + 0.5
    print("Mean test")
    print("pop_a.mean = {}\npop_b.mean = {}".format(pop_a.mean(),
                                                    pop_b.mean()))

    p_val = perm_test(
        pop_a,
        pop_b,
        stat_test=mean_test,
        perm_num=1000,
        random_seed=random_seed)
    print("permutation test p-value: {}".format(p_val))

if __name__ == "__main__":
    _experiment_01()


"""Utilities module."""

import numpy as np

def compute_tlsq(X, Y, tlsq_rank):
    """
    Compute Total Least Square.

    :param numpy.ndarray X: the first matrix;
    :param numpy.ndarray Y: the second matrix;
    :param int tlsq_rank: the rank for the truncation; If 0, the method
        does not compute any noise reduction; if positive number, the
        method uses the argument for the SVD truncation used in the TLSQ
        method.
    :return: the denoised matrix X, the denoised matrix Y
    :rtype: numpy.ndarray, numpy.ndarray

    References:
    https://arxiv.org/pdf/1703.11004.pdf
    https://arxiv.org/pdf/1502.03854.pdf
    """
    # Do not perform tlsq
    if tlsq_rank == 0:
        return X, Y

    V = np.linalg.svd(np.append(X, Y, axis=0), full_matrices=False)[-1]
    rank = min(tlsq_rank, V.shape[0])
    VV = V[:rank, :].conj().T.dot(V[:rank, :])

    return X.dot(VV), Y.dot(VV)

def compute_svd(X, svd_rank=0):
    """
    Truncated Singular Value Decomposition.

    :param numpy.ndarray X: the matrix to decompose.
    :param svd_rank: the rank for the truncation; If 0, the method computes
        the optimal rank and uses it for truncation; if positive interger,
        the method uses the argument for the truncation; if float between 0
        and 1, the rank is the number of the biggest singular values that
        are needed to reach the 'energy' specified by `svd_rank`; if -1,
        the method does not compute truncation. Default is 0.
    :type svd_rank: int or float
    :return: the truncated left-singular vectors matrix, the truncated
        singular values array, the truncated right-singular vectors matrix.
    :rtype: numpy.ndarray, numpy.ndarray, numpy.ndarray

    References:
    Gavish, Matan, and David L. Donoho, The optimal hard threshold for
    singular values is, IEEE Transactions on Information Theory 60.8
    (2014): 5040-5053.
    """
    U, s, V = np.linalg.svd(X, full_matrices=False)
    V = V.conj().T

    def omega(x):
        return 0.56 * x**3 - 0.95 * x**2 + 1.82 * x + 1.43

    if svd_rank == 0:
        beta = np.divide(*sorted(X.shape))
        tau = np.median(s) * omega(beta)
        rank = np.sum(s > tau)
    elif 0 < svd_rank < 1:
        cumulative_energy = np.cumsum(s**2 / (s**2).sum())
        rank = np.searchsorted(cumulative_energy, svd_rank) + 1
    elif svd_rank >= 1 and isinstance(svd_rank, int):
        rank = min(svd_rank, U.shape[1])
    else:
        rank = X.shape[1]

    U = U[:, :rank]
    V = V[:, :rank]
    s = s[:rank]

    return U, s, V


from typing import List
from typing import Union

import numpy as np

ArrayLike = List[Union[float, List[float]]]

def mape(y_true: ArrayLike, y_pred: ArrayLike, eps: float = 1e-15) -> float:
    """Mean absolute percentage error.

    `Wikipedia entry on the Mean absolute percentage error
    <https://en.wikipedia.org/wiki/Mean_absolute_percentage_error>`_

    Parameters
    ----------
    y_true:
        array-like of shape (n_samples,) or (n_samples, n_outputs)

        Ground truth (correct) target values.

    y_pred:
        array-like of shape (n_samples,) or (n_samples, n_outputs)

        Estimated target values.

    eps: float=1e-15
        MAPE is undefined for ``y_true[i]==0`` for any ``i``, so all zeros ``y_true[i]`` are
        clipped to ``max(eps, abs(y_true))``.

    Returns
    -------
    float
        A non-negative floating point value (the best value is 0.0).
    """
    y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

    if len(y_true_array.shape) != len(y_pred_array.shape):
        raise ValueError("Shapes of the labels must be the same")

    y_true_array = y_true_array.clip(eps)

    return np.mean(np.abs((y_true_array - y_pred_array) / y_true_array)) * 100

def smape(y_true: ArrayLike, y_pred: ArrayLike, eps: float = 1e-15) -> float:
    """Symmetric mean absolute percentage error.

    `Wikipedia entry on the Symmetric mean absolute percentage error
    <https://en.wikipedia.org/wiki/Symmetric_mean_absolute_percentage_error>`_

    .. math::
        SMAPE = \dfrac{100}{n}\sum_{t=1}^{n}\dfrac{|ytrue_{t}-ypred_{t}|}{(|ypred_{t}|+|ytrue_{t}|) / 2}

    Parameters
    ----------
    y_true:
        array-like of shape (n_samples,) or (n_samples, n_outputs)

        Ground truth (correct) target values.

    y_pred:
        array-like of shape (n_samples,) or (n_samples, n_outputs)

        Estimated target values.

    eps: float=1e-15
        SMAPE is undefined for ``y_true[i] + y_pred[i] == 0`` for any ``i``, so all zeros ``y_true[i] + y_pred[i]`` are
        clipped to ``max(eps, abs(y_true) + abs(y_pred))``.

    Returns
    -------
    float
        A non-negative floating point value (the best value is 0.0).
    """
    y_true_array, y_pred_array = np.asarray(y_true), np.asarray(y_pred)

    if len(y_true_array.shape) != len(y_pred_array.shape):
        raise ValueError("Shapes of the labels must be the same")

    return 100 * np.mean(
        2 * np.abs(y_pred_array - y_true_array) / (np.abs(y_true_array) + np.abs(y_pred_array)).clip(eps)
    )


#!/usr/bin/env python
# coding: utf-8

# # Regression

# In[2]:

get_ipython().run_line_magic('reset', '')
import numpy as np
import matplotlib.pyplot as plt
from scipy import stats
import pandas as pd

# These are some parameters to make figures nice (and big)
get_ipython().run_line_magic('matplotlib', 'inline')
get_ipython().run_line_magic('config', "InlineBackend.figure_format = 'retina'")
plt.rcParams['figure.figsize'] = 16,8
params = {'legend.fontsize': 'x-large',
          'figure.figsize': (15, 5),
         'axes.labelsize': 'x-large',
         'axes.titlesize':'x-large',
         'xtick.labelsize':'x-large',
         'ytick.labelsize':'x-large'}
plt.rcParams.update(params)

# # Simple linear regression: $\delta^{18}\text{O}$ and salinity
#
# Let's take a look at some data linking sea surface salinity and $\delta^{18}O$ in a coral from the tropical Pacific. The data comes from one of Prof. Jess Conroy's studies
# https://agupubs.onlinelibrary.wiley.com/doi/full/10.1002/2016PA003073
#
# We are looking at the relation between $\delta^{18}\text{O}$ and salinity at Palau.

# In[3]:

# read data and take a cursory look
df=pd.read_csv('/data/keeling/a/cristi/SIMLES/scratch/d18O_SSS_conroy2017.csv')
df

# In[4]:

# plot data
x=df['SSS'].values
y=df[['d18O']].values

plt.rcParams['figure.figsize'] = 16,8
fig, ax = plt.subplots(1, 1)
ax.plot(df['SSS'],df['d18O'],'o')
ax.set_xlim(left=32.5,right=34.5)
ax.set_ylim(bottom=-0.4,top=0.3)
ax.set_xlabel('Salinity (PSU)');
ax.set_ylabel('$\delta^{18}$O');

# ## Regression via sklearn
# A simple, general way to do regression is using LinearRegression model from sklearn:

# In[5]:

# import module
from sklearn.linear_model import LinearRegression

# define the x-variable (the predictor/regresso) and the y-variable. (the target/outcome)
# we'll call the values method of pandas to export the values into numpy arrays
x=df[['SSS']].values
y=df[['d18O']].values

# fit the data and obtain a prediction yhat
reg=LinearRegression().fit(x,y)
yhat=reg.predict(x);

# plot
fig, ax = plt.subplots(1, 1,figsize=[8,8])
ax.plot(x,y,'o',label='observations')
ax.plot(x,yhat,'-',label='Fit')
ax.set_xlim(left=32.5,right=34.5)
ax.set_ylim(bottom=-0.4,top=0.3)
ax.set_xlabel('Salinity (PSU)');
ax.set_ylabel('$\delta^{18}$O');
ax.legend();

print('slope='+str(reg.coef_))
print('intercept='+str(reg.intercept_))

#  Another way to look at the quality of the fit is to plot the data against the prediction, $y(x)$ vs $\hat y(x)=\hat \alpha + \hat \beta x$. For a single linear regression, the plot will actually look the same. But the place of the regression line is taken by a one-to-one line (y=x)

# In[6]:

# plot
fig, ax = plt.subplots(1, 1,figsize=[8,8])
ax.plot(yhat,y,'o')
ax.set_ylabel('$y$ = observed $\delta^{18}O$')
ax.set_xlabel('$\hat y$ predicted $\delta^{18}O$')
ax.set_xlim(left=-0.4,right=0.3)
ax.set_ylim(bottom=-0.4,top=0.3)
ax.plot([-0.4,0.3],[-0.4,0.3])

# ### Exercise: Do it by hand:
# Remember from class (or  Wikipedia ) that our best fit will be
#     $$\hat y = \hat \alpha  +\hat \beta x$$
#
# where
# $$ \hat \beta = \frac{ \overline{ (x-\overline x)(y-\overline y)}}{\overline{  (x-\overline x)^2 }}$$
#
# and
#
# $$\hat \alpha = \overline y - \hat \beta \overline x$$
#
# Compute the $\hat \alpha, \hat \beta, \hat y$ using the formulas above and redo the plot. It should look the same as what you got using sklearn

# In[7]:

# define the x-variable (the predictor/regresso) and the y-variable. (the target/outcome)
# we'll call the values method of pandas to export the values into numpy arrays
x=df[['SSS']].values
y=df[['d18O']].values

# fit the data and obtain a prediction yhat
betahat=np.mean((x-np.mean(x))*(y-np.mean(y)))/np.mean((x-np.mean(x))*(x-np.mean(x)))
alphahat=np.mean(y)-betahat*np.mean(x)
yhat=alphahat+betahat*x

#plot
fig, ax = plt.subplots(1, 1,figsize=[8,8])
ax.plot(x,y,'o',label='observations')
ax.plot(x,yhat,'-',label='Fit')
ax.set_xlim(left=32.5,right=34.5)
ax.set_ylim(bottom=-0.4,top=0.3)
ax.set_xlabel('Salinity (PSU)');
ax.set_ylabel('$\delta^{18}$O');
ax.legend();

# # Multiple linear regression
#
# Now what happens if we have more than one predictor, i.e. more than one feature that can affect the outcome?
# Let's look at some more coral geochemistry data. This time we are looking at three factors that can influence the Mg/Ca ratio in corals: Seas surface temperature (SST), sea surface salinity (SSS), and pH.  The data comes from Gray and Evans (2019): https://agupubs.onlinelibrary.wiley.com/doi/full/10.1029/2018PA003517

# In[9]:

df=pd.read_csv('/data/keeling/a/cristi/SIMLES/scratch/mgca.csv')
df.head()

# In[10]:

fig,ax=plt.subplots(1,3,figsize=[18,6])

ax[0].plot(df['SST'],np.log(df['MgCa']),'o')
ax[0].set_xlabel('SST ($^\circ C$)')
ax[0].set_ylabel('ln Mg/Ca ')

ax[1].plot(df['SSS'],np.log(df['MgCa']),'o')
ax[1].set_xlabel('SSS (psu)')
ax[1].set_ylabel('ln Mg/Ca ')

ax[2].plot(df['pH'],np.log(df['MgCa']),'o')
ax[2].set_xlabel('pH')
ax[2].set_ylabel('ln Mg/Ca')

# We see that no single feature is able to predict the Mg/Ca ratio. But what if we consider all three at the same time? We'll consider a model:
# $$\ln (\text{Mg/Ca}_n)=\beta_1 + \beta_2 \cdot SST_n + \beta_3 \cdot SSS_n + \beta_4 \cdot pH_n + \varepsilon_n,$$
# where n is the number of the measurement.
#
# In general, we can write this as:
# $$ y_n= \beta_1 x_{1,n} + \beta_2 x_{2,n} + \beta_3 x_{3,n}+ \beta_4 x_{4,n},$$
# where, in order to get an intercept we just take the first predictor to be a constant, $x_{1,n}=1$.
#
# Not that in matrix notation, this becomes:
#
# $$ \mathbf{y} =\mathbf{X}\mathbf{\beta}+\mathbf{\varepsilon}$$
#
# $$
# \left[\begin{array}{c}
# y_{1}\\
# y_{2}\\
# \vdots\\
# y_{n}
# \end{array}\right]=
# \left[\begin{array}{cccc}
# x_{1,1} & x_{1,2} & x_{1,3} & x_{1,4}\\
# x_{2,1} & x_{2,2} & x_{2,3} & x_{2,4}\\
# \vdots & \vdots & \vdots & \vdots\\
# x_{n,1} & x_{n,2} & x_{n,3} & x_{n.4}
# \end{array}\right]
# \left[\begin{array}{c}
# \beta_{1}\\
# \beta_{2}\\
# \beta_{3}\\
# \beta_{4}
# \end{array}\right]+\left[\begin{array}{c}
# \varepsilon_{1}\\
# \varepsilon_{2}\\
# \vdots\\
# \varepsilon_{n}
# \end{array}\right]
# $$
#
# In this case, the best estimate of $\beta$ is
# $$\hat \beta = \left(X^{T}X\right)^{-1}X^{T}y$$
#
# You can find a proof on  Wikipedia ). But to get some intuition you can consider the case of a single predictor:
# $$y_n=\beta x_n + \varepsilon_n,$$
# where we have shown that the least squares estimator is
# $$\hat \beta=\frac{\overline{xy}}{\overline {x^2}}= \frac{\sum_n x_n y_n}{\sum_n x_n x_n}$$
# If we write $x$ and $y$ in vector notation, than $\sum_n x_ny_n$ is just the vector dot product $\mathbf{x}^t\mathbf{y}$.
# So, we can write
# $$\hat \beta=\frac{\mathbf{x}^t\mathbf{y}}{\mathbf{x}^t\mathbf{x}}=({\mathbf{x}^t\mathbf{x}})^{-1}\mathbf{x}^t\mathbf{y}$$
#
#

# In[ ]:

# In[11]:

y=np.log(df['MgCa'].values)
X=df[['SST','SSS','pH']].values

betahat=np.matmul( np.linalg.inv(np.matmul(np.transpose(X),X)), np.matmul(np.transpose(X),y))
yhat=np.matmul(X,betahat)

plt.rcParams['figure.figsize'] = 8,8
fig, ax = plt.subplots(1, 1)
ax.plot(y,yhat,'o')
ax.set_xlim(left=1,right=3)
ax.set_ylim(bottom=1,top=3)
ax.plot([1,3],[1,3],'r-')

# ### Exercise: use sklearn
# Use sklearn (just like above), and convince yourself that you get the same results :)

# In[21]:

y=np.log(df['MgCa'].values)
X=df[['SST','SSS','pH']].values
X_exp=np.insert(X,0,1,axis=1)

from sklearn.linear_model import LinearRegression
reg=LinearRegression().fit(X_exp,y)
yhat=reg.predict(X_exp);

plt.rcParams['figure.figsize'] = 8,8
fig, ax = plt.subplots(1, 1)
ax.plot(y,yhat,'o')
ax.set_xlim(left=1,right=3)
ax.set_ylim(bottom=1,top=3)
ax.plot([1,3],[1,3],'r-')

ax.set_xlabel('observed ln Mg/Ca')
ax.set_ylabel('predicted ln Mg/Ca')


"""
Fit a line to a data set with optional weights.

Returns the parameters of the model, bo, b1:
Y = b0 + b1* X

:author: Nadia Dencheva
:version: '1.0 (2007-02-20)'

"""
import numpy as np

__version__ = '1.0'          # Release version number only
__vdate__ = '2007-02-20'     # Date of this version

def linefit(x, y, weights=None):

    """
    Parameters
    ----------
    y: 1D numpy array
        The data to be fitted
    x: 1D numpy array
        The x values of the y array. x and y must
        have the same shape.
    weights:   1D numpy array, must have the same shape as x and y
        weight values

    Examples
    --------
    >>> import numpy as N
    >>> from numpy.core import around
    >>> x = np.array([-5, -4 ,-3 ,-2 ,-1, 0, 1, 2, 3, 4, 5])
    >>> y = np.array([1, 5, 4, 7, 10, 8, 9, 13, 14, 13, 18])
    >>> around(linefit(x,y), decimals=5)
    array([9.27273, 1.43636])
    >>> x = np.array([1.3,1.3,2.0,2.0,2.7,3.3,3.3,3.7,3.7,4.,4.,4.,4.7,4.7,5.,5.3,5.3,5.3,5.7,6.,6.,6.3,6.7])
    >>> y = np.array([2.3,1.8,2.8,1.5,2.2,3.8,1.8,3.7,1.7,2.8,2.8,2.2,3.2,1.9,1.8,3.5,2.8,2.1,3.4,3.2,3.,3.,5.9])
    >>> around(linefit(x,y), decimals=5)
    array([1.42564, 0.31579])
    """
    if len(x) != len(y):
        print("Error: X and Y must have equal size\n")
        return
    n = len(x)
    w = np.zeros((n,n)).astype(float)
    if weights is None:
        for i in np.arange(n):
            w[i,i] = 1
    else:
        if len(weights) != n:
            print("Error: Weights must have the same size as X and Y.\n")
            return
        for i in np.arange(n):
            w[i,i] = weights[i]
    x = x.astype(float)
    y = y.astype(float)
    # take the weighted avg for calculatiing the covarince
    Xavg = np.sum(np.dot(w,x)) / np.sum(w.diagonal())
    Yavg = np.sum(np.dot(w,y)) / np.sum(w.diagonal())

    xm = x - Xavg
    xmt = np.transpose(xm)
    ym = y - Yavg

    b1 = np.dot(xmt,np.dot(w,ym)) / np.dot(xmt ,np.dot(w,xm))
    b0 = Yavg - b1 * Xavg

    return b0, b1


from __future__ import division
import numpy as np
from .linalg import dot_inplace_right

def eigenvalue_decomposition(S, eps=10**-10):
    r"""
    Eigenvalue decomposition of a given covariance (or scatter) matrix.

    Parameters
    ----------
    S : ``(N, N)`` `ndarray`
        Covariance/Scatter matrix
    eps : `float`, optional
        Small value to be used for the tolerance limit computation. The final
        limit is computed as ::

            limit = np.max(np.abs(eigenvalues)) * eps

    Returns
    -------
    pos_eigenvectors : ``(N, p)`` `ndarray`
        The matrix with the eigenvectors corresponding to positive eigenvalues.
    pos_eigenvalues : ``(p,)`` `ndarray`
        The array of positive eigenvalues.
    """
    # compute eigenvalue decomposition
    eigenvalues, eigenvectors = np.linalg.eigh(S)
    # sort eigenvalues from largest to smallest
    index = np.argsort(eigenvalues)[::-1]
    eigenvalues = eigenvalues[index]
    eigenvectors = eigenvectors[:, index]

    # set tolerance limit
    limit = np.max(np.abs(eigenvalues)) * eps

    # select positive eigenvalues
    pos_index = eigenvalues > 0.0
    pos_eigenvalues = eigenvalues[pos_index]
    pos_eigenvectors = eigenvectors[:, pos_index]
    # check they are within the expected tolerance
    index = pos_eigenvalues > limit
    pos_eigenvalues = pos_eigenvalues[index]
    pos_eigenvectors = pos_eigenvectors[:, index]

    return pos_eigenvectors, pos_eigenvalues

def principal_component_decomposition(X, whiten=False, centre=True,
                                      bias=False, inplace=False):
    r"""
    Apply Principal Component Analysis (PCA) on the data matrix `X`. In the case
    where the data matrix is very large, it is advisable to set
    ``inplace = True``. However, note this destructively edits the data matrix
    by subtracting the mean inplace.

    Parameters
    ----------
    X : ``(n_samples, n_features)`` `ndarray`
        Training data.
    whiten : `bool`, optional
        Normalise the eigenvectors to have unit magnitude.
    centre : `bool`, optional
        Whether to centre the data matrix. If ``False``, zero will be
        subtracted.
    bias : `bool`, optional
        Whether to use a biased estimate of the number of samples. If ``False``,
        subtracts ``1`` from the number of samples.
    inplace : `bool`, optional
        Whether to do the mean subtracting inplace or not. This is crucial if
        the data matrix is greater than half the available memory size.

    Returns
    -------
    eigenvectors : ``(n_components, n_features)`` `ndarray`
        The eigenvectors of the data matrix.
    eigenvalues : ``(n_components,)`` `ndarray`
        The positive eigenvalues from the data matrix.
    mean_vector : ``(n_components,)`` `ndarray`
        The mean that was subtracted from the dataset.
    """
    n_samples, n_features = X.shape

    if bias:
        N = n_samples
    else:
        N = n_samples - 1.0

    if centre:
        # centre data
        mean_vector = np.mean(X, axis=0)
    else:
        mean_vector = np.zeros(n_features)

    # This is required if the data matrix is very large!
    if inplace:
        X -= mean_vector
    else:
        X = X - mean_vector

    if n_features < n_samples:
        # compute covariance matrix
        # S:  n_features  x  n_features
        S = np.dot(X.T, X) / N
        # S should be perfectly symmetrical, but numerical error can creep
        # in. Enforce symmetry here to avoid creating complex
        # eigenvectors from eigendecomposition
        S = (S + S.T) / 2.0

        # perform eigenvalue decomposition
        # eigenvectors:  n_features x  n_features
        # eigenvalues:   n_features
        eigenvectors, eigenvalues = eigenvalue_decomposition(S)

        if whiten:
            # whiten eigenvectors
            eigenvectors *= np.sqrt(1.0 / eigenvalues)

        # transpose eigenvectors
        # eigenvectors:  n_samples  x  n_features
        eigenvectors = eigenvectors.T

    else:
        # n_features > n_samples
        # compute covariance matrix
        # S:  n_samples  x  n_samples
        S = np.dot(X, X.T) / N
        # S should be perfectly symmetrical, but numerical error can creep
        # in. Enforce symmetry here to avoid creating complex
        # eigenvectors from eigendecomposition
        S = (S + S.T) / 2.0

        # perform eigenvalue decomposition
        # eigenvectors:  n_samples  x  n_samples
        # eigenvalues:   n_samples
        eigenvectors_s, eigenvalues = eigenvalue_decomposition(S)

        # compute final eigenvectors
        # eigenvectors:  n_samples  x  n_features
        if whiten:
            w = (N * eigenvalues) ** -1.0
        else:
            w = np.sqrt(1.0 / (N * eigenvalues))

        dot = dot_inplace_right if inplace else np.dot
        eigenvectors = dot(eigenvectors_s.T, X)

        # whiten, and we are done.
        eigenvectors *= w[:, None]

    return eigenvectors, eigenvalues, mean_vector


from numpy import array

from sympy import symbols, Eq, Matrix, diff, Derivative, simplify, factor, expand, latex, init_printing, collect
init_printing()
from IPython.display import display, Math

x1, y1, x2, y2, x3, y3, x4, y4 = symbols('x_1, y_1, x_2, y_2, x_3, y_3, x_4, y_4')
r, s = symbols('r, s')

# Define the interpolation functions
h1 = factor(1/4*(1+r)*(1+s))
h2 = factor(1/4*(1-r)*(1+s))
h3 = factor(1/4*(1-r)*(1-s))
h4 = factor(1/4*(1+r)*(1-s))

display(Math('h_1 = ' + latex(h1)))
display(Math('h_2 = ' + latex(h2)))
display(Math('h_2 = ' + latex(h3)))
display(Math('h_2 = ' + latex(h4)))

x = h1*x1 + h2*x2 + h3*x3 + h4*x4
y = h1*y1 + h2*y2 + h3*y3 + h4*y4

display(Math('x = ' + latex(x)))
display(Math('y = ' + latex(y)))

J = Matrix([[diff(x, r), diff(y, r)],
            [diff(x, s), diff(y, s)]])

display(Math('J = (1/4)' + latex(factor(J*4))))

dH_drs = Matrix([[diff(h1, r), diff(h2, r), diff(h3, r), diff(h4, r)],
                 [diff(h1, s), diff(h2, s), diff(h3, s), diff(h4, s)]])

display(Math(r'\begin{bmatrix} \frac{dH}{dx} \\ \frac{dH}{dy} \end{bmatrix} = J^{-1}(1/4)' + latex(dH_drs*4)))
display(Math(r'\begin{bmatrix} \frac{dH}{dx} \\ \frac{dH}{dy} \end{bmatrix} = J^{-1}(1/4)' + latex(dH_drs*4)))


import numpy as np
from tqdm import tqdm

from utils import tridiag_mult, gen_mats

class PDESolver:
    '''Approximates numerically a specific PDE equation
    '''
    M_base = np.asarray([
        [1/3,1/6],
        [1/6,1/3]
    ])
    S_base = np.asarray([
        [1,-1],
        [-1,1]
    ])

    def __init__( self, t0, t1):
        self.ts = (t0, t1)

    @property
    def mats ( self ):
        '''Returns M and S matrices
        '''
        if not ( hasattr(self, '_M') and hasattr(self, '_S') ):
            self._M, self._S = gen_mats(
                self.M_base,
                self.S_base,
                mesh = self._mesh,
                d = self.d_func,
                e = self.e_func
            )
        return self._M, self._S

    def step (self, t, dt, U, ):
        '''Solve for time derivative of temperature samples that satisfy PDE equation
        Uses theta method,
        Accomadates arbitrary neumann or dirichlet boundaries,
        '''
        pass

    def combine_dU(self, U, dU):
        '''Generic of U + dU
        This function can be overwritten to allow for U to be tuple or generic
        '''
        return U + dU

    def process_sol(self, res, ts):
        '''Converts the raw loop output into a meaningful solution
        '''
        res = {
            'res': np.asarray(res),
            'time': ts
        }

        return res

    def solve( self, U0, mesh, dt, boundaries = (None, None), progress=True ):
        '''Solve the PDE system
        '''

        # Clear out any residual M and S that might be here
        for a in ('_M', '_S'):
            if hasattr(self, a):
                delattr(self, a)

        self.boundaries = list(map(
            lambda x: {'type': 'neumann', 'f': lambda t: 0.0} if x is None else x,
            boundaries
        ))

        self._mesh = mesh

        res = [U0]
        t0, t1 = self.ts
        ts  = np.arange(t0, t1+dt, dt)

        loop = ts[1:]

        # Loop across static time samples
        for t in (pbar if progress else loop):

            # Load current temperature
            U = res[-1]

            # Solve for global derivative that satisfies
            dU = self.step(
                t = t,
                dt = dt,
                U = U,
            )

            # Store the new temperature back
            res.append(self.combine_dU(U, dU))

        # Aggregate the data we used into a meaningful summary
        res = self.process_sol(res, ts)
        # Store summary
        self.solution = res

        return self.solution


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import sympy as sp

class FisExp:

	def __init__ (self, funcao, valores_conhecidos=dict(), incertezas_conhecidas=dict()):
		'''Inicializacao da funcao, em que configura a funcao principal, armazena os dicionarios e
		configura valores.
		'''
		if type(funcao) is str:
			self.__funcao = sp.sympify(funcao)
		else:
			self.__funcao = funcao

		self.__variaveis = self.__funcao.atoms(sp.Symbol)
		self.__incertezas = {}
		for simbolo in self.__variaveis:
			incerteza = sp.symbols('u_' + str(simbolo))
			self.__incertezas[simbolo] = incerteza

		self.__funcao_substituida = self.__funcao
		self.__valores_conhecidos = dict()
		self.valores_conhecidos = valores_conhecidos

		self._propagacao()
		self.__incertezas_conhecidas = dict()
		self.incertezas_conhecidas = incertezas_conhecidas

	@property
	def funcao (self):
		return self.__funcao
	@funcao.setter
	def funcao (self, nova_funcao):
		self.__init__(nova_funcao)

	@property
	def variaveis (self):
		return self.__variaveis
	@variaveis.setter
	def variaveis(self, a):
		pass

	@property
	def incertezas (self):
		return self.__incertezas
	@incertezas.setter
	def incertezas (self, a):
		pass

	@property
	def valores_conhecidos (self):
		return self.__valores_conhecidos
	@valores_conhecidos.setter
	def valores_conhecidos (self, valores_conhecidos):
		'''Gera o atributo "funcao_substituida", que é a função com os valores conhecidos para algumas variáveis
		É chamado por self.valores_conhecidos = dict com valores
		Para apagar os valores substituídos, é necessário usar self.valores_conhecidos = {}
		'''
		#for var in valores_conhecidos.keys():
		#	self.__valores_conhecidos[var] = valores_conhecidos[var]
		self.__valores_conhecidos = valores_conhecidos

		if not valores_conhecidos:
			self.__valores_conhecidos = dict()
			self.__funcao_substituida = self.__funcao
		else:
			self.__funcao_substituida = self.__funcao.subs(self.__valores_conhecidos)

	@property
	def funcao_substituida (self):
		return self.__funcao_substituida
	@funcao_substituida.setter
	def funcao_substituida (self, a):
		pass

	@property
	def incertezas_conhecidas (self):
		return self.__incertezas_conhecidas
	@incertezas_conhecidas.setter
	def incertezas_conhecidas (self, incertezas_conhecidas):
		'''Gera o atributo "propagacao_substituida", que é a função de propagação com os valores
		conhecidos para algumas variáveis.
		É chamado por self.incertezas_conhecidas = dict com valores
		'''
		#for inc in incertezas_conhecidas.keys():
		#	self.__incertezas_conhecidas[inc] = incertezas_conhecidas[inc]
		self.__incertezas_conhecidas = incertezas_conhecidas

		incertezas_chaves_corretas = dict()
		for chave_variavel in self.__variaveis:
			if str(chave_variavel) in self.__incertezas_conhecidas.keys():
				incertezas_chaves_corretas[str(self.__incertezas[chave_variavel])] = self.__incertezas_conhecidas[str(chave_variavel)]

		propagacao_substituida = self.__propagacao.subs(incertezas_chaves_corretas)
		propagacao_substituida = propagacao_substituida.subs(self.__valores_conhecidos)

		self.__propagacao_substituida = propagacao_substituida

	@property
	def propagacao (self):
		return self.__propagacao
	@propagacao.setter
	def propagacao (self, a):
		pass

	@property
	def propagacao_substituida (self):
		return self.__propagacao_substituida
	@propagacao_substituida.setter
	def propagacao_substituida (self, a):
		pass

	def _propagacao (self):
		'''Calcula a propagação de incertezas da instância do objeto. A propagação utilizada é do tipo
		"raiz quadrada da soma do produto dos quadrados das derivadas parciais e dos quadrados das incertezas
		e utiliza como nome para cada incerteza "u_VARIAVEL".
		'''
		propagacao = sp.sympify('0')
		for var in self.__variaveis:
			propagacao += self.derivar(str(var))**2*self.__incertezas[var]**2

		self.__propagacao = sp.sqrt(propagacao)

	def gerar_funcao (self, variaveis_mantidas):
		'''Gera uma funcao que tem como argumentos as variaveis em variaveis_mantidas, e permite
		avaliar a funcao principal nos pontos escolhidos.
		'''
		variaveis = list()
		for var in variaveis_mantidas:
			variaveis.append(self._gerar_simbolo(var))

		self.funcao_gerada = sp.lambdify(variaveis, self.__funcao_substituida)

	def gerar_propagacao (self, variaveis_mantidas):
		'''Gera uma funcao que tem como argumentos as variaveis em variaveis_mantidas, e permite
		avaliar a propagacao nos pontos escolhidos.
		'''
		variaveis = list()
		for var in variaveis_mantidas:
			variaveis.append(self._gerar_simbolo(var))

		self.propagacao_gerada = sp.lambdify(variaveis, self.__propagacao_substituida)

	def integrar (self, var, limites=None, substituir=False):
		'''Permite integrar facilmente a função na variável "var" (que deve vir como string), e
		se o parâmetro "limites" for utilizado, retorna	a integral avaliada nesses pontos.
		'''
		self._gerar_simbolo(var)
		if limites is None:
			if substituir:
				return self.__funcao.integrate(self._ultimo_simbolo).subs(self.valores_conhecidos)
			else:
				return self.__funcao.integrate(self._ultimo_simbolo)
		else:
			if substituir:
				return self.__funcao.integrate((self._ultimo_simbolo, limites[0], limites[1])).subs(self.valores_conhecidos)
			else:
				return self.__funcao.integrate((self._ultimo_simbolo, limites[0], limites[1]))

	def derivar (self, var, ponto_avaliado=None, indice=1, substituir=False):
		'''Permite derivar facilmente a função na variável "var" (que deve vir como string), e
		se o parâmetro "ponto_avaliado" for configurado, avalia a função derivada naquele ponto.
		Se o parâmetro "indice" for utilizado, será calculada a derivada daquele grau
		'''
		if var is not str:
			self._ultimo_simbolo = var
		else:
			self._gerar_simbolo(var)

		if ponto_avaliado is None:
			if substituir:
				return self.__funcao.diff(self._ultimo_simbolo, indice).subs(self.valores_conhecidos)
			else:
				return self.__funcao.diff(self._ultimo_simbolo, indice)
		else:
			if substituir:
				return self.__funcao.diff(self._ultimo_simbolo, indice).subs(self._ultimo_simbolo, ponto_avaliado).subs(self.valores_conhecidos)
			else:
				return self.__funcao.diff(self._ultimo_simbolo, indice).subs(self._ultimo_simbolo, ponto_avaliado)

	def _gerar_simbolo (self, var):
		'''Para permitir a interação com os símbolos internos do programa, é gerado um símbolo
		referente à "var" (que deve vir como string). Esse símbolo é guardado em "_ultimo_simbolo"
		para permitir que seja utilizado de maneira fácil
		'''
		self._ultimo_simbolo = sp.symbols(str(var))
		return self._ultimo_simbolo

	def __str__ (self):
		return str(self.__funcao)

	def __repr__ (self):
		return ('FisExp(\'' + str(self) + '\', ' + str(self.__valores_conhecidos) + ', ' + str(self.__incertezas_conhecidas) + ')')

if __name__ == "__main__":
	f = sp.sympify('x*a+b+c')
	f_a = FisExp(f)
	print ("Funcao: ", str(f_a))
	print ("Propagacao: ", f_a.propagacao)
	variaveis = {'a':4, 'b':3, 'x':2}
	asd = {'a':1, 'b':3, 'c':1, 'x':3}
	f_a.valores_conhecidos = variaveis
	f_a.incertezas_conhecidas = asd
	print ("Substituicao: ", f_a.funcao_substituida)
	f_a.funcao_substituida = 'bla'
	print("Teste de substituicao: ", f_a.funcao_substituida)
	variaveis1 = dict()
	f_a.valores_conhecidos = variaveis1
	f_a.valores_conhecidos = variaveis
	print("Teste de substituicao 2: ", f_a.funcao_substituida)
	print ('Propagacao substituida: ', f_a.propagacao_substituida)
	print ('Integral em x: ', f_a.integrar('x'))
	print ('Derivada em x: ', f_a.derivar('x'))
	f_a.gerar_funcao(['c'])
	f_a.gerar_propagacao(['c'])
	print ('Funcao gerada avaliada em c=5: ', f_a.funcao_gerada(5))
	print ('Propagacao gerada avaliada em c=5: ', f_a.propagacao_gerada(5))


import pandas as pd
import numpy as np
import math
import pdb

def average(series):

    return sum(series) / len(series)
    """
    implements the average of a pandas series from scratch
    suggested functions:
    len(list)
    sum(list)
    you should get the same result as calling .mean() on your series
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.mean.html
    See numpy documenation for implementation details:
    https://docs.scipy.org/doc/numpy/reference/generated/numpy.mean.html
    """
    pass

def standard_deviation(series):

    avg = average(series)
    ser_len = float(len(series))
    stdsum = 0
    for i in range(0, len(series)):
        stdsum += (series[i] - avg)**2

    return math.sqrt(stdsum / (ser_len - 1.0))
    """
    implements the sample standard deviation of a series from scratch
    you may need a for loop and your average function
    also the function math.sqrt
    you should get the same result as calling .std() on your data
    https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.std.html
    See numpy documenation for implementation details:
    https://docs.scipy.org/doc/numpy/reference/generated/numpy.std.html
    """
    pass

def median(series):

    ser_sort = np.sort(series)
    while len(ser_sort) > 2:
        ser_sort = ser_sort[1:len(ser_sort)-1]

    if len(ser_sort) == 1:
        return ser_sort[0]
    else:
        return (ser_sort[0] + ser_sort[1]) / 2
    """
    finds the median of the series from scratch
    you may need to sort your values and use
    modular division
    this number should be the same as calling .median() on your data
    See numpy documenation for implementation details:
    https://docs.scipy.org/doc/numpy/reference/generated/numpy.median.html
    https://pandas.pydata.org/pandas-docs/version/0.23.0/generated/pandas.Series.median.html
    """
    pass


import numpy as np

def Gaussian_data_gen(mean, variance):
    """
    Gaussian_data_gen: generate standard Gaussian data with given mean and variance via Box Muller method
    :param mean: Gaussian mean
    :param variance: Gaussian variance
    :return: number generate from standard Gaussian
    """
    # box-muller
    u = np.random.uniform(0.0, 1.0)
    v = np.random.uniform(0.0, 1.0)
    z = np.sqrt(-2.0 * np.log(u)) * np.sin(2 * np.pi * v)
    return mean + np.sqrt(variance) * z

def input_poly_basis():
    """
    input_poly_basis: input that is needed for polynomial basis linear model data generator
    :return basis num, gaussian mean, polynomial coefficients
    """
    n = int(input("Input n: "))
    a = float(input("Input a: "))
    w_str = input("Input w: ")
    w = list(map(float, w_str.split()))
    w = np.array(w).reshape(n, -1)  # [n,1]
    return n, a, w


# coding=utf-8
"""Basic linear algebra for IO Analysis."""

import numpy as np

def production_estimate(technology, demand):
    """Calculate production estimates that meets demand given technology matrix.

    Formula:
        X = (I - A) ^ -1 d

    Args;
        technology: technology matrix (ndarray).
        demand: demand vector (ndarray).

    Returns:
        production estimates (ndarray).
    """
    return leontief_inverse(technology) @ demand

def technology_matrix(flow, output):
    """Get technology matrix with given intersectoral flow matrix and output vector.

    Args:
        flow: intersectoral flow matrix (ndarray).
        output: economy output vector (ndarray).

    Returns:
        technology matrix (ndarray).
    """
    return flow / output

def mean_relative_error(ground_truth, estimate):
    """Calculate mean relative error of given estimate with respect to given ground truth vector.

    Args:
        ground_truth: true value of calculation (ndarray).
        estimate: model estimate (ndarray).

    Returns:
        mean relative error.
    """
    return np.mean((ground_truth - estimate) / ground_truth)

def leontief_inverse(technology):
    """Calculate Leontief inverse.

    Formula:
        L = (I - A) ^ -1

    Args:
        technology: technology matrix (ndarray).

    Returns:
        Leontief inverse (ndarray).
    """
    return np.linalg.inv(np.identity(technology.shape[0]) - technology)

def taylor_series_estimate(technology, demand, converge_threshold=0.1**10):
    """Calculate Leontief inverse estimate using matrix power series.

    Formula:
        (I - A) ^ -1 = I + A + A^2 + A^3 + A^4 + ...

    Args:
        technology: technology matrix (ndarray).
        demand: demand vector (ndarray).
        converge_threshold: level of convergence when algorithm will stop.

    Returns:
        n - number of rounds economy needs to converge with given threshold to balanced state.
    """
    convex_result = production_estimate(technology, demand)

    print("Calculating Taylor series estimate of Leontief inverse.")
    n = 0
    inverse_estimate = np.identity(tech.shape[0])
    taylor_expansion_term = tech
    while mean_relative_error(convex_result, inverse_estimate @ demand) > converge_threshold:
        inverse_estimate += taylor_expansion_term
        taylor_expansion_term = taylor_expansion_term @ tech
        print(n, convex_result, inverse_estimate @ demand)
        n += 1
    return n

if __name__ == '__main__':
    print("Linear Algebra for IO Analysis.")

    tech = np.array([[10/27, 2/15], [4/27, 2/15]])
    dem = np.array([15, 9])
    taylor_series_estimate(tech, dem)

    tech = np.array([[0.05, 0.2, 0.025], [0.05, 0.1, 0.025], [0.6, 0.6, 0.15]])
    dem = np.array([35 * 3 / 4, 36, 25 * 0.95])
    print(production_estimate(tech, dem))


import numpy as np
import scipy.spatial.distance as dist
from scipy import stats
from sklearn import neighbors
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis

class Question1(object):
    # lab1.ex5 (x^T)Ay = np.dot(np.dot(X, A), Y.T)
    # lab1.ex6 (x^T)Ax = np.sum(np.dot(X, A)*X, axis=1)
    def bayesClassifier(self, data, pi, means, cov):
        # labels = None
        invC = np.linalg.pinv(cov)
        objectiveFunction = np.log(pi) + np.dot(
            np.dot(means, invC), data.T).T - 0.5*np.sum(np.dot(means, invC)*means, axis=1).T
        labels = np.argmax(objectiveFunction, axis=1)
        return labels

    def classifierError(self, truelabels, estimatedlabels):
        boolean = truelabels != estimatedlabels
        # print(np.count_nonzero(boolean))
        error = (np.count_nonzero(boolean)) / len(truelabels)
        return error

class Question2(object):
    def trainLDA(self, trainfeat, trainlabel):
        # Assuming all labels up to nlabels exist.
        nlabels = int(trainlabel.max())+1
        pi = np.zeros(nlabels)            # Store your prior in here
        # Store the class means in here
        means = np.zeros((nlabels, trainfeat.shape[1]))
        # Store the covariance matrix in here
        cov = np.zeros((trainfeat.shape[1], trainfeat.shape[1]))
        # Put your code below
        for i in range(nlabels):
            boolean = i == trainlabel
            pi[i] = np.count_nonzero(boolean) / len(trainlabel)
            means[i] = np.sum(trainfeat[boolean], axis=0) / np.count_nonzero(boolean)
            # print(trainfeat[boolean])
            for j in trainfeat[boolean]:
                # print((j-pi[i]).shape)
                # print((j-pi[i]).T.shape)
                # print(np.dot((j-pi[i]), (j-pi[i]).T))
                cov += np.dot((j-means[i]).reshape((trainfeat.shape[1], 1)), (j-means[i]).reshape((trainfeat.shape[1], 1)).T)
        cov /= (len(trainlabel) - nlabels)

        # Don't change the output!
        return (pi, means, cov)

    def estTrainingLabelsAndError(self, trainingdata, traininglabels):
        q1 = Question1()
        # You can use results from Question 1 by calling q1.bayesClassifier(...), etc.
        # If you want to refer to functions under the same class, you need to use self.fun(...)
        pi, means, cov = self.trainLDA(trainingdata, traininglabels)
        esttrlabels = q1.bayesClassifier(trainingdata, pi, means, cov)
        trerror = q1.classifierError(traininglabels, esttrlabels)
        return (esttrlabels, trerror)

    def estValidationLabelsAndError(self, trainingdata, traininglabels, valdata, vallabels):
        q1 = Question1()
        # You can use results from Question 1 by calling q1.bayesClassifier(...), etc.
        # If you want to refer to functions under the same class, you need to use self.fun(...)
        pi, means, cov = self.trainLDA(trainingdata, traininglabels)
        estvallabels = q1.bayesClassifier(valdata, pi, means, cov)
        valerror = q1.classifierError(vallabels, estvallabels)
        # Don't change the output!
        return (estvallabels, valerror)

class Question3(object):
    def kNN(self, trainfeat, trainlabel, testfeat, k):
        distance = dist.cdist(testfeat, trainfeat, metric="euclidean")
        knn = np.argpartition(distance, k)[:, :k]
        labels = stats.mode(trainlabel[knn], axis=1)[0].reshape((testfeat.shape[0], ))
        # print(testfeat.shape)
        # print(labels.shape)
        return labels

    def kNN_errors(self, trainingdata, traininglabels, valdata, vallabels):
        q1 = Question1()
        trainingError = np.zeros(4)
        validationError = np.zeros(4)
        k_array = [1, 3, 4, 5]

        for i in range(len(k_array)):
            # Please store the two error arrays in increasing order with k
            # This function should call your previous self.kNN() function.
            # Put your code below
            esiTraLabels = self.kNN(trainingdata, traininglabels, trainingdata, k_array[i])
            esiValLabels = self.kNN(trainingdata, traininglabels, valdata, k_array[i])
            trainingError[i] = q1.classifierError(traininglabels, esiTraLabels)
            validationError[i] = q1.classifierError(vallabels, esiValLabels)

        # Don't change the output!
        return (trainingError, validationError)

class Question4(object):
    def sklearn_kNN(self, traindata, trainlabels, valdata, vallabels):
        classifier, valerror, fitTime, predTime = (None, None, None, None)
        q1 = Question1()
        import time
        classifier = neighbors.KNeighborsClassifier(n_neighbors=1, algorithm="ball_tree")
        fitTimeStart = time.time()
        classifier.fit(traindata, trainlabels)
        fitTimeEnd = time.time()
        predTimeStart = time.time()
        estimatedlabels = classifier.predict(valdata)
        predTimeEnd = time.time()
        valerror = q1.classifierError(vallabels, estimatedlabels)
        fitTime = fitTimeEnd - fitTimeStart
        predTime = predTimeEnd - predTimeStart
        # Don't change the output!
        return (classifier, valerror, fitTime, predTime)

    def sklearn_LDA(self, traindata, trainlabels, valdata, vallabels):
        classifier, valerror, fitTime, predTime = (None, None, None, None)
        q1 = Question1()
        import time
        classifier = LinearDiscriminantAnalysis()
        fitTimeStart = time.time()
        classifier.fit(traindata, trainlabels)
        fitTimeEnd = time.time()
        predTimeStart = time.time()
        estimatedlabels = classifier.predict(valdata)
        predTimeEnd = time.time()
        valerror = q1.classifierError(vallabels, estimatedlabels)
        fitTime = fitTimeEnd - fitTimeStart
        predTime = predTimeEnd - predTimeStart
        # Don't change the output!
        return (classifier, valerror, fitTime, predTime)

###


import numpy as np
import matplotlib.pylab as plt

a = np.array([1010, 1000, 990])

c = np.max(a) #1010

print(a - c)

print(np.exp(a - c)/np.sum(np.exp(a - c)))

#implement softmax function avoid overflow
def softmax(a):
    c = np.max(a)
    exp_a = np.exp(a - c) #aovid overflow
    sum_exp_a = np.sum(exp_a)
    y = exp_a / sum_exp_a

    return y

b = np.array([0.3, 2.9, 4.0])
y = softmax(b)
print(y)
print(np.sum(y))


#!env python
#coding=utf-8
#Author:  joshua_zero@outlook.com

import numpy as np
import scipy as sp
from scipy.optimize import leastsq
import matplotlib.pyplot as plt
import matplotlib
"""
class static_learn_cls1():
    def __init__(self):
        print("init")

    def real_func(self, x):
        return np.sin(2*np.pi*x)

    def fit_func(self, p, x):
        f = np.ploy1d(p)
        return f(x)

    def residuals_func(self, p, x, y):
        ret = fit_func(p,x) - y
        return ret
"""

def real_func(x):
    return np.sin(2*np.pi*x)

def fit_func(p, x):
    f = np.ploy1d(p)
    return f(x)

def residuals_func(p, x, y):
    ret = fit_func(p, x) - y
    return ret

def fitting(x, y, points, M=0):
    """
      M 为多项式的次数
    """
    #随机初始化多项式参数
    p_init = np.random.rand(M+1)
    #最小二乘法
    p_lsq = leastsq(residuals_func,p_init,args=(x,y)）
    print("Fitting Parameters: {}".format(p_lsq[0]))

    #show
    plt.plot(points, real_func(points), label='real')
    plt.plot(points, fit_func(p_lsq[0], points), label='fitted curve')
    plt.plot(x, y, 'bo', label='noise')
    plt.legend()
    plt.show()
    return p_lsq

def demo_test1():
    x = np.linspace(0,1,10)
    x_points = np.linspace(0,1,1000)
    y_ = real_func(x)
    y = [np.random.normal(0,0.1) + y1 for y1 in y_ ]
    """
      M 为多项式的次数
    """
    #随机初始化多项式参数
    p_init = np.random.rand(M+1)
    #最小二乘法
    p_lsq = leastsq(residuals_func,p_init,args=(x,y)）
    print("Fitting Parameters: {}".format(p_lsq[0]))

    #show
    plt.plot(points, real_func(points), label='real')
    plt.plot(points, fit_func(p_lsq[0], points), label='fitted curve')
    plt.plot(x, y, 'bo', label='noise')
    plt.legend()
    plt.show()
    return p_lsq
    #fitting(x,y,x_points, 0)

if __name__ == "__main__":
    demo_test1()


""" Test 1-D Karhunen-Loeve approximation: Brownian motion"""
import numpy as np
from simulation import KL_1DNys
import matplotlib.pyplot as plt
# some tunable parameters
N = 200 # order of the KL expansion
M = 200 # number of quadrature points
a, b = 0., 1. # interval endpoints
def Bm(s,t):# Brownian motion covariance
    return np.minimum(s,t)
X, phi, L = KL_1DNys(N,M,a,b,Bm)# simulation
#plot eigenvalues against exact: pi/L = (k-0.5)**2
L_exct = [(k + 0.5)**2 for k in range(N)] #+1 for python indexing
L_apprx = 1./(L[:N]*np.pi**2)
plt.plot(L_exct, label = "exact eigenvalues")
plt.plot(L_apprx,'x', label = "numerical eigenvalues")
plt.legend()
plt.ylabel(r' $\frac{1}{\lambda_k\pi^2}$')
plt.title(' Eigenvalues')
plt.show()
plt.close()
t= np.linspace(a,b,M)
#plot the first 6 eigenfunctions
for i in range(6):
    plt.subplot(2,3,i+1).set_title(r'$\phi_k$, k = {}'/format(i+1))
    exact = np.sqrt(2)*np.sin((k+0.5)*np.pi*t) # exact eigenfunc.
    apprx = np.abs(phi[:,k])*np.sign(exact) # approximate eigen, same sign as exact
    plt.plot(t, apprx,'x',label = "Numerical")
    plt.plot(t, exact, label = "exact")
    plt.legend()
plt.show()
plt.close()
plt.plot(t,X) # plot the Brownian motion
plt.title("Brownian motion KL expansion: order N = {]".format(N))
plt.show()


#!/usr/bin/env python3

import numpy as np
import abc
from scipy.special import expit

AVAILABLE_ACTIVATIONS = ["identity", "sigmoid", "relu", "tanh", "heaviside"]

AVAILABLE_OUTPUT_ACTIVATIONS = [
    "identity", "sigmoid", "softmax"]

AVAILABLE_COST_FUNCTIONS = ["mse", "log_loss", "exponential_cost",
                            "hellinger_distance",
                            "kullback_leibler_divergence",
                            "generalized_kullback_leibler_divergence",
                            "itakura_saito_distance"]

AVAILABLE_REGULARIZATIONS = ["l1", "l2", "elastic_net"]

# =============================================================================
# ACTIVATION FUNCTIONS
# =============================================================================

def sigmoid(x):
    """Sigmoidal activation function.

    Args:
        x (ndarray): weighted sum of inputs
    """
    return expit(x)
    # return 1.0/(1.0 + np.exp(-x))

def sigmoid_derivative(z):
    s = sigmoid(z)
    # print(s)
    return s*(1-s)

def identity(x):
    """Identity activation function. Input equals output.

    Args:
        x (ndarray): weighted sum of inputs.
    """
    return x

def identity_derivative(x):
    """Simply returns the derivative, 1, of the identity."""
    return 1.0

def softmax(z):
    """The Softmax activation function. Assures that no outliers can
    dominate too much.

    Numerically stable sosftmax
    https://eli.thegreenplace.net/2016/the-softmax-function-and-its-derivative/

    Args:
        x (ndarray): weighted sum of inputs.
    """

    # z_exp = np.exp(x)
    z_exp = np.exp((z - np.max(z)))
    return z_exp/np.sum(z_exp)

def softmax_derivative(x):
    """The derivative of the Softmax activation function.

    Args:
        x (ndarray): weighted sum of inputs.
    """

    S = softmax(x).reshape(-1,1)
    # return S - np.einsum("i...,j...->i...", S, S)
    return (np.diagflat(S) - np.dot(S, S.T)).sum(axis=1,keepdims=True)

def heaviside(x):
    """The Heaviside activation function.

    Args:
        x (ndarray): weighted sum of inputs
    """
    return np.where(x >= 0, 1, 0)

def heaviside_derivative(x):
    """The derivative of the Heaviside activation function.

    Args:
        x (ndarray): weighted sum of inputs
    """
    return np.zeros(x.shape)

def relu(x):
    """The rectifier activation function. Only activates if argument x is
    positive.

    Args:
        x (ndarray): weighted sum of inputs
    """
    # np.clip(x, 0, np.finfo(x.dtype).max, out=x)
    # return x
    return np.where(x >= 0, x, 0)

def relu_derivative(x):
    """The derivative of the tangens hyperbolicus activation function.

    Args:
        x (ndarray): weighted sum of inputs
    """
    return np.where(relu(x) > 0, 1, 0)

def tanh_(x):
    """The tangens hyperbolicus activation function.

    Args:
        x (ndarray): weighted sum of inputs
    """
    return np.tanh(x)

def tanh_derivative(x):
    """The derivative of the tangens hyperbolicus activation function.

    Args:
        x (ndarray): weighted sum of inputs
    """
    return 1 - tanh_(x)**2

# =============================================================================
# COST FUNCTIONS
# =============================================================================

class _BaseCost:
    """Base cost function class."""
    @staticmethod
    @abc.abstractmethod
    def __call__(a, y):
        """Returns cost function.

        Args:
            a (ndarray): layer output.
            y (ndarray): true output.
        Return:
            (float): cost function output.
        """
        return None

    @staticmethod
    @abc.abstractmethod
    def delta(a, y, x):
        return None

class MSECost(_BaseCost):
    @staticmethod
    def __call__(a, y):
        """Returns cost function.

        Args:
            a (ndarray): all layer outputs.
            y (ndarray): all true outputs.
        Return:
            (float): cost function output.
        """
        return 0.5*np.mean(np.linalg.norm(a - y, axis=1)**2, axis=0)

    @staticmethod
    def delta(a, y, a_prime):
        return (a - y)# * a_prime

class LogEntropyCost(_BaseCost):
    @staticmethod
    def __call__(a, y):
        """Returns cost function.

        Args:
            a (ndarray): layer output.
            y (ndarray): true output.
        Return:
            (float): cost function output.
        """
        return - np.mean(y*np.log(a))# + (1 - y)*np.log(1 - a))

    @staticmethod
    def delta(a, y, a_prime):
        return a - y # For softmax
        # return (- y / a) * a_prime # General expr, still a bit bugged smh

class ExponentialCost(_BaseCost):
    """Exponential cost function."""
    @staticmethod
    def cost(a, y, tau=0.1):
        """Returns cost function.

        Args:
            a (ndarray): layer output.
            y (ndarray): true output.
        Return:
            (float): cost function output.
        """
        return tau*np.exp(1/tau * np.sum((y-y_true)**2))

    @staticmethod
    def delta(a, y, a_prime, tau=0.1):
        """Exponential cost function gradient."""
        return 2/tau * (y-y_true)*exponential_cost(y, y_true, tau) * a_prime

# =============================================================================
# REGULARIZATIONS
# =============================================================================

def _l1(weights):
    """The L1 norm."""
    return np.linalg.norm(weights, ord=1)

def _l1_derivative(weights):
    """The derivative of the L1 norm."""
    # NOTE: Include this in report
    # https://math.stackexchange.com/questions/141101/minimizing-l-1-regularization
    return np.sign(weights)

def _l2(weights):
    """The L2 norm."""
    return 0.5*np.dot(weights, weights)

def _l2_derivative(weights):
    """The derivative of the L2 norm."""
    # NOTE: Include this in report
    # https://math.stackexchange.com/questions/2792390/derivative-of-
    # euclidean-norm-l2-norm
    return weights

def _elastic_net(weights):
    """The elastic net regularization, L_en = L1 + L2."""
    return np.linalg.norm(weights, ord=1) + 0.5*np.dot(weights, weights)

def _elastic_net_derivative(weights):
    """Derivative of elastic net is just L1 and L2 derivatives combined."""
    return np.sign(weights) + weights

class _BaseRegularization:
    """Base cost function class."""
    @staticmethod
    @abc.abstractmethod
    def __call__(weights):
        """Returns the regularization."""
        return None

    @staticmethod
    @abc.abstractmethod
    def delta(weights):
        return None

class L1Regularization(_BaseRegularization):
    @staticmethod
    def __call__(weights):
        """Returns the regularization."""
        return np.linalg.norm(weights, ord=1)

    @staticmethod
    def derivative(weights):
        return np.sign(weights)

class L2Regularization(_BaseRegularization):
    @staticmethod
    def __call__(weights):
        """Returns the regularization."""
        return 0.5*np.dot(weights, weights)

    @staticmethod
    def derivative(weights):
        return weights

class ElasticNetRegularization(_BaseRegularization):
    @staticmethod
    def __call__(weights):
        """Returns the regularization."""
        return np.linalg.norm(weights, ord=1) + 0.5*np.dot(weights, weights)

    @staticmethod
    def derivative(weights):
        return np.sign(weights) + weights


import copy
import itertools
import numpy as np

# Euclidean distance
def euclidean(A, B):
    """
    Calculate Euclidean distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = euclidean(A, B)
    """

    tmp = np.sum(np.square(A - B))
    return np.sqrt(tmp)

# Manhattan distance
def manhattan(A, B):
    """
    Calculate Manhattan (Taxicab) distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = manhattan(A, B)
    """

    tmp = np.sum(np.abs(A - B))
    return tmp

# for Hausdorff distance
def hausdorff_distance(A, B):
    min_h = np.inf
    for i, j in itertools.product(range(len(A)), range(len(B))):
        d = euclidean(A[i], B[j])
        if d < min_h:
            min_h = d
            min_ind = j

    max_h = -np.inf
    for i in range(len(A)):
        d = euclidean(A[i], B[min_ind])
        if d > max_h:
            max_h = d

    return max_h

# Hausdorff distance
def hausdorff(A, B):
    """
    Calculate Hausdorff distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = hausdorff(A, B)
    """

    ah = hausdorff_distance(A, B)
    bh = hausdorff_distance(B, A)
    return max(ah, bh)

# Correlation distance
def correlation(A, B):
    """
    Calculate Correlation distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = correlation(A, B)
    """

    numerator = np.sum((A - np.mean(A)) * (B - np.mean(B)))
    denominator = np.sqrt(np.sum((A - np.mean(A)) ** 2)) * np.sqrt(np.sum((B - np.mean(B)) ** 2))
    if denominator == 0:
        denominator = 1
    return 1 - (numerator / denominator)

# Chebyshev distance
def chebyshev(A, B):
    """
    Calculate Chebyshev distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = chebyshev(A, B)
    """

    max_h = -np.inf
    for i, j in itertools.product(range(len(A)), range(len(B))):
        d = np.abs(A[i] - B[j])
        if d > max_h:
            max_h = d

    return max_h

# Standardized euclidean distance
def std_euclidean(A, B):
    """
    Calculate Standardized Euclidean distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = std_euclidean(A, B)
    """

    tab_std = np.vstack((A, B))
    stdv = np.sum(np.square(tab_std - np.mean(tab_std, axis = 0)), axis = 0)
    stdv = np.sqrt(stdv / tab_std.shape[0])
    stdv[stdv == 0] = 1
    tmp = np.sum(np.square((A - B) / stdv))
    return np.sqrt(tmp)

# Cosine distance
def cosine(A, B):
    """
    Calculate Cosine distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = cosine(A, B)
    """

    numerator = np.sum(A * B)
    denominator = (np.sqrt(np.sum(np.square(A)))) * (np.sqrt(np.sum(np.square(B))))
    if denominator == 0:
        denominator = 1
    return 1 - (numerator / denominator)

# Cosine similarity measure
def csm(A, B):
    """
    Calculate Cosine similarity measure of distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ---------
    >>> distance = csm(A, B)
    """

    numerator = np.sum(A * B)
    denominator = (np.sqrt(np.sum(A))) * (np.sqrt(np.sum(B)))
    if denominator == 0:
        denominator = 1
    return numerator / denominator

# Squared Euclidean distance
def squared_euclidean(A, B):
    """
    Calculate Squared Euclidean distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = squared_euclidean(A, B)
    """

    tmp = np.sum(np.square(A - B))
    return tmp

# Sorensen or Bray-Curtis distance
def bray_curtis(A, B):
    """
    Calculate Bray-Curtis distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = bray_curtis(A, B)
    """

    numerator = np.sum(np.abs(A - B))
    denominator = np.sum(A + B)
    if denominator == 0:
        denominator = 1
    return numerator / denominator

# Canberra distance
def canberra(A, B):
    """
    Calculate Canberra distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = canberra(A, B)
    """

    numerator = np.abs(A - B)
    denominator = A + B
    denominator[denominator == 0] = 1
    tmp = np.sum(numerator / denominator)
    return tmp

# Lorentzian distance
def lorentzian(A, B):
    """
    Calculate Lorentzian distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = lorentzian(A, B)
    """

    tmp = np.sum(np.log(1 + np.abs(A - B)))
    return tmp

# Jaccard distance
def jaccard(A, B):
    """
    Calculate Jaccard distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    -----------
    >>> distance = jaccard(A, B)
    """

    numerator = np.sum(np.square(A - B))
    denominator = np.sum(A ** 2) + np.sum(B ** 2) - np.sum(A * B)
    if denominator == 0:
        denominator = 1
    return numerator / denominator

# Dice distance
def dice(A, B):
    """
    Calculate Dice distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = dice(A, B)
    """

    numerator = np.sum(np.square(A - B))
    denominator = np.sum(A ** 2) + np.sum(B ** 2)
    if denominator == 0:
        denominator = 1
    return numerator / denominator

# Bhattacharyya distance
def bhattacharyya(A, B):
    """
    Calculate Bhattacharyya distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ---------
    >>> distance = bhattacharyya(A, B)
    """

    value = (np.sum(np.sqrt(A * B)))**2
    if value == 0:
        tmp = 0
    else:
        tmp = -np.log(value)
    return tmp

# Hellinger distance
def hellinger(A, B):
    """
    Calculate Hellinger distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    -----------
    >>> distance = hellinger(A, B)
    """

    value = 1 - np.sum(np.sqrt(A * B))
    if value < 0:
        value = 0
    return 2 * np.sqrt(value)

# Matusita distance
def matusita(A, B):
    """
    Calculate Matusita distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = matusita(A, B)
    """

    value = 2 - 2 * (np.sum(np.sqrt(A * B)))
    if value < 0:
        value = 0
    return np.sqrt(value)

# Squared-chord distance
def squared_chord(A, B):
    """
    Calculate Squared-Chord distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ---------
    >>> distance = squared_chord(A, B)
    """

    tmp = np.sum(np.square(np.sqrt(A) - np.sqrt(B)))
    return tmp

# Pearson chi-square distance
def pearson_chi_square(A, B):
    """
    Calculate Pearson Chi Square distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ---------
    >>> distance = pearson_chi_square(A, B)
    """

    numerator = np.square(A - B)
    denominator = copy.deepcopy(B)
    denominator[denominator == 0] = 1
    tmp = np.sum(numerator / denominator)
    return tmp

# Squared chi-square distance
def squared_chi_square(A, B):
    """
    Calculate Squared Chi Sqaure distance between two vectors `A` and `B`.

    Parameters
    -----------
        A : ndarray
            First vector containing values
        B : ndarray
            Second vector containing values

    Returns
    --------
        float
            distance value between two vectors

    Examples
    ----------
    >>> distance = squared_chi_square(A, B)
    """

    numerator = np.square(A - B)
    denominator = A + B
    denominator[denominator == 0] = 1
    tmp = np.sum(numerator / denominator)
    return tmp


'''
This module contains tools for estimating numerical integrals.
'''

import numpy as np
import matplotlib.pyplot as plt

def visualizeRectangles(f, xmin, xmax, N):
    """

    Plot "N" rectangles between "xmin" and "xmax",
    to visualize what happens when we calculate
    a rectangular integral of the function "f".

    Arguments
        f    : a function to be integrated
        xmin : lower limit of integration
        xmax : upper limit of integration
        N    : number of rectangles to use

    Returns
        (nothing), but it makes a plot

    Example:
        def integrand(x)
            return x**2
        plotRectangles(integrand, 0, 2, 5)
    """

    # first, make a high-resolution plot of the function
    xsmooth = np.linspace(xmin, xmax, 500)
    plt.plot(xsmooth, f(xsmooth), linewidth=10, alpha=0.3, color='orchid')

    # next, make a grid spanning xmin to xmax, with N+1 elements
    x = np.linspace(xmin, xmax, N+1)

    # calculate the offsets from x[:-1] to x[1:]
    dx = np.diff(x)

    # loop over rectangles
    for i in range(N):

        # where are the left and right edges of the rectangle?
        left = x[i]
        right = x[i] + dx[i]

        # set the rectangle plotting style
        plotkw = dict(color='black', linewidth=2)

        # plot vertical line from 0 up to curve (left edge of rectangle)
        plt.plot([left, left], [0, f(x[i])], **plotkw)

        # plot another vertical line (right edge of rectangle)
        plt.plot([right, right], [0, f(x[i])], **plotkw)

        # plot horizontal line (top of rectangle)
        plt.plot([left, right], [f(x[i]), f(x[i])], **plotkw)

    # set the x and y labels
    plt.xlabel('x')
    plt.ylabel('f(x)')

    # make sure the plot displays to the screen
    plt.show()

def integrateRectangles(f, xmin, xmax, N):
    """

    Approximate the area under the function "f",
    from "xmin" to "xmax", using N rectangles.

    Arguments
        f    : a function to be integrated
        xmin : lower limit of integration
        xmax : upper limit of integration
        N    : number of rectangles to use

    Returns
        the numerical integral of "f"

    Example:
        def integrand(x)
            return x**2
        integrateRectangles(integrand, 0, 1, 5)
    """

    # These x values represent edges of rectangles.
    # We need (N+1) edges for N rectangles.
    x = np.linspace(xmin, xmax, N + 1)

    # The widths of the rectangles:
    dx = np.diff(x)  # equivalently x[1:] - x[:-1]

    # The heights of the rectangles (using the left of each pair of points)
    h = f(x[:-1])

    # The areas of each rectangle.
    A = dx * h

    # Add up all those areas.
    return np.sum(A)