# https://towardsdatascience.com/gradient-descent-the-learning-rate-and-the-importance-of-feature-scaling-6c0b416596e1

# linear regression model y = b + wx + e
# in our model use feature(x) to predict value of a label y
# 3 elements in our model:
# parameter b, bias or intercept, tells us the expected value of y when x is 0
# parameter w, the weight or slope which tells us how much y increases, on average, if we increase x by one unit
# e which is there to account for inherent noise

# Generate synthetic data: Let b = 1 and w = 2
# Generate feature x: use numpy rand to randomly generate 100(N) points between 0 and 1
# Then, we plug our feature (x) and our parameters b and w into our equation to compute our labels (y). But we need to add some Gaussian noise (epsilon) as well; otherwise, our synthetic dataset would be a perfectly straight line.

# We can generate noise using Numpy’s randn method, which draws samples from a normal distribution (of mean 0 and variance 1), and then multiplying it by a factor to adjust for the level of noise.
# Since I don’t want to add so much noise, I picked 0.1 as my factor.

#%%
import numpy as np

true_b = 1
true_w = 2
N = 100

# Data Generation
np.random.seed(42)
x = np.random.rand(N, 1)
epsilon = (.1 * np.random.randn(N, 1))
y = true_b + true_w * x + epsilon
# %%
# Train-Validation-Test Split

# Shuffles the indices
idx = np.arange(N)
np.random.shuffle(idx)

# Uses first 80 random indices for train
train_idx = idx[:int(N*.8)]
# Uses the remaining indices for validation
val_idx = idx[int(N*.8):]

# Generates train and validation sets
x_train, y_train = x[train_idx], y[train_idx]
x_val, y_val = x[val_idx], y[val_idx]

import matplotlib.pyplot as plt
plt.plot(x_train,y_train,'bo')
plt.xlabel("x_train")
plt.ylabel("y_train")
plt.show()

plt.plot(x_val,y_val,'ro')
plt.xlabel("x_val")
plt.ylabel("y_val")
plt.show()

# #%%
# Random Initialization
# In our example, we already know the true values of the parameters, but this will obviously never happen in real life: if we knew the true values, why even bother to train a model to find them?!
# OK, given that we’ll never know the true values of the parameters, we need to set initial values for them. How do we choose them? It turns out; a random guess is as good as any other.
# So, we can randomly initialize the parameters/weights (we have only two, b and w).
# %%
# Initializes parameters "b" and "w" randomly
np.random.seed(42)
b = np.random.randn(1)
w = np.random.randn(1)

# Our randomly initialized parameters are: b = 0.49 and w = -0.13. Are these parameters any good?
import numpy as np
import matplotlib.pyplot as plt

plt.plot(x_train,y_train,'bo')
plt.xlabel("x_train")
plt.ylabel("y_train")
plt.plot(x_train, w*x_train + b)
plt.show()

#Obviously not… but, exactly how bad are they? That's what the loss is for. Our goal will be to minimize it.
# %%
#Loss Surface
# After choosing a random starting point for our parameters, we use them to make predictions, compute the corresponding errors, and aggregate these errors into a loss. Since this is a linear regression, we’re using Mean Squared Error (MSE) as our loss. The code below performs these steps:
# Computes our model's predicted output - forward pass
yhat = b + w * x_train

# Computing the loss
# We are using ALL data points, so this is BATCH gradient
# descent. How wrong is our model? That's the error!
error = (yhat - y_train)

# It is a regression, so it computes mean squared error (MSE)
loss = (error ** 2).mean()

print(loss)
# %%
# We have just computed the loss (2.74) corresponding to our randomly initialized parameters (b = 0.49 and w = -0.13).
# Now, what if we did the same for ALL possible values of b and w?
# Well, not all possible values, but all combinations of evenly spaced values in a given range?

# We could vary b between -2 and 4, while varying w between -1 and 5, for instance, each range containing 101 evenly spaced points.
# If we compute the losses corresponding to each different combination of the parameters b and w inside these ranges,
# the result would be a grid of losses, a matrix of shape (101, 101).

# These losses are our loss surface, which can be visualized in a 3D plot,
# where the vertical axis (z) represents the loss values. If we connect the combinations
# of b and w that yield the same loss value, we’ll get an ellipse. Then, we can draw this
# ellipse in the original b x w plane (in blue, for a loss value of 3).
# This is, in a nutshell, what a contour plot does. From now on, we’ll always use the contour plot, instead of the corresponding 3D version.

# This is one of the nice things about tackling a simple problem like a linear
# regression with a single feature: we have only two parameters, and thus we can compute and visualize the loss surface.

# Cross-Sections
# Another nice thing is that we can cut a cross-section in the loss surface to check what the loss looks like if the other parameter were held constant.

# Updating Parameters
# Finally, we use the gradients to update the parameters.
# Since we are trying to minimize our losses,
# we reverse the sign of the gradient for the update.
# There is still another (hyper-)parameter to consider:
# the learning rate, denoted by the Greek letter eta (that looks like the letter n),
# which is the multiplicative factor that we need to apply to the gradient for the parameter update.

# We can also interpret this a bit differently: each parameter is going to have its
# value updated by a constant value eta (the learning rate), but this constant is going to be weighted by how much that parameter contributes to minimizing the loss (its gradient).
# Honestly, I believe this way of thinking about the parameter update makes more sense:
# first, you decide on a learning rate that specifies your step size, while the gradients
# tell you the relative impact (on the loss) of taking a step for each parameter.
# Then you take a given number of steps that’s proportional to that relative impact:
# more impact, more steps.

# Too big, for a learning rate, is a relative concept: it depends on how steep the curve is or,
# in other words, it depends on how big the gradient is.
# We do have many curves, many gradients: one for each parameter.
# But we only have one single learning rate to choose (sorry, that’s the way it is!).
# It means that the size of the learning rate is limited by the steepest curve.
# All other curves must follow suit, meaning, they’d be using a sub-optimal learning rate,
# given their shapes.
# The reasonable conclusion is: it is best if all the curves are equally steep, so the learning rate is closer to optimal for all of them!

# How do we achieve equally steep curves? I’m on it!
# First, let’s take a look at a slightly modified example, which I am calling the “bad” dataset:

# Scaling / Standardizing / Normalizing
# Different how? There is this beautiful thing called the StandardScaler, which transforms a feature in such a way that it ends up with zero mean and unit standard deviation.
# How does it achieve that? First, it computes the mean and the standard deviation of a given feature (x) using the training set (N points):

see link above for further details


import numpy as np
import accpm
import pandas as pd
import scipy.optimize as opt
from scipy.special import expit # The logistic sigmoid function

def cost(w, X, y, c=0):
    """
    Returns the cross-entropy error function with (optional) sum-of-squares regularization term.

    w -- parameters
    X -- dataset of features where each row corresponds to a single sample
    y -- dataset of labels where each row corresponds to a single sample
    c -- regularization coefficient (default = 0)
    """
    outputs = expit(X.dot(w)) # Vector of outputs (or predictions)

    return -( y.transpose().dot(np.log(outputs)) + (1-y).transpose().dot(np.log(1-outputs)) ) + c*0.5*w.dot(w)

def grad(w, X, y, c=0):
    """
    Returns the gradient of the cross-entropy error function with (optional) sum-of-squares regularization term.
    """
    outputs = expit(X.dot(w))
    return X.transpose().dot(outputs-y) + c*w

def train(X, y,c=0):
    """
    Returns the vector of parameters which minimizes the error function via the BFGS algorithm.
    """
    initial_values = np.zeros(X.shape[1]) # Error occurs if inital_values is set too high
    return opt.minimize(cost, initial_values, jac=grad, args=(X,y,c),
                        method='BFGS', options={'disp' : False, 'gtol' : 1e-03}).x

def predict(w, X):
    """
    Returns a vector of predictions.
    """
    return expit(X.dot(w))

def compute_accuracy(predictions, y):
    """
    predictions -- dataset of predictions (or outputs) from a model
    y -- dataset of labels where each row corresponds to a single sample
    """
    predictions = predictions.round()
    size = predictions.shape[0]
    results = predictions == y
    correct = np.count_nonzero(results)
    accuracy = correct/size
    return accuracy

def compute_weights(X_training, Y_training, iterations):
    weights = []

    size = X_training.shape[0]
    index = np.arange(size)
    np.random.shuffle(index)

    for i in range(1, iterations + 1):
        X_i, Y_i =  X_training[index[:i]], Y_training[index[:i]]
        weight = train(X_i, Y_i)
        weights.append(weight)

    return weights

def weights_matrix(n, iterations, X_training, Y_training):
    matrix_of_weights = []
    np.random.seed(1)
    for i in range(n):
        weights = compute_weights(X_training, Y_training, iterations)
        matrix_of_weights.append(weights)
    return np.array(matrix_of_weights)

def experiment(n, iterations, X_testing, Y_testing, X_training, Y_training):

    matrix_of_weights = weights_matrix(n, iterations, X_training, Y_training)
    matrix_of_accuracies = []

    for weights in matrix_of_weights:
        accuracies = []
        for weight in weights:
            predictions = predict(weight, X_testing)
            accuracy = compute_accuracy(predictions, Y_testing)
            accuracies.append(accuracy)
        matrix_of_accuracies.append(accuracies)

    matrix_of_accuracies = np.array(matrix_of_accuracies)
    sum_of_accuracies = matrix_of_accuracies.sum(axis=0)
    average_accuracies = sum_of_accuracies/n
    return average_accuracies


# Import functions and libraries
import cv2
import os
import numpy as np
import matplotlib.pyplot as plt
from scipy.fft import dct, idct

# set image file to ../data/00.bmp
# you are free to point to any other image files
IMG_FILE = os.path.join("..", "data", "zelda.bmp")

# read image file, img is a gray scale image
img_gray = cv2.imread(IMG_FILE, cv2.IMREAD_GRAYSCALE)
print(f"Reading {IMG_FILE}, img size(width,height): {img_gray.shape}")

def dct2(a):
    # 2D dct conversion
    # convert image a from spatial domain to frequency domain
    return dct(dct(a.T, norm='ortho').T, norm='ortho')

def idct2(a):
    # 2D idct converstion
    # convert image from freuqency domain back to spatial domain
    return idct(idct(a.T, norm='ortho').T, norm='ortho')

# create a variable to hold dct coefficients
img_size = img_gray.shape
# for forward 2d DCT on 8x8 block
dct_8x8 = np.zeros(img_size)
for i in np.r_[:img_size[0]:8]:
    for j in np.r_[:img_size[1]:8]:
        # Apply DCT to the image every 8x8 block of it.
        dct_8x8[i:(i+8), j:(j+8)] = dct(img_gray[i:(i+8), j:(j+8)])

# now inverse 2d DCT on 8x8 block
dct_8x8_reconstructed = np.zeros(img_size)
for i in np.r_[:img_size[0]:8]:
    for j in np.r_[:img_size[1]:8]:
        # Apply inverse DCT to the DCT results every 8x8 block of it.
        dct_8x8_reconstructed[i:(i+8), j:(j+8)
                              ] = idct(dct_8x8[i:(i+8), j:(j+8)])

# Threshold (TRY YOUR THRESHOLD!!!!)
thresh = 0.01
# discard those coefficients below threshold
dct_thresh = dct_8x8 * (abs(dct_8x8) > (thresh*np.max(dct_8x8)))
percent_nonzeros = np.sum(dct_thresh != 0.0) / (img_size[0]*img_size[1]*1.0)
print(f"Keeping only {percent_nonzeros*100.0} of the DCT coefficients")

# now inverse 2d DCT on 8x8 block for thie threashold-ed DCT coefficients
dct_8x8_reconstructed_th = np.zeros(img_size)
for i in np.r_[:img_size[0]:8]:
    for j in np.r_[:img_size[1]:8]:
        # Apply inverse DCT to the DCT results every 8x8 block of it.
        dct_8x8_reconstructed_th[i:(i+8), j:(j+8)
                                 ] = idct(dct_thresh[i:(i+8), j:(j+8)])

# specify a position
pos = 20
print(f"Image data at pos {pos, pos} to {pos+8, pos+8}")
print(img_gray[pos:pos+8, pos:pos+8])
print(f"DCT coefficients at pos {pos, pos} to {pos+8, pos+8}")
print(dct_8x8[pos:pos+8, pos:pos+8])
print(f"DCT coefficients with threshold at pos {pos, pos} to {pos+8, pos+8}")
print(dct_thresh[pos:pos+8, pos:pos+8])

plt.gray()
plt.subplot(131)
plt.imshow(img_gray)
plt.title('original image')
plt.subplot(132)
plt.imshow(dct_8x8_reconstructed)
plt.title('reconstructed image (DCT+IDCT) 8x8 block')
plt.subplot(133)
plt.imshow(dct_8x8_reconstructed_th)
plt.title('reconstructed image with threshold')
plt.show()

# Question 1:
# think about changing threshold of DCT?
# change thresh at Line 47 to see how this value affect final output


import scipy.stats
import numpy as np

def main():
    a = [[81, 82, 79, 81, 78, 89, 92, 87, 85, 86],
         [71, 72, 72, 66, 72, 77, 81, 77, 73, 79],
         [76, 79, 77, 76, 78, 89, 87, 84, 87, 87]]
    alpha = 0.05
    group_num = len(a)
    ingroup_num = len(a[0])
    group_mean = [np.mean(ele) for ele in a]
    print("group_mean=", group_mean)
    grand_mean = np.mean(group_mean)
    print("grand_mean=", grand_mean)
    # SSA处理效应
    SSA = sum([ingroup_num * ((ele - grand_mean) ** 2) for ele in group_mean])
    # MSA
    MSA = SSA / (group_num - 1)

    print('df1=', group_num - 1)
    print('SSA=', SSA)
    print('MSA=', MSA)

    SSE = 0
    for i in range(group_num):
        for j in range(ingroup_num):
            SSE += (a[i][j] - group_mean[i]) ** 2

    MSE = SSE / (group_num * (ingroup_num - 1))
    print(group_num * (ingroup_num - 1))
    print('MSE=', MSE)
    F_practical = MSA / MSE
    print('F_practical=', F_practical)
    F_expected = scipy.stats.f.ppf(1 - alpha, group_num - 1, group_num * (ingroup_num - 1))
    print('F_expected=', F_expected)
    print('df2', group_num * (ingroup_num - 1))
    # 计算P
    P = scipy.stats.f.cdf(F_practical, group_num - 1, group_num * (ingroup_num - 1))
    print('p', P)
    if F_practical > F_expected:
        return False
    return True

if __name__ == '__main__':
    print(main())


import numpy as np
import matplotlib.pyplot as plt

np.random.seed(42)

X = 2*np.random.rand(100, 1)
y = 4+3*X+np.random.randn(100, 1)

X_b = np.c_[np.ones((100, 1)), X]

X_new = np.array(([0], [2]))
X_new_b = np.c_[np.ones((2, 1)), X_new]

theta_path_bgd = []

def plot_gradient_descent(theta, eta, theta_path=None):
    m = len(X_b)
    plt.plot(X, y, "b.")
    n_iterations = 1000
    for iteration in range(n_iterations):
        if iteration < 10:
            y_predict = X_new_b.dot(theta)
            style = "b-" if iteration > 0 else "r--"
            plt.plot(X_new, y_predict, style)
        gradients = 2/m * X_b.T.dot(X_b.dot(theta) - y)
        theta = theta - eta * gradients
        if theta_path is not None:
            theta_path.append(theta)
    plt.xlabel("$x_1$", fontsize=18)
    plt.axis([0, 2, 0, 15])
    plt.title(r"$\eta = {}$".format(eta), fontsize=16)

np.random.seed(42)
theta = np.random.randn(2, 1)  # random initialization

plt.figure(figsize=(10, 4))
plt.subplot(131)
plot_gradient_descent(theta, eta=0.02)
plt.ylabel("$y$", rotation=0, fontsize=18)
plt.subplot(132)
plot_gradient_descent(theta, eta=0.1, theta_path=theta_path_bgd)
plt.subplot(133)
plot_gradient_descent(theta, eta=0.5)
plt.show()


# Import functions and libraries
import cv2
import os
import numpy as np
import matplotlib.pyplot as plt
from scipy.fft import dct, idct

# set image file to ../data/00.bmp
# you are free to point to any other image files
IMG_FILE = os.path.join("..", "data", "zelda.bmp")

# read image file, img is a gray scale image
img_gray = cv2.imread(IMG_FILE, cv2.IMREAD_GRAYSCALE)
print(f"Reading {IMG_FILE}, img size(width,height): {img_gray.shape}")

def dct2(a):
    # 2D dct conversion
    # convert image a from spatial domain to frequency domain
    return dct(dct(a.T, norm='ortho').T, norm='ortho')

def idct2(a):
    # 2D idct converstion
    # convert image from freuqency domain back to spatial domain
    return idct(idct(a.T, norm='ortho').T, norm='ortho')

# create a variable to hold dct coefficients
img_size = img_gray.shape
# for forward 2d DCT on 8x8 block
dct_8x8 = np.zeros(img_size)
for i in np.r_[:img_size[0]:8]:
    for j in np.r_[:img_size[1]:8]:
        # Apply DCT to the image every 8x8 block of it.
        dct_8x8[i:(i+8), j:(j+8)] = dct(img_gray[i:(i+8), j:(j+8)])

# specify a position
pos = 400
# Extract a block from image
plt.subplot(1, 2, 1)
plt.imshow(img_gray[pos:pos+8, pos:pos+8], cmap='gray')
print(f"Image data at pos {pos, pos} to {pos+8, pos+8}")
print(img_gray[pos:pos+8, pos:pos+8])
plt.title("An 8x8 Image block")

# Display the dct of that block
plt.subplot(1, 2, 2)
plt.imshow(dct_8x8[pos:pos+8, pos:pos+8], cmap='gray',
           vmax=np.max(dct_8x8)*0.01, vmin=0)
print(f"DCT coefficients at pos {pos, pos} to {pos+8, pos+8}")
print(dct_8x8[pos:pos+8, pos:pos+8])
plt.title("An 8x8 DCT block")
plt.show()

# Display entire DCT
plt.figure()
plt.imshow(dct_8x8, cmap='gray', vmax=np.max(dct_8x8)*0.01, vmin=0)
plt.title("8x8 DCTs of the image")
plt.show()

# Question 1:
# think about the other postion?
# change Pos at Line 39 to see image pixels and DCT coefficients in other block
# make sure pos+8 is less than 512 as our image size is 512


import sys
import time
#import csv
import numpy as np
from numpy.linalg import inv

''' Returns the minimum weigth vector and the offset '''
def Regression(X, Y, lambda_value) :
    ''' Adding the columns of Ones '''
    col_Ones = np.ones((len(X), 1))
    col_Ones.shape
    X = np.append(col_Ones, X, 1)
    X.shape

    #print(len(X))

    I = np.identity(len(X[0]))
    #print(len(I))
    I[0][0] = 0

    temp_1 = np.dot(np.transpose(X), X)
    temp_2 = lambda_value*I
    temp_3 = np.dot(np.transpose(X), Y)

    w = np.dot(inv(temp_1 + temp_2), temp_3)
    b = w[0][0]
    w = np.delete(w, (0), axis=0)
    return {'w': w, 'b': b}

''' Returns the square error of a given data set with a trained w and b '''
def Loss_Function(X, Y, w, b) :
    L = 0

    temp_1 = np.dot(X, w)
    total = 0

    for i in range(0, len(X)) :
        ele = temp_1[i][0]
        curr = ele+b
        curr = (Y[i][0] - curr)
        curr = curr * curr
        total = total + curr

    L = total / len(X)
    return {'L': L}

''' Loading the data and slicing it to X and Y componenets '''
data = np.loadtxt(sys.argv[1]) #$Input

''' Splitting the data up '''
X = data[:,:len(data[0])-1] #Data
Y = data[:, len(data[0])-1:] #Class

X_train = X[0: int(.80*len(X)), :]
Y_train = Y[0: int(.80*len(Y)), :]

X_test = X[int(.80*len(X))+1 : len(X)+1,:]
Y_test = Y[int(.80*len(Y))+1 : len(Y)+1,:]

''' Start Timer '''
start = time.time()
''' Start Timer '''

''' Function Calls '''
Regression_Result = Regression(X_train, Y_train, float(sys.argv[2]))
Loss_Result = Loss_Function(X_test, Y_test, Regression_Result["w"], Regression_Result["b"])

''' End Timer '''
end = time.time()
''' End Timer '''

total = start - end

''' Outputs '''
print("--- Running Linear Regression ---")
print("--- Last updated June 8th, 2017 ---")
print("--- By Patrick Le @ lepatrick714 ---")

print("\n\n")

print("---- WEIGHTS ----")
print(Regression_Result["w"])

print("\n\n")

print("---- OFFSET ----")
print(Regression_Result["b"])

print("\n\n")

print("---- ACCURACY ---")
print("{} % Correct".format( 1*(100 - Loss_Result["L"]) ))

print("\n\n")

print("---- INACCURACY ----")
print("{} % Incorrect".format(Loss_Result["L"]))
print("\n\n")

print("--- END OF PROGRAM ---")


import numpy as np

from numba import jit
from numba import float64

@jit((float64[:], float64), nopython=True, nogil=True)
def ewma(arr_in, halflife):
    r"""Exponentialy weighted moving average specified by a decay ``halflife``
    to provide better adjustments for small windows via:

        y[t] = (x[t] + (1-a)*x[t-1] + (1-a)^2*x[t-2] + ... + (1-a)^n*x[t-n]) /
               (1 + (1-a) + (1-a)^2 + ... + (1-a)^n).

    Parameters
    ----------
    arr_in : np.ndarray, float64
        A single dimenisional numpy array
    halflife : float64
        Specify decay in terms of half-life,
        :math:`\alpha = 1 - exp(log(0.5) / halflife),\text{ for } halflife > 0`

    Returns
    -------
    np.ndarray
        The EWMA vector, same length / shape as ``arr_in``

    Examples
    --------
    >>> import pandas as pd
    >>> a = np.arange(5, dtype=float)
    >>> exp = pd.DataFrame(a).ewm(halflife=2, adjust=True).mean()
    >>> np.isclose(ewma(a, 2), exp.values.ravel())  # isclose for floating point err
    True
    """
    n = arr_in.shape[0]
    ewma = np.empty(n, dtype=float64)
    decay_coefficient = 1 - np.exp(np.log(0.5) / halflife)
    w = 1
    ewma_old = arr_in[0]
    ewma[0] = ewma_old
    for i in range(1, n):
        w += (1 - decay_coefficient)**i
        ewma_old = ewma_old * (1 - decay_coefficient) + arr_in[i]
        ewma[i] = ewma_old / w
    return ewma

@jit((float64[:], float64), nopython=True, nogil=True)
def _ewma_infinite_hist(arr_in, halflife):
    r"""Exponentialy weighted moving average specified by a decay ``halflife``
    assuming infinite history via the recursive form:

        (2) (i)  y[0] = x[0]; and
            (ii) y[t] = a*x[t] + (1-a)*y[t-1] for t>0.

    This method is less accurate that ``_ewma`` but
    much faster:

        In [1]: import numpy as np, bars
           ...: arr = np.random.random(100000)
           ...: %timeit bars._ewma(arr, 2)
           ...: %timeit bars._ewma_infinite_hist(arr, 2)
        3.74 ms ± 60.2 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
        262 µs ± 1.54 µs per loop (mean ± std. dev. of 7 runs, 1000 loops each)

    Parameters
    ----------
    arr_in : np.ndarray, float64
        A single dimenisional numpy array
    halflife : float64
        Specify decay in terms of half-life,
        :math:`\alpha = 1 - exp(log(0.5) / halflife),\text{ for } halflife > 0`

    Returns
    -------
    np.ndarray
        The EWMA vector, same length / shape as ``arr_in``

    Examples
    --------
    >>> import pandas as pd
    >>> a = np.arange(5, dtype=float)
    >>> exp = pd.DataFrame(a).ewm(halflife=2, adjust=False).mean()
    >>> np.array_equal(_ewma_infinite_hist(a, 2), exp.values.ravel())
    True
    """
    n = arr_in.shape[0]
    ewma = np.empty(n, dtype=float64)
    decay_coefficient = 1 - np.exp(np.log(0.5) / halflife)
    ewma[0] = arr_in[0]
    for i in range(1, n):
        ewma[i] = arr_in[i] * decay_coefficient + ewma[i - 1] * (1 - decay_coefficient)
    return ewma

def ewma_2d(x, halflife):
    """
    Exponentially Weighted Moving Average,
    optimised for 2D data sets.
    """
    assert x.ndim == 2
    assert np.isfinite(halflife) and halflife > 0

    decay_coefficient = np.exp(np.log(0.5) / halflife)
    out = np.empty_like(x, dtype=np.float64)

    for i in range(out.shape[0]):
        if i == 0:
            out[i, :] = x[i, :]
            sum_prior = 1
            first_weight = 1
        else:
            first_weight *= decay_coefficient
            sum_i = sum_prior + first_weight

            for j in range(x.shape[1]):
                out[i, j] = (decay_coefficient * out[i - 1, j] * sum_prior + x[i, j]) / sum_i

            sum_prior = sum_i

    return out

if __name__ == '__main__':
    import pytest
    pytest.main([__file__])


'''
Computing the variance
100xp

It is important to have some understanding of what commonly-used functions are doing
under the hood. Though you may already know how to compute variances, this is a beginner
course that does not assume so. In this exercise, we will explicitly compute the variance
of the petal length of Iris veriscolor using the equations discussed in the videos. We will
then use np.var() to compute it.

Instructions
-Create an array called differences that is the difference between the petal lengths
(versicolor_petal_length) and the mean petal length. The variable versicolor_petal_length
is already in your namespace as a NumPy array so you can take advantage of NumPy's vectorized operations.
-Square each element in this array. For example, x**2 squares each element in the array x.
Store the result as diff_sq.
-Compute the mean of the elements in diff_sq using np.mean(). Store the result as variance_explicit.
-Compute the variance of versicolor_petal_length using np.var(). Store the result as variance_np.
-Print both variance_explicit and variance_np in one print call to make sure they are consistent.
'''
import numpy as np

versicolor_petal_length = np.array([4.7,  4.5,  4.9,  4.,  4.6,  4.5,  4.7,  3.3,  4.6,  3.9,  3.5,
                                    4.2,  4.,  4.7,  3.6,  4.4,  4.5,  4.1,  4.5,  3.9,  4.8,  4.,
                                    4.9,  4.7,  4.3,  4.4,  4.8,  5.,  4.5,  3.5,  3.8,  3.7,  3.9,
                                    5.1,  4.5,  4.5,  4.7,  4.4,  4.1,  4.,  4.4,  4.6,  4.,  3.3,
                                    4.2,  4.2,  4.2,  4.3,  3.,  4.1])

# Array of differences to mean: differences
differences = np.array(versicolor_petal_length -
                       np.mean(versicolor_petal_length))

# Square the differences: diff_sq
diff_sq = differences ** 2

# Compute the mean square difference: variance_explicit
variance_explicit = np.mean(diff_sq)

# Compute the variance using NumPy: variance_np
variance_np = np.var(versicolor_petal_length)

# Print the results
print(variance_explicit, variance_np)


"""
# !/usr/bin/env python
# -*- coding: utf-8 -*-
@Time        : 2022/2/23 16:43
@Author      : shengdl999links@gmail.com
@ProjectName : udacity-program_self_driving_car_engineer_v1.0_source.0
@File        : iou.py
"""

import numpy as np
from utils import get_data, check_results

def calculate_iou(gt_bbox, pred_bbox):
    """
    calculate iou
    args:
    - gt_bbox [array]: 1x4 single gt bbox
    - pred_bbox [array]: 1x4 single pred bbox
    returns:
    - iou [float]: iou between 2 bboxes
    参考： https://blog.csdn.net/briblue/article/details/91366128
    """
    x_min = np.max([gt_bbox[0], pred_bbox[0]])
    y_min = np.max([gt_bbox[1], pred_bbox[1]])
    x_max = np.min([gt_bbox[2], pred_bbox[2]])
    y_max = np.min([gt_bbox[3], pred_bbox[3]])

    # calculate the intersection between two area
    # 两个区域面积的交集
    intersection = max(0, x_max - x_min) * max(0, y_max - y_min)

    # calculate the area
    gt_area = (gt_bbox[2] - gt_bbox[0]) * (gt_bbox[3] - gt_bbox[1])
    # calculate the predication area
    pred_area = (pred_bbox[2] - pred_bbox[0]) * (pred_bbox[3] - pred_bbox[1])

    # 两个区域面积的并集
    union_area = gt_area + pred_area - intersection
    # calculate the iou score [0, 1]
    iou = intersection / union_area
    return iou

def calculate_ious(gt_bboxes, pred_bboxes):
    """
    calculate ious between 2 sets of bboxes
    args:
    - gt_bboxes [array]: Nx4 ground truth array
    - pred_bboxes [array]: Mx4 pred array
    returns:
    - iou [array]: NxM array of ious
    """

    ious = np.zeros((gt_bboxes.shape[0], pred_bboxes.shape[0]))
    for i, gt_bbox in enumerate(gt_bboxes):
        for j, pred_bbox in enumerate(pred_bboxes):
            ious[i, j] = calculate_iou(gt_bbox, pred_bbox)
    return ious

if __name__ == "__main__":
    ground_truth, predictions = get_data()
    # get bboxes array
    filename = 'segment-1231623110026745648_480_000_500_000_with_camera_labels_38.png'
    gt_bboxes = [g['boxes'] for g in ground_truth if g['filename'] == filename][0]
    gt_bboxes = np.array(gt_bboxes)
    pred_bboxes = [p['boxes'] for p in predictions if p['filename'] == filename][0]
    pred_boxes = np.array(pred_bboxes)

    ious = calculate_ious(gt_bboxes, pred_boxes)
    check_results(ious)


import numpy as np
import matplotlib.pyplot as plt
PI = np.pi

def sinc_interp(xin, r=1.0):
    xin = np.array(xin)
    N = xin.size
    M = int(N * r)

    u = np.linspace(0, N, M)
    v = np.linspace(0, N, N)
    xout = []
    for i in u:
        xout.append(np.sum(xin * sinc(v - i)))
    return np.array(xout), np.array(u)

def sinc(x):
    x = np.array(x)
    eps = 1.0e-8
    y = np.where(np.abs(PI * x) < eps, 1.0,
                 np.sin(PI * x + eps) / (PI * x + eps))
    return y

Ns = 100.0
T = 64.0
Fs = Ns / T
Ts = 1.0 / Fs

t0 = np.linspace(0, T, Ns)

a = 0.9
x0 = t0 * a**t0

r = 8.0
t1 = np.linspace(0, T, Ns * r)

x1 = np.interp(t1, t0, x0)

x2, n2 = sinc_interp(x0, r=r)

t2 = n2 * Ts

plt.figure()
plt.plot(t0, x0, '-ob')
plt.plot(t1, x1, '-g')
plt.plot(t2, x2, '-r')
plt.grid()
plt.xlabel('Time/s')
plt.ylabel('Amplitude')
plt.title('interpolation')
plt.legend(['original', 'np.interp', 'sinc_interp'])

plt.show()


#########################
##                     ##
## Irving Gomez Mendez ##
##     May 04, 2021    ##
##                     ##
#########################

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.linear_model import LogisticRegression
from scipy.stats import norm

# Data from Hosmer, D.W. & Lemeshow, S. (1989) Applied logistic regression. Wiley
# Data are ages (in years) and indicator of significant damage in the coronary of 100 people
dat = pd.read_csv('Heart_Disease_vs_Age.csv')
n = dat.shape[0]
X = np.vstack([np.ones(n), dat['age']]).T
y = dat['chd']
alpha = 0.05

# Logistic regression by hand
b = [0,0] # initial values

tolm   = 1e-6       # tolerance (minimum norm of the difference of the betas)
iterm  = 100        # maximum number of iterations
tolera = 1          # initialize tolera
itera  = 0          # initialize ittera
histo  = b          # initialize beta upgrade

while((tolera > tolm) and (itera < iterm)):
  p      = 1/(1+np.exp(-X @ b))
  W      = np.diag(p*(1-p))
  delta  = np.linalg.solve(X.T @ W @ X, X.T @ (y-p))
  b      = b+delta
  tolera = np.sqrt(sum(delta**2))
  histo  = np.vstack([histo, b])
  itera  = itera+1

n0 = 50
x0 = np.linspace(dat['age'].min(), dat['age'].max(), n0)
X0 = np.vstack([np.ones(n0), x0]).T
p0 = 1/(1+np.exp(-X0 @ b))

aux_conf = np.sqrt(np.diag(X0 @ np.linalg.solve(X.T @ W @ X, X0.T)) * (p0*(1-p0))**2)

upp_conf = np.minimum(p0 + norm.ppf(1-alpha/2)*aux_conf, 1)
low_conf = np.maximum(p0 - norm.ppf(1-alpha/2)*aux_conf, 0)

plt.figure(figsize=(10,7.5))
plt.fill_between(x0, low_conf, upp_conf, facecolor='yellow', alpha=0.5, label='Confidence interval')
plt.plot(dat['age'], dat['chd'], 'o')
plt.plot(x0, p0, 'r'),
plt.xlabel('Age')
plt.ylabel('Prob. of Coronary Heart Disease')

# Using LogisticRegression
logreg = LogisticRegression(penalty='none')
logreg.fit(np.array(dat['age'])[:,None],y)

b_star = logreg.coef_[0]
b_star
logreg.intercept_

plt.figure(figsize=(10,7.5))
plt.plot(dat['age'], dat['chd'], 'o')
plt.plot(x0, (logreg.predict_proba(x0[:,None]).T)[1,], 'r'),
plt.xlabel('Age')
plt.ylabel('Prob. of Coronary Heart Disease')

###


'''
 Perform the following operations on an array of mobile phones prices 6999, 7500, 11999,
27899, 14999, 9999.
a. Create a 1d-array of mobile phones prices
b. Convert this array to float type
c. Append a new mobile having price of 13999 Rs. to this array
d. Reverse this array of mobile phones prices
e. Apply GST of 18% on mobile phones prices and update this array.
f. Sort the array in descending order of price
g. What is the average mobile phone price
h. What is the difference b/w maximum and minimum price

'''

import numpy as np

arr = np.array([6999, 7500, 11999,
27899, 14999, 9999]);  #Create a 1d-array of mobile phones prices

print(arr)

arr = arr.astype(np.float); #Convert this array to float type

print(arr)

arr2 = np.append(arr , 13999) #Append a new mobile having price of 13999 Rs. to this array

print(arr2)

arr2 = arr2[::-1] #Reverse this array of mobile phones prices

print(arr2)

m = (arr2 != 0)

arr2[m] = arr2[m] - (arr2[m] * (18 /100)) #Apply GST of 18% on mobile phones prices and update this array.

print(arr2)

arr2 = np.sort(arr2)[::-1] #Sort the array in descending order of price

print(arr2)

print(np.average(arr2)) #What is the average mobile phone price

print(arr2.max() - arr2.min()) #What is the difference b/w maximum and minimum price

'''
Output

[ 6999  7500 11999 27899 14999  9999]
[ 6999.  7500. 11999. 27899. 14999.  9999.]
[ 6999.  7500. 11999. 27899. 14999.  9999. 13999.]
[13999.  9999. 14999. 27899. 11999.  7500.  6999.]
[11479.18  8199.18 12299.18 22877.18  9839.18  6150.    5739.18]
[22877.18 12299.18 11479.18  9839.18  8199.18  6150.    5739.18]
10940.439999999999
17138.0

'''


###  this function to calculate probability density function for multivariate hypergeometric function
import numpy as np
import scipy.special as sc
def mhygepdf(m,n):
    #MHYGEPDF Multivariate hypergeometric probability density function.
    # X = MHYGEPDF(M,N) returns the multivariate hypergeometric probability
    # density function at M with integer parameters in N. Note: The density
    # function is zero unless the elements in M are integers.
    #
    # If there are m_i elements of class i in a population and you take n
    # elements at random without replacement, then the number of elements of
    # each class in the sample (x_1,x_2,...,x_c) has the multivariate
    # hypergeometric distribution. This has the same relationship to the
    # multinomial distribution that the hypergeometric distribution has to the
    # binomial distribution--the multinomial distribution is the
    # 'with-replacement' distribution and the multivariate hypergeometric is
    # the 'without-replacement' distribution.
    #
    # Syntax: function x = mhygepdf(m,n)
    #
    # Inputs:
    # m - list of total (population) elements of class i
    # n - list of sampled elements of class i
    #
    # Output:
    # x - multivariate hypergeometric probability for values x_1,x_2,...,x_c
    #
    # Example. From the example given on the web Wikipedia: The Free
    #  Encyclopedia [http://en.wikipedia.org/wiki/Hypergeometric_distribution].
    #  Suppose there are 5 black, 10 white, and 15 red marbles in an urn. You
    #  reach in and randomly select six marbles without replacement. What is
    #  the probability that you pick exactly two of each color?
    #
    #             m = [5,10,15]; n = [2,2,2];
    #
    # Calling on Matlab the function:
    #             x = mhygepdf(m,n)
    #
    # Answer is: (in format long)
    #
    # x =
    #
    #   0.07957559681698
    # Creted by vikash Pandey

    ##### test 'm and must be a list of non-negative and integers.'

    m_int=[isinstance(item, int) for item in m ]
    n_int=[isinstance(item, int) for item in n ]
    if (not all(m_int)):
        print ('M must be a list of non-negative and integers.')
        exit()

    if (not all(n_int)):
        print ('N must be a list of non-negative and integers.')
        exit()
    ### We are going to multivariate compute hypergeometric pdf

    M = np.sum(m)
    N = np.sum(n)
    o = len(m)

    ###
    A = []
    for i in range(o):
        a = np.exp(sc.gammaln(m[i] + 1) - sc.gammaln(n[i] +1) - sc.gammaln(m[i] - n[i] + 1));
        A.append(a)
    ###
    B = np.exp(sc.gammaln(M + 1) - sc.gammaln(N +1) - sc.gammaln(M - N + 1));

    x = max(np.cumprod(A))/B;

    return x


import matplotlib.pyplot as plt  # 加载matplotlib用于数据的可视化
from sklearn.decomposition import PCA  # 加载PCA算法包
from sklearn.datasets import load_iris

import numpy as np
from sklearn import manifold, datasets

def pca_visualize_tutorial():
    data = load_iris()
    y = data.target
    x = data.data
    pca = PCA(n_components=3)  # 加载PCA算法，设置降维后主成分数目为2
    reduced_x = pca.fit_transform(x)  # 对样本进行降维

    red_x, red_y = [], []
    blue_x, blue_y = [], []
    green_x, green_y = [], []

    for i in range(len(reduced_x)):
        if y[i] == 0:
            red_x.append(reduced_x[i][0])
            red_y.append(reduced_x[i][1])

        elif y[i] == 1:
            blue_x.append(reduced_x[i][0])
            blue_y.append(reduced_x[i][1])

        else:
            green_x.append(reduced_x[i][0])
            green_y.append(reduced_x[i][1])

    # 可视化
    plt.scatter(red_x, red_y, c='r', marker='x')
    plt.scatter(blue_x, blue_y, c='b', marker='D')
    plt.scatter(green_x, green_y, c='g', marker='.')
    plt.show()

def tsne_visualize_tutorial():

    digits = datasets.load_digits(n_class=6)
    X, y = digits.data, digits.target
    n_samples, n_features = X.shape

    '''显示原始数据'''
    n = 20  # 每行20个数字，每列20个数字
    img = np.zeros((10 * n, 10 * n))
    for i in range(n):
        ix = 10 * i + 1
        for j in range(n):
            iy = 10 * j + 1
            img[ix:ix + 8, iy:iy + 8] = X[i * n + j].reshape((8, 8))
    plt.figure(figsize=(8, 8))
    plt.imshow(img, cmap=plt.cm.binary)
    plt.xticks([])
    plt.yticks([])
    plt.show()

    '''t-SNE'''
    tsne = manifold.TSNE(n_components=2, init='pca', random_state=501)
    X_tsne = tsne.fit_transform(X)

    print("Org data dimension is {}. Embedded data dimension is {}".format(X.shape[-1], X_tsne.shape[-1]))

    '''嵌入空间可视化'''
    x_min, x_max = X_tsne.min(0), X_tsne.max(0)
    X_norm = (X_tsne - x_min) / (x_max - x_min)  # 归一化
    plt.figure(figsize=(8, 8))
    for i in range(X_norm.shape[0]):
        plt.text(X_norm[i, 0], X_norm[i, 1], str(y[i]), color=plt.cm.Set1(y[i]),
                 fontdict={'weight': 'bold', 'size': 9})
    plt.xticks([])
    plt.yticks([])
    plt.show()

if __name__ == '__main__':
    # pca_visualize_tutorial()
    tsne_visualize_tutorial()


import math
import numpy as np

### Square

print('\nExample of taking the square')
print(math.pow(2,2))
print(np.square(2))

print('\nExample of taking the cube')
print(math.pow(2,3))
print(np.power(2,3))

print('\nExample of taking the square root')
print(math.sqrt(4))
print(np.sqrt(4))

print('\nExample of taking the exponent')
print(math.exp(3))
print(np.exp(3))

# Using numpy with lists

print('\nExample of squaring elements in a list')
print(np.square([1, 2, 3, 4, 5]))

print('\nExample of taking the square root of a list')
print(np.sqrt([1, 4, 9, 16, 25]))

print('\nExamples of taking the cube of a list')
print(np.power([1, 2, 3, 4, 5], 3))

# Using numpy in a function

def numpy_example(x):
    return np.exp(x)

x = [1, 2, 3, 4, 5]
print(numpy_example(x))


import numpy as np

def accuracy_score(y_true, y_pred):
    return np.mean(y_true == y_pred)

def r2_score(y_true, y_pred):
    sse = np.sum(np.square(y_true - y_pred))
    s_yy = np.sum(np.square(y_true - np.mean(y_true)))
    return 1 - sse / s_yy

def mse_score(y_true, y_pred):
    return np.mean(np.square(y_true - y_pred))

def rmse_score(y_true, y_pred):
    return np.sqrt(np.mean(np.square(y_true - y_pred)))

def mae_score(y_true, y_pred):
    return np.mean(np.abs(y_true - y_pred))

def poisson_score(y_true, y_pred):
    return np.mean(y_true * np.log(y_true / y_pred) - y_true + y_pred)

def binary_cross_entropy(a, y):
    return -np.mean(np.nan_to_num(y * np.log(a) + (1 - y) * np.log(1 - a)))

def cross_entropy(a, y, axis):
    return -np.mean(np.sum(y * np.nan_to_num(np.log(a)), axis=axis))

def gini_score(p):
    return 1 - np.sum(np.square(p))

def entropy_score(p):
    return -np.sum(p * np.log(p))

def misclassification_score(p):
    return 1 - np.max(p)


import math
from numba import jit
import time, timeit

def length(a, c):
    return (2 * (a*c)**(1/2) * (4*a*c+1)**(1/2) + math.asinh(2 * (a*c)**(1/2))) / (2*a)

@jit
def f1(a, c, l):
    '''Returns f(a).'''
    return (2 * (a*c)**(1/2) * (4*a*c+1)**(1/2) + math.asinh(2 * (a*c)**(1/2))) - 2*a*l

@jit
def f2(a, c, l):
    '''Returns f'(a).'''
    return 2 * ((c * (1/a+4*c))**(1/2) - l)

@jit
def find_a(c, l):
    '''Approximates a using Newton's method.'''

    # The initial guess for a.
    a = 1.0

    # The error term.
    error = 0

    # Run 30 iterations of Newton's method
    for i in range(30):
        a1 = a - f1(a, c, l) / f2(a, c, l)
        error = abs(a1 - a)
        a = a1

        # Convergence completed early.
        if error < 3e-10:
            return a

    return a

def generate(c, l):
    '''Returns a lambda function for quick evaluation of z(x).'''

    a = find_a(c, l)
    return lambda x: a * x**2 - c

l = 5
print('Actual length:', l)

c = 0.5
a = find_a(c, l)

print('Computed a:', a)
guessed = length(a, c)
print('Result length:', guessed)
print('Error:', (guessed-l)/l * 100)


from sympy import sympify, Symbol

x = Symbol('x') #Se comvierte la variable a simbolico

#Funcion principal del metodo
def metodo_nuevo(x_0, f, tol, iter_max):

    if tol <= 0:
        raise  ValueError('Tolerancia no debe ser cero.') #Error al ingresar una tolerancia con las condiones no aptas
    f = sympify(f)
    k = 0
    x_k = x_0
    error = tol+1

    D = []
    A = []

    # Cliclo de iteraciones segun tolerancia o maximo de iteraciones
    while error > tol and k < iter_max:
        x_k = calc_sgte(x_k, f)
        error = calc_error(x_k, f)

        D.append(k)
        A.append(error)

        k+=1

    return [x_k, error, k, D, A]

#Calcula el error
def calc_error(x_k, f):
    return abs(f.subs(x, x_k))

#Calcula la variable Z al ser una expresion tan grande
def calc_z(x_k, f):
    f_k = f.subs(x, x_k)
    den = f.subs(x, x_k + f_k) - f.subs(x, x_k - f_k)
    z = x_k + (2* (f_k**2)/den)
    return z

#Realiza el calculo de Xk+1
def calc_sgte(x_k, f):
    f_k = f.subs(x, x_k)
    z = calc_z(x_k, f)#Invoca la funcion para calcular el valor de Z
    den = f.subs(x, x_k + f_k) - f.subs(x, x_k - f_k)
    x_sgte = x_k - 2*(f_k)*(f.subs(x,z) - f_k)/den
    return round(x_sgte,6) #Restinge la cantidad de decimales a 5

if __name__ == '__main__':
    # Cada caso de prueba recibe: valor de X0, Ecuacion y numero de iteraciones maximas
    # Test 1. Ejemplo (b)
    result = metodo_nuevo(1.5, 'x^3+4*x^2-10', iter_max=7)
    print(result)

    # Test 2. Ejemplo (h)
    result = metodo_nuevo(0.7, 'x^2-exp(x)-3*x+2', iter_max=7)
    print(result)

    # Test 3. Ejemplo (e)
    result = metodo_nuevo(2, 'x^3-10', iter_max=7)
    print(result)

    # Test 4. Ejemplo (c)
    result = metodo_nuevo(1, 'cos(x)-x', iter_max=6)
    print(result)


# Demonstration of L2 regularization
# Also comparison between maximum likelihood estimation(MLE) & maxiumum a posteriori solution(MAP) is done
# All equations are from theory notebook

# Code Flow:
    # 1. Import all relevant libraries.
    # 2. Generate sample data with some outliers.
    # 3. Plot the generated data.
    # 4. Add bias term.
    # 5. Plot the MLE solution.
    # 6. Plot the L2-Reg solution & compare it with MLE.

# 1. Import all relevant libraries:
import numpy as np
import matplotlib.pyplot as plt

# 2. Generate sample data:
N = 50
X = np.linspace(0,10,N)
Y = 0.5*X + np.random.randn(N)

# make outliers
Y[-1] += 30
Y[-2] += 30

# 3.Plot the data
plt.scatter(X, Y)
plt.show()

# 4.Add bias term
X = np.vstack([np.ones(N), X]).T

# 5. Plot the MLE solution.
w_ml = np.linalg.solve(X.T.dot(X), X.T.dot(Y))
Yhat_ml = X.dot(w_ml)
plt.figure(1)
plt.scatter(X[:,1], Y,label = 'data',c = 'green')
plt.plot(X[:,1], Yhat_ml,label = 'MLE',c = 'red')
plt.xlabel('X')
plt.ylabel('Y')
plt.legend()
plt.title('MLE Solution')

# 6. Plot the regularized solution
# probably don't need an L2 regularization this high in many problems
# everything in this example is exaggerated for visualization purposes
l2 = 1000.0
w_map = np.linalg.solve(l2*np.eye(2) + X.T.dot(X), X.T.dot(Y))
Yhat_map = X.dot(w_map)
plt.figure(2)
plt.scatter(X[:,1], Y,label = 'data',c = 'green')
plt.plot(X[:,1], Yhat_ml, label='MLE',c = 'red')
plt.plot(X[:,1], Yhat_map, label='MAP', c = 'black')
plt.xlabel('X')
plt.ylabel('Y')
plt.legend()
plt.title('MLE vs. MAP')

1.Linear Regression/1.Code - Using Theory/4.L2-Regularization.py


# Cross-validation for parameter tuning, model selection, and feature selection
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.neighbors import KNeighborsClassifier
from sklearn import metrics

# read dataset
iris = load_iris()

# create X and y data
X, y = iris.data, iris.target

# train/test split
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=3)

# check classification of KNN with K=5
knn = KNeighborsClassifier(n_neighbors=5)
knn.fit(X_train,y_train)
print(metrics.accuracy_score(y_test,knn.predict(X_test)))

# rerun different train/test split
# initial (state=4) = 97.36%
# next (state=1) = 100%
# final (state=3) = 94.74%

# Steps for k-fold Cross-validation
# choose a number for k partitions of equal size (folds)
# use fold 1 as testing and rest as training selection
# train model and calc testing accuracy_score
# report k times with different fold each time

# GOAL: select best tuning parametes for KNN on iris dataset
from sklearn.model_selection import cross_val_score
# 10-fold cross-validation with K=5 for KNN
knn = KNeighborsClassifier(n_neighbors=5)
scores = cross_val_score(knn, X, y, cv=10, scoring='accuracy')
print(scores)

# use the average as an estimator for out-of-sample accuracy
print(scores.mean())

# search for an optimal value of k for KNN
k_range = range(1,31)
k_scores = []
for k in k_range:
    knn = KNeighborsClassifier(n_neighbors=k)
    scores = cross_val_score(knn, X, y, cv=10, scoring='accuracy')
    k_scores.append(scores.mean())
print(k_scores)

# plot with matplotlib
import matplotlib.pyplot as plt

plt.plot(k_range, k_scores)

# GOAL: compare best KNN to logreg
knn = KNeighborsClassifier(n_neighbors=20)
print(cross_val_score(knn, X, y, cv=10, scoring='accuracy').mean())

# 10-fold cross validation with logreg
from sklearn.linear_model import LogisticRegression
logreg = LogisticRegression()
print(cross_val_score(logreg, X, y, cv=10, scoring='accuracy').mean())

# cross validation for feature selection
import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression

# load data
df = pd.read_csv('http://www-bcf.usc.edu/~gareth/ISL/Advertising.csv', index_col=0)

# feature col list
feature_cols = ['TV', 'radio', 'newspaper']
# select X and y data
X = df[feature_cols]
y = df.sales

# 10-fold cross validation with all three features
ln = LinearRegression()
scores = cross_val_score(ln, X, y, cv=10, scoring='mean_squared_error')
print(scores)
# fix the signs of MSE scores
mse_scores = -scores
print(mse_scores)
# convert from mse to RMSE
rmse_scores = np.sqrt(mse_scores)
print(rmse_scores)
print(rmse_scores.mean())

# 10-fold cross val for two features
feature_cols = ['TV', 'radio']
X = df[feature_cols]
print(np.sqrt(-cross_val_score(ln,X,y,cv=10,scoring='mean_squared_error')).mean())


"""
Created by : Rafay Khan
Twitter: @RafayAK
"""

import numpy as np

"""
Contains a bunch of Cost functions.
This file implementations of :
    - Binary Cross_entropy Cost function
        * compute_binary_cost(Y, P_hat) -> "unstable"
        * compute_stable_bce_cost(Y, Z) -> "stable"
        * computer_keras_like_bce_cost(Y, P_hat, from_logits=False) -> stable
    - Mean Squared Error Cost function
"""

def compute_bce_cost(Y, P_hat):
    """
    This function computes Binary Cross-Entropy(bce) Cost and returns the Cost and its
    derivative.
    This function uses the following Binary Cross-Entropy Cost defined as:
    => (1/m) * np.sum(-Y*np.log(P_hat) - (1-Y)*np.log(1-P_hat))

    Args:
        Y: labels of data
        P_hat: Estimated output probabilities from the last layer, the output layer

    Returns:
        cost: The Binary Cross-Entropy Cost result
        dP_hat: gradient of Cost w.r.t P_hat

    """
    m = Y.shape[1]  # m -> number of examples in the batch

    cost = (1/m) * np.sum(-Y*np.log(P_hat) - (1-Y)*np.log(1-P_hat))
    cost = np.squeeze(cost)  # remove extraneous dimensions to give just a scalar (e.g. this turns [[17]] into 17)

    dP_hat = (1/m) * (-(Y/P_hat) + ((1-Y)/(1-P_hat)))

    return cost, dP_hat

def compute_stable_bce_cost(Y, Z):
    """
    This function computes the "Stable" Binary Cross-Entropy(stable_bce) Cost and returns the Cost and its
    derivative w.r.t Z_last(the last linear node) .
    The Stable Binary Cross-Entropy Cost is defined as:
    => (1/m) * np.sum(max(Z,0) - ZY + log(1+exp(-|Z|)))
    Args:
        Y: labels of data
        Z: Values from the last linear node

    Returns:
        cost: The "Stable" Binary Cross-Entropy Cost result
        dZ_last: gradient of Cost w.r.t Z_last
    """
    m = Y.shape[1]

    cost = (1/m) * np.sum(np.maximum(Z, 0) - Z*Y + np.log(1+ np.exp(- np.abs(Z))))
    dZ_last = (1/m) * ((1/(1+np.exp(- Z))) - Y)  # from Z computes the Sigmoid so P_hat - Y, where P_hat = sigma(Z)

    return cost, dZ_last

def compute_keras_like_bce_cost(Y, P_hat, from_logits=False):
    """
    This function computes the Binary Cross-Entropy(stable_bce) Cost function the way Keras
    implements it. Accepting either probabilities(P_hat) from the sigmoid neuron or values direct
    from the linear node(Z)

    Args:
        Y: labels of data
        P_hat: Probabilities from sigmoid function
        from_logits: flag to check if logits are being provided or not(Default: False)

    Returns:
        cost: The "Stable" Binary Cross-Entropy Cost result
        dZ_last: gradient of Cost w.r.t Z_last

    """
    if from_logits:
        # assume that P_hat contains logits and not probabilities
        return compute_stable_bce_cost(Y, Z=P_hat)

    else:
        # Assume P_hat contains probabilities. So make logits out of them

        # First clip probabilities to stable range
        EPSILON = 1e-07
        P_MAX = 1- EPSILON  # 0.9999999

        P_hat = np.clip(P_hat, a_min=EPSILON, a_max=P_MAX)

        # Second, Convert stable probabilities to logits(Z)
        Z = np.log(P_hat/(1-P_hat))

        # now call compute_stable_bce_cost
        return compute_stable_bce_cost(Y, Z)

def compute_mse_cost(Y, Y_hat):
    """
    This function computes Mean Squared Error(mse) Cost and returns the Cost and its derivative.
    This function uses the Squared Error Cost defined as follows:
    => (1/2m)*sum(Y - Y_hat)^.2

    Args:
        Y: labels of data
        Y_hat: Predictions(activations) from a last layer, the output layer

    Returns:
        cost: The Squared Error Cost result
        dY_hat: gradient of Cost w.r.t Y_hat

    """
    m = Y.shape[1]  # m -> number of examples in the batch

    cost = (1 / (2 * m)) * np.sum(np.square(Y - Y_hat))
    cost = np.squeeze(cost)  # remove extraneous dimensions to give just a scalar (e.g. this turns [[17]] into 17)

    dY_hat = -1 / m * (Y - Y_hat)  # derivative of the squared error cost function

    return cost, dY_hat


import numpy as np
from numba_stats import truncnorm, norm

def test_pdf():
    x = np.linspace(-1, 5, 10)
    xmin = 1
    xmax = 4
    mu = 2
    sigma = 3
    got = truncnorm.pdf(x, xmin, xmax, mu, sigma)
    expected = norm.pdf(x, mu, sigma) / (
        norm.cdf(xmax, mu, sigma) - norm.cdf(xmin, mu, sigma)
    )
    expected[x < xmin] = 0
    expected[x > xmax] = 0
    np.testing.assert_allclose(got, expected)

def test_cdf():
    x = np.linspace(-1, 5, 10)
    xmin = 1
    xmax = 4
    mu = 2
    sigma = 3
    got = truncnorm.cdf(x, xmin, xmax, mu, sigma)
    expected = (norm.cdf(x, mu, sigma) - norm.cdf(xmin, mu, sigma)) / (
        norm.cdf(xmax, mu, sigma) - norm.cdf(xmin, mu, sigma)
    )
    expected[x < xmin] = 0
    expected[x > xmax] = 1
    np.testing.assert_allclose(got, expected)

def test_ppf():
    expected = np.linspace(0, 1, 10)
    xmin = 1
    xmax = 4
    mu = 2
    sigma = 3
    x = truncnorm.ppf(expected, mu, sigma, xmin, xmax)
    got = truncnorm.cdf(x, mu, sigma, xmin, xmax)
    np.testing.assert_allclose(got, expected, atol=1e-14)


import numpy as np

def elm_fit(X, t, h):
    W = np.random.uniform(-.5, .5, (h, len(X[0])))
    Hi = X @ W.T
    H = 1 / (1 + np.exp(-Hi))
    Ht = H.T
    Hp = np.linalg.inv(Ht @ H) @ Ht
    b = Hp @ t
    y = H @ b
    mape = sum(abs(y - t) / t * 100) / len(t)

    return W, b, mape

def elm_predict(X, W, b):
    Hi = X @ W.T
    H = 1 / (1 + np.exp(-Hi))
    y = H @ b

    return y

def test():
    X_train = [[1, 1, 1],
               [1, 0, 1],
               [1, 1, 0],
               [1, 1, 0],
               [0, 1, 0],
               [0, 0, 0],
               [0, 1, 0],
               [1, 1, 0],
               [0, 0, 0]]

    y_train = [1, 1, 1, 2, 2, 2, 3, 3, 3]
    h = 3

    W, b, mape = elm_fit(X_train, y_train, h)

    X_test = [[1, 0, 1],
              [1, 1, 0],
              [0, 1, 0]]

    y_test = [1, 1, 3]
    y_predict = elm_predict(X_test, W, b)
    mape = sum(abs(y_predict - y_test) / y_test * 100) / len(y_test)

    print(y_test)
    print(y_predict)
    print(mape)

if __name__ == '__main__':
    test()


import numpy as np
import matplotlib.pyplot as plt
import math
#Utility functions for two-dimensional vectors

def convert_angle_to_plus_minus_pi(angle):
    return(np.arctan2(np.sin(angle),np.cos(angle)))

def direction_sign(v0, v1):
    #Input: unit vectors in 2D
    #Check whether vector 1 has the same direction as vector 2 (the angle between them is smaller than 90 degrees)
    return np.sign(np.dot(v0,v1))

def angle_between_lines(v0,v1):
    # Input: unit vectors describing lines in 2D
    # Compute angle between lines (directionless). Output range is from -pi/2 to pi/2
    sin_theta = np.cross(v0,v1)
    return np.arcsin(sin_theta)

def angle_between_vectors(v0,v1):
    # Input: unit vectors in 2D
    # Compute angle between vectors (with direction) Output range is from -pi to pi
    sin_theta = np.cross(v0,v1)
    cos_theta = np.dot(v0,v1)
    return math.atan2(sin_theta,cos_theta)

def signed_distance_point_to_line(point, line_point, line_vector):
    #Input: 2D points, 2D unit vector
    point_vector = point-line_point
    d = np.cross(line_vector,point_vector)
    return d

def closest_point(x0,y0,xs,ys):
    #Compute distance between points, given as lists or np arrays of x and y coordinates
    x0 = np.repeat(x0,len(xs))
    y0 = np.repeat(y0,len(ys))
    sum_squared_error = 0.5*np.sqrt((x0-xs)**2 +(y0-ys)**2)
    ind = np.argmin(sum_squared_error)

    return ind

def line_to_next_point(point_ind, xs, ys, step = 1):
    #Line segment directly from current to next point
    #Returns line on point, unit vector form
    point = np.array([xs[point_ind],ys[point_ind]])
    next_point = np.array([xs[point_ind+step],ys[point_ind+step]])
    vector = next_point-point
    vector = vector/np.linalg.norm(vector)

    return point,vector

def line_fit_from_points(point_ind,xs,ys,forward_window=20, backward_window=0):
    #Fit line on points from sliding window in a list
    xs = np.array(xs)
    ys = np.array(ys)

    start_ind = point_ind-backward_window
    stop_ind = point_ind + forward_window+1

    if start_ind >= 0 and stop_ind <= len(xs):
        xpoints = xs[start_ind : stop_ind]
        ypoints = ys[start_ind : stop_ind]
    elif start_ind < 0:
        xpoints = xs[point_ind : stop_ind]
        ypoints = ys[point_ind : stop_ind]
    elif stop_ind > len(xs):
        xpoints = xs[start_ind : point_ind]
        ypoints = ys[start_ind : point_ind]

    #Compute linear fit
    coef = np.polyfit(xpoints,ypoints,deg=1)
    fit_fn = np.poly1d(coef)

    # Get fitted line on point,vector form
    line_point = np.array([xs[point_ind],fit_fn(xs[point_ind])])
    if point_ind + 1 < len(xs):
        next_line_point = np.array([xs[point_ind+1],fit_fn(xs[point_ind+1])])
        line_vector = next_line_point - line_point
    else:
        next_line_point = np.array([xs[point_ind-1],fit_fn(xs[point_ind-1])])
        line_vector = line_point - next_line_point
    line_vector = line_vector/np.linalg.norm(line_vector)

    return line_point,line_vector

def line_XY_intersection(point, direction):
    """
    Finds intersection (x,y) between XY plane and a line.

    point: some point on the line (e.g. camera position)
    direction: some vector pointing along the line

    Assumes numpy arrays.
    """
    r = point[2]/direction[2]
    xy = point[0:2] - r*direction[0:2]
    return xy

def orient2d(a, b, c):
    """
    The Orient2D geometric predicate.

    c can be 2x1 or 2xN: c[0] is x values, c[1] is y values

    The output is a scalar number which is:
        > 0 if abc forms an angle in (0, pi), turning left,
        < 0 if abc forms an angle in (0, -pi), turning right,
        = 0 if abc forms an angle equal to 0 or pi, or is straight.

    Alternatively, it can be interpreted as:
        > 0 if c is to the left of the line ab,
        < 0 if c is to the right of the line ab,
        = 0 if c is on the line ab,
    in all cases seen from above, and along ab.

    The algorithm do not use exact arithmetics, and may fail within
    machine tolerances.
    """
    return (a[0]-c[0])*(b[1]-c[1]) - (a[1]-c[1])*(b[0]-c[0])


import numpy as np

# counts the number of iterations until the function diverges or
# returns the iteration threshold that we check until
def mandelbrot(c, threshold):
    z = complex(0, 0)
    for iteration in range(threshold):
        z = (z * z) + c
        if abs(z) > 2:
            return iteration
    return 0

# takes the iteration limit before declaring function as convergent and
# takes the density of the atlas
# create atlas, plot mandelbrot set, display set
def mandelbrot_set(xmin=-2.25, xmax=0.75, ymin=-1.5, ymax=1.5,
                   threshold=120, xn=1000):
    yn = int((ymax - ymin) / (xmax - xmin) * xn)
    # location and size of the atlas rectangle
    real_axis = np.linspace(xmin, xmax, xn)
    imaginary_axis = np.linspace(ymin, ymax, yn)
    # 2-D array to represent mandelbrot atlas
    atlas = np.empty((yn, xn))

    # color each point in the atlas depending on the iteration count
    for ix, cx in enumerate(real_axis):
        for iy, cy in enumerate(imaginary_axis):
            c = complex(cx, cy)
            atlas[iy, ix] = mandelbrot(c, threshold)
    return atlas

if __name__ == '__main__':
    import sys
    import matplotlib.pyplot as plt
    # time to party!!
    # -0.748768, -0.748718, 0.0650619375, 0.0650900625, threshold=2048, xn=1920
    atlas = mandelbrot_set(threshold=255, xn=1024)
    if len(sys.argv) > 1:
        # save mandelbrot set
        fn = sys.argv[1]
        plt.imsave(f'{fn}.jpg', atlas, cmap='gnuplot2')
    else:
        # plot and display mandelbrot set
        plt.imshow(atlas, cmap='gnuplot2')
        plt.show()


import numpy as np
def elimination(a,b):
    row, col = np.shape(a)
    if row == col:        #condition to check square matrix
        n = len(b)
        x  = np.zeros(n, float)
        for p in range(n-1):       #index the pivots
            if a[p,p] < 1e-12: #putting threshold condition to check pivot not to be zero
                for i in range(p+1,n):  #index elements under pivot
                    if np.fabs(a[i,p]) > np.fabs(a[p,p]):  #non zero rows under pivot element
                        a[[p,i]] = a[[i,p]]   #interchange rows and columns
                        b[[p,i]] = b[[i,p]]
                        break
            for i in range(p+1,n):
                if a[i,p] == 0:    #skip the row
                    continue
                factor = a[p,p]/a[i,p]
                for j in range(p,n):
                    a[i,j] = a[p,j] - a[i,j]*factor
                b[i] = b[p] - b[i]*factor     #elimination
        '''check rank of matrix to determine consistency'''
        rank_a = np.linalg.matrix_rank(a)
        aug_matrix = np.column_stack((a,b.T))
        aug_matrix_rank = np.linalg.matrix_rank(aug_matrix)
        if rank_a == aug_matrix_rank:
            if aug_matrix_rank == np.shape(b)[0]:
                print("System of equations has unique solution")
                print("Augmented Matrix \n" ,np.column_stack((a,b.T)))
                '''Back Substitution'''
                x[n-1] = b[n-1] / a[n-1,n-1]
                for i in range(n-2,-1, -1):
                    sum_ax = 0
                    for j in range(i+1, n):
                        sum_ax += a[i, j]* x[j]
                    x[i] = (b[i] - sum_ax) / a[i,i]

                print('The solution of the system: ')
                print(x)
                numpy_solution = np.linalg.solve(a, b)
                print("Solution by inbuilt function", numpy_solution)
            elif aug_matrix_rank < np.shape(b)[0]:
                print("system has infinitely many solutions")
        elif rank_a < aug_matrix_rank:
            print("System is inconsistent")
    else:
        print("Number of rows should be equal to number of columns")

if __name__ == "__main__":
    print("E.g. 1")
    a = np.array([[1,-2,1],
                [2,-5,4],
                [1,-4,6]], float)
    b = np.array([5,-3,10], float)
    elimination(a,b)

    print("E.g. 2")
    a = np.array([[1,-2,1,-1,1],
                [2,-5,4,1,-1],
                [1,-4,6,2,-1]], float)
    b = np.array([5,-3,10], float)
    elimination(a,b)
    print("E.g. 3")
    a = np.array([[1,-5,4],
                [1,-5,3],
                [2,-10,13]], float)
    b = np.array([3,6,5], float)
    elimination(a,b)

    print("E.g. 4")
    a = np.array([[12,-3],
                [-12,3]], float)
    b = np.array([6,-6], float)
    elimination(a,b)


from autograd import grad
from scipy.optimize import fsolve
from scipy.optimize import minimize

def objective(X):
    x, y, z = X
    return x**2 + y**2 + z**2

def eq(X):
    x, y, z = X
    return 2 * x - y + z - 3

sol = minimize(objective, [1, -0.5, 0.5],
               constraints={'type': 'eq', 'fun': eq})

print(sol)

def F(L):
    'Augmented Lagrange function'
    x, y, z, lmd = L
    return objective([x, y, z]) - lmd * eq([x, y, z])

dfdL = grad(F, 0)  # Gradients of the Lagrange function

def obj(L):
    # Find L that returns all zeros in this function.
    x, y, z, _ = L
    dFdx, dFdy, dFdz, dFdlam = dfdL(L)
    return [dFdx, dFdy, dFdz, eq([x, y, z])]

x, y, z, _ = fsolve(obj, [0.0, 0.0, 0.0, 1.0])
print(f'The answer is at {x, y, z}')


# -*- coding: utf-8 -*-
"""
Created on Sun Jan 17 12:07:50 2021

@author: zayn
"""

"""
Linear regression implementation.

"""

import numpy as np

class LReg:
    """ implementation of multi variable Linear Regression,
        with results derived using steepest descent and
        theoretical normal equations.
    """

    def __init__(self, training_data=[], use_norm=True):
        """Create a linear regression classifier.
        :param training_data: training feature data.
        :param use_norm: Whether to use normalizing when calculating linear regression.
        """
        if use_norm:
            self.normed_training_data, self.mean, self.std = self.featureNormalize(training_data)
        else:
             self.normed_training_data= training_data
             self.mean=[]
             self.std=[]

        self.use_norm=use_norm

    def computeCost(self, X, y, theta):
        m=len(y)
        preiction=X@theta
        err_abs=(preiction-y).T@(preiction-y)
        J=err_abs/(2*m)
        return J
    def gradientDescent(self, X, y, theta, alpha, num_iters):

        # Initialize some useful values
        m=len(y)
        J_history=np.zeros([num_iters, 1])
        for iter in range(num_iters):
            J_history[iter]=self.computeCost(X, y, theta) #Save the cost J in every iteration
            preiction=X@theta
            err=(preiction-y)
            update=X.T@err
            theta=theta-alpha/m*update

        return theta, J_history
    def featureNormalize(self, X):
        X=np.asarray(X)
        XT=X.T;
        XnT=np.zeros(XT.shape)
        xmeanv=np.zeros(XT.shape[0])
        xstdv=np.zeros(XT.shape[0])
        for ii in range(XT.shape[0]):
            xmean=np.mean(XT[ii,:]);
            xstd=np.std(XT[ii,:])
            Xub=XT[ii,:]-xmean;
            XnT[ii,:]=Xub/xstd
            xmeanv[ii]=xmean
            xstdv[ii]=xstd
        Xn=XnT.T

        return Xn, xmeanv, xstdv

    def normalEqn(self, X, y):
        XT=X.T
        XTX=XT@X
        pinv=np.linalg.pinv(XTX)
        XTy=XT@y
        theta=pinv@XTy
        return theta


import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import norm
import math
import time

def getSampleMeanVariance(sample):
	sum = 0.00
	N = len(sample)
	for i in sample:
		sum += i

	avg = sum/(N)

	cnt = 0.00
	for i in sample:
		cnt += (i-avg)**2

	variance = cnt/(N-1)
	return avg, variance

def mean_variance(sample, type, N):
	if type == 0:
		print("For N(0, 1), ")

	if type == 1:
		print("For N(0, 5), ")

	if type == 2:
		print("For N(5, 5), ")

	sampleX = []
	sampleY = []

	for x, y in sample:
		sampleX.append(x)
		sampleY.append(y)

	m1, v1 = getSampleMeanVariance(sampleX)
	m2, v2 = getSampleMeanVariance(sampleY)

	print('Mean of {} numbers of z1 is {}'.format(N, m1))
	print('Mean of {} numbers of z2 is {}'.format(N, m2))

	print('Variance of {} numbers of z1 is {}'.format(N, v1))
	print('Variance of {} numbers of z2 is {}'.format(N, v2))

def boxMuller():
	u1 = np.random.uniform(0.00, 1.00)
	u2 = np.random.uniform(0.00, 1.00)

	R = -2.00*math.log(u1)
	V = 2.00*math.pi*u2

	Z1 = math.sqrt(R)*math.cos(V)
	Z2 = math.sqrt(R)*math.sin(V)

	return Z1, Z2

def marsgliaBray():
	X = 100.00
	u1 = 1.00
	u2 = 1.00

	rejected = 0
	total = 0
	while X > 1.00 :
		u1 = np.random.uniform(0.00, 1.00)
		u2 = np.random.uniform(0.00, 1.00)

		u1 = 2*u1 - 1.00
		u2 = 2*u2 - 1.00

		X = u1**2 + u2**2
		rejected += 1
		total += 1

	rejected -= 1
	Y = math.sqrt( (-2.00*math.log(X)) / X)
	Z1 = u1*Y
	Z2 = u2*Y

	return Z1, Z2, rejected, total

def plotCDF(sample, N, method):
	for i in range(N):
		sample[i][0] *= math.sqrt(5)
		sample[i][1] *= math.sqrt(5)

	data = []
	for x, y in sample:
		data.append(x)
		data.append(y)

	data_size = len(data)

	data_set = sorted(set(data))
	bins = np.append(data_set, data_set[-1]+1)

	counts, bin_edges = np.histogram(data, bins=bins, density=False)
	counts = counts.astype(float)/data_size
	cdf = np.cumsum(counts)

	# Mean = 0, Variance = 5, Color = Blue
	plt.plot(bin_edges[0:-1], cdf, linestyle='--', marker='o', color='m')
	plt.ylim((0, 1))
	plt.ylabel("CDF")
	plt.xlabel("x")
	plt.grid(True)

	for i in range(N):
		sample[i][0] += 5
		sample[i][1] += 5

	data = []
	for x, y in sample:
		data.append(x)
		data.append(y)

	data_size = len(data)

	data_set = sorted(set(data))
	bins = np.append(data_set, data_set[-1]+1)

	counts, bin_edges = np.histogram(data, bins=bins, density=False)
	counts = counts.astype(float)/data_size
	cdf = np.cumsum(counts)

	# Mean = 5, Variance = 5, Color = Red
	plt.plot(bin_edges[0:-1], cdf, linestyle='--', marker='o', color='c')
	plt.legend(["N(0, 5)", "N(5, 5)"])
	plt.ylim((0, 1))
	plt.ylabel("CDF")
	plt.grid(True)

	plt.title("For N = {}, Method = {}".format(N, method))

	plt.show()

def plotHistogram(sample, N,  method):
	print()
	data = []
	for x, y in sample:
		data.append(x)
		data.append(y)

	mean = 0
	variance = 1
	# Plot between -10 and 10 with .001 steps.
	x_axis = np.arange(-5, 5, 0.001)
	plt.plot(x_axis, norm.pdf(x_axis, mean, math.sqrt(variance)), color='m')

	hist, bins, _ = plt.hist(data, density=True, bins=100, color='c')
	plt.xlabel('x')
	plt.ylabel('y')
	plt.legend(['Actual Distribution for N(0, 1)'], loc = "upper right")

	plt.title('Frequency v/s Sample for N = {}, Method = {}'.format(N, method))
	plt.show()

def execute(N):
	sample_1 = []
	sample_2 = []
	rejected = 0
	total = 0

	# For N(0, 1)
	time_start = time.process_time_ns()
	for i in range(N):
		x, y = boxMuller()
		sample_1.append([x, y])
	tBoxMuller = (time.process_time_ns() - time_start)

	time_start = time.process_time_ns()
	for i in range(N):
		x, y, r, t = marsgliaBray()
		rejected += r
		total += t
		sample_2.append([x, y])
	tMarsgliaBray = (time.process_time_ns() - time_start)

	plotHistogram(sample_1, N, "Box-Muller")
	plotHistogram(sample_2, N, "Marsglia and Bray" )

	print('Box-Muller method: ')
	mean_variance(sample_1, 0, N)
	print()

	print('Marsglia and Bray method: ')
	mean_variance(sample_2, 0, N)
	print()

	plotCDF(sample_1, N, "Box-Muller")
	plotCDF(sample_2, N, "Marsglia and Bray")

	# Output Handling
	print("Time for Box-Muller (N = {}) {} ns".format(N, tBoxMuller))
	print("Time for Marsglia and Bray (N = {}) {} ns".format(N, tMarsgliaBray))
	print()

	print("{} {}".format(rejected, total))
	print("For N = {}, Required Comparison = {}".format(N, abs(1-math.pi/4 - rejected/total)))

	print("\n\n")

def main():
	execute(100)
	execute(10000)

if __name__ == '__main__':
	main()


__all__ = ['unsigned_volume','signed_volume']

from scipy import sqrt,inner,shape,asarray
from scipy.misc import factorial
from scipy.linalg import det

def unsigned_volume(pts):
    """Unsigned volume of a simplex

    Computes the unsigned volume of an M-simplex embedded in N-dimensional
    space. The points are stored row-wise in an array with shape (M+1,N).

    Parameters
    ----------
    pts : array
        Array with shape (M+1,N) containing the coordinates
        of the (M+1) vertices of the M-simplex.

    Returns
    -------
    volume : scalar
        Unsigned volume of the simplex

    Notes
    -----
    Zero-dimensional simplices (points) are assigned unit volumes.

    Examples
    --------
    >>> # 0-simplex point
    >>> unsigned_volume( [[0,0]] )
    1.0
    >>> # 1-simplex line segment
    >>> unsigned_volume( [[0,0],[1,0]] )
    1.0
    >>> # 2-simplex triangle
    >>> unsigned_volume( [[0,0,0],[0,1,0],[1,0,0]] )
    0.5

    References
    ----------
    [1] http://www.math.niu.edu/~rusin/known-math/97/volumes.polyh

    """

    pts = asarray(pts)

    M,N = pts.shape
    M -= 1

    if M < 0 or M > N:
        raise ValueError('array has invalid shape')

    if M == 0:
        return 1.0

    A = pts[1:] - pts[0]
    return sqrt(abs(det(inner(A,A))))/factorial(M)

def signed_volume(pts):
    """Signed volume of a simplex

    Computes the signed volume of an M-simplex embedded in M-dimensional
    space. The points are stored row-wise in an array with shape (M+1,M).

    Parameters
    ----------
    pts : array
        Array with shape (M+1,M) containing the coordinates
        of the (M+1) vertices of the M-simplex.

    Returns
    -------
    volume : scalar
        Signed volume of the simplex

    Examples
    --------
    >>> # 1-simplex line segment
    >>> signed_volume( [[0],[1]] )
    1.0
    >>> # 2-simplex triangle
    >>> signed_volume( [[0,0],[1,0],[0,1]] )
    0.5
    >>> # 3-simplex tetrahedron
    >>> signed_volume( [[0,0,0],[3,0,0],[0,1,0],[0,0,1]] )
    0.5

    References
    ----------
    [1] http://www.math.niu.edu/~rusin/known-math/97/volumes.polyh

    """

    pts = asarray(pts)

    M,N = pts.shape
    M -= 1

    if M != N:
        raise ValueError('array has invalid shape')

    A = pts[1:] - pts[0]
    return det(A)/factorial(M)


#!/usr/bin/env python
# coding: utf-8

# # ml lab1

# In[2]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

pd.set_option('display.max_rows', 10)

# ### 1. read data

# In[104]:

data1 = np.genfromtxt('data/ex1data1.txt', delimiter=',')
rest = pd.DataFrame(data1, columns=['Population', 'Income'])
print(rest)

# ### 2. plot data

# In[264]:

def get_plot():
    plt.figure(figsize=(10, 6))
    plt.grid(True)
    plt.ylabel('Income, $10.000')
    plt.xlabel('Population of City, 10.000')
    plt.plot(
        rest['Population'],
        rest['Income'],
        'rx', markersize=10, label='Training Data')
    return plt

plot = get_plot()
plot.show()

# ### 3. cost function J(θ)

# In[289]:

# Linear hypothesis function
def h(X, theta):
    return np.dot(X, theta)

# J = compute_cost(X, y, theta)
# computes the cost of using theta as the parameter for linear regression
# to fit the data points in X and y
def compute_cost(X, y, theta):
    m = y.size
    loss = h(X, theta) - y
    return np.sum(np.square(loss)) / (2. * m)

# In[290]:

(_, n) = rest.shape
theta = np.zeros((1, n)).T

X1 = rest[['Population']]
X1.insert(0, 'theta_0', 1)
y1 = rest[['Income']]

J = compute_cost(X1, y1, theta)
print(f'theta:\t{theta.ravel()}\nJ:\t{float(J)}')

# ### 4. gradient descent

# In[291]:

# Performs gradient descent to learn theta
def gradient_descent(X, y, theta, alpha=0.01, iterations=1500):
    m = y.size
    J_history = []
    XT = X.T

    for i in range(iterations):
        loss = h(X, theta) - y
        gradient = np.dot(XT, loss) / m
        theta -= alpha * gradient
        J_history.append(compute_cost(X, y, theta))
    return theta, J_history

theta, j_history = gradient_descent(X1, y1, theta)
print(f'computed theta: {theta.ravel()}')

# In[246]:

sample_population = [3, 11, 15, 16, 18.5]
predicted_income = [np.dot([1, x], theta).sum() for x in sample_population]
pd.DataFrame(
    zip(sample_population, predicted_income),
    columns=['Sample Population', 'Predicted Income'])

# In[277]:

h_values = [np.dot(x, theta).sum() for x in X1.to_numpy()]

plot = get_plot()
plot.plot(rest['Population'], h_values, 'b-', label='Hypothesis')
plot.legend()
plot.show()

# ###  5. visualizing J(θ)

# In[284]:

# grid coordinates for plotting
xvals = np.linspace(-10, 10, 50)
yvals = np.linspace(-1, 4, 50)
xx, yy = np.meshgrid(xvals, yvals, indexing='xy')
Z = np.zeros((xvals.size, yvals.size))

# calculate Z-values (Cost) based on grid of coefficients
for (i, j), v in np.ndenumerate(Z):
    Z[i, j] = compute_cost(X1, y1, theta=[[xx[i, j]], [yy[i, j]]])

# In[294]:

from mpl_toolkits.mplot3d import axes3d

fig = plt.figure(figsize=(15, 6))
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 2, projection='3d')

# left plot
CS = ax1.contour(xx, yy, Z, np.logspace(-2, 3, 20), cmap=plt.cm.jet)
ax1.scatter(*theta, c='r')

# right plot
ax2.plot_surface(xx, yy, Z, rstride=1, cstride=1, alpha=0.6, cmap=plt.cm.jet)
ax2.set_zlabel('Cost')
ax2.set_zlim(Z.min(), Z.max())
ax2.view_init(elev=15, azim=230)

# settings common to both plots
for ax in fig.axes:
    ax.set_xlabel(r'$\theta_0$', fontsize=17)
    ax.set_ylabel(r'$\theta_1$', fontsize=17)

# ### 6. read ex2 data

# In[364]:

data2 = np.genfromtxt('data/ex1data2.txt', delimiter=',')
houses = pd.DataFrame(data2, columns=['Area', 'Bedrooms', 'Price'])
print(houses)

# ### 7. features normalization

# In[487]:

# Normalizes the features in X:
#   returns a normalized version of X where
#   the mean value of each feature is 0 and the standard deviation is 1
def feature_normalization(X):
    mu = X.mean(axis=0)
    sigma = X.std(axis=0)
    return (X - mu) / sigma, mu, sigma

# In[488]:

X2 = houses[['Area', 'Bedrooms']]
X2_norm, mu, sigma = feature_normalization(X2)
X2_norm.describe()

# In[489]:

y2 = houses[['Price']]
_, n2 = houses.shape

X2.insert(0, 'theta_0', 1)
X2_norm.insert(0, 'theta_0', 1)

# In[490]:

t1 = np.zeros((1, n2)).T
t2 = np.zeros((1, n2)).T
(theta1, j_history) = gradient_descent(X2, y2, t1, 0.00000001, 50)
(theta2, j_norm_history) = gradient_descent(X2_norm, y2, t2, 0.1, 50)
print(f'theta1:\t{theta1.ravel()}\ntheta2:\t{theta2.ravel()}')

# In[435]:

p1 = plt.plot(range(len(j_history)), j_history, color='black')
p2 = plt.plot(range(len(j_norm_history)), j_norm_history, color='red')
plt.legend((p1[0], p2[0]), ('raw', 'normalized'))
plt.show()

# ### 8. multi gradient descent with vectorizing

# In[436]:

alpha = 0.01
iterations = 400

(theta_mul, _) = gradient_descent(X2_norm, y2, np.zeros((1, n2)).T, alpha, iterations)
print(f'theta_mul:\t{theta_mul.ravel()}')

# ### 9. execution time

# In[437]:

from timeit import default_timer

iterations = 1000
alpha = 0.02

start = default_timer()
(theta_timer, _) = gradient_descent(X2_norm.to_numpy(), y2.to_numpy(), np.zeros((1, n2)).T, alpha, iterations)
end = default_timer()
print(f'theta_timer:\t{theta_timer.ravel()}\ttime:{end - start}')

# ### 10. ɑ varying plot

# In[458]:

def draw_alphas(iterations):
    alphas = np.linspace(0.1, 0.001, num=7)
    plots = []
    for alpha in alphas:
        (theta, j_hist) = gradient_descent(
            X2_norm.to_numpy(),
            y2.to_numpy(),
            np.zeros((1, n2)).T,
            alpha,
            iterations)
        p = plt.plot(range(len(j_hist)), j_hist)
        plots.append(p[0])

    plt.title(f'iterations: {iterations}')
    plt.legend(plots, [f'Alpha: {a:.3f}' for a in alphas])
    plt.show()

# In[462]:

draw_alphas(30)
draw_alphas(60)
draw_alphas(100)

# ### 11. least squares

# In[548]:

# computes the closed-form solution to linear regression using the normal equations
def normal_eqn(X, y):
    XX = np.asmatrix(X)
    XT = XX.T
    return np.array([float(el) for el in ((XT @ XX).I @ XT) @ y])

# In[557]:

theta_sq = normal_eqn(X2.to_numpy(), y2.to_numpy())
print(f'theta_sq:\t{theta_sq.ravel()}\ntheta_gd:\t{theta_mul.ravel()}')

# In[560]:

AREA = 1890
ROOMS = 4

price_sq = np.array([1, AREA, ROOMS]) @ theta_sq.T
price_gd = (np.array([1, (AREA - mu[0]) / sigma[0], (ROOMS - mu[1]) / sigma[1]]) @ theta_mul)[0]
print(f'price_sq:\t{price_sq}\nprice_gd:\t{price_gd}')


import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

k = 2
X = np.genfromtxt("data/data-dimred-X.csv", dtype=float, delimiter=',').T
y = np.genfromtxt("data/data-dimred-y.csv", dtype=float, delimiter=',')

#from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
#clf = LinearDiscriminantAnalysis(n_components=2)
#clf.fit(X, y)
#z = clf.transform(X)

#print(z.shape)
#plt.scatter(z[:,0], z[:,1], c=y)
#plt.savefig("pca_2d.png", bbox_inches="tight", pad_inches=0)
#plt.show()

#fig = plt.figure()
#ax = plt.axes(projection='3d')
#ax.scatter3D(z[:,0], z[:,1], z[:,2], c=y);
#plt.savefig("pca_3d.png", bbox_inches="tight", pad_inches=0)
#plt.show()

n_examples = X.shape[0]
n_features = X.shape[1]
n_classes = 3

n_examples_class = np.zeros(n_classes)
for i in range(n_classes):
  n_examples_class[i] = y[np.where(y==i+1)].size

u = np.zeros((n_classes, n_features))
cov = np.zeros((n_classes, n_features, n_features))
for i in range(n_classes):
  indices = np.where(y==i+1)[0]
  u[i, :] = np.mean(X[indices, :], axis=0)
  cov[i, :, :] = np.cov(X[indices, :].T)

S_w = np.zeros((n_features, n_features))
for i in range(n_classes):
  S_w = S_w + cov[i, :, :]

u_all = np.mean(X, axis=0)

S_b = np.zeros((n_features, n_features))
for i in range(n_classes):
  S_b = S_b + np.outer(u[i,:]-u_all, u[i,:]-u_all)

C = np.dot(np.linalg.pinv(S_w), S_b)

eival, eivec = np.linalg.eigh(C)
eivec = eivec.T[::-1]
z = np.dot(X, eivec[:3].T)
z = z.T

plt.scatter(z[0], z[1], c=y)
plt.savefig("out/03/lda/lda_2d.png", bbox_inches="tight", pad_inches=0)
plt.show()

# plot in 2D
fig = plt.figure()
ax = plt.axes(projection='3d')
ax.scatter3D(z[0], z[1], z[2], c=y);
plt.savefig("out/03/lda/lda_3d.png", bbox_inches="tight", pad_inches=0)
plt.show()


# -*- coding: utf-8 -*-
"""
Created on Wed Apr 18 12:47:56 2020
@author: Sila
"""

# Monte Carlo methods, or Monte Carlo experiments, are a broad class of computational algorithms
# that rely on repeated random sampling to obtain numerical results.

import numpy as np
import math
import random
import matplotlib.pyplot as plt

# Initialization
square_size = 1
points_inside_circle = 0
points_inside_square = 0
sample_size = 1000
arc = np.linspace(0, np.pi / 2, 100)

# A function called generate_points which generates random points inside the square.
def generate_points(size):
    x = random.random() * size
    y = random.random() * size
    return (x, y)

# A function called is_in_circle which will check if the point we generated falls within the circle.
def is_in_circle(point, size):
    return math.sqrt(point[0] ** 2 + point[1] ** 2) <= size

# square size = r^2 = 1, circle size = r^2*pi / 4 = 1^2* pi / 4 = pi / 4.
# circle_size / square_size = pi / 4
# The program keeps track of how many points it's picked so far (N) and how many of those points fell inside the circle (M).
# Pi is then approximated as follows:
#
#
#  pi =  4*M / N
#
#
def compute_pi(points_inside_circle, points_inside_square):
    return 4 * (points_inside_circle / points_inside_square)

plt.axes().set_aspect('equal')
plt.plot(1 * np.cos(arc), 1 * np.sin(arc))

for i in range(sample_size):
    point = generate_points(square_size)
    plt.plot(point[0], point[1], 'c.')
    points_inside_square += 1

    if is_in_circle(point, square_size):
        points_inside_circle += 1

plt.show()

print("Approximate value of pi is {}".format(compute_pi(points_inside_circle, points_inside_square)))
print("Awesome")


import numpy as np
from matplotlib import pyplot as plt

def mc_int(f, mins, maxs, numPoints=500, numIters=100):
    '''Use Monte-Carlo integration to approximate the integral of f
    on the box defined by mins and maxs.

    INPUTS:
    f         - A function handle. Should accept a 1-D NumPy array
    	 	as input.
    mins      - A 1-D NumPy array of the minimum bounds on integration.
    maxs      - A 1-D NumPy array of the maximum bounds on integration.
    numPoints - An integer specifying the number of points to sample in
    		the Monte-Carlo method. Defaults to 500.
    numIters - An integer specifying the number of times to run the
    		Monte Carlo algorithm. Defaults to 100.

    ALGORITHM:
    Run the Monte-Carlo algorithm `numIters' times and return the average
    of these runs.

    EXAMPLES:
    >>> f = lambda x: np.hypot(x[0], x[1]) <= 1
    >>> # Integral over the square [-1,1] x [-1,1] should be pi
    >>> mc_int(f, np.array([-1,-1]), np.array([1,1]))
    3.1290400000000007
    '''
    side_lengths = maxs-mins
    dims = mins.shape[0]

    answers = np.empty(numIters)

    for i in xrange(numIters):
        points = np.random.rand( numPoints, dims )
        points = points*side_lengths + mins
        total = np.sum(np.apply_along_axis(f, 1, points))
        answers[i] = np.prod(side_lengths)*float(total)/numPoints

    return answers.mean()


import numpy as np
from scipy import ndimage
import cv2 as cv
import sys, os
import matplotlib.pyplot as plt

#Lista de algunos kernels que andan por ahí

kernelBlur =[[1.0/9,1.0/9,1.0/9],
			[1.0/9,1.0/9,1.0/9],
			[1.0/9,1.0/9,1.0/9]]

kernelGaussianBlur =[[1.0/16,2.0/16,1.0/16],
					[2.0/16,4.0/16,2.0/16],
					[1.0/16,2.0/16,1.0/16]]

kernelGaussianBlurMax =[[1.0/256,4.0/256,6.0/256,4.0/256,1.0/256],
						[4.0/256,16.0/256,24.0/256,16.0/256,1.0/256],
						[6.0/256,24.0/256,36.0/256,24.0/256,6.0/256],
						[4.0/256,16.0/256,24.0/256,16.0/256,1.0/256],
						[1.0/256,4.0/256,6.0/256,4.0/256,1.0/256]]

kernelEdge1= [[1,0,-1],
			[0,0,0],
			[-1,0,1]]

kernelEdge3= [[-1,-1,-1],
			[-1,8,-1],
			[-1,-1,-1]]

kernelLaplacian = [[0,1,0],
				[1,-4,1],
				[0,1,0]]

kernelSharpen= [[0,-1,0],
				[-1,5,-1],
				[0,-1,0]]

kernelSobelX = [[1,0,-1],
				[2,0,-2],
				[1,0,-1]]

kernelSobelY = [[1,2,1],
				[0,0,0],
				[-1,-2,-1]]

kernelPrewittX = [[-1,0,1],
					[-1,0,1],
					[-1,0,1]]

kernelPrewittY = [[1,1,1],
					[0,0,0],
					[-1,-1,-1]]

kernelRobertsX = [[1,0],
				[0,-1]]

kernelRobertsY = [[0,1],
				[-1,0]]

######## Algunas funciones auxiliares #######

#Cargar custom kernel
def cargarKernel(ruta):
	#TODO: Verificar que sea cuadrada + verificar ruta y eso
	archivo_kernel = open (ruta , 'r')
	contenido = [[float(num) for num in line.split(',')] for line in archivo_kernel ]
	#TODO: Parsear para que acepte floats del tipo x/y
	return contenido

#Ayuda
def mostrarAyuda():
	print("\
Para procesar alguna imagen:\n\
\n\
	python procesador.py [nombre_imagen] [opciones] [argumento_extra]\n\
	\n\
	*Las imagenes tienen que estar en el mismo directorio\n\
	\n\
Opciones:\n\
	\n\
	h: Ayuda\n\
	k: Usar kernel desde un archivo, en [argumento_extra] se debe colocar el nombre del mismo.\n\
	b: Blur simple\n\
	gb: Blur Gaussiano 3x3\n\
	gbm: Blur Gaussiano 5x5\n\
	e1: Algun detector de bordes desconocido\n\
	e3: Algun detector de bordes desconocido 2\n\
	l: Laplaciano\n\
	sh: Filtro sharpen\n\
	s: Sobel\n\
	p: Prewitt\n\
	r: Roberts\n\
	mediana: Filtro mediana\n\
	nsp: Ruido sal y pimienta\n\
	ni: Ruido impulsivo\n\
	ng: Ruido Gaussiano\n\
	nr: Rudio Rayleigh\n\n")
	sys.exit()

#Abrir imagen para procesar
def abrir(nombre_imagen):
	directorio_actual = os.getcwd()
	ruta_imagen = directorio_actual + '/' +  nombre_imagen
	imagen = cv.imread(ruta_imagen,0)
	#Chequear si existe
	if imagen is None:
		print("Imagen no encontrada\n")
		mostrarAyuda()
		sys.exit()
	else:
		alto, ancho = imagen.shape
		salida = np.empty([alto,ancho])
		#Hago esto porque opencv o numpy tiene un mecanismo muy gracioso, no puedo copiarlo directamente
		for i in range(alto):
			for j in range(ancho):
				salida[i,j] = imagen[i,j]
		return salida

#Guardar
def guardar(filtro,imagen):
	cv.imwrite(filtro+".png",imagen)

#Guardar y salir
def salir(filtro,imagen):
	cv.imwrite("result/"+filtro+".png",imagen)
	plt.imshow(imagen, cmap='gray')
	sys.exit()

def normalizar(imagen):
	#Normaliza a valores entre 0 y 255
	alto, ancho = imagen.shape
	im_normal = np.empty([alto,ancho])
	maximo = np.amax(imagen)
	minimo = np.amin(imagen)
	shrink = 255/(maximo - minimo)
	return ((imagen - minimo) * shrink).astype(int)

####### Programa principal #######

if sys.argv[1] == "h":
	mostrarAyuda()
else:
	imagen = abrir(sys.argv[1])

#Empezar a procesar
alto, ancho = imagen.shape
filtrada = np.empty([alto,ancho])

#Ver opciones y procesar
#TODO: Mejorar logica de parametros por consola

#Custom kernel
if('k' in sys.argv):
	if len(sys.argv)>3:
		kernelCustom = np.array(cargarKernel(sys.argv[3]))
		filtrada = ndimage.convolve(imagen, kernelCustom).astype(int)
		salir("custom_kernel",filtrada)
	else:
		print("Falta ruta del archivo con el kernel")

if('b' in sys.argv):
	filtrada = ndimage.convolve(imagen, kernelBlur)
	salir("blur",filtrada)

if('gb' in sys.argv):
	filtrada = ndimage.convolve(imagen, kernelGaussianBlur)
	salir("gaussian_blur",filtrada)

if('gbm' in sys.argv):
	filtrada = ndimage.convolve(imagen, kernelGaussianBlurMax)
	salir("gaussian_blur_max",filtrada)

if('e1' in sys.argv):
	filtrada = ndimage.convolve(imagen, kernelEdge1)
	salir("edge1",filtrada)

if('e3' in sys.argv):
	filtrada = ndimage.convolve(imagen, kernelEdge3)
	salir("edge3",filtrada)

if('l' in sys.argv):
	filtrada = ndimage.convolve(imagen, kernelLaplacian)
	salir("laplacian",filtrada)

if('sh' in sys.argv):
	filtrada = ndimage.convolve(imagen, kernelSharpen)
	salir("sharpen",filtrada)

##### Edge detection mas trabajada: Sobel, Prewitt y Roberts

if('s' in sys.argv):
	umbral = 30
	if len(sys.argv)>3:
		umbral = int(sys.argv[3])
	derivadax = np.empty([alto,ancho])
	derivadax = ndimage.convolve(imagen, kernelSobelX)
	derivaday = np.empty([alto,ancho])
	derivaday = ndimage.convolve(imagen, kernelSobelY)
	#Normalizar antes de guardar
	guardar("sobel_x",normalizar(derivadax))
	guardar("sobel_y",normalizar(derivaday))
	filtrada = normalizar(np.hypot(derivadax,derivaday))
	salir("sobel" + "-" + str(umbral),((filtrada>umbral) * filtrada))

if('p' in sys.argv):
	umbral = 30
	if len(sys.argv)>3:
		umbral = int(sys.argv[3])
	derivadax = np.empty([alto,ancho])
	derivadax = ndimage.convolve(imagen, kernelPrewittX)
	derivaday = np.empty([alto,ancho])
	derivaday = ndimage.convolve(imagen, kernelPrewittY)
	#Normalizar antes de guardar
	guardar("prewitt_x",normalizar(derivadax))
	guardar("prewitt_y",normalizar(derivaday))
	filtrada = normalizar(np.hypot(derivadax,derivaday))
	salir("prewitt" + "-" + str(umbral),((filtrada>umbral) * filtrada))

if('r' in sys.argv):
	umbral = 30
	if len(sys.argv)>3:
		umbral = int(sys.argv[3])
	derivadax = np.empty([alto,ancho])
	derivadax = ndimage.convolve(imagen, kernelRobertsX)
	derivaday = np.empty([alto,ancho])
	derivaday = ndimage.convolve(imagen, kernelRobertsY)
	#Normalizar antes de guardar
	guardar("roberts_x",normalizar(derivadax))
	guardar("roberts_y",normalizar(derivaday))
	filtrada = normalizar(np.hypot(derivadax,derivaday))
	salir("roberts" + "-" + str(umbral),((filtrada>umbral) * filtrada))

##### Filtro mediana

if('mediana' in sys.argv):
	#Hice un filtro de mediana con opencv, numpy y un filtro de mediana de scipy
	#Por alguna razon es bastante lento (porque tiene que ordenar 9 elementos por cada pixel)
	salir("mediana",ndimage.median_filter(imagen, 3))

##### Ruidos

#TODO: no iterar para detectar los umbrales en ruido impulsivo y sal-pimienta
if('nsp' in sys.argv):
	#Condimentar
	umbralSal = 0.999
	umbralPimienta = 0.0001
	if len(sys.argv)>3:
		#Nos da 1000 valores de umbral en total (si solo usamos enteros)
		umbralSal = (1000-float(sys.argv[3]))/1000
		umbralPimienta = (float(sys.argv[3]))/1000
	for i in range(alto):
			for j in range(ancho):
				valor = np.random.random_sample()
				if valor>umbralSal:
					filtrada[i,j] = 255
				elif valor<umbralPimienta:
					filtrada[i,j] = 0
				else:
					filtrada[i,j] = imagen[i,j]
	salir("ruido_sal-pimienta" + "-" + str(umbralSal) + "-" + str(umbralPimienta),filtrada)

if('ni' in sys.argv):
	#Salar
	umbral = 0.999
	if len(sys.argv)>3:
		#Nos da 1000 valores de umbral en total (si solo usamos enteros)
		umbral = (1000-float(sys.argv[3]))/1000
	for i in range(alto):
			for j in range(ancho):
				if np.random.random_sample()>umbral:
					filtrada[i,j] = 255
				else:
					filtrada[i,j] = imagen[i,j]
	salir("ruido_impulsivo" + "-" + str(umbral),filtrada)

if('ng' in sys.argv):
	#Ruido gaussiano
	media = 0
	desviacion = 1
	if len(sys.argv)>3:
		desviacion = float(sys.argv[3])
	salir("ruido_gaussiano" + "-" + str(media) + "-" + str(desviacion),\
		(imagen + np.random.normal(media,desviacion,(alto,ancho))))

if('nr' in sys.argv):
	#Ruido Rayleigh
	escala = 1
	if len(sys.argv)>3:
		escala = float(sys.argv[3])
	salir("ruido_rayleigh" + str(escala),imagen + np.random.rayleigh(escala,(alto,ancho)))


import numpy as np

T_a = np.array([[ .5,  .1,  .1,  .15,  .15],
               [  1,   0,   0,    0,    0],
               [.75,   0,   0,  .25,    0],
               [.25,   0,   0,  .75,    0],
               [ .3,   0,   0,    0,   .7]])

T_b =np.array( [  [ .1,  .1,  .1,  .1,  .1,  .1,  .1,  .1,  .2],
                 [1./3.,   0, 1./3.,   0,   0,   0, 1./3.,   0,   0],
                 [  0,   0, 1/3,   0,   0, 2/3,   0,   0,   0],
                 [1/4,   0,   0,   0,   0,   0, 3/4,   0,   0],
                 [0.3, 0.4,   0,   0, 0.3,   0,   0,   0,   0],
                 [0.1,   0,   0,   0,   0, 0.9,   0,   0,   0],
                 [0.2,    0, 0.2,  0, 0.2,   0, 0.2,   0, 0.2],
                 [0.5,    0,   0,  0,   0,   0,   0,   0, 0.5],
                 [  0,  0.3,  0,0.05,0.05,   0, 0.3, 0.2, 0.1]   ] )

p_a = np.zeros((1,5))
p_a[0,0]=1
#print(p_a)

p_b = np.zeros((1,9))#creating random matrix 1 X 9
p_b[0,0] = 1
#print(p_b,"pb")

#print(p_b.dot(T_b))

#def SetTransitionMatrix_PartA():
#todo
#use a function to set it or by hand up top, your choice.

'''
#in class
T = np.zeros((10,10))

Slides = [(0,2),(0,4)]

prop = 1./7.
for s in Slides:
    T[s] = prob

p = np.zeros(10)
p.dot(T)

for in range nlah
    p^(k+1)= p^k * T

    return some p vector

if p<10 snd p>20 :
    do this
elif p<20 and p<30:
    do this
else:
'''


import numpy as np

def Gini_index(Y_data):
    gini = 0
    #=========    Edit here    ==========
    gini = 1 - np.sum([(count / len(Y_data)) ** 2 for _, count in zip(*np.unique(Y_data, return_counts=True))])
    #====================================
    return gini

def Entropy(Y_data):
    entropy = 0
    # =====    Edit here    ========
    arr = np.array([(count / len(Y_data)) for _, count in zip(*np.unique(Y_data, return_counts=True))])
    entropy = -(arr * np.log2(arr)).sum()
    # ==============================
    return entropy

def impurity_func(Y_data, criterion):

    if criterion == 'gini':
        return Gini_index(Y_data)

    elif criterion == 'entropy':
        return Entropy(Y_data)

def Finding_split_point(df, feature, criterion):

    col_data = df[feature]
    Y = df.values[:, -1]
    distinct_data = np.unique(col_data)

    split_point = distinct_data[0]
    min_purity = 1

    for idx, val in enumerate(distinct_data):
        less_idx = (col_data < val)

        y0 = Y[less_idx]
        y1 = Y[~less_idx]

        p0 = len(y0) / len(Y)
        p1 = len(y1) / len(Y)

        purity = np.sum([p0 * impurity_func(y0, criterion), p1 * impurity_func(y1, criterion)])

        if min_purity > purity:
            min_purity = purity
            split_point = val

    return split_point

def Gaussian_prob(x, mean, std):
    '''
    :param x: input value: X
    :param mean: the mean of X
    :param std: the standard deviation of X
    :return: probaility (X) ~ N(μ, σ^2)
    '''
    ret = 0
    # ========      Edit here         =========
    z = (x - mean) / std
    ret = np.exp(z ** 2 / -2) / (std * np.sqrt(2 * np.pi))
    # =========================================
    return ret


"""
Example of a sampling based gradient descent (ascent) implementation in tensorflow to find the
parameters of a probability distribution that maximizes the expected value of a given utility function
Author: Sebastian Gottwald
Project: https://github.com/sgttwld/gradientdescent
"""

import numpy as np
import tensorflow as tf
import os
import math
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'

# parameters
numSamples = 1000
lr = .01
precision = 1e-7
numEp = 20000

## placeholder
x = tf.placeholder(tf.float64, shape=[None])

## utility function
U = -(x**2-2)**2/4 + 1

## variables
theta = tf.Variable(np.ones(2),dtype=tf.float64)
mu = tf.exp(theta[0])
sigma = tf.exp(theta[1])

## parametrized (gaussian) density evaluated at x
p = tf.exp(-(x-mu)**2/(2*sigma**2))/tf.sqrt(2*math.pi*sigma**2)

## objective for gradient descent
## (logp makes the auto-derivative correct in case of sampling)
obj = tf.reduce_mean(tf.log(p)*U)

## true objective (only for evaluation)
obj_true = tf.reduce_mean(U)

## optimizer and training operator
optimizer = tf.train.AdamOptimizer(learning_rate=lr,beta1=.9, beta2=.999,epsilon=1e-08,name='Adam')
# optimizer = tf.train.GradientDescentOptimizer(learning_rate=lr)
train_op = optimizer.minimize(-obj)

## initializer
init = tf.global_variables_initializer()

with tf.Session() as sess:

    ## initializing
    sess.run(init)
    obj0 = 0

    for n in range(0,numEp):

        ## data generation
        mu_curr = mu.eval(session=sess)
        sigma_curr = sigma.eval(session=sess)
        data = np.random.normal(loc=mu_curr, scale=sigma_curr, size=numSamples)

        ## run gradient descent
        sess.run(train_op,feed_dict={x: data})

        ## evaluation
        obj_curr = obj_true.eval(session=sess,feed_dict={x: data})
        if n % 50 == 0:
            print('ep:',n,'E_p[U] = ', obj_curr)
        if abs(obj_curr-obj0) < precision:
            break
        else:
            obj0 = obj_curr

    mu_fin = mu.eval(session=sess)
    sigma_fin = sigma.eval(session=sess)

## discretizing the distribution for visualization
xx = np.linspace(0,2,50)
dens = np.exp(-(xx-mu_fin)**2/(2*sigma_fin**2))/np.sqrt(2*math.pi*sigma_fin**2)

## for command line visualization:
import hipsterplot as hplt

def show(v,stretch=2):
    hplt.plot(np.concatenate([[val]*stretch for val in v]), num_x_chars=stretch*len(v),num_y_chars=10)

print('utility')
show(-(xx**2-2)**2 / 4 + 1,stretch=1)

print('probability density', ' mu =',mu_fin, ' sigma =',sigma_fin)
show(dens,stretch=1)


import numpy as np

class L1_Norm(object):
    def __init__(self, coeff=1.):
        self.coeff = coeff

    def func_eval(self, x):
        """! Compute the function value of the \f$\ell_1\f$ - norm

        Parameters
        ----------
        @param x : input vector

        Returns
        ----------
        @retval : function value
        """
        return self.coeff*np.sum(np.abs(x))

    def prox_eval(self, x, prox_param):
        """! Compute the proximal operator of the \f$\ell_1\f$ - norm

        \f$ prox_{\lambda \|.\|_1} = {arg\min_x}\left\{\|.\|_1 + \frac{1}{2\lambda}\|x - w\|^2\right\} \f$

        Parameters
        ----------
        @param w : input vector
        @param prox_param : penalty paramemeter

        Returns
        ----------
        @retval : output vector
        """

        prox_param *= self.coeff
        return np.sign( x ) * np.maximum( np.abs( x ) - prox_param, 0 )