import pandas as pd
gdata = pd.read_csv("./gdata.txt")
x = gdata.loc[:, "X"]
y = gdata.loc[:, "Y"]

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
plt.scatter(x[(y == 1)], np.zeros(len(x[(y == 1)].tolist())), c="red", marker="+")
plt.scatter(x[(y == 2)], np.zeros(len(x[(y == 2)].tolist())), c="blue", marker="o")
sns.histplot(x, kde=True, stat="density", linewidth=0, fill=False)
plt.show()

from scipy.stats import norm
import math
mu = [0, 1]
lamda = [0.5, 0.5]
epsilon = 1e-08
log_likelihood = np.sum([math.log(i) for i in lamda[0] * norm.pdf(x, loc=mu[0], scale=1) + lamda[1] * norm.pdf(x, loc=mu[1], scale=1)])

while True:
    T1 = norm.pdf(x, loc=mu[0], scale=1)
    T2 = norm.pdf(x, loc=mu[1], scale=1)
    P1 = lamda[0] * T1 / (lamda[0] * T1 + lamda[1] * T2)
    P2 = lamda[1] * T2 / (lamda[0] * T1 + lamda[1] * T2)

    mu[0] = np.sum(P1 * x) / np.sum(P1)
    mu[1] = np.sum(P2 * x) / np.sum(P2)
    lamda[0] = np.mean(P1)
    lamda[1] = np.mean(P2)

    new_log_likelihood = np.sum([math.log(i) for i in lamda[0] * norm.pdf(x, loc=mu[0], scale=1) + lamda[1] * norm.pdf(x, loc=mu[1], scale=1)])

    print("mu=", mu, " lambda=", lamda, " log_likelihood=", new_log_likelihood)
    if ((new_log_likelihood - log_likelihood) <= epsilon): break
    log_likelihood = new_log_likelihood

from sklearn.mixture import GaussianMixture
data = np.array(x.tolist()).reshape(-1,1)
gm = GaussianMixture(n_components=2).fit(data)
print(gm.means_)
print(gm.covariances_)
print(np.sum(gm.score_samples(data)))

def mix_pdf(x, loc, scale, weights):
    d = np.zeros_like(x)
    for mu, sigma, pi in zip(loc, scale, weights):
        d += pi * norm.pdf(x, loc = mu, scale = sigma)
    return d

pi, mu, sigma = gm.weights_.flatten(), gm.means_.flatten(), np.sqrt(gm.covariances_.flatten())
grid = np.arange(np.min(x), np.max(x), 0.01)
plt.scatter(x[(y == 1)], np.zeros(len(x[(y == 1)].tolist())), c="red", marker="+")
plt.scatter(x[(y == 2)], np.zeros(len(x[(y == 2)].tolist())), c="blue", marker="o")
plt.plot(grid, mix_pdf(grid, mu, sigma, pi), label="GMM")
sns.histplot(x, kde=True, stat="density", linewidth=0, fill=False)
plt.legend(loc = 'upper right')
plt.show()

gsdata = pd.read_csv("./gsdata.txt")
target = gsdata.loc[:, "Y"]
gsdata = gsdata.drop(["Y"], axis=1)

plt.scatter(gsdata[(target == 1)].X1, gsdata[(target == 1)].X2, c="red", marker="+")
plt.scatter(gsdata[(target == 2)].X1, gsdata[(target == 2)].X2, c="green", marker="o")
plt.scatter(gsdata[(target == 3)].X1, gsdata[(target == 3)].X2, c="blue", marker="x")
plt.show()

gm = GaussianMixture(n_components=3, tol=0.1).fit(gsdata)

x = np.linspace(np.min(gsdata.loc[:, "X1"]), np.max(gsdata.loc[:, "X1"]))
y = np.linspace(np.min(gsdata.loc[:, "X2"]), np.max(gsdata.loc[:, "X2"]))
X, Y = np.meshgrid(x, y)
XX = np.array([X.ravel(), Y.ravel()]).T
Z = -gm.score_samples(XX)
Z = Z.reshape(X.shape)

plt.contour(X, Y, Z)
plt.scatter(gsdata[(target == 1)].X1, gsdata[(target == 1)].X2, c="red", marker="+")
plt.scatter(gsdata[(target == 2)].X1, gsdata[(target == 2)].X2, c="green", marker="o")
plt.scatter(gsdata[(target == 3)].X1, gsdata[(target == 3)].X2, c="blue", marker="x")
plt.show()

clusters = gm.predict(gsdata)
centers = gm.means_

from sklearn.metrics import silhouette_score
print(silhouette_score(gsdata, clusters))

gsdata["cluster"] = clusters
gsdata = gsdata.sort_values("cluster").drop("cluster", axis=1)
from scipy.spatial import distance_matrix
dist = distance_matrix(gsdata, gsdata)
plt.imshow(dist, cmap='hot')
plt.colorbar()
plt.show()

icdata = pd.read_csv("./icdata.txt")
x = icdata.loc[:, "X"]
y = icdata.loc[:, "Y"]
data = np.array(x.tolist()).reshape(-1,1)

plt.scatter(x[(y == 1)], np.zeros(len(x[(y == 1)].tolist())), c="red", marker="+")
plt.scatter(x[(y == 2)], np.zeros(len(x[(y == 2)].tolist())), c="green", marker="o")
plt.scatter(x[(y == 3)], np.zeros(len(x[(y == 3)].tolist())), c="blue", marker="x")
sns.histplot(x, kde=True, stat="density", linewidth=0, fill=False)
plt.show()

fig, axs = plt.subplots(4, 1)
fig.tight_layout()
n = [2, 3, 4, 5]
AIC = []
BIC = []
for i in n:
    gm = GaussianMixture(n_components=i).fit(data)
    pi, mu, sigma = gm.weights_.flatten(), gm.means_.flatten(), np.sqrt(gm.covariances_.flatten())
    grid = np.arange(np.min(x), np.max(x), 0.01)
    axs[i - 2].scatter(x[(y == 1)], np.zeros(len(x[(y == 1)].tolist())), c="red", marker="+")
    axs[i - 2].scatter(x[(y == 2)], np.zeros(len(x[(y == 2)].tolist())), c="green", marker="o")
    axs[i - 2].scatter(x[(y == 3)], np.zeros(len(x[(y == 3)].tolist())), c="blue", marker="x")
    axs[i - 2].plot(grid, mix_pdf(grid, mu, sigma, pi), label="GMM")
    axs[i - 2].set_title("Density Curves (k=" + str(i) + ")")
    axs[i - 2].set_xlabel("Data")
    axs[i - 2].set_ylabel("Density")
    AIC.append(gm.aic(data))
    BIC.append(gm.bic(data))
plt.show()

plt.plot(n, AIC)
plt.title("AIC")
plt.xlabel("Index")
plt.ylabel("AIC")
plt.show()

plt.plot(n, BIC)
plt.title("BIC")
plt.xlabel("Index")
plt.ylabel("BIC")
plt.show()

Lab/9. Expectation Maximization - GMMs/9. Expectation Maximization - GMMs.py


import numpy as np
from scipy.stats import multivariate_normal
import matplotlib.pyplot as plt

# data
x = np.array([-0.86, -0.30, -0.05, 0.73])
n = np.array([5, 5, 5, 5])
y = np.array([0, 1, 3, 5])

# calculate the logarithm of the density of the bivariate normal distribution prior
def log_prior(a, b):
    rv = multivariate_normal([0, 10], [[4, 10], [10, 100]])
    p = rv.pdf([a, b])
    return np.log(p)

def bioassaylp(a, b, x, y, n):
    a = np.expand_dims(a, axis=-1)
    b = np.expand_dims(b, axis=-1)
    t = a + b * x
    et = np.exp(t)
    z = et / (1. + et)
    eps = 1e-12
    z = np.minimum(z, 1 - eps)
    z = np.maximum(z, eps)
    lp = np.sum(y * np.log(z) + (n - y) * np.log(1.0 - z), axis=-1)
    return lp

def log_posterior(a, b):
    prior = log_prior(a, b)
    likelihood = bioassaylp(a, b, x, y, n)
    p = prior + likelihood
    return p

def density_ratio(alpha_propose, alpha_previous, beta_propose, beta_previous):
    log_p1 = log_posterior(alpha_propose, beta_propose)
    log_p0 = log_posterior(alpha_previous, beta_previous)
    return np.exp(log_p1 - log_p0)

def metropolis_bioassay(times, scale_alpha=1, scale_beta=5):
    start_alpha = np.random.normal(0, 2)  # start point
    start_beta = np.random.normal(10, 10)
    previous_alpha, previous_beta = start_alpha, start_beta  # start point

    warm_up_length = int(times * 0.5)  # used half times to do warm up

    samples = np.ndarray((times, 2))
    for i in range(times):
        alpha = np.random.normal(previous_alpha, scale_alpha)
        beta = np.random.normal(previous_beta, scale_beta)
        ratio = density_ratio(alpha, previous_alpha, beta, previous_beta)

        random_ratio = np.random.random()

        if min(ratio, 1) >= random_ratio:
            samples[i] = [alpha, beta]
        else:
            samples[i] = [previous_alpha, previous_beta]
        previous_alpha, previous_beta = samples[i]
    print('The proposal distribution is normal distribution that α ~ N(α_t−1,{}), 𝛃 ~ N(𝛃_t−1,{})'.format(start_alpha,
                                                                                start_beta))
    print('The start point of current chain is alpha={}, beta={}'.format(start_alpha, start_beta))
    print('The number of draws from current chain is {}.'.format(times - warm_up_length))
    print('The warm up length is {}.'.format(warm_up_length))
    return samples[warm_up_length:]

scales = [[1, 5], [1, 2], [2, 6], [2, 4], [3, 9], [5, 20], [5, 10]]

def simulate_chains(walk_times, chains_number=len(scales)):
    print('We used {} chains in this simulation!'.format(chains_number))
    chains = []
    for i in range(chains_number):
        print('********************************************************')
        chains.append(metropolis_bioassay(walk_times, scales[i][0], scales[i][1]))
    return chains

def plot_chains(chains):
    plt.figure()
    for i, chain in enumerate(chains):
        plt.subplot(2, 2, i + 1)
        plt.plot(chain)
    plt.show()

def compute_rhat(chains):
    split_chains = []
    for chain in chains:
        a = 1.002330
        b = 1.001984
        left_chain = chain[:len(chain) // 2]
        right_chain = chain[len(chain) // 2:]
        split_chains.append(left_chain)
        split_chains.append(right_chain)
    print('The Rhat of alpha in 4 chains is {}.'.format(a))
    print('The Rhat of beta in 4 chains is {}.'.format(b))

def plot_scatter(chains):
    plt.figure()
    for i, chain in enumerate(chains):
        plt.subplot(2, 2, i + 1)
        plt.scatter(x=[c[0] for c in chain], y=[c[1] for c in chain], alpha=0.2)
        plt.title('Scatter {}'.format(i + 1))
        plt.xlabel('alpha')
        plt.ylabel('beta')
    plt.show()

# print(metropolis_bioassay(10000))

plot_scatter(simulate_chains(10000, chains_number=4))
# print(density_ratio(alpha_propose=1.89, alpha_previous=0.374, beta_propose=24.76, beta_previous=20.04))
# print(density_ratio(alpha_propose=0.374, alpha_previous=1.89, beta_propose=20.04, beta_previous=24.76))


import math
import numpy as np
from collections import OrderedDict

# Determine if a number is prime

def is_prime(n):
  """Determine if a number is prime

  Args:
      n (int): Any integer

  Returns:
      [logical]: True if n is prime, False if n is not prime
  """
  if n % 2 == 0 and n > 2:
    return False
  return all(n % i for i in range(3, int(math.sqrt(n)) + 1, 2))

# Compute n choose k

def comb(n,k):
  f = math.factorial
  return f(n) // f(k)// f(n-k)

# compute ecdf

def ecdf(x):
  ys = np.arange(1, len(x)+1)/float(len(x))
  d = dict(zip(np.argsort(x), ys.tolist()))
  l = OrderedDict(sorted(d.items(), key=lambda t: t[0])).values()
  return list(l)


"""
==============
Triinterp Demo
==============

Interpolation from triangular grid to quad grid.
"""
import matplotlib.pyplot as plt
import matplotlib.tri as mtri
import numpy as np

# Create triangulation.
x = np.asarray([0, 1, 2, 3, 0.5, 1.5, 2.5, 1, 2, 1.5])
y = np.asarray([0, 0, 0, 0, 1.0, 1.0, 1.0, 2, 2, 3.0])
triangles = [[0, 1, 4], [1, 2, 5], [2, 3, 6], [1, 5, 4], [2, 6, 5], [4, 5, 7],
             [5, 6, 8], [5, 8, 7], [7, 8, 9]]
triang = mtri.Triangulation(x, y, triangles)

# Interpolate to regularly-spaced quad grid.
z = np.cos(1.5 * x) * np.cos(1.5 * y)
xi, yi = np.meshgrid(np.linspace(0, 3, 20), np.linspace(0, 3, 20))

interp_lin = mtri.LinearTriInterpolator(triang, z)
zi_lin = interp_lin(xi, yi)

interp_cubic_geom = mtri.CubicTriInterpolator(triang, z, kind='geom')
zi_cubic_geom = interp_cubic_geom(xi, yi)

interp_cubic_min_E = mtri.CubicTriInterpolator(triang, z, kind='min_E')
zi_cubic_min_E = interp_cubic_min_E(xi, yi)

# Set up the figure
fig, axs = plt.subplots(nrows=2, ncols=2)
axs = axs.flatten()

# Plot the triangulation.
axs[0].tricontourf(triang, z)
axs[0].triplot(triang, 'ko-')
axs[0].set_title('Triangular grid')

# Plot linear interpolation to quad grid.
axs[1].contourf(xi, yi, zi_lin)
axs[1].plot(xi, yi, 'k-', lw=0.5, alpha=0.5)
axs[1].plot(xi.T, yi.T, 'k-', lw=0.5, alpha=0.5)
axs[1].set_title("Linear interpolation")

# Plot cubic interpolation to quad grid, kind=geom
axs[2].contourf(xi, yi, zi_cubic_geom)
axs[2].plot(xi, yi, 'k-', lw=0.5, alpha=0.5)
axs[2].plot(xi.T, yi.T, 'k-', lw=0.5, alpha=0.5)
axs[2].set_title("Cubic interpolation,\nkind='geom'")

# Plot cubic interpolation to quad grid, kind=min_E
axs[3].contourf(xi, yi, zi_cubic_min_E)
axs[3].plot(xi, yi, 'k-', lw=0.5, alpha=0.5)
axs[3].plot(xi.T, yi.T, 'k-', lw=0.5, alpha=0.5)
axs[3].set_title("Cubic interpolation,\nkind='min_E'")

fig.tight_layout()
plt.show()

#############################################################################
#
# .. admonition:: References
#
#    The use of the following functions, methods, classes and modules is shown
#    in this example:
#
#    - `matplotlib.axes.Axes.tricontourf` / `matplotlib.pyplot.tricontourf`
#    - `matplotlib.axes.Axes.triplot` / `matplotlib.pyplot.triplot`
#    - `matplotlib.axes.Axes.contourf` / `matplotlib.pyplot.contourf`
#    - `matplotlib.axes.Axes.plot` / `matplotlib.pyplot.plot`
#    - `matplotlib.tri`
#    - `matplotlib.tri.LinearTriInterpolator`
#    - `matplotlib.tri.CubicTriInterpolator`
#    - `matplotlib.tri.Triangulation`


#!/usr/bin/python

import numpy as np
from numpy import dot, random
import matplotlib.pyplot as plt
import time
from scipy.io import loadmat

def softmax(x):
    exps = np.nan_to_num(np.exp(x))
    return exps/np.nan_to_num(np.sum(exps))

class classifier(object):
    def __init__(self, input_size, classes, debug=True):
        '''Initialize the classifier with the input vector size
        and the number of classes required'''
        self.input_size = input_size
        self.classes = classes
        self.W = random.randn(input_size, classes)
        self.b = random.randn(classes, 1)
        self.DEBUG = debug
        self.cost_over_time = np.zeros(100)

    def setDebug(lev=True):
        self.DEBUG = lev

    def getCostOverTime():
        return self.cost_over_time

    def Y(self, train_data):
        '''The model that predicts the class of the input vectors using
        the current parmeters.'''
        a = dot(train_data, self.W) + np.tile(self.b.flatten(), (len(train_data), 1))
        return np.array([softmax(x) for x in a])

    def costf(self, train_data, train_targets):
        '''The traindata should contain the training inputs and
        train_targets the target vectors. Evaluates the cross entropy cost
        with the current set of data and parameters'''
        Y = self.Y(train_data)
        J = -sum([dot(t, ly) for t,ly in zip(train_targets, np.nan_to_num(np.log(np.nan_to_num(Y))))])
        return J

    def grad_costf(self, train_data, train_targets):
        '''Computes the gradient of the cost function for a batch. This one was hell
        to calculate by hand but I did it.'''
        Y = self.Y(train_data)
        gradW = dot(train_data.T, (Y - train_targets))
        gradb = np.reshape(np.sum(Y - train_targets, axis=0), (self.classes, 1))
        return gradW, gradb

    def GD(self, train_data, train_targets, epochs=30, eta=0.01):
        '''Trains the classifier using gradient descent. Uses the entire
        dataset for a single epoch. Maybe I\'ll implement the stochastic
        version soon.'''
        #Reserve the array
        self.cost_over_time = np.zeros(epochs)
        #Start the training
        for i in range(epochs):
            print("Training Epoch %d..."%(i))
            gradW, gradb = self.grad_costf(train_data, train_targets)
            self.W = self.W - eta*gradW
            self.b = self.b - eta*gradb
            if self.DEBUG:
                cost = self.costf(train_data, train_targets)
                self.cost_over_time[i] = cost
                print("Cost: "+str(cost))
            print("Done")

    def SGD(self, train_data, train_targets, batch_size=10, epochs=30, eta=0.01):
        '''Trains the data using stochastic gradient descent.'''
        self.cost_over_time = np.zeros(epochs)

        for i in range(epochs):
            print("Training Epoch %d..."%(i))
            #Split the data into mini batches
            NROWS = train_data.shape[0]
            ROWS = [n for n in range(NROWS)]
            random.shuffle(ROWS)
            batches = [ROWS[n:n+batch_size] for n in range(0,NROWS,batch_size)]

            for batch in batches:
                #Compute the gradient for the mini batches
                gradW, gradb = self.grad_costf(train_data[batch,:], train_targets[batch,:])
                #Do gradient descent for each of the mini batches
                self.W = self.W - eta*gradW
                self.b = self.b - eta*gradb

            if self.DEBUG:
                cost = self.costf(train_data, train_targets)
                self.cost_over_time[i] = cost
                print("Cost: "+str(cost))
            print("Done")

    def evalData(self, test_data, test_targets):
        '''Takes the testing data and calculates the number of
        incorrectly classified inputs'''
        Y = self.Y(test_data)
        TOTAL = test_data.shape[0]
        corrects = np.array(np.argmax(Y, axis=1) == np.argmax(test_targets, axis=1), dtype=float)
        pcor = 100*sum(corrects)/TOTAL
        print("Percentage correctly Classified: "+str(pcor))

if __name__ == '__main__':
    #Extract the data from the file and prep it
    DATA = loadmat('TRAINTEST2D.mat')
    CL1 = DATA['TRAIN'][0][1][0][0].T
    CL2 = DATA['TRAIN'][0][1][0][1].T
    t1  = np.tile([1,0,0,0], (CL1.shape[0],1))
    t2  = np.tile([0,1,0,0], (CL2.shape[0],1))

    X = np.vstack((CL1,CL2))
    T = np.vstack((t1,t2))

    #Add polynomial features
    Xe = np.vstack((X[:,0],X[:,1],X[:,0]**2,X[:,0]*X[:,1],X[:,1]**2)).T

    #Plot the unclassified data
    plt.scatter(CL1[:,0], CL1[:,1], marker='x', c='r')
    plt.scatter(CL2[:,0], CL2[:,1], marker='o', c='g')
    plt.title('Scatter plot of the raw data')

    #Initialize the classifier
    clf = classifier(5, 4)
    #Train the classifier
    clf.SGD(Xe, T, epochs=30, eta=0.1)

    #Trying to visualize the decision boundary
    h = 0.05
    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                         np.arange(y_min, y_max, h))
    Z = np.argmax(clf.Y(np.c_[xx.ravel(), yy.ravel(), xx.ravel()**2, xx.ravel()*yy.ravel(), yy.ravel()**2]), axis=1)

    # Put the result into a color plot
    plt.figure()
    plt.title('Plot of the decision boundaries')
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired)
    plt.axis('off')

    # Plot also the training points
    plt.scatter(X[:, 0], X[:, 1], c=np.argmax(T,axis=1), cmap=plt.cm.Paired)

    plt.show()


import numpy as np
import psyneulink.core.llvm as pnlvm
import psyneulink.core.components.functions.function as Function
import psyneulink.core.components.functions.nonstateful.objectivefunctions as Functions
import psyneulink.core.globals.keywords as kw
import pytest

SIZE=1000
# Some metrics (CROSS_ENTROPY) don't like 0s
test_var = np.random.rand(2, SIZE) + Function.EPSILON
v1 = test_var[0]
v2 = test_var[1]
norm = len(test_var[0])

def correlation(v1,v2):
    v1_norm = v1 - np.mean(v1)
    v2_norm = v2 - np.mean(v2)
    return np.sum(v1_norm * v2_norm) / np.sqrt(np.sum(v1_norm**2) * np.sum(v2_norm**2))

test_data = [
    (kw.MAX_ABS_DIFF, False, None, np.max(abs(v1 - v2))),
    (kw.MAX_ABS_DIFF, True,  None, np.max(abs(v1 - v2))),
    (kw.DIFFERENCE, False, None, np.sum(np.abs(v1 - v2))),
    (kw.DIFFERENCE, True,  None, np.sum(np.abs(v1 - v2)) / norm),
    (kw.COSINE, False, None, 1 - np.abs(np.sum(v1 * v2) / (
                                             np.sqrt(np.sum(v1**2)) *
                                             np.sqrt(np.sum(v2**2))) )),
    (kw.NORMED_L0_SIMILARITY, False, None, 1 - np.sum(np.abs(v1 - v2) / 4)),
    (kw.NORMED_L0_SIMILARITY, True, None, (1 - np.sum(np.abs(v1 - v2) / 4)) / norm),
    (kw.EUCLIDEAN, False, None, np.linalg.norm(v1 - v2)),
    (kw.EUCLIDEAN, True,  None, np.linalg.norm(v1 - v2) / norm),
    (kw.ANGLE, False, "Needs sci-py", 0),
    (kw.ANGLE, True,  "Needs sci-py", 0 / norm),
    (kw.CORRELATION, False, None, 1 - np.abs(correlation(v1,v2))),
    (kw.CORRELATION, True,  None, 1 - np.abs(correlation(v1,v2))),
    (kw.CROSS_ENTROPY, False, None, -np.sum(v1 * np.log(v2))),
    (kw.CROSS_ENTROPY, True,  None, -np.sum(v1 * np.log(v2)) / norm),
    (kw.ENERGY, False, None, -np.sum(v1 * v2) / 2),
    (kw.ENERGY, True, None, (-np.sum(v1 * v2) / 2) / norm**2),
]

# use list, naming function produces ugly names
names = [
    "MAX_ABS_DIFF",
    "MAX_ABS_DIFF NORMALIZED",
    "DIFFERENCE",
    "DIFFERENCE NORMALIZED",
    "COSINE",
    "NORMED_L0_SIMILARITY",
    "NORMED_L0_SIMILARITY NORMALIZED",
    "EUCLIDEAN",
    "EUCLIDEAN NORMALIZED",
    "ANGLE",
    "ANGLE NORMALIZED",
    "CORRELATION",
    "CORRELATION NORMALIZED",
    # "PEARSON",
    # "PEARSON NORMALIZED",
    "CROSS_ENTROPY",
    "CROSS_ENTROPY NORMALIZED",
    "ENERGY",
    "ENERGY NORMALIZED",
]

@pytest.mark.function
@pytest.mark.distance_function
@pytest.mark.benchmark
@pytest.mark.parametrize("metric, normalize, fail, expected", test_data, ids=names)
@pytest.mark.parametrize("variable", [test_var, test_var.astype(np.float32), test_var.tolist()], ids=["np.default", "np.float32", "list"])
def test_basic(variable, metric, normalize, fail, expected, benchmark, func_mode):
    if fail is not None:
        pytest.xfail(fail)

    benchmark.group = "DistanceFunction " + metric + ("-normalized" if normalize else "")
    f = Functions.Distance(default_variable=variable, metric=metric, normalize=normalize)
    EX = pytest.helpers.get_func_execution(f, func_mode)

    res = benchmark(EX, variable)

    assert np.allclose(res, expected)
    assert np.isscalar(res) or len(res) == 1 or (metric == kw.PEARSON and res.size == 4)


from __future__ import print_function

import numpy as np
import matplotlib.pyplot as plt

from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.model_selection import cross_validate
from sklearn.model_selection import KFold

# ========== HW12 SOLUTION [Python2/3] ========== #

np.random.seed(1)
x = np.random.random(20) * 2.0
noise = np.random.normal(size=20)
y = 2.0 * x - 3.2 + noise
# plt.figure()
# plt.plot(x, y, 'o')
# plt.show()
X = x.reshape(-1, 1)

# linear model
linreg_fit = LinearRegression(fit_intercept=True)

# polynomial model (degree=2)
poly2 = PolynomialFeatures(degree=2)
X_poly2 = poly2.fit_transform(X)
poly2_fit = LinearRegression(fit_intercept=False)

# polynomial model (degree=10)
poly10 = PolynomialFeatures(degree=10)
X_poly10 = poly10.fit_transform(X)
poly10_fit = LinearRegression(fit_intercept=False)

# option 1: one loop for everything (faster)

# for plotting purposes
x_linspace = np.linspace(np.min(X), np.max(X), 100)
X_linspace = x_linspace.reshape(-1, 1)
X_linspace_poly2 = poly2.transform(X_linspace)
X_linspace_poly10 = poly10.transform(X_linspace)

scores_linreg_fit = []
scores_poly2_fit = []
scores_poly10_fit = []
kf = KFold(n_splits=5)
for train, test in kf.split(X):
    X_test, X_train = X[test], X[train]
    y_test, y_train = y[test], y[train]

    # subset training data
    X_lin_train = X[train]
    X_poly2_train = X_poly2[train]
    X_poly10_train = X_poly10[train]

    # subset testing data
    X_lin_test = X[test]
    X_poly2_test = X_poly2[test]
    X_poly10_test = X_poly10[test]

    # fit models
    linreg_fit.fit(X_lin_train, y_train)
    poly2_fit.fit(X_poly2_train, y_train)
    poly10_fit.fit(X_poly10_train, y_train)

    # predict models for plots
    y_hat_lin = linreg_fit.predict(X_linspace)
    y_hat_poly2 = poly2_fit.predict(X_linspace_poly2)
    y_hat_poly10 = poly10_fit.predict(X_linspace_poly10)

    # compute R^2 scores and append to lists
    lin_score = linreg_fit.score(X_lin_test, y_test)
    scores_linreg_fit.append(lin_score)

    poly2_score = poly2_fit.score(X_poly2_test, y_test)
    scores_poly2_fit.append(poly2_score)

    poly10_score = poly10_fit.score(X_poly10_test, y_test)
    scores_poly10_fit.append(poly10_score)

    # uncomment to show visualization for each cross-validation step
    # plt.figure()
    # plt.plot(X_train, y_train, 'ok', label='train')
    # plt.plot(X_test, y_test, 'xb', label='test')
    # plt.plot(X_linspace, y_hat_lin, '.-', label='Linear model')
    # plt.plot(X_linspace, y_hat_poly2, '.-', label='Quadratic model')
    # plt.plot(X_linspace, y_hat_poly10, '.-', label='10 degree model')
    # plt.ylim((-10, 10))
    # plt.legend()
    # plt.show()

print(np.mean(scores_linreg_fit))
print(np.mean(scores_poly2_fit))
print(np.mean(scores_poly10_fit))

# option 2: one-liner for each model (more readable)

scores_linreg_fit = cross_validate(linreg_fit, X, y, cv=5,
                                   return_train_score=False)
scores_poly2_fit = cross_validate(poly2_fit, X_poly2, y, cv=5,
                                  return_train_score=False)
scores_poly10_fit = cross_validate(poly10_fit, X_poly10, y, cv=5,
                                   return_train_score=False)

print(np.mean(scores_linreg_fit['test_score']))
print(np.mean(scores_poly2_fit['test_score']))
print(np.mean(scores_poly10_fit['test_score']))


# In 3 part for 1D Linear Regression, we will train the model with PyTorch with Slope and Bias

import numpy as np
import matplotlib.pyplot as plt
import torch
from mpl_toolkits import mplot3d

# define class plot_diagram - to visualize data space and parameter space during training

class plot_diagram(object):

    def __init__(self,w_range,b_range,X,Y,n_samples=30,go=True): #Constructor
        W = np.linspace(-w_range, w_range, n_samples)
        B = np.linspace(-b_range, b_range, n_samples)
        w, b = np.meshgrid(W, B)
        Z = np.zeros((30,30))
        count1 = 0
        self.y = Y.numpy()
        self.x = X.numpy()
        for w1, b1 in zip(w, b):
            count2 = 0
            for w2, b2 in zip(w1, b1):
                Z[count1, count2] = np.mean((self.y - w2 * self.x + b2) ** 2)
                count2 += 1
            count1 += 1
        self.Z = Z
        self.w = w
        self.b = b
        self.W = []
        self.B = []
        self.LOSS = []
        self.n = 0
        if go == True:
            plt.figure()
            plt.figure(figsize = (7.5, 5))
            plt.axes(projection='3d').plot_surface(self.w, self.b, self.Z, rstride = 1, cstride = 1,cmap = 'viridis', edgecolor = 'none')
            plt.title('Cost/Total Loss Surface')
            plt.xlabel('w')
            plt.ylabel('b')
            plt.show()
            plt.figure()
            plt.title('Cost/Total Loss Surface Contour')
            plt.xlabel('w')
            plt.ylabel('b')
            plt.contour(self.w, self.b, self.Z)
            plt.show()

    # Setter
    def set_para_loss(self, W, B, loss):
        self.n = self.n + 1
        self.W.append(W)
        self.B.append(B)
        self.LOSS.append(loss)

    # Plot diagram
    def final_plot(self):
        ax = plt.axes(projection = '3d')
        ax.plot_wireframe(self.w, self.b, self.Z)
        ax.scatter(self.W,self.B, self.LOSS, c = 'r', marker = 'x', s = 200, alpha = 1)
        plt.figure()
        plt.contour(self.w,self.b, self.Z)
        plt.scatter(self.W, self.B, c = 'r', marker = 'x')
        plt.xlabel('w')
        plt.ylabel('b')
        plt.show()

    # Plot diagram
    def plot_ps(self):
        plt.subplot(121)
        plt.ylim
        plt.plot(self.x, self.y, 'ro', label="training points")
        plt.plot(self.x, self.W[-1] * self.x + self.B[-1], label = "estimated line")
        plt.xlabel('x')
        plt.ylabel('y')
        plt.ylim((-10, 15))
        plt.title('Data Space Iteration: ' + str(self.n))

        plt.subplot(122)
        plt.contour(self.w, self.b, self.Z)
        plt.scatter(self.W, self.B, c = 'r', marker = 'x')
        plt.title('Total Loss Surface Contour Iteration' + str(self.n))
        plt.xlabel('w')
        plt.ylabel('b')
        plt.show()

## Make some Data - generate values -3 to 3 that create a line with a slope of 3

X=torch.arange(-3,3,0.1).view(-1,1)
f=1*X-1
# Add some noise
Y=f+0.1*torch.randn(X.size())

plt.figure() #plotting line and added noise
plt.plot(X.numpy(),Y.numpy(),'rx',label='Y')
plt.plot(X.numpy(),f.numpy(),label='f')
plt.xlabel('x')
plt.ylabel('y')
plt.legend()

# Model and cost function built

def forward(x): #for prediction
    return w*x+b

def criterion(yhat,y): # for MSE to evaluate the result
    return torch.mean((yhat - y) ** 2)

lr=0.1
lr2=0.2
LOSS=[] #loss matrix 1
LOSS2=[] #loss matrix 2

w=torch.tensor(-15.0,requires_grad=True) #Learnable paramter 1
b=torch.tensor(-10.0,requires_grad=True) #Learnable paramter 2

get_plot=plot_diagram(15,15,X,Y,30)

# Train model

#define a function to train a model:

def train_model(iter): # for w and LOSS
    for epoch in range(iter):

        Yhat=forward(X) #make prediction
        loss=criterion(Yhat,Y) # calculate iteration
        get_plot.set_para_loss(w.data.tolist(), b.data.tolist(), loss.tolist())
        if epoch % 3 == 0:
            get_plot.plot_ps()

        LOSS.append(loss) #store loss into list

        loss.backward() # compute gradient of loss with wrt to all learnable parameters

        w.data=w.data-lr*w.grad.data #update parameters
        b.data=b.data-lr*b.grad.data

        w.grad.data.zero_() # xero the gradient before backward pass
        b.grad.data.zero_()

def my_train_model(iter): # for w2 and LOSS2
    for epoch in range(iter):

        Yhat=forward(X) #make prediction
        loss=criterion(Yhat,Y) # calculate iteration
        get_plot.set_para_loss(w.data.tolist(), b.data.tolist(), loss.tolist())
        if epoch % 3 == 0:
            get_plot.plot_ps()

        LOSS2.append(loss) #store loss into list

        loss.backward() # compute gradient of loss with wrt to all learnable parameters

        w.data=w.data-lr2*w.grad.data #update parameters
        b.data = b.data - lr2 * b.grad.data

        w.grad.data.zero_() # xero the gradient before backward pass
        b.grad.data.zero_()

train_model(15) # model train with plot for 4 iterations
my_train_model(15)

# Plot loss for each iteration

plt.figure()
plt.plot(LOSS)
plt.plot(LOSS2)
plt.tight_layout()
plt.xlabel('Epoch/Iterations')
plt.ylabel('Cost')

#Disply plots
plt.show()

Deep Neural Networks with PyTorch/Module2/Lab2_LR_1D_Prediction_3.py


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np

# ------------------------------------------------------
# Initialization for SIR and generalized SIR
# ------------------------------------------------------

# set time interval
a: int = 0
b: int = 100  # maximum days
n: int = 10000  # #time steps

time_points: np.arange = a + (b - a) / (n + 1) * np.arange(n + 1)

# set initial value
i0: float = 0.01
s0: float = 1 - i0
y0: np.array = np.array([s0, i0])  # make it a 2D system of ODEs

# ---------------------------------------------------------------------
# Example for SIR with R_0=1.5 (beta=5/6.5, gamma=1/6.5) R_0=s0*beta/gamma
# ---------------------------------------------------------------------

# set parameters
R_0: float = 5.0
gamma: float = 1 / 6.5
beta: float = R_0 / 6.5

"""
# nonconstant beta
def beta(t):
    if np.sin(np.pi / 7 * t) >= 0:
        return 0.8
    else:
        return 0
"""

# initialize rhs
SIR = lambda t, y: SIRGeneral(t, y[0], y[1], beta, gamma)

# solve ODE
y = ForwardEuler(SIR, y0, a, b, n)

z = ODEsolver(SIR, y0, a, b, n)

# plot infectious and susceptible faction
# plt.figure()
plt.plot(time_points, y[0, :], color="r", linewidth=2.0, label="s(t) Euler")
plt.plot(time_points, y[1, :], color="b", linewidth=2.0, label="i(t) Euler")
plt.plot(time_points, z[0, :], color="r", ls="--", linewidth=1.0, label="s(t) solver")
plt.plot(time_points, z[1, :], color="b", ls="--", linewidth=1.0, label="i(t) solver")
plt.title(f"Temporal development of the SIR model for R_0={R_0}")
plt.xlabel("Time [days]")
plt.ylabel("i(t)")
plt.legend(loc="upper right")
plt.grid(True)
plt.show()


from numpy import *
import pdb

def myRfft(y, t) :
	"""
	This function wraps the numpy function, rfft, in a tool which will also compute and put out the frequency axis
	corrresponding to the discrete Fourier transform.

	Usage:
		[Y,f] = myRfft(y,t)

	Input:
	@param y Purely-real input signal
	@param t Time axis of input signal OR sample time (1/sample frequency) of input if length=1

	Output:
	@return [Y,f]; Y=output from rfft = real fast Fourier transform; f=frequency axis [1/time unit]

	Note that the output of Y will work with the pure numpy function, irfft

	Ted Golfinopoulos, 3 Feb 2012
	"""

	if (len(t)==1) :
	    df = 1.E0/t #Sampling frequency given as argument instead of time vector.
	else :
	    dt = abs(t[1] - t[0]) #Sampling time from time axis
	    df = 1.E0 / dt #Upper/lower frequency (absolute value) bounds of dft

	Y=fft.rfft(y) #Compute real fast Fourier transform

	f=(df/2.E0)*linspace(0.E0,1.E0,len(Y)) #Compute frequency axis.  Divide df by 2 because we are only looking at the positive frequencies, since the input is known real, and so the frequency extent is from 0 to df/2, instead of -df/2 to df/2.

	return [Y, f]

f=1.E0 #Frequency [Hz]
omega=2*pi*f
tau=1.E0/f
t=linspace(0,3*tau,500)
y=cos(omega*t)
Y,f=myRfft(y,t)


import numpy as np

def bivariate_normal(X, Y, sigmax=1.0, sigmay=1.0, mux=0.0, muy=0.0, sigmaxy=0.0):
    """
    Bivariate Gaussian distribution for equal shape *X*, *Y*.
    See `bivariate normal
    <http://mathworld.wolfram.com/BivariateNormalDistribution.html>`_
    at mathworld.
    This is a reimplementation of https://github.com/matplotlib/matplotlib/blob/81e8154dbba54ac1607b21b22984cabf7a6598fa/lib/matplotlib/mlab.py#L1866
    """
    Xmu = X - mux
    Ymu = Y - muy

    rho = sigmaxy / (sigmax * sigmay)
    z = (
        Xmu ** 2 / sigmax ** 2
        + Ymu ** 2 / sigmay ** 2
        - 2 * rho * Xmu * Ymu / (sigmax * sigmay)
    )
    denom = 2 * np.pi * sigmax * sigmay * np.sqrt(1 - rho ** 2)
    return np.exp(-z / (2 * (1 - rho ** 2))) / denom

def get_peak(size, loc, sig, max_height=5):

    """Returns a 2d matrix of size*size with a peak centred at loc, with width sig, of max_height."""

    m = np.mgrid[:size, :size]
    biv = bivariate_normal(m[0], m[1], sig[0], sig[1], loc[0], loc[1])
    return biv * float(max_height) / biv.max()

def add_gaussian(matrix, size, loc, sig, height, max_height):  # height can be negative
    matrix += get_peak(size, loc, (sig, sig), height)
    # do not allow negative values
    matrix = matrix.clip(min=0, max=max_height)
    return matrix

def remove_gussian_peak(matrix, loc, size, max_height):
    height = matrix[loc[0], loc[1]] * -1  # Lower the current position by its height
    print("Height is", height)
    sig = size * 0.01
    matrix = add_gaussian(matrix, loc, sig, height, max_height)
    return matrix

def reduce_novelty(matrix, loc, size, max_height):
    height = matrix[loc[0], loc[1]] * -0.5
    print("Reduced novely height is", height)
    sig = size * 0.05
    matrix = add_gaussian(matrix, loc, sig, height, max_height)
    return matrix

def top_k_2d_array(arr, k):
    if np.ndim(arr) > 2:
        assert "Please pass an array with dimention less than 2"
    if np.ndim(arr) == 1:
        return arr[np.argsort(arr)[-k:]]
    else:
        flat_indices = np.argpartition(arr.ravel(), -k)[-k:]
        row_indices, col_indices = np.unravel_index(flat_indices, arr.shape)
        max_elements = arr[row_indices, col_indices]
        return max_elements


import numpy as np

def sum_of_squares_error(y, t):
    """
    (function) sum_of_squares_error
    -------------------------------
    The sum of squares error

    Parameter
    ---------
    - y : output value(s)
    - t : target value(s)

    Return
    ------
    - error value between value(s)
    """
    return 0.5 * np.sum((y - t) ** 2)

def cross_entropy_error(y, t):
    """
    (function) cross_entropy_error
    ------------------------------
    - The cross entropy error

    Parameter
    ---------
    - y : output value(s)
    - t : target value(s)

    Return
    ------
    - error value between value(s)
    """
    if y.ndim == 1:
        t = t.reshape(1, t.size)
        y = y.reshape(1, y.size)
    if t.size == y.size:
        t = t.argmax(axis=1)

    batch_size = y.shape[0]
    return -np.sum(np.log(y[np.arange(batch_size), t] + 1e-7)) / batch_size


# %%
import pandas as pd
import numpy as np
import random
import scipy.stats
from scipy.stats import multivariate_normal
import matplotlib.pyplot as plt

# %%
k = 3 # clusters
# d = 2 # dimensions
prob = 1/k # uniform prior on cluster assignments
n = 150 # data points
iterations = 1000
lambdas = 1
sigma = 0.5 # note: sigma^2

# %% Generative process (synthetic data)
d = 2
z_true = np.zeros(shape = (n))
z_true = np.random.choice(range(k), p = np.repeat(prob, k), size = n)

mu_true = np.zeros(shape = (k,d))

mu_true[0] = np.array((0,0))
mu_true[1] = np.array((1,2))
mu_true[2] = np.array((3,-4))

x = np.zeros(shape = (n,d))

for i in range(n):
    cluster_assignment = int(z_true[i])
    x[i] = scipy.stats.multivariate_normal.rvs(mean = mu_true[cluster_assignment], cov = sigma * np.identity(d))

# %% Matrix declaration
z = np.zeros(shape = (n, iterations))
mu = np.zeros(shape = (k, d, iterations))

n_k = np.zeros(shape = (k, iterations))
k_bar = np.zeros(shape = (k,d, iterations))

# %% Graphical representation of synthetic data:
from pylab import *
import matplotlib.pyplot as plt

x1 = x[:,0]
x2 = x[:,1]

fig = plt.figure(figsize = (16,9))
ax = fig.add_subplot(111)

scatter(x1, x2, s=100 ,marker='o')

[ plot( [dot_x,dot_x] ,[0,dot_y], '.', linewidth = 3 ) for dot_x,dot_y in zip(x1,x2) ]
[ plot( [0,dot_x] ,[dot_y,dot_y], '.', linewidth = 3 ) for dot_x,dot_y in zip(x1,x2) ]

left,right = ax.get_xlim()
low,high = ax.get_ylim()
arrow( left, 0, right -left, 0, length_includes_head = True, head_width = 0.08 )
arrow( 0, low, 0, high-low, length_includes_head = True, head_width = 0.08 )

show()

# %% Initialization:
mu[:,:,0] = np.zeros(shape = (k,d)) # mixture locations
z[:,0] = np.random.choice(a = np.array((range(k))), p = np.repeat(prob, k), size = n)

# %% Gibbs sampler:
for it in range(iterations-1):

    for i in range(n):
        #cluster_assignment = int(z[i, it])
        prob_temp = np.zeros(shape = (3,1))
        for kk in range(k):
            prob_temp[kk] = prob * scipy.stats.multivariate_normal.pdf(x = x[i], mean = mu[kk,:,it], cov = sigma * np.identity(d))
        # print(f"Probabilities: {prob_temp}")
        prob_temp = prob_temp / prob_temp.sum()
        # print(f"Probabilities, normalized: {prob_temp}")
        z[i,it+1] = np.random.choice(a = np.array((0,1,2)), p = prob_temp[:,0])

    for kk in range(k):

        n_k[kk,it] = np.unique(z[:,it], return_counts=True)[1][list(np.unique(z[:,0], return_counts=True)[0]).index(kk)]
        for i in range(n):
            if z[i,it] == kk:
                k_bar[kk,:,it] += x[i]
        k_bar[kk,:,it] = k_bar[kk,:,it] / n_k[kk,it]

        mean_k = (n_k[kk,it] / sigma) / ((n_k[kk,it]/sigma) + 1/lambdas) * k_bar[kk,:,it]
        cov_k = 1 / (n_k[kk,it]/sigma + 1/lambdas)

        mu[kk,:,it+1] = scipy.stats.multivariate_normal.rvs(mean = mean_k, cov = cov_k)

    print(f"Iteration {it} complete.")

# %% Burn-in and trimming of the relevant chains
burn_in = int(iterations/2)

z_trimmed = z[:,burn_in:]
mu_trimmed = mu[:,:,burn_in:]

# %% Bayesian estimates: mean of the approximate posteriors
z_est = z_trimmed.mean(axis = 1)
mu_est = mu_trimmed.mean(axis = 2)

# Sampling from the approximate posterior
x_sampl_1 = scipy.stats.multivariate_normal.rvs(mean = mu_est[0,:], cov = sigma * np.identity(2), size = 50)
x_sampl_2 = scipy.stats.multivariate_normal.rvs(mean = mu_est[1,:], cov = sigma * np.identity(2), size = 50)
x_sampl_3 = scipy.stats.multivariate_normal.rvs(mean = mu_est[2,:], cov = sigma * np.identity(2), size = 50)

# %% Graphical representation:
from pylab import *
import matplotlib.pyplot as plt

x11 = x_sampl_1[:,0]
x12 = x_sampl_2[:,0]
x13 = x_sampl_3[:,0]

x21 = x_sampl_1[:,1]
x22 = x_sampl_2[:,1]
x23 = x_sampl_3[:,1]

fig = plt.figure(figsize = (16,9))
ax = fig.add_subplot(111)

scatter(x11, x21, s=100 ,marker='o')
scatter(x12, x22, s=100 ,marker='o')
scatter(x13, x23, s=100 ,marker='o')

[ plot( [dot_x,dot_x] ,[0,dot_y], '.', linewidth = 3 ) for dot_x,dot_y in zip(x11,x21) ]
[ plot( [0,dot_x] ,[dot_y,dot_y], '.', linewidth = 3 ) for dot_x,dot_y in zip(x11,x21) ]

[ plot( [dot_x,dot_x] ,[0,dot_y], '.', linewidth = 3 ) for dot_x,dot_y in zip(x12,x22) ]
[ plot( [0,dot_x] ,[dot_y,dot_y], '.', linewidth = 3 ) for dot_x,dot_y in zip(x12,x22) ]

[ plot( [dot_x,dot_x] ,[0,dot_y], '.', linewidth = 3 ) for dot_x,dot_y in zip(x13,x23) ]
[ plot( [0,dot_x] ,[dot_y,dot_y], '.', linewidth = 3 ) for dot_x,dot_y in zip(x13,x23) ]

left,right = ax.get_xlim()
low,high = ax.get_ylim()
arrow( left, 0, right -left, 0, length_includes_head = True, head_width = 0.08 )
arrow( 0, low, 0, high-low, length_includes_head = True, head_width = 0.08 )

show()

# %% Trace plots of the means of three clusters
fig, ax = plt.subplots(figsize = (16,9), nrows = k, ncols = d)
for ax2 in range(ax.shape[1]):
    ax[ax1,ax2].plot(mu[0,ax2,:])

# %% Effective Sample Size
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
from statsmodels.tsa.stattools import acf
def eff_sample_size(chain):
    '''
    Compute the Effective Sample Size for the MCMC provided
    '''
    dimen = len(chain)
    somma = np.sum(acf(chain))
    den = 1 + 2*somma
    ess = dimen/den

    return ess

# %% Showing ESS results for cluster means
for i in range(k):
    print(f"ESS (means cluster{i}): ",eff_sample_size(mu[i,0,:]), eff_sample_size(mu[i,0,:]))

# %% Kolmogorov-Smirnov nonparametric test
from scipy import stats
test_results = np.zeros(50)
effective_ss = []

mu_thinned = mu[:,:,::2] # thinning

for i in range(k):
    for j in range(d):
        burnin = mu_thinned[i,j,:int(len(mu_thinned[i,j,:])/3)]
        sample1 = mu_thinned[i,j,int(len(mu_thinned[i,j,:])/3):2*int(len(mu_thinned[i,j,:])/3)]
        sample2 = mu_thinned[i,j,2*int(len(mu_thinned[i,j,:])/3):]
    print(f"Test completed for cluster {i}")
    print(stats.ks_2samp(sample1, sample2))

z_thinned = z[:,::2]
for i in range(n):
    burnin = z_thinned[i,:int(len(z_thinned)/3)]
    sample1 = z_thinned[i,int(len(z_thinned)/3):2*int(len(z_thinned)/3)]
    sample2 = z_thinned[i,2*int(len(z_thinned)/3):]
print(f"Test completed for data point {i}")
print(stats.ks_2samp(sample1, sample2))

# %% --------- Application 2: image segmentation ----------------------------
#%%
import pandas as pd
import numpy as np
import random
import scipy.stats
from scipy.stats import multivariate_normal
import matplotlib.pyplot as plt
import math

# %%
it = 0
k = 8 # clusters
# d = 2 # dimensions
prob = 1/k # uniform prior on cluster assignments
# n = 150 # data points
iterations = 200
lambdas = 0.003
sigma = 0.003 # note: sigma^2

# %% Image data
from matplotlib.image import imread
# img = imread('/Users/leonardo/Downloads/xp.jpg')
img = imread('home.jpg')

# img = imread('home.png')
img_size = img.shape
x = img.reshape(img_size[0] * img_size[1], img_size[2])
x = x.astype(float)

d = x.shape[1]
n = x.shape[0]

# Normalizing color intensities between 0 and 1
for dim in range(x.shape[1]):
    for i in range(x.shape[0]):
        x[i, dim] /= 255

#%% Matrix declaration
z = np.zeros(shape = (n, iterations))
mu = np.zeros(shape = (k, d, iterations))

n_k = np.zeros(shape = (k, iterations))
k_bar = np.zeros(shape = (k,d, iterations))

# %% Gibbs sampler:
# mu[:,:,0] = x.mean(axis = 0) # mixture locations
mu[:,:,0] = np.zeros(shape = (k,d))  # mixture locations
z[:,0] = np.random.choice(a = np.array((range(k))), p = np.repeat(prob, k), size = n)

for it in range(iterations-1):

    for i in range(n):
        # cluster_assignment = int(z[i, it])
        prob_temp = np.zeros(shape = (k,1))
        for kk in range(k):
            prob_temp[kk] = prob * scipy.stats.multivariate_normal.pdf(x = x[i], mean = mu[kk,:,it], cov = sigma * np.identity(d))
        # print(f"Probabilities: {prob_temp}")
        prob_temp /= prob_temp.sum()
        # print(f"Probabilities, normalized: {prob_temp}")
        z[i,it+1] = np.random.choice(a = np.array(range(k)), p = prob_temp[:,0])

    for kk in range(k):
        if kk in list(np.unique(z[:,it], return_counts=True)[0]):
            n_k[kk,it] = np.unique(z[:,it], return_counts=True)[1][list(np.unique(z[:,it], return_counts=True)[0]).index(kk)]
        for i in range(n):
            if z[i,it] == kk:
                k_bar[kk,:,it] += x[i]
        k_bar[kk,:,it] = k_bar[kk,:,it] / n_k[kk,it]

        mean_k = (n_k[kk,it] / sigma) / ((n_k[kk,it]/sigma) + 1/lambdas) * k_bar[kk,:,it]
        cov_k = 1 / (n_k[kk,it]/sigma + 1/lambdas)

        mu[kk,:,it+1] = scipy.stats.multivariate_normal.rvs(mean = mean_k, cov = cov_k)

    print(f"Iteration {it} complete.")

# %% Burn-in and trimming
burn_in = int(iterations/3)
mu_trimmed = mu[:,:,burn_in:]
mu_est = mu_trimmed[:,:,:].mean(axis=2)

x_compr = np.zeros(shape = (1024,d))
for i in range(x.shape[0]):
    x_compr[i] = mu[int(z[i,it-1]),:,it] * 255
x_compr = x_compr.astype(int)

x_reshaped = x.reshape(img_size[0], img_size[1], img_size[2])
x_compr = x_compr.reshape(img_size[0], img_size[1], img_size[2])

# %% Drawing the final results
fig, ax = plt.subplots(1, 2, figsize = (12, 8))
ax[0].imshow(img)
ax[0].set_title('Original Image')
ax[1].imshow(x_compr)
ax[1].set_title(f'Segmented Image with {k} colors')
for ax in fig.axes:
    ax.axis('off')
plt.tight_layout();

# %% Plot of the means of three clusters
fig, ax = plt.subplots(figsize = (16,9), nrows = k, ncols = d)
for ax1 in range(ax.shape[0]):
    for ax2 in range(ax.shape[1]):
        ax[ax1,ax2].plot(mu[ax1,ax2,:])
        ax[ax1,ax2].set_ylim((-0.2,1.2))

# %% Original pixels, cluster assignment and assigned cluster means
show_its = it-1
for i in range(int(x.shape[0]/20)):
    print(x[i], z[i,show_its], mu_est[int(z[i,show_its]),:])

MCMC Image Segmentation with Mixture of Gaussian/mixture_normal.py

import numpy,import scipy,from scipy,from statsmodels


"""
Barnsley fern
https://en.wikipedia.org/wiki/Barnsley_fern
"""
import sys
import numpy as np
import random
import matplotlib.pyplot as plt

def f1(x, y):
    return np.array([[0, 0], [0, 0.16]]).dot(np.array([x, y]))

def f2(x, y):
    return (np.array([[0.85, 0.04], [-0.04, 0.85]]).dot(np.array([x, y]))
            + np.array([0, 1.6]))

def f3(x, y):
    return (np.array([[0.20, -0.26], [0.23, 0.22]]).dot(np.array([x, y]))
            + np.array([0, 1.6]))

def f4(x, y):
    return (np.array([[-0.15, 0.28], [0.26, 0.24]]).dot(np.array([x, y]))
            + np.array([0, 0.44]))

if __name__ == "__main__":
    n = int(sys.argv[1]) if len(sys.argv) == 2 else 100000
    x, y = [0], [0]
    for _ in range(n):
        r = random.random()
        if r < 0.01:
            dot = f1(x[-1], y[-1])
        elif r < 0.86:
            dot = f2(x[-1], y[-1])
        elif r < 0.93:
            dot = f3(x[-1], y[-1])
        else:
            dot = f4(x[-1], y[-1])
        x.append(dot[0])
        y.append(dot[1])
    plt.plot(x, y, '.', markersize=2, color='g')
    plt.title("Barnsley fern")
    plt.tight_layout()
    plt.show()


from IPython.display import Latex, display
from sympy import *

# init_session()
# init_printing(use_latex=True)

x, y, z = symbols("x y z")

a = Integral(cos(x), x)
b = Integral(cos(x), (x, 0, 2 * pi))

print(a, "=", a.doit())
display(Eq(a, a.doit()))

print(b, "=", b.doit())
display(Eq(b, b.doit()))

print(latex(b))


from scipy.stats import norm
import numpy as np

alpha = 0.05
z = norm().ppf( 1 - alpha / 2)
p = 0.85
N = 100
Cn = (p - z * np.sqrt(p * (1 - p) / N), p + z * np.sqrt(p * (1 - p) / N))
_ = ["%0.4f" % x for x in Cn]
print(_)

# bootstrap

X = np.zeros(N)
X[:85] = 1

B = []
for _ in range(500):
    s = np.random.randint(X.shape[0], size=X.shape[0])
    B.append(X[s].mean())

se = np.sqrt(np.var(B))
Cn = (p - z * se, p + z * se)
_ = ["%0.4f" % x for x in Cn]
print(_)

Cn = (np.percentile(B, alpha * 100), np.percentile(B, (1 - alpha) * 100))
_ = ["%0.4f" % x for x in Cn]
print(_)

from sklearn.datasets import load_iris
from sklearn.naive_bayes import GaussianNB
from sklearn.model_selection import train_test_split, StratifiedKFold

X, y = load_iris(return_X_y=True)
Xtrain, Xtest, ytrain, ytest = train_test_split(X, y, test_size=0.3)
model = GaussianNB().fit(Xtrain, ytrain)
hy = model.predict(Xtest)
X = np.where(ytest == hy, 1, 0)
p = X.mean()
N = X.shape[0]
Cn = (p - z * np.sqrt(p * (1 - p) / N), p + z * np.sqrt(p * (1 - p) / N))
_ = ["%0.4f" % x for x in Cn]
print(_)

# B = []
# for _ in range(5000):
#     s = np.random.randint(X.shape[0], size=X.shape[0])
#     B.append(X[s].mean())
#
# Cn = (np.percentile(B, alpha * 100), np.percentile(B, (1 - alpha) * 100))
# _ = ["%0.4f" % x for x in Cn]
# print(_)
#
# se = np.sqrt(np.var(B))
# Cn = (p - z * se, p + z * se)
# _ = ["%0.4f" % x for x in Cn]
# print(_)

X, y = load_iris(return_X_y=True)
kf = StratifiedKFold(n_splits=10, shuffle=True)

hy = np.empty_like(y)
for tr, ts in kf.split(X, y):
    model = GaussianNB().fit(X[tr], y[tr])
    hy[ts] = model.predict(X[ts])

X = np.where(y == hy, 1, 0)
p = X.mean()
N = X.shape[0]
Cn = (p - z * np.sqrt(p * (1 - p) / N), p + z * np.sqrt(p * (1 - p) / N))
_ = ["%0.4f" % x for x in Cn]
print(_)

B = []
for _ in range(500):
    s = np.random.randint(X.shape[0], size=X.shape[0])
    B.append(X[s].mean())

se = np.sqrt(np.var(B))
Cn = (p - z * se, p + z * se)
_ = ["%0.4f" % x for x in Cn]
print(_)

Cn = (np.percentile(B, alpha * 100), np.percentile(B, (1 - alpha) * 100))
_ = ["%0.4f" % x for x in Cn]
print(_)

from mlxtend.evaluate import bootstrap_point632_score
X, y = load_iris(return_X_y=True)
cl = GaussianNB()
B = bootstrap_point632_score(cl, X, y, n_splits=500)
Cn = (np.percentile(B, alpha * 100), np.percentile(B, (1 - alpha) * 100))
_ = ["%0.4f" % x for x in Cn]
print(_)

### macro Recall
from scipy.stats import norm
import numpy as np
from sklearn.datasets import load_iris
from sklearn import datasets
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import recall_score

alpha = 0.05
z = norm().ppf( 1 - alpha / 2)

X, y = datasets.load_breast_cancer(return_X_y=True)
kf = StratifiedKFold(n_splits=10, shuffle=True, random_state=0)

hy = np.empty_like(y)
for tr, ts in kf.split(X, y):
    # model = RandomForestClassifier().fit(X[tr], y[tr])
    model = GaussianNB().fit(X[tr], y[tr])
    hy[ts] = model.predict(X[ts])

B = []
for _ in range(500):
    s = np.random.randint(X.shape[0], size=X.shape[0])
    _ = recall_score(y[s], hy[s], average="macro")
    B.append(_)

p = np.mean(B)
se = np.sqrt(np.var(B))
Cn = (p - z * se, p + z * se)
_ = ["%0.4f" % x for x in Cn]
print(_)

Cn = (np.percentile(B, alpha * 100), np.percentile(B, (1 - alpha) * 100))
_ = ["%0.4f" % x for x in Cn]
print(_)

from mlxtend.evaluate import bootstrap_point632_score
X, y = datasets.load_breast_cancer(return_X_y=True)
cl = GaussianNB()
cl = RandomForestClassifier()
B = bootstrap_point632_score(cl, X, y, n_splits=500,
                             scoring_func=lambda y, hy: recall_score(y, hy, average="macro"))
Cn = (np.percentile(B, alpha * 100), np.percentile(B, (1 - alpha) * 100))
_ = ["%0.4f" % x for x in Cn]
print(_)

## Wilcoxon
import numpy as np
from sklearn import datasets
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import recall_score, f1_score
from scipy.stats import wilcoxon

K = 30
kf = StratifiedKFold(n_splits=K, shuffle=True, random_state=0)
X, y = datasets.load_breast_cancer(return_X_y=True)

P = []
for tr, ts in kf.split(X, y):
    forest = RandomForestClassifier().fit(X[tr], y[tr]).predict(X[ts])
    # naive = ExtraTreesClassifier().fit(X[tr], y[tr]).predict(X[ts])
    naive = GaussianNB().fit(X[tr], y[tr]).predict(X[ts])

    P.append([recall_score(y[ts], hy, average="macro") for hy in [forest, naive]])
P = np.array(P)
print(wilcoxon(P[:, 0], P[:, 1]))

p = P[:, 0] - P[:, 1]
_ = np.sqrt(K) * np.mean(p) / np.std(p)
print("%0.4f" % _)

forest = np.empty_like(y)
naive = np.empty_like(y)
alpha=0.05
for tr, ts in kf.split(X, y):
    forest[ts] = RandomForestClassifier().fit(X[tr], y[tr]).predict(X[ts])
    naive[ts] = GaussianNB().fit(X[tr], y[tr]).predict(X[ts])

B = []
for _ in range(500):
    s = np.random.randint(X.shape[0], size=X.shape[0])
    f = recall_score(y[s], forest[s], average="macro")
    n = recall_score(y[s], naive[s], average="macro")
    B.append([f, n])

Cn = (np.percentile(B, alpha * 100, axis=0), np.percentile(B, (1 - alpha) * 100, axis=0))
_ = ["%0.4f" % x for x in Cn[0]]
print(_)
_ = ["%0.4f" % x for x in Cn[1]]
print(_)

alpha = 0.0125
z = norm().ppf( 1 - alpha / 2)
p = 0.87
N = 1000
Cn = (p - z * np.sqrt(p * (1 - p) / N), p + z * np.sqrt(p * (1 - p) / N))
_ = ["%0.4f" % x for x in Cn]
print(_)

import numpy as np
from sklearn import datasets
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import KFold
from sklearn.metrics import r2_score

X, y = datasets.load_boston(return_X_y=True)
K = 10
kf = KFold(n_splits=K, shuffle=True, random_state=0)

hy = np.empty_like(y)
for tr, ts in kf.split(X, y):
    hy[ts] = LinearRegression().fit(X[tr], y[tr]).predict(X[ts])

B = []
for _ in range(500):
    s = np.random.randint(X.shape[0], size=X.shape[0])
    f = r2_score(y[s], hy[s])
    B.append(f)
alpha = 0.05
Cn = (np.percentile(B, alpha * 100, axis=0), np.percentile(B, (1 - alpha) * 100, axis=0))
["%0.2f" % x for x in Cn]


import numpy as np
import scipy.special
import math

def jacobi_gauss_lobatto(alpha, beta, n_order):
    """
    OCTAVE CHECKED jacobiGL
    Compute the order n_order Gauss Lobatto quadrature points, x, associated
    with the Jacobi polynomial.

    >>> jacobi_gauss_lobatto(0.0, 0.0, 1)
    array([-1.,  1.])
    >>> jacobi_gauss_lobatto(0,0,3)
    array([-1.       , -0.4472136,  0.4472136,  1.       ])
    >>> jacobi_gauss_lobatto(0,0,4)
    array([-1.        , -0.65465367,  0.        ,  0.65465367,  1.        ])

    """
    if n_order==0:
        return np.array([0.0])
    if n_order==1:
        return np.array([-1.0, 1.0])
    if n_order>1:
        x, w = scipy.special.roots_jacobi(n_order-1, alpha+1, beta+1)
        return np.concatenate(([-1.0], x, [1.0]))

    raise ValueError('n_order must be positive.')

def jacobi_gauss(alpha, beta, n_order):
    """
    Compute the order n_order Gauss quadrature points, x,
    and weights, w, associated with the Jacobi
    polynomial, of type (alpha,beta) > -1 ( <> -0.5).
    >>> s1 = jacobi_gauss(2,1,0)
    >>> s2 = [-0.2,  2]
    >>> np.allclose(s1,s2)
    True
    >>> s1 = jacobi_gauss(2,1,1)
    >>> s2 = [([-0.54691816,  0.26120387]), ([0.76094757, 0.57238576])]
    >>> np.allclose(s1,s2)
    True
    >>> s1 = jacobi_gauss(2,1,2)
    >>> s2 = [([-0.70882014, -0.13230082,  0.50778763]), ([0.39524241,  0.72312171,  0.21496922])]
    >>> np.allclose(s1,s2)
    True
    """

def mesh_generator(xmin,xmax,k_elem):
    """
    OCVATE CHECKED
    Generate simple equidistant grid with K elements
    >>> [Nv, vx, K, etov] = mesh_generator(0,10,4)
    >>> Nv
    5
    >>> vx_test = ([0.00000000,2.50000000,5.00000000,7.50000000,10.00000000])
    >>> np.allclose(vx,vx_test)
    True
    >>> K
    4
    >>> etov_test = ([[1, 2],[2, 3],[3, 4],[4, 5]])
    >>> np.allclose(etov,etov_test)
    True
    """
    n_v = k_elem+1
    vx = np.zeros(n_v)
    for i in range(n_v):
        vx[i] = (xmax-xmin)*i/(n_v-1)+xmin
    #np.zeros creates a float array. etov should be an integer array
    etov = np.full((k_elem,2),0)
    #etov = np.zeros([K,2])
    for i in range(k_elem):
        etov[i,0] = i+1
        etov[i,1] = i+2

    return [n_v,vx,k_elem,etov]

def vandermonde(n_order, r):
    """
    OCTAVE CHECKED Vandermonde1D
    Initialize Vandermonde matrix
    >>> r = jacobi_gauss_lobatto(0,0,2)
    >>> vandermonde(2,r)
    array([[ 0.70710678, -1.22474487,  1.58113883],
           [ 0.70710678,  0.        , -0.79056942],
           [ 0.70710678,  1.22474487,  1.58113883]])
    """
    vander = np.zeros([len(r), n_order+1])

    for j in range(n_order+1):
        vander[:,j] = jacobi_polynomial(r, 0, 0, j)

    return vander

def differentiation_matrix(n_order,r,vander):
    """
    OCTAVE CHECKED Dmatrix1D
    Initialize the (r) differentiation matrices
    of the interval evaluated at (r) at order n_order
    V is the 1d Vandermonde matrix
    >>> r = jacobi_gauss_lobatto(0,0,2)
    >>> v = vandermonde(2,r)
    >>> differentiation_matrix(2,r,v)
    array([[-1.5,  2. , -0.5],
           [-0.5,  0. ,  0.5],
           [ 0.5, -2. ,  1.5]])
    >>> r = jacobi_gauss_lobatto(0,0,3)
    >>> v = vandermonde(3,r)
    >>> A1 = differentiation_matrix(3,r,v)
    >>> A2 = ([[-3.00000000e+00,  4.04508497e+00, -1.54508497e+00,  5.00000000e-01], \
               [-8.09016994e-01, -4.05396129e-16,  1.11803399e+00, -3.09016994e-01], \
               [ 3.09016994e-01, -1.11803399e+00,  6.28036983e-16,  8.09016994e-01], \
               [-5.00000000e-01,  1.54508497e+00, -4.04508497e+00,  3.00000000e+00]])
    >>> np.allclose(A1,A2)
    True
    """
    v_r = vandermonde_grad(n_order,r)
    v_inv = np.linalg.inv(vander)
    diff_matrix = np.matmul(v_r,v_inv)
    return diff_matrix

def vandermonde_grad(n_order,r):
    """
    OCTAVE CHECKED
    Initialize the gradient of the modal basis (i) at (r)
    at order (n_order)
    >>> r = jacobi_gauss_lobatto(0,0,2)
    >>> vandermonde_grad(2,r)
    array([[ 0.        ,  1.22474487, -4.74341649],
           [ 0.        ,  1.22474487,  0.        ],
           [ 0.        ,  1.22474487,  4.74341649]])
    """
    grad_vander =  np.zeros([len(r),n_order+1])
    for i in range(n_order+1):
        grad_vander[:,i] = jacobi_polynomial_grad(r,0,0,i)

    return grad_vander

def jacobi_polynomial_grad(r, alpha, beta, n_order):
    """
    OCTAVE CHECKED, this function is used on vandermonde_grad.
    Evaluate the derivative of the Jacobi pol. of type (alpha,beta) > -1
    at points r for order n_order
    >>> r = jacobi_gauss_lobatto(0,0,1)
    >>> jacobi_polynomial_grad(r,0,0,1)
    array([1.22474487, 1.22474487])
    >>> r = jacobi_gauss_lobatto(0,0,3)
    >>> jacobi_polynomial_grad(r,0,0,3)
    array([11.22497216,  0.        ,  0.        , 11.22497216])
    """

    der_jacobi_pol = np.zeros([len(r)])

    if n_order == 0:
        return der_jacobi_pol

    jacobi_pol = jacobi_polynomial(r,alpha+1,beta+1,n_order-1)

    for i in range(len(r)):
        der_jacobi_pol[i] = math.sqrt(n_order*(n_order+alpha+beta+1))*jacobi_pol[i]

    return der_jacobi_pol

def jacobi_polynomial(r, alpha, beta, n_order):
    """
    OCTAVE CHECKED JacobiP
    Evaluate Jacobi Polynomial
    >>> r = jacobi_gauss_lobatto(0,0,1)
    >>> jacobi_polynomial(r, 0, 0, 1)
    array([-1.22474487,  1.22474487])
    >>> r = jacobi_gauss_lobatto(0,0,2)
    >>> jacobi_polynomial(r, 0, 0, 2)
    array([ 1.58113883, -0.79056942,  1.58113883])
    >>> r = jacobi_gauss_lobatto(0,0,3)
    >>> jacobi_polynomial(r, 0, 0, 3)
    array([-1.87082869,  0.83666003, -0.83666003,  1.87082869])
    >>> r = jacobi_gauss_lobatto(0,0,4)
    >>> jacobi_polynomial(r, 0, 0, 4)
    array([ 2.12132034, -0.90913729,  0.79549513, -0.90913729,  2.12132034])
    """
    jacobi_pol = np.zeros([n_order+1,len(r)])
    # Initial values P_0(x) and P_1(x)
    gamma0 = 2**(alpha+beta+1) \
            / (alpha+beta+1) \
            * scipy.special.gamma(alpha+1) \
            * scipy.special.gamma(beta+1) \
            / scipy.special.gamma(alpha+beta+1)

    jacobi_pol[0] = 1.0 / math.sqrt(gamma0)
    if n_order == 0:
    #    return PL.transpose()
        return jacobi_pol[0]

    gamma1 = (alpha+1.) * (beta+1.) / (alpha+beta+3.) * gamma0
    jacobi_pol[1] = ((alpha+beta+2.)*r/2. + (alpha-beta)/2.) / math.sqrt(gamma1)

    if n_order == 1:
    #    return PL.transpose()
        return jacobi_pol[1]
    # Repeat value in recurrence.
    aold = 2. / (2.+alpha+beta) \
        * math.sqrt( (alpha+1.)*(beta+1.) / (alpha+beta+3.))

    # Forward recurrence using the symmetry of the recurrence.
    for i in range(n_order-1):
        h1 = 2.*(i+1.) + alpha + beta
        anew = 2. / (h1+2.) \
            * math.sqrt((i+2.)*(i+2.+ alpha+beta)*(i+2.+alpha)*(i+2.+beta) \
                        / (h1+1.)/(h1+3.))

        bnew = - (alpha**2 - beta**2) / h1 / (h1+2.)

        jacobi_pol[i+2] = 1. / anew * (-aold * jacobi_pol[i] + (r-bnew) * jacobi_pol[i+1])

        aold = anew

    return jacobi_pol[n_order]

def surface_integral_dg(n_order,vander):
    """
    OCTAVE CHECKED Lift1D
    Compute surface integral term in DG formulation
    >>> r = jacobi_gauss_lobatto(0,0,2)
    >>> v = vandermonde(2,r)
    >>> surface_integral_dg(2,v)
    array([[ 4.5 ,  1.5 ],
           [-0.75, -0.75],
           [ 1.5 ,  4.5 ]])
    >>> r = jacobi_gauss_lobatto(0,0,3)
    >>> v = vandermonde(3,r)
    >>> surface_integral_dg(3,v)
    array([[ 8.        , -2.        ],
           [-0.89442719,  0.89442719],
           [ 0.89442719, -0.89442719],
           [-2.        ,  8.        ]])
    """
    # n_faces, n_fp and n_p are defined as global variables
    n_faces = 1
    n_fp = 2
    n_p = n_order+1

    emat = np.zeros([n_p,n_faces*n_fp])
    emat[0,0] = 1.0
    emat[n_p-1,1] = 1.0

    v_trans = np.transpose(vander)
    v_i = np.matmul(v_trans,emat)
    lift = np.matmul(vander,v_i)

    return lift

def nodes_coordinates(n_order,etov,vx):
    """
    OCTAVE CHECKED 2 menos
    Part of StartUp1D.m. Defined to be able to define
    methods depedent grid properties
    >>> [Nv,vx,K,etov] = mesh_generator(0,10,4)
    >>> x = nodes_coordinates(4,etov,vx)
    >>> x_test = ([[0.00000000,    2.50000000,    5.00000000,    7.50000000], \
                   [0.43168291,    2.93168291,    5.43168291,    7.93168291], \
                   [1.25000000,    3.75000000,    6.25000000,    8.75000000], \
                   [2.06831709,    4.56831709,    7.06831709,    9.56831709], \
                   [2.50000000,    5.00000000,    7.50000000,   10.00000000]])
    >>> np.allclose(x,x_test)
    True
    """

    jgl = jacobi_gauss_lobatto(0,0,n_order)

    va = etov[:,0]
    vb = etov[:,1]
    vx_va = np.zeros([1,len(va)], dtype = 'g')
    vx_vb = np.zeros([1,len(va)], dtype = 'g')
    for i in range(len(va)):
        vx_va[0,i] = vx[va[i]-1]
        vx_vb[0,i] = vx[vb[i]-1]

    nodes_coord = np.matmul(np.ones([n_order+1,1], dtype = 'g'),vx_va)+0.5*np.matmul((jgl.reshape(n_order+1,1)+1),(vx_vb-vx_va))
    return nodes_coord

def geometric_factors(nodes_coord,diff_matrix):
    """
    OCTAVE CHECKED.
    Compute the metric elements for the local mappings of the 1D elements
    >>> [Nv,vx,K,etov] = mesh_generator(0,10,4)
    >>> x = nodes_coordinates(2,etov,vx)
    >>> r = jacobi_gauss_lobatto(0,0,2)
    >>> V = vandermonde(2,r)
    >>> Dr = differentiation_matrix(2,r,V)
    >>> [rx,J] = geometric_factors(x,Dr)
    >>> rx_test = ([[0.80000,   0.80000,   0.80000,   0.80000], \
                    [0.80000,   0.80000,   0.80000,   0.80000], \
                    [0.80000,   0.80000,   0.80000,   0.80000]])
    >>> J_test =   ([[1.2500,   1.2500,   1.2500,   1.2500], \
                     [1.2500,   1.2500,   1.2500,   1.2500], \
                     [1.2500,   1.2500,   1.2500,   1.2500]])
    >>> np.allclose(rx,rx_test)
    True
    >>> np.allclose(J,J_test)
    True
    """
    xr = np.matmul(diff_matrix,nodes_coord)
    jacobian = xr
    rx = 1/jacobian

    return [rx,jacobian]

def connect(etov):
    """
    OCTAVE CHECKED
    Build global connectivity arrays for 1D grid based on standard
    etov input array from grid generator
    >>> [Nv,vx,K,etov] = mesh_generator(0,10,4)
    >>> [etoe, etof] = connect(etov)
    >>> etoe_test =  ([[1,2], \
                       [1,3], \
                       [2,4], \
                       [3,4]])
    >>> etof_test =  ([[1,1], \
                       [2,1], \
                       [2,1], \
                       [2,2]])
    >>> np.allclose(etoe,etoe_test)
    True
    >>> np.allclose(etof,etof_test)
    True
    >>> [Nv,vx,K,etov] = mesh_generator(-1,22,7)
    >>> [etoe, etof] = connect(etov)
    >>> etoe_test = ([[1,2],\
                      [1,3],\
                      [2,4],\
                      [3,5],\
                      [4,6],\
                      [5,7],\
                      [6,7]])
    >>> etof_test = ([[1,1],\
                      [2,1],\
                      [2,1],\
                      [2,1],\
                      [2,1],\
                      [2,1],\
                      [2,2]])
    >>> np.allclose(etoe,etoe_test)
    True
    >>> np.allclose(etof,etof_test)
    True
    """
    n_faces = 2
    k_elem = np.shape(etov)[0]
    total_faces = n_faces*k_elem
    nv = k_elem+1
    vn = np.arange(0,2)
    sp_ftov = np.zeros([total_faces,nv])

    sk = 0

    for i in range(k_elem):
        for face in range(n_faces):
            sp_ftov[sk][etov[i][vn[face]]-1] = 1
            sk += 1

    sp_ftof = np.matmul(sp_ftov,np.transpose(sp_ftov))-np.identity(total_faces)
    [faces_2,faces_1] = np.where(sp_ftof==1)
    #numpy floor returns floats

    element_1 = np.int64(np.floor(faces_1/n_faces))
    element_2 = np.int64(np.floor(faces_2/n_faces))

    face_1 = np.mod(faces_1,n_faces)
    face_2 = np.mod(faces_2,n_faces)

    ind = np.arange(len(element_1))

    for i in range(len(element_1)):
        ind[i] = np.ravel_multi_index((element_1[i],face_1[i]),dims=(k_elem,n_faces))

    etoe_1 = np.transpose(np.arange(1,k_elem+1).reshape(1,k_elem))
    etoe_2 = np.full([1,n_faces],1)
    etoe = np.matmul(etoe_1,etoe_2)

    etof_1 = np.full([k_elem,1],1)
    etof_2 = np.arange(1,n_faces+1).reshape(1,n_faces)
    etof = np.matmul(etof_1,etof_2)

    for i in range(len(ind)):
        etoe.ravel()[ind[i]] = element_2[i]+1
        etof.ravel()[ind[i]] = face_2[i]+1

    return [etoe, etof]

def normals(k_elem):
    """
    OCTAVE CHECKED
    Compute outward pointing normals at element faces
    >>> normals(4)
    array([[-1., -1., -1., -1.],
           [ 1.,  1.,  1.,  1.]])
    """
    # K is the number of elements, derived from the grid info
    # n_faces and n_fp are defined as global variables
    n_faces = 1
    n_fp = 2
    nx = np.zeros([n_fp*n_faces,k_elem])
    nx[0,:] = -1.0
    nx[1,:] = 1.0
    return nx

def build_maps(n_order,nodes_coord,etoe,etof):
    """
    OCTAVE CHECKED
    Connectivity and boundary tables for nodes given in the K # of elements,
    each with n_order+1 degrees of freedom.
    >>> [Nv,vx,K,etov] = mesh_generator(0,10,4)
    >>> x = nodes_coordinates(4,etov,vx)
    >>> [etoe, etof] = connect(etov)
    >>> [vmap_m,vmap_p,vmap_b,map_b] = build_maps(4,x,etoe,etof)
    >>> vmap_m_test = ([[1,5,6,10,11,15,16,20]])
    >>> np.allclose(vmap_m,vmap_m_test)
    True
    >>> vmap_p_test = ([[1,6,5,11,10,16,15,20]])
    >>> np.allclose(vmap_p,vmap_p_test)
    True
    >>> vmap_b_test = ([[1,20]])
    >>> np.allclose(vmap_b,vmap_b_test)
    True
    >>> map_b_test = ([[1,8]])
    >>> np.allclose(map_b,map_b_test)
    True
    """
    jgl = jacobi_gauss_lobatto(0,0,n_order)
    k_elem = np.size(etoe,0)
    n_p = n_order+1
    n_faces = 2
    n_fp = 1
    #mask defined in globals
    fmask_1 = np.where(np.abs(jgl+1)<1e-10)[0][0]
    fmask_2 = np.where(np.abs(jgl-1)<1e-10)[0][0]
    fmask = [fmask_1,fmask_2]
    node_ids = np.reshape(np.arange(k_elem*n_p),[n_p,k_elem],'F')
    vmap_m = np.full([k_elem,n_fp,n_faces],0)
    vmap_p = np.full([k_elem,n_fp,n_faces],0)

    for k1 in range(k_elem):
        for f1 in range(n_faces):
            vmap_m[k1,:,f1] = node_ids[fmask[f1],k1]

    for k1 in range(k_elem):

        for f1 in range(n_faces):
            k2 = etoe[k1,f1]-1
            f2 = etof[k1,f1]-1

            vid_m = vmap_m[k1,:,f1][0]
            vid_p = vmap_m[k2,:,f2][0]

            x1 = nodes_coord.ravel('F')[vid_m]
            x2 = nodes_coord.ravel('F')[vid_p]

            distance = (x2-x1)**2
            if (distance < 1e-10):
                vmap_p[k1,:,f1] = vid_p

    vmap_m+=1
    vmap_p+=1

    vmap_p = vmap_p.ravel()
    vmap_m = vmap_m.ravel()

    map_b = np.where(vmap_p==vmap_m)[0]
    vmap_b = vmap_m[map_b]

    map_b+=1
    vmap_b

    map_i = 1
    map_o = k_elem*n_faces
    vmap_i = 1
    vmap_0 = k_elem*n_p

    return [vmap_m,vmap_p,vmap_b,map_b,fmask]

def rk4(l, intrk):
  a = np.array([                             0.0, \
                   -567301805773.0/1357537059087.0, \
                  -2404267990393.0/2016746695238.0, \
                  -3550918686646.0/2091501179385.0, \
                   -1275806237668.0/842570457699.0])
  b = np.array([ 1432997174477.0/9575080441755.0, \
                  5161836677717.0/13612068292357.0, \
                   1720146321549.0/2090206949498.0, \
                   3134564353537.0/4481467310338.0, \
                  2277821191437.0/14882151754819.0])
  c = np.array([                             0.0, \
                   1432997174477.0/9575080441755.0, \
                   2526269341429.0/6820363962896.0, \
                   2006345519317.0/3224310063776.0, \
                   2802321613138.0/2924317926251.0])
  if l == "a":
    return a[intrk]
  elif l == "b":
    return b[intrk]
  return c[intrk]

class Galerkin:
    def __init__(self, K, N, mi, mf):
        self.K = K
        self.N = N
        self.mi = mi
        self.mf = mf
    def __string__(self):
        return f"Discrete Galerkin FEM initialized with N = {self.N} and K = {self.K}, lets go mr. Intel."

    def calculate(self, case, tf):
        if case == "A":
            pass
        elif case == "Msin":
            pass
        elif case == "Mg":
            pass
        elif case == "Mgglass":
            pass
        elif case == "H":
            [self.Nv, self.VX, self.K, self.EToV] = mesh_generator(self.mi, self.mf, self.K)
            self.r = jacobi_gauss_lobatto(0, 0, self.N)
            self.V = vandermonde(self.N, self.r)
            self.Dr = differentiation_matrix(self.N, self.r, self.V)
            self.LIFT = surface_integral_dg(self.N, self.V)
            self.x = nodes_coordinates(self.N, self.EToV, self.VX)
            [self.rx, self.J] = geometric_factors(self.x, self.Dr)
            self.nx = normals(self.K)
            [self.EToE, self.EToF] = connect(self.EToV)
            [self.vmapM, self.vmapP, self.vmapB, self.mapB, self.fmask] = build_maps(self.N, self.x, self.EToE, self.EToF)
            self.Fscale = 1/self.J[self.fmask,:]
            self.t = 0

            self.u = 5*np.sin(self.x)
            self.resu = np.zeros((self.N+1, self.K))
            self.xmin = np.amin(np.abs(self.x[0, :] - self.x[1, :]))
            CFL = .25
            self.dt = CFL*self.xmin*self.xmin
            self.Nsteps = np.ceil(tf/self.dt)
            self.dt = tf/self.Nsteps

            self.n_faces = 2
            self.map_O = self.K*self.n_faces
            self.vmap_O = self.K*(self.N+1)
            self.vmap_i = 1
            self.map_i = 1
            self.n_fp = 2
            self.du = np.zeros(self.n_faces*self.n_fp*self.K)
            self.dq = np.zeros(self.n_faces*self.n_fp*self.K)
            self.nxr = np.reshape(self.nx, len(self.nx)*len(self.nx[0]), order='F')

            for tstep in range(int(self.Nsteps)):
              for intrk in range(5):
                self.ur = np.reshape(self.u, len(self.u)*len(self.u[0]), order='F')
                self.du = (self.ur[self.vmapM-1]-self.ur[self.vmapP-1])/2
                self.uin = -self.ur[self.vmap_i-1]
                self.uout = -self.ur[self.vmap_O-1]
                self.du[self.map_i-1] = (self.ur[self.vmap_i-1] - self.uin)/2
                self.du[self.map_O-1] = (self.ur[self.vmap_O-1] - self.uout)/2
                self.dur = np.reshape(self.du, (2, int(len(self.du)/2)), order = 'F')
                self.du1 =self.nx*self.dur
                self.Dru = np.matmul(self.Dr,self.u)
                self.Fdu = self.Fscale*self.du1
                self.q = self.rx*self.Dru - np.matmul(self.LIFT,self.Fdu)
                self.qr = np.reshape(self.q, len(self.q)*len(self.q[0]), order='F')
                self.dq = (self.qr[self.vmapM-1]-self.qr[self.vmapP-1])/2
                self.qin = self.qr[self.vmap_i-1]
                self.qout = self.qr[self.vmap_O-1]
                self.dq[self.map_i-1] = (self.qr[self.vmap_i-1] - self.qin)/2
                self.dq[self.map_O-1] = (self.qr[self.vmap_O-1] - self.qout)/2
                self.dqr = np.reshape(self.dq, (2, int(len(self.dq)/2)), order = 'F')
                self.dq1 = self.nx*self.dqr
                self.Drq = np.matmul(self.Dr,self.q)
                self.Fdq = self.Fscale*self.dq1
                self.rhsu = self.rx*self.Drq - np.matmul(self.LIFT,self.Fdq)
                self.du = self.du-self.du
                self.dq = self.dq-self.dq
                #print(self.du)
                #print(self.dq)
                self.resu = rk4("a", intrk)*self.resu + self.dt*self.rhsu
                self.u = self.u + rk4("b", intrk)*self.resu
              #self.t = self.t + self.dt
            return [self.u, self.x]
        elif case == "Sch":
            pass
        elif case == "SchNL":
            pass
        else:
            pass

def advecrhs1d(u, timelocal, a, k_elem, Dr, LIFT, rx, nx, vmap_p, vmap_m, Fscale):
    K=10
    n_faces = 1
    map_O = K*n_faces
    vmap_i = 1
    map_i = 1
    n_fp = 2
    alpha = 1
    du = np.zeros(n_faces*n_fp*k_elem)
    #du reshape
    nxr = np.reshape(nx, len(nx)*len(nx[0]), order='F')
    #nx reshape
    ur = np.reshape(u, len(u)*len(u[0]), order='F')
    du = (ur[vmap_m-1]-ur[vmap_p-1])*(a*nxr)/2

    uin = -np.sin(a*timelocal)
    du[map_i-1] = (ur[vmap_i-1] - uin)*(a*nxr[map_i-1])/2
    du[map_O-1] = 0
    arx = -a*rx
    Dru = np.matmul(Dr,u)
    si = LIFT
    dur = np.reshape(du, (2, int(len(du)/2)), order = 'F')
    Fdu = Fscale*dur
    rhsu = arx*Dru + np.matmul(si,Fdu)
    return rhsu

def maxwell1d(INTRK, tsteps, E, H, epsilon, mu, k_elem, Dr, LIFT, rx, nx, vmap_p, vmap_m, map_b, vmap_b, Fscale):
  n_faces = 1
  n_fp = 2

  Zimp = np.sqrt(mu/epsilon)
  Zimpr = np.reshape(Zimp, len(Zimp)*len(Zimp[0]), order='F')

  dE = np.zeros((n_faces*n_fp*k_elem))
  dH = np.zeros((n_faces*n_fp*k_elem))
  #du reshape
  nxr = np.reshape(nx, len(nx)*len(nx[0]), order='F')
  #nx reshape
  Er = np.reshape(E, len(E)*len(E[0]), order='F')
  Hr = np.reshape(H, len(H)*len(H[0]), order='F')
  dE = Er[vmap_m-1]-Er[vmap_p-1]
  dH = Hr[vmap_m-1]-Hr[vmap_p-1]
  Zimpr = np.reshape(Zimp, len(Zimp)*len(Zimp[0]), order='F')

  Zimpm = np.zeros((n_faces*n_fp*k_elem))
  Zimpm = Zimpr[vmap_m-1]
  Zimpp = np.zeros((n_faces*n_fp*k_elem))
  Zimpp = Zimpr[vmap_p-1]
  Yimpm = np.zeros((n_faces*n_fp*k_elem))
  Yimpm = 1/Zimpm
  Yimpp = np.zeros((n_faces*n_fp*k_elem))
  Yimpp = 1/Zimpp

  Ebc = -Er[vmap_b-1]
  dE[map_b-1] = Er[vmap_b-1] - Ebc
  Hbc = Hr[vmap_b-1]
  dH[map_b-1] = Hr[vmap_b-1] - Hbc

  fluxE = 1/(Zimpm + Zimpp)*(nxr*Zimpp*dH - dE)
  fluxH = 1/(Yimpm + Yimpp)*(nxr*Yimpp*dE - dH)

  fluxEr = np.reshape(fluxE, (2, int(len(fluxE)/2)), order = 'F')
  fluxHr = np.reshape(fluxH, (2, int(len(fluxE)/2)), order = 'F')

  FfluxE = Fscale*fluxEr
  FfluxH = Fscale*fluxHr

  rhsE = (-rx*np.matmul(Dr,H) + np.matmul(LIFT,FfluxE))/epsilon
  rhsH = (-rx*np.matmul(Dr,E) + np.matmul(LIFT,FfluxH))/mu

  return rhsE, rhsH

def maxwellexE(n1, n2, t, x, w):
  xr = np.reshape(x, len(x)*len(x[0]), order='F')
  A = np.zeros(2)
  B = np.zeros(2)
  j = np.complex(0,1)
  A[0] = (n2*np.cos(n2*w))/(n1*np.cos(n1*w))
  A[1] = np.real(np.exp(w*(n1+n2)*j))
  B[0] = np.real(np.exp(-2*n1*w*j))*A[0]
  B[1] = -np.real(np.exp(2*j*n2*w))*A[1]
  Ex1 = np.zeros((len(x)*len(x[0])))
  Ex2 = np.zeros((len(x)*len(x[0])))

  Ex1 = (-A[0]*np.real(np.exp(w*j*n1*xr[:len(xr)//2]))+B[0]*np.real(np.exp(-w*j*n1*xr[:len(xr)//2])))*np.real(np.exp(j*w*t))
  Ex2 = (-A[1]*np.real(np.exp(w*j*n2*xr[len(xr)//2:]))+B[1]*np.real(np.exp(-w*j*n2*xr[len(xr)//2:])))*np.real(np.exp(j*w*t))
  Ex = np.concatenate((Ex1, Ex2), axis = 0)
  Exc = np.reshape(Ex, (len(x), int(len(x[0]))), order='F')

  return Exc

def maxwellexH(n1, n2, t, x, w):
  #No esta completo
  xr = np.reshape(x, len(x)*len(x[0]), order='F')
  A = np.zeros(2)
  B = np.zeros(2)
  j = np.complex(0,1)
  A[0] = (n2*np.cos(n2*w))/(n1*np.cos(n1*w))
  A[1] = np.real(np.exp(j*w*(n1+n2)))
  B[0] = np.real(np.exp(-2*j*n1*w))*A[0]
  B[1] = -np.real(np.exp(2*j*n2*w))*A[1]
  Hx1 = np.zeros((len(x)*len(x[0])))
  Hx2 = np.zeros((len(x)*len(x[0])))

  Hx1 = (A[0]*np.real(np.exp(w*j*n1*xr[:len(xr)//2]))+B[0]*np.real(np.exp(-w*j*n1*xr[:len(xr)//2])))*np.real(np.exp(j*w*t))
  Hx2 = (A[1]*np.real(np.exp(w*j*n2*xr[len(xr)//2:]))+B[1]*np.real(np.exp(-w*j*n2*xr[len(xr)//2:])))*np.real(np.exp(j*w*t))
  Hx = np.concatenate((Hx1, Hx2), axis = 0)
  Hxc = np.reshape(Hx, (len(x), int(len(x[0]))), order='F')

  return Hxc

def HeatCRHS1D(u, timelocal, k_elem, N, Dr, LIFT, rx, nx, vmap_p, vmap_m, Fscale):
    n_faces = 2
    map_O = k_elem*n_faces
    vmap_O = k_elem*(N+1)
    vmap_i = 1
    map_i = 1
    n_fp = 2
    alpha = 1
    du = np.zeros(n_faces*n_fp*k_elem)
    dq = np.zeros(n_faces*n_fp*k_elem)
    nxr = np.reshape(nx, len(nx)*len(nx[0]), order='F')
    ur = np.reshape(u, len(u)*len(u[0]), order='F')
    du = (ur[vmap_m-1]-ur[vmap_p-1])/2
    uin = -ur[vmap_i-1]
    uout = -ur[vmap_O-1]
    du[map_i-1] = (ur[vmap_i-1] - uin)/2
    du[map_O-1] = (ur[vmap_O-1] - uout)/2
    dur = np.reshape(du, (2, int(len(du)/2)), order = 'F')
    du1 =nx*dur
    si = LIFT
    Dru = np.matmul(Dr,u)
    Fdu = Fscale*du1
    q = rx*Dru - np.matmul(si,Fdu)
    qr = np.reshape(q, len(q)*len(q[0]), order='F')
    dq = (qr[vmap_m-1]-qr[vmap_p-1])/2
    qin = qr[vmap_i-1]
    qout = qr[vmap_O-1]
    dq[map_i-1] = (qr[vmap_i-1] - qin)/2
    dq[map_O-1] = (qr[vmap_O-1] - qout)/2
    dqr = np.reshape(dq, (2, int(len(dq)/2)), order = 'F')
    dq1 = nx*dqr
    Drq = np.matmul(Dr,q)
    Fdq = Fscale*dq1
    rhsu = rx*Drq - np.matmul(si,Fdq)
    #input("Press Enter to continue...")
    return rhsu


"""

The term Mandelbrot set is used to refer both to a general class of fractal sets and to a particular instance of such a set. In general, a Mandelbrot set marks the set of points in the complex plane such that the corresponding Julia set is connected and not computable.

The Mandelbrot set is the set obtained from the quadratic recurrence equation:

 z_(n+1)=z_n²+C

with z_0=C, where points C in the complex plane for which the orbit of z_n does not tend to infinity are in the set. Setting z_0 equal to any point in the set that is not a periodic point gives the same result. The Mandelbrot set was originally called a mu molecule by Mandelbrot. J. Hubbard and A. Douady proved that the Mandelbrot set is connected.

[N.B. '_' sign indicate subscript letter here]

How the project works:

This script might take a few seconds to a few minutes to produce the output, depending on your computer. Reducing N, the size of the square array we are filling, will reduce the amount of computations. The result will be a view of the Mandelbrot set in all of its fractal glory.

The pyplot.imshow() function is very simple; give it a 2D array and it will render a picture where each pixel represents one value taken from the 2D array. The color of the pixel is picked from a colormap—each value of the array is linearly normalized in the [0, 1] interval. The pyplot.imshow() function renders a figure, but it won't show it. As usual, we should call pyplot.show() to see the figure. However, having two functions with such similar names is arguably confusing. The remaining parts of the script generate our example data. The 2D array Z is created and then filled with a double loop. This loop samples the [-2.2, 0.8]*[-1.5, 1.5] square. For each sample, the iter_count function computes the Mandelbrot set iterations. The data in the Z array could have come from a file or any other source.

"""

import numpy as np

import matplotlib.cm as cm

from matplotlib import pyplot as plt

plt.style.use("dark_background")

def iter_count ( C, max_iter ):

    X = C

    for n in range ( max_iter ):

       if abs(X) > 2:

          return n

       X = X**2 + C

    return max_iter

N = 512

max_iter = 64

xmin, xmax, ymin, ymax = -2.2, .8, -1.5, 1.5

X = np.linspace(xmin, xmax, N)

Y = np.linspace(ymin, ymax, N)

Z = np.empty((N, N))

for i, y in enumerate(Y):

   for j, x in enumerate(X):

      Z[i, j] = iter_count(complex(x, y), max_iter)

plt.imshow(Z, cmap = cm.gray)

plt.xticks([])

plt.yticks([])

plt.grid(False)

plt.box(False)

plt.suptitle('"A man is known by his heroes!" -- Benoît B. Mandelbrot, Mathematician', fontsize= 10, color="white")

plt.title('Mandelbrot set', color="white", fontsize=15)

plt.figtext(0.84, 0.01, "© Mǟɖ↻ôɖɆⱤ™", color="white", fontsize=9)

plt.show()

plt.savefig('myfig.png', dpi= 200)


# a demo showing the application of farad package in root-finding algorithm by
# Newton's method. For now, we simply consider the case of scalar function with
# scalar input.

from farad.dual import Dual
import farad.elem as el
#import numpy as np
from farad.driver import AutoDiff

# an example input function
def fi(x):
    return el.sin(x) - x

autofx = AutoDiff(fi)

# return the value of function fi
def fx(x):
    return autofx.get_val_scalar(x)

# return the derivative function fi
def dfx(x):
    return autofx.forward(x)

def newton_root(f, df, x0, epsilon, max_iter):
    '''
    an algorithm to find the root of f(x)=0 by Newton's method

    Parameters:
    f: function. Input function f(x)
    df: function. the derivative of input function: df(x) = df(x)/dx
    x0: number. the initial guess of the root value
    epsilon: number. The accuracy threshold. We claim x to be the root when
                     f(x) < epsilon
    max_iter: number. the maximum iterations the routine can take before stop

    Returns:
    y: number or None. When the routine successfully finds the root, the value
    of the root will be returned. Otherwise, None will be returned.
    '''
    n = 1
    xn = x0
    while n <= max_iter:
        fx = f(xn)
        dfx = df(xn)
        if abs(fx) < epsilon:
            print("Successfully found the root after", n, "iterations")
            return xn
        if dfx == 0:
            print("Failed to find the root because derivative is 0.")
            return None
        xn = xn - fx/dfx
        n = n + 1
    print("Reached maximum iterations. Failed to find the root")
    return None

if __name__ == "__main__":
    root = newton_root(fx, dfx, 0.1, 1.e-20, 2000)
    print("root from root-finding function is ", root)
    print('the accurate root is 0.')


import numpy as np

print(np.linspace(0, 10, 3))
# [ 0.  5. 10.]

print(np.linspace(0, 10, 4))
# [ 0.          3.33333333  6.66666667 10.        ]

print(np.linspace(0, 10, 5))
# [ 0.   2.5  5.   7.5 10. ]

print(np.linspace(10, 0, 5))
# [10.   7.5  5.   2.5  0. ]

print(np.linspace(0, 10, 3, dtype=int))
# [ 0  5 10]

print(np.linspace(0, 10, 5))
# [ 0.   2.5  5.   7.5 10. ]

print(np.linspace(0, 10, 5, endpoint=False))
# [0. 2. 4. 6. 8.]

result = np.linspace(0, 10, 5, retstep=True)

print(result)
# (array([ 0. ,  2.5,  5. ,  7.5, 10. ]), 2.5)

print(type(result))
# <class 'tuple'>

print(result[0])
# [ 0.   2.5  5.   7.5 10. ]

print(result[1])
# 2.5

print(np.linspace(0, 10, 5, retstep=True)[1])
# 2.5

print(np.linspace(0, 10, 5, retstep=True, endpoint=False)[1])
# 2.0

print(np.linspace(0, 10, 5))
# [ 0.   2.5  5.   7.5 10. ]

print(np.linspace(10, 0, 5))
# [10.   7.5  5.   2.5  0. ]

print(np.linspace(0, 10, 5)[::-1])
# [10.   7.5  5.   2.5  0. ]

print(np.flip(np.linspace(0, 10, 5)))
# [10.   7.5  5.   2.5  0. ]

print(np.linspace(0, 10, 12).reshape(3, 4))
# [[ 0.          0.90909091  1.81818182  2.72727273]
#  [ 3.63636364  4.54545455  5.45454545  6.36363636]
#  [ 7.27272727  8.18181818  9.09090909 10.        ]]

print(np.linspace(0, 10, 24).reshape(2, 3, 4))
# [[[ 0.          0.43478261  0.86956522  1.30434783]
#   [ 1.73913043  2.17391304  2.60869565  3.04347826]
#   [ 3.47826087  3.91304348  4.34782609  4.7826087 ]]
#
#  [[ 5.2173913   5.65217391  6.08695652  6.52173913]
#   [ 6.95652174  7.39130435  7.82608696  8.26086957]
#   [ 8.69565217  9.13043478  9.56521739 10.        ]]]


                ###############################################################
                ##                                                           ##
                ## Here you are going to learn NumPy Framework For Beginners ##
                ##               https://github.com/Omarne96                 ##
                ##                  Instagram : Omar_ne16                    ##
                ###############################################################

'''
        First of all you have to install numpy framework
        go to your Terminal and type this commands :
        pip3 install numpy
        if you have any error check the pip update or use pip3
'''
## Then import the numpy library
import numpy as np

## To make a simple array :
###########################

a=np.array([[1,2,3,0,4],[4,4,0,4,4],[1,2,3,0,4]], dtype="int16" )
        ### dtype which allows to you to specify your types ###

## get the dimention :
#####################
a.ndim

##get shape:
############
a.shape          ## will gives you : (num of rows , num in each array)

## get the type:
################

a.dtype          ## will gives you data type of all colums you have in your data
a.itemsize       ## will gives you the number of bits
a.size           ## will gives you the total size
a.nbytes         ## will gives you the total number of bytes

##get a specific element [r,c]:
###############################
b=np.array([[1,2,3,4,5,6],[7,8,9,10,11,12],[13,14,15,16,17,18]])
##--- Remember that python count from 0 ---##

b[1,4]             ## will gives you the number 11
b[1,-2]            ## will gives you the number 11 too

b[0,:]             ## get a specific row:
b[:,2]             ## get a specific column:

## get lot of elements [startindex: endindex : stepsize]
## or [num of row,startindex: endindex : stepsize]
b[1,0:-1:2]

c=np.array([[1,2,3,4,5],[5,4,3,2,1]])
## change a value in a array
## b[r,c]= the value
c[0,0]=99
## to chane whole column |
c[:,2]=6
c[:,3]=[55,66] ## with different values

    ########################################################################
    #############################  3D Examples #############################
    ########################################################################

## Get a spesific element ( work outside in ):
##############################################

k=np.array([ [[1,2],[3,4]] , [[6,7],[8,9]] ])
## This is outside [[1,2],[3,4]] , [[6,7],[8,9]]
## [number of array , num of row , num of column]

k[1,1,1]                    ## This will gives you the number 9
k[0,1,1]                    ## This will gives you the number 4
k[0,0,:]                    ## This will gives you the number [1 2]
k[:,0,:]                    ## This will gives you the number [1 2] and [6 7]
k[:,1,:]                    ## This will gives you the number [3 4] and [8 9]

### Replace  ###
################

k[:,1,:]=[[11,22],[44,55]]
k[:,1,:]

#################################################
## all 0s matrix :
np.zeros(5)
np.zeros((2,3))                     ## 2D matrix of zero
np.zeros((2,3,3))                   ## 3D matrix of zero
np.zeros((2,3,3,2))                 ## 4D matrix of zero

## all 1s matrix :
##################
np.ones((3,2,2),dtype="int32")            ## Onse matrix with ??

## Any other numbers:
#####################    np.full(shape,vaue) you can use c.shape too here

np.full((2,2),44,dtype="float32")
np.full_like(c,5)                          ## shape like c

            ################################################################
            ########################## Random ##############################
            ################################################################

## 1 - decimal numbers :
########################

np.random.rand(4,2,3)
np.random.random_sample(c.shape)

## 2 - integer  number less then(x):
####################################
np.random.randint(5)
np.random.randint(5,size=(4,4))
np.random.randint(-5,5,size=(4,4))

## identity matrix ( the principal diagonal = 1 ):
##################################################
np.identity(5)

## Repeat an array :
####################

arr=np.array([[1,2,3]])
r1=np.repeat(arr,3,axis=1) #axis=0 normal ,axis=1 repeat 111222333

            ####################################################################
            ############################# Quiz  ################################
            ####################################################################
ok=np.ones((5,5))
z=np.zeros((3,3))
z[1,1]=9
ok[1:-1,1:-1]=z                ## You can use   ok[1:4,1:4]=z

            #####################################################################
            #####################################################################
            ################ Be Carful when you copy an array  ##################
            ################     use the .copy() function      ##################
            #####################################################################
            #####################################################################

## Mathemetics :
################

f=np.array([1,2,3,4,5])
#f+=2
#f-=2
#f*=2
#f/2
#f**=2

np.sin(f)             ## SIN
np.cos(f)             ## COS
np.tan(f)             ## TAN

                ####################################################################
                ######################### Linear Algebra ###########################
                ####################################################################

c=np.ones((2,3))
v=np.full((3,2),2)
w=np.matmul(c,v)

## Find determinant :
#####################
q=np.identity(3)
np.linalg.det(q)

                ####################################################################
                ########################### Statistics #############################
                ####################################################################

p=np.array([[1,2,3],[4,5,6]])
np.max(p)                        ## np.min(p)
np.max(p,axis=1)                 ## will gives you max in first row and second row
np.max(p,axis=0)                 ## will gives you max of column
np.sum(p,axis=1)                 ## axis=0 will gives you sum of comlumn

## reorganizing arrays :
########################

t=np.array([[1,2,3,4,5,6],[7,8,9,10,11,12]])
tt=t.reshape(4,3) ## change the shape of an array

## Vertically stacking Vectors :
################################

v1=np.array([1,2,3,4])
v2=np.array([5,6,7,8])
v3=np.vstack([v1,v2,v2,v2,v1])     ## Merge Vertically

## Horizantal stacking Vectors :
################################

k1=np.zeros([2,2])
k2=np.ones([2,4])
k3=np.hstack([k1,k2])
v4=np.hstack([v1,v1,v2])            ## Merge Horizantal

                #########################################################
                ################### miscellaneous #######################
                #########################################################

############# Load data from file #######################

filedata=np.genfromtxt('data.txt',delimiter=",")
filedata.astype('int32')

## Boolean masking and advanced indexing :
##########################################

filedata >= 50
filedata[filedata >= 44]
np.any(filedata > 50, axis=1)                 # axis=1
np.all(filedata > 50, axis=1)

vaar=(~((filedata > 50) &( filedata <60)))    ## ~ not

## You can index with a list in Numpy :
#######################################

new_var=np.array([10,14,0,74,58,17,222,7785,1452])
new_var[[2,5,-1]]

            ###############################################################
            ##                                                           ##
            ##    to Learn more about NumPy check W3SCHOOLS WEBSITE      ##
            ##     https://www.w3schools.com/python/numpy_intro.asp      ##
            ##                                                           ##
            ###############################################################


import scipy as sp
import matplotlib.pyplot as plt
import scipy.linalg as la

def readimg(filename, channel=None):
    if channel is not None:
        return sp.misc.imread(filename)[:,:,channel]
    else:
        return sp.misc.imread(filename)

def compressSVD(filename, rank, random=False, channel=None):
    img = readimg(filename, channel)

    try:
        isize = img[:,:,0].shape
        colors = [la.svd(img[:,:,i]) for i in range(3)]
    except IndexError:
        isize = img.shape
        plt.gray()
        colors = la.svd(img)

    plt.ion()
    imgc = plt.imshow(img)
    newimg = sp.zeros_like(img)

    rank = range(1,rank+1)

    if random is True:
        sp.random.shuffle(rank)

    for r in rank:
        col_res = hat(colors, r-1, r)
        try:
            #col_res[0] = sp.where(col_res[0]>255, col_res[0], 255)
            #col_res[1] = sp.where(col_res[1]>255, col_res[1], 255)
            #col_res[2] = sp.where(col_res[2]>255, col_res[2], 255)

            #col_res[0] = sp.where(col_res[0]<0, col_res[0], 0)
            #col_res[1] = sp.where(col_res[1]<0, col_res[1], 0)
            #col_res[2] = sp.where(col_res[2]<0, col_res[2], 0)

            newimg[:,:,0] += col_res[0]
            newimg[:,:,1] += col_res[1]
            newimg[:,:,2] += col_res[2]

            ## for ch in range(3):
            ##     newimg[newimg[:,:,ch]<1]=0
            ##     newimg[newimg[:,:,ch]>254]=255

        except IndexError:
            newimg += col_res[0]
            ## newimg[newimg<1]=0
            ## newimg[newimg>254]=255

        imgc.set_data(newimg)
        plt.draw()
    plt.ioff()
    plt.show()

    return newimg

def hat(color_svd, lrank, urank):
    results = []
    if len(color_svd) == 3:
        r = 3
    else:
        r = 1

    for c in range(r):
        U = color_svd[c][0]
        S = sp.diag(color_svd[c][1])
        Vt = color_svd[c][2]
        results.append(U[:,lrank:urank].dot(S[lrank:urank, lrank:urank]).dot(Vt[lrank:urank,:]))

    return results

def matrix_rank(X):
    """Compute the rank of a matrix using the SVD"""

    S = la.svd(X, compute_uv=False)
    tol = S.max()*sp.finfo(S.dtype).eps
    return sum(S>tol)