#!/usr/bin/python3

"""
    By listing the first six prime numbers: 2, 3, 5, 7, 11, and 13, we can see that the 6th prime is 13.

What is the 10001st prime number?
"""

import numpy as np

def isprime(num: int) -> bool:
    for i in range(2, int(np.sqrt(num))+1):
        if num%i == 0:
            return False
    return True

def euler7() -> int:
    tot = 1
    i = 3
    while True:
        if isprime(i):
            tot = tot + 1
            if tot == 10001:
                break
        i = i + 2
    return i

tot = euler7()
print(tot)


import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import binom,beta,norm
from crypto_quandl_loader import load_data
from qa import quadratic_approximation
import price_changes

data = load_data('BTER/VTCBTC')
list_of_price_changes = price_changes.compute(data)

c1 = list_of_price_changes.count(1)
c0 = list_of_price_changes.count(0)

sample_size = len(list_of_price_changes)
p1 = c1 / sample_size
p0 = c0 / sample_size

print('likehood(goes up, binom):' + str(binom.pmf(c1, sample_size, p1)))
print('likehood(goes down, binom):%f' % binom.pmf(c0, sample_size, p0))

priors = np.random.uniform(0., 1., sample_size)
priors.sort()

mean_q, std_q = quadratic_approximation(sample_size,c1)

plt.plot(priors, beta.pdf(priors, c1 + 1, sample_size - c1 + 1), label='beta posterior')
plt.plot(priors, norm.pdf(priors, mean_q['p'], std_q), label='quadratic approximation')
plt.legend(loc=0, fontsize=13)
plt.xlabel('chance VTC-BTC goes up in price')
plt.ylabel('density')
plt.show()


import numpy as np
import math

def logistic_sigmoid(x):
	return 1.0 / (1.0 + np.math.exp(-x))

def logistic_sigmoid_derivative(x):
	z = logistic_sigmoid(x)
	return z * (1.0 - z)

def normal_distribution(mu, sigma_squared, x):
	return 1.0/(math.sqrt(2*math.pi*sigma_squared))*math.exp(-(x-mu)**2/(2*sigma_squared))

def one_1_at(size, index):
	res = np.zeros(size)
	res[index] = 1.0
	return res


from math import cos, sin
import numpy as np

def calculate_rotation_matrix_extrinsic(alpha_rad, beta_rad, gamma_rad):
    """
    Applies extrinsic rotation defined by Euler angles alpha, beta, gamma.
    Counter-clockwise transformation for angles is used.

    Rotational matrix is a multiplication of the basic rotations over one axis.

    R = Rz(gamma)*Ry(beta)*Rx(alpha)

    where basic rotations are:

                 1    0        0
    Rx(alpha) =  0 cos(alpha) -sin(alpha)
                 0 sin(alpha)  cos(alpha)

                cos(beta)    0  sin(beta)
    Ry(beta) =     0         1     0
                -sin(beta)   0  cos(beta)

                cos(gamma) -sin(gamma)  0
    Rz(gamma) = sin(gamma)  cos(gamma)  0
                0              0        1

    The order of angles transformations is the very important,
    pay attention here we first transform gamma around z,
    then beta around y and then alpha around x.

    :param
    alpha: rotation around x axis (radians)
    beta:  rotation around y axis (radians)
    gamma: rotation around z axis (radians)
    :return:
    3x3 rotational matrix
    """

    Rx = [[1,    0,             0    ],
          [0, cos(alpha_rad), -sin(alpha_rad)],
          [0, sin(alpha_rad),  cos(alpha_rad)]]

    Ry = [[cos(beta_rad),  0, sin(beta_rad) ],
          [    0,          1,      0        ],
          [-sin(beta_rad), 0, cos(beta_rad)]]

    Rz = [[cos(gamma_rad), -sin(gamma_rad), 0],
          [sin(gamma_rad),  cos(gamma_rad), 0],
          [    0,                  0,       1]]

    R = np.dot(np.dot(Rz, Ry), Rx)

    return R


import numpy as np

def MAPE(y_real, y_predicted, aggregate):
	'''
	MAPE: mean absolute percentage error
	'''
	error = y_real - y_predicted
	absolute_error = np.abs(error)

	percentage_error = 100*absolute_error/y_real

	MAPE = aggregate(percentage_error)
	average_deviation = np.std(percentage_error)

	return MAPE, average_deviation

def MASE(y_real, y_predicted, r=14):
	'''
	MASE: Mean absoulte scaled error. r is the seasonality
	'''
	error = y_real - y_predicted
	absolute_error = np.abs(error)
	sum_absolute_error = np.sum(absolute_error)

	real_minus_r = np.zeros(len(y_real))
	real_minus_r[r:] = y_real[0:-r]

	sum_scaled = np.sum(np.abs(y_real - real_minus_r))

	return sum_absolute_error/sum_scaled

def MSE(y_real, y_predicted):
	'''
	MSE: Mean squared error
	'''
	error = y_real - y_predicted
	sse = np.dot(error, error)

	T = len(y_real)

	return sse/T


import sys
sys.path.insert(0,"../robotModel/")
import trHomogene
import sympy as sp

sp.init_printing()

theta = sp.symbols('theta')
a     = sp.symbols('a')
d     = sp.symbols('d')
alpha = sp.symbols('alpha')

print ""
print "rotX :"
print trHomogene.rotX(theta)

print ""
print "rotY :"
print trHomogene.rotY(theta)

print ""
print "rotZ :"
print trHomogene.rotZ(theta)

print ""
print "dhMatrix :"
print trHomogene.dhMatrix(theta,a,d,alpha)


import csv
from math import sqrt
import numpy as np
import sys
import utils

def kNN(dataset, new_data, dfn, c_index=-1, k=1):
    dist = []
    for data in dataset:
        if c_index == -1:
            without_class = data[:-1]
        else:
            without_class = data[0:c_index] + data[(c_index+1):]
        dist.append((dfn(without_class, new_data), data[c_index]))

    dist.sort(key=lambda tup: tup[0])

    scoreboard = {}

    for vote in dist[:k]:
        if vote[1] not in scoreboard:
            scoreboard[vote[1]] = 0
        scoreboard[vote[1]] += 1

    winner = (0, None)
    for key, value in scoreboard.items():
        if value > winner[0]:
            winner = (value, key)

    return winner[1]

def hamming_dist(v1, v2):
    return np.sum(abs(np.array(v1) - np.array(v2)))

def euclidian_dist(v1, v2):
    return np.sqrt(np.sum((np.array(v1) - np.array(v2)) ** 2))

# def hamming_dist(v1, v2):
#     dist = 0
#     for i in range(len(v1)):
#         dist += abs(v1[i] - v2[i])
#     return dist

# def euclidian_dist(v1, v2):
#     dist = 0
#     for i in range(len(v1)):
#         dist += (v1[i] - v2[i]) ** 2
#     return np.sqrt(dist)

def linear_mahalanobis(training_set, pixels, callback):
    r = []
    g = []
    b = []
    r_sum = 0
    g_sum = 0
    b_sum = 0

    for position in training_set:
        (red, green, blue) = training_set[position]
        r.append(red)
        g.append(green)
        b.append(blue)

        r_sum += red
        g_sum += green
        b_sum += blue

    size = len(training_set)
    center = (r_sum / size, g_sum / size, b_sum / size)
    A = np.cov(np.array([r, g, b]))
    try:
        Ainv = np.linalg.inv(A)
    except:
        print("Error: failed to invert the pixel matrix")
        sys.exit(0)

    distance_map = {}
    for position in pixels:
        pixel_color = pixels[position]
        delta = utils.tuple_difference(pixel_color, center)
        deltaT = np.array([delta]).transpose()
        result = np.dot([delta], Ainv)
        result = np.dot(result, deltaT)
        distance = sqrt(result[0][0])
        distance_map[position] = distance

    max_dist = distance_map[max(distance_map)]
    for position in distance_map:
        distance = 255 * (1 - distance_map[position] / max_dist)
        callback(position, pixels[position], distance)

def quadratic_mahalanobis(training_set, pixels, callback):
    r = []
    g = []
    b = []
    r2 = []
    g2 = []
    b2 = []
    rg = []
    rb = []
    gb = []
    r_sum = 0
    g_sum = 0
    b_sum = 0
    r2_sum = 0
    g2_sum = 0
    b2_sum = 0
    rg_sum = 0
    rb_sum = 0
    gb_sum = 0

    for position in training_set:
        (red, green, blue) = training_set[position]
        r.append(red)
        g.append(green)
        b.append(blue)
        r2.append(red * red)
        g2.append(green * green)
        b2.append(blue * blue)
        rg.append(red * green)
        rb.append(red * blue)
        gb.append(green * blue)

        r_sum += r[-1]
        g_sum += g[-1]
        b_sum += b[-1]
        r2_sum += r2[-1]
        g2_sum += g2[-1]
        b2_sum += b2[-1]
        rg_sum += rg[-1]
        rb_sum += rb[-1]
        gb_sum += gb[-1]

    size = len(training_set)
    center = (r_sum / size, g_sum / size, b_sum / size,\
              r2_sum / size, g2_sum / size, b2_sum / size,\
              rg_sum / size, rb_sum / size, gb_sum / size)
    A = np.cov(np.array([r, g, b, r2, g2, b2, rg, rb, gb]))
    try:
        Ainv = np.linalg.inv(A)
    except:
        print("Error: failed to invert the pixel matrix")
        sys.exit(0)

    distance_map = {}
    for position in pixels:
        (r, g, b) = pixels[position]
        pixel = (r, g, b, r * r, g * g, b * b, r * g, r * b, g * b)
        delta = utils.tuple_difference(pixel, center)
        deltaT = np.array([delta]).transpose()
        result = np.dot([delta], Ainv)
        result = np.dot(result, deltaT)
        distance = sqrt(result[0][0])
        distance_map[position] = distance

    max_dist = distance_map[max(distance_map)]
    for position in distance_map:
        distance = 255 * (1 - distance_map[position] / max_dist)
        callback(position, pixels[position], distance)

def l2norm(training_set, pixels, callback):
    r_sum = 0
    g_sum = 0
    b_sum = 0

    for position in training_set:
        (red, green, blue) = training_set[position]
        r_sum += red
        g_sum += green
        b_sum += blue

    size = len(training_set)
    center = (r_sum / size, g_sum / size, b_sum / size)

    distance_map = {}
    for position in pixels:
        pixel_color = pixels[position]
        distance = euclidian_dist(pixel_color, center)
        distance_map[position] = distance

    max_dist = distance_map[max(distance_map)]
    for position in distance_map:
        distance = 255 * (1 - distance_map[position] / max_dist)
        callback(position, pixels[position], distance)


import pulp as pl
import numpy as np
# create model and solver
model = pl.LpProblem("1-1tiny-problem", pl.LpMinimize)

month_num = 8
product_num = 1

# parameters
q = [5000]  # product fix cost
p = [100]  # cost per product
d = np.array([400, 400, 800, 800, 1200, 1200, 1200, 1200]).reshape(
    (1, -1))  # demand of product by month
s_init = [200]  # initial stock
h = [5]  # stock cost per product month

# variables
x = pl.LpVariable.dicts('x', (range(product_num), range(month_num)), 0)
s = pl.LpVariable.dicts('s', (range(product_num), range(month_num)), 0)
y = pl.LpVariable.dicts('y', (range(product_num), range(month_num)),
                        cat='Binary')

# objective
model += pl.lpSum(pl.lpSum(p[i]*x[i][j] + q[i]*y[i][j] \
            for i in range(product_num)) for j in range(month_num)) \
            + pl.lpSum(pl.lpSum(h[i]*s[i][j] for i in range(product_num)) \
             for j in range(month_num)) \
            + pl.lpSum(h[i]*s[i][month_num-1]/2 for i in range(product_num))

# constraints
for i in range(product_num):
    # init stock
    model += s_init[i] + x[i][0] == d[i][0] + s[i][0]
    # first month max limit of x
    model += x[i][0] <= pl.lpSum(d[i][k]
                                 for k in range(0, month_num)) * y[i][0]
    for j in range(1, month_num):
        # stock balance
        model += s[i][j - 1] + x[i][j] == d[i][j] + s[i][j]
        # max limit of x
        model += x[i][j] <= pl.lpSum(d[i][k]
                                     for k in range(j, month_num)) * y[i][j]

# solver = pl.PULP_CBC_CMD()
# solver = pl.GUROBI_CMD(path=r"C:\gurobi911\win64\bin\gurobi_cl.exe")
solver = pl.SCIP_CMD(path=r"C:\Program Files\SCIPOptSuite 7.0.2\bin\scip.exe")
# solver = pl.GLPK_CMD(path=r"C:\Personal\Binary\glpk465\glpsol.exe")
# solver = pl.MOSEK()
# solver = pl.GUROBI()
result = model.solve(solver)
print("x val is")
for i in range(product_num):
    for j in range(month_num):
        print(pl.value(x[i][j]), end=",")
    print("\n")


import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

from sklearn.linear_model import LinearRegression, RANSACRegressor

df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/housing/housing.data', header=None, sep='\s+')
df.columns = ['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX',
              'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'B', 'LSTAT', 'MEDV']

x = df[['RM']].values
y = df['MEDV'].values

# ransac -  Radom Sample Consensus
ransac = RANSACRegressor(LinearRegression(), max_trials=100, min_samples=50,
                         residual_metric=lambda x: np.sum(np.abs(x), axis=1),
                         residual_threshold=5.0, random_state=0)
ransac.fit(x, y)

inliner_mask = ransac.inlier_mask_
outliner_mask = np.logical_not(inliner_mask)
line_x = np.arange(3, 10, 1)
line_y_ransac = ransac.predict(line_x[:, np.newaxis])

plt.scatter(x[inliner_mask], y[inliner_mask], c='blue', marker='o', label='inliners')
plt.scatter(x[outliner_mask], y[outliner_mask], c='lightgreen', marker='s', label='outliners')
plt.plot(line_x, line_y_ransac, color='red')
plt.xlabel('Average number of rooms[RM]')
plt.ylabel('Price in $1000\'s [MEDV]')
plt.legend(loc='upper left')
plt.show()

print('Slope: %.3f' % ransac.estimator_.coef_[0])
print('Intercept: %.3f' % ransac.estimator_.intercept_)


import numpy as np

def _legDVR(start,stop,num):
    """
    Calculate Legendre DVR grid and operators

    Parameters
    ----------
    start : float
        Grid start value
    stop : float
        Grid stop value. Must be larger than start
    num : int
        Number of grid points. Must be >= 2.

    Returns
    -------
    grid : ndarray
        Array of DVR grid points
    D : ndarray
        First derivative operator in DVR basis, shape (num, num)
    D2 : ndarray (num,num)
        Symmetrized derivative operator (d^dagger * d) in DVR basis, shape (num, num)

    Notes
    -----
    The Legendre DVR is defined over a finite integration domain with
    non-periodic boundary conditions. The derivative operator is thus not
    stricty anti-Hermitian because of non-zero boundary terms. The `D2`
    matrix returned is the representation of the explicitly symmetrized
    :math:`-\partial^\dagger \cdot \partial`
    operator, which is *not* equivalent to :math:`\partial \cdot \partial`

    """

    # Construct coordinate operator in FBR
    XFBR = np.zeros((num,num))
    for p in range(num-1):

        val = np.sqrt(2*p+1) * np.sqrt(2*p+3) * (stop-start)/4.0 \
            * 2*(p+1) / ( (2*p+1)*(2*p+3))
        XFBR[p,p+1] = val
        XFBR[p+1,p] = val

    XFBR += np.eye(num)*(stop+start)/2.0

    # Calculate derivative operator matrix elements
    DFBR = np.zeros((num,num))
    D2FBR = np.zeros((num,num)) # -d^dagger * d
    for n in range(num):
        for m in range(num):

            if (m-n) % 2 == 1 and n <= m:
                DFBR[n,m] = np.sqrt( (2*n+1) * (2*m+1) ) * 2.0/(stop-start)

            if (m-n) % 2 == 0 :
                p = min(m,n)
                D2FBR[m,n] = -np.sqrt(2*m+1) * np.sqrt(2*n+1) \
                    * 2.0 / (stop-start)**2 * p*(p+1)

    # Diagonalize coordinate matrix in FBR
    w,v = np.linalg.eigh(XFBR)
    # Adjust phase of eigenfunctions
    for j in range(num):
        if v[0,j] < 0:
            v[:,j] *= -1.0
    # Calculate derivative operators in DVR
    D = v.transpose() @ DFBR @ v
    D2 = v.transpose() @ D2FBR @ v

    grid = w

    return grid, D, D2

def _legDVRwfs(q,start,stop,num):
    """
    Calculate Legendre DVR wavefunctions.

    Parameters
    ----------
    q : ndarray
        A 1D array of coordinate values.
    start : float
        DVR grid start value.
    stop : float
        DVR grid stop value.
    num : int
        The number of DVR grid points

    Returns
    -------
    wfs : ndarray
        A (`q`.size, `num`) shaped array with
        the DVR wavefunctions evaluated at grid points `q`.

    """

    if np.ndim(q) != 1:
        raise ValueError("q must be 1-dimensional")
    if stop <= start:
        raise ValueError("stop value must be larger than start value")
    if num < 2 :
        raise ValueError("num must be >= 2")

    nq = q.size

    # Construct coordinate operator in FBR
    XFBR = np.zeros((num,num))
    for p in range(num-1):

        val = np.sqrt(2*p+1) * np.sqrt(2*p+3) * (stop-start)/4.0 \
            * 2*(p+1) / ( (2*p+1)*(2*p+3))
        XFBR[p,p+1] = val
        XFBR[p+1,p] = val

    XFBR += np.eye(num)*(stop+start)/2.0

    # Diagonalize coordinate matrix in FBR
    w,v = np.linalg.eigh(XFBR)
    # Adjust phase of eigenfunctions
    for j in range(num):
        if v[0,j] < 0:
            v[:,j] *= -1.0

    z = (q - (start+stop)/2.0) * 2.0/(stop-start)

    # Calculate the standard Legendre polynomials
    # via the recurrence relation
    #  n*P_n = (2n-1) z P_{n-1}  - (n-1) P_{n-2}
    #
    leg = np.ndarray((nq,num), dtype = q.dtype)
    leg[:,0] = 1
    leg[:,1] = z
    for n in range(2,num):
        leg[:,n] = (2*n-1)/n * z*leg[:,n-1]  -  (n-1)/n * leg[:,n-2]
    # Now, normalize the functions for (start, stop)
    for n in range(0,num):
        leg[:,n] *= np.sqrt((2*n+1)/(stop-start))

    wfs = leg @ v

    return wfs


import numpy as np

class Stochastic_gradient_descent():

    def __init__(self,learning_rate=0.1):
        self.learning_rate=learning_rate

    def fit(self,X,Y):
        '''
        first finding the size of given features.

        m=no. of given training sets
        n=no.features in each training set

        after that we will assign the theta to
        the random values.

        '''
        m,n=X.shape
        self.m,self.n=X.shape

        # now initializing the theta.

        self.theta=np.random.random((self.n+1,1))
        self.X=X
        self.Y=Y
        '''
        we are adding a column vector having value 1,because
        the hypothesis function for stochastic gradient descent
        algo is:h(theta)=theta(0)+theta(1)*X(1)+theta(2)*X(2)...........
        so, In order to vectorize the hypothesis fn we will add column vector.
        Here is breif explanation: X.shape=(m,n) and theta.shape=(n+1,1)
        so, in order to multiply the both matrix, we have to add one column to
        X, so that X will become: (m,n+1)

        '''
        ones=np.ones(self.m,int).reshape(-1,1)
        self.X=np.concatenate((ones,self.X),axis=1)
        self.gradient_descent()
        return self

    def gradient_descent(self):

        #updating theta.
        for i in range(self.m):
            grad_cost=self.X[i,:]@self.theta-self.Y[i,:]
            self.theta=self.theta-(self.learning_rate*grad_cost*(self.X[i,:].reshape(-1,1)))
        return self

    def predict(self,X):
        m,n=X.shape
        ones=np.ones(m,int).reshape(-1,1)
        X=np.concatenate((ones,X),axis=1)
        return (X@self.theta)


# QR factorization algorithm

import numpy as np
from qr_factor import qr_factor

# Overdetermined matrix
A = np.matrix('-1 -1  1;'
              ' 1  3  3;'
              '-1 -1  5;'
              ' 1  3  7 ', float)
# QR factorization (deep-copy of A because we need it or the "check")
(Q,R) = qr_factor(np.matrix(A))
# Results check
if np.allclose(A, Q*R) == False:
    raise Exception('QR factorizzation test failure')


"""
This is our script to post process predictions generated by our classifier on images, as well as any other data that needs post-processing.

Further documentation found in each function.

-Blake Edwards / Dark Element
"""
import sys
import numpy as np
from scipy.sparse import csr_matrix, eye
from scipy.sparse.linalg import bicg

def normalize_adjacency_matrix(adjacency_matrix):
    """
    Helper function for denoise_predictions.

    Arguments:
        adjacency_matrix: A matrix of size h * w,
            https://en.wikipedia.org/wiki/Adjacency_matrix

            For this type of normalization, should be symmetric - i.e. from an undirected graph.

    Returns:
        normalized_adjacency_matrix: A normalized version of our adjacency matrix,
            such that each entry represents:
                1) If the source and dest node are connected, and if so, then
                2) how connected the source node is
                3) how connected the destination node is
        If the input is symmetric, this output will still be symmetric.
        Though it could also be normalized in other ways.
    """
    """
    We sum over all the rows of this matrix to get the degrees,
        since it is a symmetric adjacency matrix this is the same as if we summed over the columns.

    Once we have the degrees vector, we use it as the data to create a new sparse matrix,
        of the same shape as our adjacency matrix, and with the vector elements on the diagonal
        via using (row, col) pairs of (0,0), (1,1), ..., (h*w-1, h*w-1) with (np.arange(h*w), np.arange(h*w))
    """
    """
    We get our h*w via the length of the matrix, since we don't need individual h and w values.
    """
    adjacency_matrix_len = adjacency_matrix.shape[0]
    degrees = adjacency_matrix.sum(axis=1)
    degrees = np.array(degrees).flatten()
    degree_matrix = csr_matrix((degrees, (np.arange(adjacency_matrix_len), np.arange(adjacency_matrix_len))), shape=(adjacency_matrix_len, adjacency_matrix_len))

    """
    We use the adjacency matrix and degree matrix to make our normalized adjacency matrix, via

        M = D^(-1/2) * A * D^(-1/2)

    Where
        D = degree matrix,
        A = adjacency matrix,
        M = normalized adjacency result

    Since our degree matrix is a diagonal matrix,
        the elementwise matrix power of it is the same
        as the normal matrix power, so we can quickly
        use the built-in csr_matrix function for elementwise power
        to compute D^(-1/2).

    We then do the dot product in the normal way.
    """
    degree_matrix = degree_matrix.power(-1./2)
    normalized_adjacency_matrix = degree_matrix.dot(adjacency_matrix.dot(degree_matrix))#DAD
    return normalized_adjacency_matrix

def denoise_predictions(predictions, neighbor_weight):
    """
    Arguments:
        predictions: np array of shape (h, w, class_n), the predictions.
            This will be referred to as an image of size h x w, as this makes
            notation much easier.
        neighbor_weight:
            How much importance to put on the neighbor values.
            This could also be thought of as a smoothing factor.
            Should be between 0 and 1

    Returns:
        Each "prediction" is an element in our predictions input,
            and is a vector of length class_n. There are h * w of these predictions
            in our entire predictions array.

        For each prediction vector in our predictions,
            we produce a new denoised prediction vector,
            taking into account both the nearby prediction vectors to this element,
            and the original prediction vector.

        Returns a new predictions array, of the same shape (h, w, class_n) as the original.
    """
    """
    First we need to create an adjacency matrix for each entry in our predictions array,
        as if each node was a prediction vector, so that the graph is a rectangle of shape h x w,
        and each node is connected to all of it's nearby nodes.

    Ex, with h = 3, w = 4:

    . . . .
    . . . . (I think it may be easier to just imagine the lines, due to limitations of text art)
    . . . .

    Such that the minimum amount of connections a node will have (unless there is a really really small h and w)
        will be 3, and the maximum will be 8.

    Since adjacency matrices are sparse (mostly zeros), we can save memory by using different compression algorithms to
        only store the nonzero elements. I'll be using scipy's CSR element, as this algorithm offers easy matrix-vector
        dot products, and as a result of this can be extended to matrix-matrix dot products - CSR is efficient when
        you need to perform linear algebra ops on your sparse matrices.

    Since the adjacency matrix will be of size (h*w, h*w), it quickly becomes absolutely necessary to use a compression format,
        or else numpy won't even be able to hold it in memory. For an example of h = 300, w = 400, the raw adjacency matrix
        would contain h*w * h*w elements, = 300*400 * 300*400 = 120000 * 120000 = 1.44e10 .

    Due to the simple nature of our graph, undirected and where each node is connected in a simple and deterministic manner to its neighbors,
        we can construct it with O(n) complexity. I designed a way to incrementally create each row in this matrix given the h and w,
        as this is all that's necessary given our knowledge of how the graph is constructed. I could loop through every element and construct it
        the way it would be done by hand, however this would have O(n^2) complexity, and since n = h*w, that is really really bad. For the sake
        of optimizing down to O(n), I had to observe and replicate the patterns in adjacency matrices of this type. It is not as obviously intuitive
        as the by-hand method, however it will run at O(n), able to loop through and construct rows of the adjacency matrix at a time.
    """
    """
    Prepare these lists for constructing our CSR matrix
    """
    rows = []
    cols = []
    data = []

    """
    Get this data from our shape
    """
    h, w, class_n = predictions.shape

    def ins(i, j, x):
        """
        Arguments:
            i: row index of x
            j: col index of x
            x: data to add
        Returns:
            Adds i, our row index, to the row indices list
            Adds j, our col index, to the col indices list
            Adds x, our data, to the data list

            This is just a quick function to save time in the next steps.
        """
        if j >= 0 and j < h*w:
            rows.append(i)
            cols.append(j)
            data.append(x)

    """
    Loop through and construct our rows
    """
    for i in range(h*w):
        if i % w == w-1:
            ins(i, i-(w+1), 1)
            ins(i, i-(w), 1)
            ins(i, i-1, 1)
            ins(i, i+(w-1), 1)
            ins(i, i+(w), 1)

        elif i % w == 0:
            ins(i, i-(w), 1)
            ins(i, i-(w-1), 1)
            ins(i, i+1, 1)
            ins(i, i+(w), 1)
            ins(i, i+(w+1), 1)

        else:
            ins(i, i-(w+1), 1)
            ins(i, i-(w), 1)
            ins(i, i-(w-1), 1)
            ins(i, i-1, 1)
            ins(i, i+1, 1)
            ins(i, i+(w-1), 1)
            ins(i, i+(w), 1)
            ins(i, i+(w+1), 1)

    """
    Now that lists are finished changing in size,
        we cast to nparrays
    """
    rows = np.array(rows)
    cols = np.array(cols)
    data = np.array(data)

    """
    Using our data and row and column indices, we create our adjacency matrix as a CSR Sparse matrix
    """
    adjacency_matrix = csr_matrix((data, (rows, cols)), shape=(h*w, h*w))

    """
    We then normalize this adjacency matrix
    """
    normalized_adjacency_matrix = normalize_adjacency_matrix(adjacency_matrix)

    """
    Then we use the following equation, from this paper:

        http://journals.plos.org/plosone/article?id=10.1371/journal.pone.0051947

        (I - l*M)*f = (1-l)*y

    In order to solve for new values of y (our predictions), so that we have
        new predictions in f.

    That paper has the explanation for that equation.
    Once we have that equation, we can simplify so that:

        A = (I - l * M)
        x = f
        b = (1-l)*y

    Our equation is now:

        Ax = b ,

    Which is just the setup for a least squares approximation of x:

        https://en.wikipedia.org/wiki/Least_squares
        https://www.youtube.com/watch?v=MC7l96tW8V8 (Khan Academy)

    So we prepare those components first.
    """
    A = eye(h*w) - neighbor_weight * normalized_adjacency_matrix

    predictions = np.reshape(predictions, (-1, class_n))
    b = (1.-neighbor_weight) * predictions

    """
    Unfortunately, least squares usually expects x and b to be vectors.
    Right now, x is going to be of shape (h*w, class_n), and b is also of shape (h*w, class_n).
    These are very much not vectors, but matrices instead. So how can we extend least squares to work in this setting?

    It's actually very simple, due to one property.

    For the system Ax = b, where
        A is a matrix of size n x n
        x is a matrix of size n x m
        b is a matrix of size n x m,

    Let us look at a column vector in matrix b, at position i.
    Each element in this vector is determined entirely by a column vector in matrix x,
        at position i,
        as well as the entire matrix A.
    For an understanding of this, think about how the elements of this column vector in b are
        obtained through the dot product of A and x.

    What this means is that we can solve for the entire matrix x by solving for each of the columns
        in x independently, looping across both the columns we are solving for in x and the other matching column
        in b, and solving using normal least squares.

    Remember that x is our new prediction values, so this is the last math-heavy step.
    """
    x = np.zeros(b.shape, dtype=np.float32)
    for i in range(class_n):
        col = b[:,i]
        col = np.reshape(col, (-1, 1))
        """
        Solve for this column using Bi-Conjugate Gradient Iteration from scipy,
            and set the result column into our resulting x matrix.
        """
        x[:,i], flags = bicg(A, col)

    """
    At this point our entire x result matrix has been constructed,
        however it is of shape (h*w, class_n).
    So we reshape it into (h, w, class_n), and rename it to the more
        appropriate denoised_predictions - since these are our denoised predictions.
    """
    denoised_predictions = np.reshape(x, (h, w, class_n))
    return denoised_predictions


""" Distance Correlation

Notes
-----
Snippet adapted from: https://gist.github.com/Satra/aa3d19a12b74e9ab7941

"""
# Author: Avraam Marimpis <avraam.marimpis@gmail.com>

import numpy as np
from scipy.spatial.distance import pdist, squareform  # type: ignore

def dcorr(x: np.ndarray, y: np.ndarray) -> float:
    """ Distance Correlation

    Parameters
    ----------
    x : array-like, shape(n_samples)
        Input time series.

    y : array-like, shape(N)
        Input time series.

    Returns
    -------
    val : float
        The computed distance correlation.
    """
    lx = len(x)
    ly = len(y)
    if lx != ly:
        raise Exception("")

    X = np.atleast_1d(x)  # type: ignore
    Y = np.atleast_1d(y)  # type: ignore

    if np.prod(X.shape) == len(X):
        X = X[:, None]
    if np.prod(Y.shape) == len(Y):
        Y = Y[:, None]
    X = np.atleast_2d(X)
    Y = np.atleast_2d(Y)
    n = X.shape[0]

    a = squareform(pdist(X))
    b = squareform(pdist(Y))

    A = a - a.mean(axis=0)[None, :] - a.mean(axis=1)[:, None] + a.mean()
    B = b - b.mean(axis=0)[None, :] - b.mean(axis=1)[:, None] + b.mean()

    dcov2_xy = (A * B).sum() / float(n * n)
    dcov2_xx = (A * A).sum() / float(n * n)
    dcov2_yy = (B * B).sum() / float(n * n)
    dcor = np.sqrt(dcov2_xy) / np.sqrt(np.sqrt(dcov2_xx) * np.sqrt(dcov2_yy))

    return dcor


"""Module contains functions for compute similarity."""

from numba import njit
import numpy as np

@njit
def imk(dist, alpha):
    """
    Inverse Multiquadratic Kernel (IMK).

    Parameters
    ----------
    dist : float or numpy.ndarray, shape=(m,)
        Distances.

    alpha : float
        Control the decay of the function.

    Results
    -------
    sim : float or numpy.ndarray, shape=(m)
        Similatity result.

    """
    return 1 / np.sqrt(1 + alpha ** 2 * dist ** 2)

@njit
def wsf(dist, alpha):
    """
    Wegerich Similarity Function (WSF).

    Parameters
    ----------
    dist : float or numpy.ndarray, shape=(m,)
        Distances.

    alpha : float
        Control the decay of the function.

    Returns
    -------
    sim : float or numpy.ndarray, shape=(m)
        Similatity result.

    """
    return 1 / (1 + alpha * dist)

@njit
def cck(dist, alpha):
    """
    Cauchy Kernel Function (CCK).

    Parameters
    ----------
    dist : float or numpy.ndarray, shape=(m,)
        Distances.

    alpha : float
        Control the decay of the function.

    Returns
    -------
    sim : float or numpy.ndarray, shape=(m)
        Similatity result.

    """
    return 1 / (1 + (alpha ** 2) * (dist ** 2))

@njit
def lk(dist, alpha):
    """
    Exponential or Laplace Kernel (LK).

    Parameters
    ----------
    dist : float or numpy.ndarray, shape=(m,)
        Distances.

    alpha : float
        Control the decay of the function.

    Results
    -------
    sim : float or numpy.ndarray, shape=(m)
        Similatity result.

    """
    return np.exp(-alpha * dist)

@njit
def rbf(dist, alpha):
    """
    Radial Basis Function (RBF).

    Parameters
    ----------
    dist : float or numpy.ndarray, shape=(m,)
        Distances.

    alpha : float
        Control the decay of the function.

    Results
    -------
    sim : float or numpy.ndarray, shape=(m)
        Similatity result.

    """
    return np.exp(-alpha * dist ** 2)

def sto(dist, d, epsilon=1e-10):
    """
    Saturated Triangular Operator (STO).

    Eq.:
        d - dist, if dist <= d + epsilon
        epsilon, if dist > d + epsilon

    Parameters
    ----------
    dist : float or numpy.ndarray, shape=(m,)
        Distances.

    d : float
        Threshold based on observartions varince.

    epsilon : float
        Value for similarity when the distance is greater than `d`.

    Results
    -------
    sim : float or numpy.ndarray, shape=(m)
        Similatity result.

    """
    less_eq_d = np.less_equal(dist, d + epsilon)

    sim = dist.copy()
    sim[less_eq_d] = 1 - dist[less_eq_d] / d  # Adjust to interval [0, 1]
    sim[~less_eq_d] = epsilon

    return sim


# Basic OLS regression using base python (and numpy)

import numpy as np
import matplotlib.pyplot as plt

# Load data
dataset = np.genfromtxt('../data/regression_heart.csv', delimiter=",")

x = dataset[:,1:]
y = dataset[:,0]
y = np.reshape(y, (y.shape[0],1))  # Reshape to a column vector

# Scale (standardize) data for smoother gradient descent
x = (x - np.mean(x, axis=0)) / np.std(x, axis=0)
x = np.insert(x, 0, 1, axis=1)  # Add 1's for bias

# Learning parameters
alpha = 0.01  # Learning rate
iterations = 1000

# Notes
# Cost function for linear regression: 1/2m(sum((theta0 + theta1(x) - y) ^ 2))
# Partial derivative wrt theta0: 1/m(sum(theta0 + theta1 - y))
# Partial derivative wrt theta1: 1/m(sum(theta0 + theta1 - y)) * x
# Parameter update: theta = theta - alpha(partial derivative)

# Training
theta = np.ones((x.shape[1],1))  # Initial weights
m = y.shape[0]  # Number of training examples. Equivalent to X.shape[0]
cost_history = np.zeros(iterations)  # Initialize array of cost history values

for i in xrange(iterations):  # Batch gradient descent
    residuals = np.dot(x, theta) - y
    squared_error = np.dot(residuals.T, residuals)
    cost = 1.0/(2*m) * squared_error  # Quadratic loss

    gradient = 1.0/m * np.dot(residuals.T, x).T  # Calculate derivative
    theta -= (alpha * gradient)  # Update weights

    cost_history[i] = cost  # Store the cost for this iteration

    if (i+1) % 100 == 0:
        print "Iteration: %d | Cost: %f" % (i+1, cost)

# Plot training curve
plt.plot(range(1, len(cost_history)+1), cost_history)
plt.grid(True)
plt.xlim(1, len(cost_history))
plt.ylim(0, max(cost_history))
plt.title("Training Curve")
plt.xlabel("Iteration #")
plt.ylabel("Cost")


import random
import sys
import numpy as np
import skfuzzy as fuzz
from skfuzzy import control as ctrl
import matplotlib.pyplot as plt

def fuzzing(vel, dis, flag = 0):
    # Fuzzificacao
    velocidade = ctrl.Antecedent(np.arange(0, 101, 1), 'velocidade')
    distancia = ctrl.Antecedent(np.arange(0, 101, 1), 'distancia')
    pressao = ctrl.Consequent(np.arange(0, 101, 1), 'pressao')

    velocidade.automf(5)
    distancia.automf(5)

    # pressao['low'] = fuzz.trimf(pressao.universe, [0, 0, 50])
    # pressao['medium'] = fuzz.trimf(pressao.universe, [0, 50, 100])
    # pressao['high'] = fuzz.trimf(pressao.universe, [50, 100, 100])

    pressao['poor'] = fuzz.gaussmf(pressao.universe, 0, 10)
    pressao['mediocre'] = fuzz.gaussmf(pressao.universe, 25, 10)
    pressao['average'] = fuzz.gaussmf(pressao.universe, 50, 10)
    pressao['decent'] = fuzz.gaussmf(pressao.universe, 75, 10)
    pressao['good'] = fuzz.gaussmf(pressao.universe, 100, 10)

    # Regras
    regras = []

    regras.append(ctrl.Rule(velocidade['mediocre'] & distancia['average'], pressao['poor']))
    regras.append(ctrl.Rule(velocidade['mediocre'] & distancia['mediocre'], pressao['mediocre']))
    regras.append(ctrl.Rule(velocidade['mediocre'] & distancia['poor'], pressao['average']))
    regras.append(ctrl.Rule(velocidade['average']  & distancia['average'], pressao['average']))
    regras.append(ctrl.Rule(velocidade['average']  & distancia['mediocre'], pressao['decent']))
    regras.append(ctrl.Rule(velocidade['average']  & distancia['poor'], pressao['good']))
    regras.append(ctrl.Rule(velocidade['decent'] & distancia['good'], pressao['average']))
    regras.append(ctrl.Rule(velocidade['decent'] & distancia['decent'], pressao['decent']))

    regras.append(ctrl.Rule(velocidade['poor'], pressao['poor']))
    # regras.append(ctrl.Rule(velocidade['mediocre'] | distancia['good'], pressao['decent']))
    regras.append(ctrl.Rule(velocidade['good'], pressao['good']))

    pressao_ctrl = ctrl.ControlSystem(regras)
    pressao_sim = ctrl.ControlSystemSimulation(pressao_ctrl)

    pressao_sim.input['velocidade'] = vel
    pressao_sim.input['distancia'] = dis

    pressao_sim.compute()
    print(pressao_sim.output['pressao'])

    # distancia.view()
    # velocidade.view()
    pressao.view(sim=pressao_sim)
    plt.show()

if __name__ == '__main__':
    fuzzing(float(sys.argv[1]), float(sys.argv[2]))


import tensorflow as tf
import numpy as np
import sys
import math

# Polynomial basis functions
def basis0(x):
    return tf.convert_to_tensor([x])

def basis1(x):
    return tf.convert_to_tensor([-0.5 * (x - 1.0), 0.5 * (x + 1.0)])

# quadratic polynomial
def basis2(xIn):
    x = tf.convert_to_tensor(xIn)
    return tf.convert_to_tensor(
        [0.5 * x * (x - 1), -1.0 * (x + 1) * (x - 1), 0.5 * x * (x + 1)])

# cubic polynomial
def basis3(xIn):
    x = xIn
    eta = x
    powEta2 = eta * x
    powEta3 = powEta2 * x

    temp = tf.convert_to_tensor([-0.6666666666666666 * powEta3 + 0.6666666666666666 * powEta2 +
                                 0.1666666666666667 * eta - 0.1666666666666667,
                                 1.333333333333333 * powEta3 - 0.6666666666666666 * powEta2 -
                                 1.333333333333333 * eta + 0.6666666666666666,
                                 -1.333333333333333 * powEta3 - 0.6666666666666666 * powEta2 +
                                 1.333333333333333 * eta + 0.6666666666666666,
                                 0.6666666666666666 * powEta3 + 0.6666666666666666 * powEta2 -
                                 0.1666666666666667 * eta - 0.1666666666666667])
    return temp

# quartic polynomial
def basis4(xIn):
    x = xIn

    eta = x
    powEta2 = x * x
    powEta3 = powEta2 * x
    powEta4 = powEta3 * x

    temp = tf.convert_to_tensor([powEta4 -
                                 powEta3 -
                                 0.5 *
                                 powEta2 +
                                 0.5 *
                                 eta, -
                                 2.0 *
                                 powEta4 +
                                 1.414213562373095 *
                                 powEta3 +
                                 2.0 *
                                 powEta2 -
                                 1.414213562373095 *
                                 eta, 1 -
                                 3 *
                                 powEta2 +
                                 2 *
                                 powEta4, -
                                 2 *
                                 powEta4 -
                                 1.414213562373095 *
                                 powEta3 +
                                 2.0 *
                                 powEta2 +
                                 1.414213562373095 *
                                 eta, powEta4 +
                                 powEta3 -
                                 0.5 *
                                 powEta2 -
                                 0.5 *
                                 eta])

    return temp

# quintic polynomial

def basis5(xIn):
    #x = np.tanh(xIn)
    x = xIn
    eta = x
    powEta2 = eta * eta
    powEta3 = eta * powEta2
    powEta4 = eta * powEta3
    powEta5 = eta * powEta4

    temp = tf.convert_to_tensor([(0.1 -
                                  0.1 *
                                  eta -
                                  1.2 *
                                  powEta2 +
                                  1.2 *
                                  powEta3 +
                                  1.6 *
                                  powEta4 -
                                  1.6 *
                                  powEta5), 3.2 *
                                 powEta5 -
                                 2.588854381999832 *
                                 powEta4 -
                                 3.505572809000084 *
                                 powEta3 +
                                 2.83606797749979 *
                                 powEta2 +
                                 0.3055728090000842 *
                                 eta -
                                 0.247213595499958, -
                                 3.2 *
                                 powEta5 +
                                 0.9888543819998319 *
                                 powEta4 +
                                 5.294427190999916 *
                                 powEta3 -
                                 1.63606797749979 *
                                 powEta2 -
                                 2.094427190999916 *
                                 eta +
                                 0.6472135954999582, 3.2 *
                                 powEta5 +
                                 0.9888543819998319 *
                                 powEta4 -
                                 5.294427190999916 *
                                 powEta3 -
                                 1.63606797749979 *
                                 powEta2 +
                                 2.094427190999916 *
                                 eta +
                                 0.6472135954999582, -
                                 3.2 *
                                 powEta5 -
                                 2.588854381999832 *
                                 powEta4 +
                                 3.505572809000084 *
                                 powEta3 +
                                 2.83606797749979 *
                                 powEta2 -
                                 0.3055728090000842 *
                                 eta -
                                 0.247213595499958, (0.1 +
                                                     0.1 *
                                                     eta -
                                                     1.2 *
                                                     powEta2 -
                                                     1.2 *
                                                     powEta3 +
                                                     1.6 *
                                                     powEta4 +
                                                     1.6 *
                                                     powEta5)])

    return temp

'''
Discontinuous polynomial, 2 parts
'''
def basisDG(x, basis) :
    xr = tf.where(x > 0, 2.0 * (x - 0.5), 0 * x)
    xl = tf.where(x <= 0, 2.0 * (x + 0.5), 0 * x)
    pos = tf.where(x > 0, 1.0, 0.0)
    neg = tf.where(x <= 0, 1.0, 0.0)

    fr = pos*basis(xr)
    fl = neg*basis(xl)
    f = tf.concat([fl,fr],axis=0)

    return f

def basis1DG(x):
    return basisDG(x, basis1)

def basis2DG(x):
    return basisDG(x, basis2)

def basis3DG(x):
    return basisDG(x, basis3)

def basis4DG(x):
    return basisDG(x, basis4)

def basis5DG(x):
    return basisDG(x, basis5)

'''
Continuous piecewise polynomials
'''
def basisCG(x, basis, width) :
    #Add one additional dimension for the batch, I believe.
    dims = len(x.get_shape().as_list())+1

    lR = [ [0]*2 for _ in range(dims) ]
    lL = [ [0]*2 for _ in range(dims) ]

    lR[0][0] = width-1
    lL[0][1] = width-1

    paddingsR = tf.convert_to_tensor(lR)
    paddingsL = tf.convert_to_tensor(lL)

    xr = tf.where(x > 0, 2.0 * (x - 0.5), 0 * x)
    xl = tf.where(x <= 0, 2.0 * (x + 0.5), 0 * x)
    pos = tf.where(x > 0, 1.0, 0.0)
    neg = tf.where(x <= 0, 1.0, 0.0)

    fr = tf.pad(pos*basis(xr),paddingsR)
    fl = tf.pad(neg*basis(xl),paddingsL)

    f = fr+fl

    return f

def basis1CG(x):
    return basisCG(x, basis1, 2)

def basis2CG(x):
    return basisCG(x, basis2, 3)

def basis3CG(x):
    return basisCG(x, basis3, 4)

def basis4CG(x):
    return basisCG(x, basis4, 5)

def basis5CG(x):
    return basisCG(x, basis5, 6)


"""
this python file includes some basic functions of Black-Scholes-Merton formulas
"""
def bsm_call_value(S0,K,T,r,sigma):
    '''
    Valuation of European call option in BSM model
    :param S0: initial stock level
    :param K: strike price
    :param T: maturity date
    :param r: constant risk-free short rate
    :param sigma: volatility factor in diffusion term
    :return: the call value
    '''

    from math import log,sqrt,exp
    from scipy import stats

    S0 = float(S0)
    d1 = (log(S0 / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * sqrt(T))
    d2 = (log(S0 / K) + (r - 0.5 * sigma ** 2) * T) / (sigma * sqrt(T))
    value = (S0 * stats.norm.cdf(d1,0.0,1.0)) - K * exp(-r * T) * stats.norm.cdf(d2,0.0,1.0)
    return value

def bsm_vega(S0,K,T,r,sigma):
    '''
    calculating Vega of European option in BSM model
    :param S0: initial stock level
    :param K: strike price
    :param T: maturity date
    :param r: constant risk-free short rate
    :param sigma: volatility factor in diffusion term
    :return: Vega
    '''

    from math import log,sqrt
    from scipy import stats

    S0 = float(S0)
    d1 = (log(S0 / K) + (r + 0.5 * sigma ** 2) * T) / (sigma * sqrt(T))
    vega = S0 * stats.norm.cdf(d1,0.0,1.0) * sqrt(T)
    return vega

def bsm_call_imp_vol(S0,K,T,r,C0,sigma_est, it=100):
    '''
    implied volatility of European call option in BSM model
    :param S0: initial stock level
    :param K: strike price
    :param T: maturity date
    :param r: constant risk-free short rate
    :param C0:
    :param sigma_est: estimate of impl. volatility
    :param it: number of iterations
    :return:
    '''

    for i in range(it):
        sigma_est -= ((bsm_call_value(S0,K,T,r,sigma_est)-C0)/bsm_vega(S0,K,T,r,sigma_est))
    return sigma_est


import numpy as np

"""
Conceito: Z score é uma medida estatística que calcula quantos
desvios padrão um determinado dado está afastado da media da distribuição a qual pertence.

https://www.investopedia.com/terms/z/zscore.asp#:~:text=A%20Z%2Dscore%20is%20a,identical%20to%20the%20mean%20score.
"""

def z_score(distribuicao):

    """
    >>> distribuicao = [1, 3, 5, 7]
    >>> z_score(distribuicao)
    [-1.3416407864998738, -0.4472135954999579, 0.4472135954999579, 1.3416407864998738]

    :param distribuição: sequência python com qualquer distribuição de dados de tipo quantitativo discreto ou contínuo
    :return: uma lista com os z scores da distribuição
    """

    media = np.mean(distribuicao) #Media da distribuição

    desvio_padrao = np.std(distribuicao) #Desvio padrão da distribuição

    # Calculando o z_scores de cada valor da distribuição com list comprehension
    z_scores = [(x - media) / desvio_padrao for x in distribuicao]

    return z_scores # lista com os z_scores calculados


import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.svm import SVC
from math import ceil,floor

rs = np.random.RandomState(1234)

# Generate some fake data.
n_samples = 200
# X is the input features by row.
X = np.zeros((200,3))
n_s_2=floor(n_samples/2)
X[:n_s_2] = rs.multivariate_normal( np.ones(3), np.eye(3), size=n_s_2)
X[n_s_2:] = rs.multivariate_normal(-np.ones(3), np.eye(3), size=n_s_2)
# Y is the class labels for each row of X.
Y = np.zeros(n_samples); Y[n_s_2:] = 1

# Fit the data with an svm
svc = SVC(kernel='linear')
svc.fit(X,Y)

# The equation of the separating plane is given by all x in R^3 such that:
# np.dot(svc.coef_[0], x) + b = 0. We should for the last coordinate to plot
# the plane in terms of x and y.

z = lambda x,y: (-svc.intercept_[0]-svc.coef_[0][0]*x-svc.coef_[0][1]) / svc.coef_[0][2]

tmp = np.linspace(-2,2,51)
x,y = np.meshgrid(tmp,tmp)

# Plot stuff.
fig = plt.figure()
ax  = fig.add_subplot(111, projection='3d')
ax.plot_surface(x, y, z(x,y))
ax.plot3D(X[Y==0,0], X[Y==0,1], X[Y==0,2],'ob')
ax.plot3D(X[Y==1,0], X[Y==1,1], X[Y==1,2],'sr')
plt.show()


from scipy import integrate
import numpy as np
import matplotlib.pyplot as plt

def sys(q,t):
    yi = q[0]
    xi = q[1]
    f0 = xi
    f1 = (g - ((cD/m)*(xi**2)))
    return [f0,f1]

# Constants
g =  9.80665
cD =  0.2028
m = 80

# ICS
x0 = 0
y0 = 0
ics = [y0,x0]

t = np.linspace(0,100,10000)
sol = integrate.odeint(sys,ics,t)
y = sol[:,0]
x = sol[:,1]

plt.figure(1)
plt.plot(t,y, label='Position')
plt.plot(t,x, label='Velocity')
plt.ylim(0,5000)
plt.legend(loc=0)
plt.plot()

print "Time to fall 5000m is between:"
t5000_b1 = np.min(np.where(y>5000))-1
t5000_b2 = np.min(np.where(y>5000))
print str((t5000_b1/100)) + " seconds; here y = " + str(y[t5000_b1])
print "and"
print str((t5000_b2/100)) + " seconds; here y = " + str(y[t5000_b2])

plt.figure(2)
plt.plot(t,x, label='Velocity')
plt.legend(loc=0)
plt.title('Skydiver velocity vs time')
plt.xlabel(r'$t$')
plt.ylabel(r'$v$')
plt.plot()

xmax = np.max(x)
print "maximum speed is " + str(xmax)

# and we are dooooon

ExamPrep/Shit Comp/ShitComp/(Past paper) 1314/Q1.py


import numpy as np
###################################################################################
D2R = np.pi/180.
R2D = 1./D2R
###################################################################################

def sind(degrees):
    return np.sin(D2R*degrees)

def cosd(degrees):
    return np.cos(D2R*degrees)

def sincosd(deg):
    r = np.deg2rad(deg)
    return np.sin(r), np.cos(r)

def rot2deg(deg):
    '''Returns a 2-d rotation matrix by angle 'deg'.'''
    sa,ca=sincosd(deg)
    return np.array([[ca,-sa],[sa,ca]])

def tand(degrees):
    return np.tan(D2R*degrees)

def put360(degrees):  # Puts an angle into range 0 to 360.
    return degrees % 360

def arctan2d(y, x):
    return put360(np.arctan2(y, x)*R2D)

def arcsind(x):
    return np.arcsin(x)*R2D

def getXYZ(lon,lat):  # Convert spherical angles (in degrees) into xyz triplet
    return np.array([cosd(lon)*cosd(lat), sind(lon)*cosd(lat), sind(lat)])

def getNorm(xyz):
    return np.sqrt(np.sum(xyz**2,axis=0))

def getNormalized(xyz):
    return xyz/getNorm(xyz)

def average_angles_deg(a1, a2):
    '''Computes the average of two angles (in degrees),
    handling wrap-around.  That is, if the two angles are thought of as
    directions in the plane, this gives the midpoint between them.'''
    x = cosd(a1) + cosd(a2)
    y = sind(a1) + sind(a2)
    return arctan2d(y, x)


import numpy as np

class PCA:

    def __init__(self, n_components):
        self.n_components = n_components
        self.components = None
        self.mean = None

    def fit(self, X):
        self.mean = np.mean(X, axis=0)
        X = X - self.mean

        # covariance, function needs samples as columns
        cov = np.cov(X.T)

        # eigenvalues, eigenvectors
        eigenvalues, engenvectors = np.linalg.eig(cov)

        # -> eigenvector v = [:,i] column vector, transpose for easier calculations
        # sort eigenvectors
        engenvectors = engenvectors.T
        idxs = np.argsort(eigenvalues)[::-1]
        eigenvalues = eigenvalues[idxs]
        eigenvectors = engenvectors[idxs]

        # sort first n eigenvectors
        self.components = engenvectors[0:self.n_components]

    def transform(self, X):
        # project data
        X = X - self.mean
        return np.dot(X, self.components.T)


import numpy as np

class MSELoss:
    def forward(self, y, target):
        self.diff = diff = y - target
        return np.sum(np.square(diff)) / diff.size

    def backward(self):
        assert hasattr(self, 'diff'), "The values passed to forward were not saved"
        diff = self.diff
        return (2 * diff) / diff.size

class Linear:
    def __init__(self, in_features, out_features):
        # Initialization: TODO: Research the best way.
        bound = 3 / np.sqrt(in_features)
        self.W = np.random.uniform(-bound, bound, (out_features, in_features))
        bound = 1 / np.sqrt(in_features)
        self.b = np.random.uniform(-bound, bound, out_features)

        self.grad_W = None
        self.grad_b = None

    def forward(self, x):
        self.x = x
        return np.matmul(x, self.W.transpose()) + self.b

    def backward(self, dy):
        assert hasattr(self, 'x'), "The values passed to forward were not saved"
        assert dy.ndim == 2 and dy.shape[1] == self.W.shape[0], f'{dy.ndim} != 2 or {dy.shape[1]} != {self.W.shape[0]}'
        self.grad_W = np.matmul(dy.transpose(), self.x)
        self.grad_b = np.sum(dy, axis=0)
        dx = np.matmul(dy, self.W)
        return dx

class Tanh:
    def forward(self, x):
        self.x = x
        return np.tanh(x)

    def backward(self, dy):
        assert hasattr(self, 'x'), "The values passed to forward were not saved"
        return dy * (1.0 - np.square(np.tanh(self.x)))

class MLP:
    "NumPy MLP"
    def __init__(self):
        self.layer_input = Linear(in_features=1, out_features=10)
        self.tanh_1 = Tanh()
        self.layer_hidden_1 = Linear(in_features=10, out_features=11)
        self.tanh_2 = Tanh()
        self.layer_output = Linear(in_features=11, out_features=1)

    def forward(self, x):
        x = self.layer_input.forward(x)
        x = self.tanh_1.forward(x)
        x = self.layer_hidden_1.forward(x)
        x = self.tanh_2.forward(x)
        x = self.layer_output.forward(x)
        return x

    def backward(self, dy):
        dx = self.layer_output.backward(dy)
        dx = self.tanh_2.backward(dx)
        dx = self.layer_hidden_1.backward(dx)
        dx = self.tanh_1.backward(dx)
        dx = self.layer_input.backward(dx)
        return dx

class Adam:
    "TODO: Proper implementation"
    def __init__(self, lr):
        self.lr = lr

    def step(self, model):
        for module in model.__dict__.values():
            if isinstance(module, Linear):
                module.W = module.W - module.grad_W * self.lr
                module.b = module.b - module.grad_b * self.lr

def train(X, Y, epochs, learning_rate):
    model = MLP()
    optimizer = Adam(lr=learning_rate)
    criterion = MSELoss()
    for i in range(epochs):
        # Forward computations
        Y_pred = model.forward(X)
        loss = criterion.forward(Y_pred, Y)

        # Backward computations
        dy = criterion.backward()
        dx = model.backward(dy)

        # Gradient decsent
        optimizer.step(model)

    return model

def predict(model, X):
    return model.forward(X)


import math
import numpy as np
from scipy import stats

class DataCollector:
    def __init__(self, name, unit, sigfig = None):
        self.name = name
        self.unit = unit
        self.collection = np.array([])
        self.summary = None
        self.sigfig = sigfig

    def insert(self, data):
        self.collection = np.append(self.collection, data)

    def generate_summary(self):
        if len(self.collection) == 0:
            print('\t[WARNING] ', self.name, 'unable to compute other metrics because the array is empty')
            self.summary = {}
        else:
            mean = np.mean(self.collection)
            std = np.std(self.collection)
            CI_scale = std / math.sqrt(len(self.collection))

            self.summary = {
                "total" : np.sum(self.collection),
                "minimum" : np.min(self.collection),
                "maximum" : np.max(self.collection),
                "average" : mean,
                "standard deviation" : std,
                "median" : np.percentile(self.collection, 50),
                "P90" : np.percentile(self.collection, 90),
                "P95" : np.percentile(self.collection, 95),
                "P99" : np.percentile(self.collection, 99),
                "90% confidence interval" : list(stats.norm.interval(0.90, loc=mean, scale=CI_scale)),
                "95% confidence interval" : list(stats.norm.interval(0.95, loc=mean, scale=CI_scale)),
                "99% confidence interval" : list(stats.norm.interval(0.99, loc=mean, scale=CI_scale)),
            }

        if self.sigfig is not None:
            for key in list(self.summary.keys()):
                if type(self.summary[key]) is list:
                    for i in range(len(self.summary[key])):
                        self.summary[key][i] = round(self.summary[key][i], self.sigfig)
                else:
                    self.summary[key] = round(self.summary[key], self.sigfig)

    def print_summary(self):
        print('< summary for {} ( unit : {} ) >'.format(self.name, self.unit))
        print('\tlength : ', len(self.collection))
        if len(self.collection) == 0:
            print('\tunable to compute other metrics because the array is empty')
            return
        if self.summary is None:
            self.generate_summary()
        for key, val in self.summary.items():
            print('\t', key, ' - ', val)

    def get_summary(self):
        if self.summary is None:
            self.generate_summary()
        return self.summary

    def get_len(self):
        return len(self.collection)

    def combine(self, data_collector):
        self.collection = np.add(self.collection, data_collector.collection);

    def get_collection(self):
        return self.collection

    def set_collection(self, collection):
        self.collection = collection


import numpy as np

def softmax_loss(x, no_of_samples, y, reg):
	logs = -np.log(x[range(no_of_samples), y])
	data_loss = np.sum(logs)/no_of_samples
	reg_loss = 0.5*reg * np.sum(W1*W1) # ......... Do this term for all weights in network
	loss = data_loss + reg_loss
	return loss

def mean_squared_error(scores, y, der=False):

	if der:
		return y - scores

	return np.mean(np.squared(y - scores))


import numpy as np
import scipy
from scipy import sparse
import matplotlib
import matplotlib.pyplot as plt
import cPickle as pickle
import time
import math

def sparse_matrix(edges, shape, symmetric=True):
    if symmetric:
        edges = edges[edges[:,0]!=edges[:,1],:]
        edges = np.vstack([ edges, edges[:,[1,0]] ])
    A = scipy.sparse.csc_matrix(
    (np.ones(edges.shape[0]), (edges[:,0], edges[:,1])), shape=shape
    )
    return scipy.sparse.csc_matrix((np.ones(A.nnz), A.nonzero()), shape=shape)

def sample_entries(Pi, m):
    nrow, ncol = Pi.shape
    probs = np.reshape(Pi, (1,-1))[0]/Pi.sum() # row by row
    idx = np.random.choice(len(probs), p=probs, size=(m,1))
    return np.hstack([idx/ncol, idx%ncol])

def sample_edges(P, m):
    nrow, ncol = P[0].shape[0], P[0].shape[1]
    edges = np.zeros((m,2))
    mul = np.repeat([[nrow, ncol]], m, axis=0)
    for i in xrange(len(P)):
        print '\t\tStep', i+1, 'of', len(P), '..'
        edges = edges*mul
        edges += sample_entries(P[i], m)
    return edges

def sample_graph(P0, M, noise=0, symmetric=True):
    if P0 is None:
        return None
    m = int(math.ceil(np.power(P0.sum(), M)))
    P = [P0 * (1 + noise*(np.random.random(P0.shape) - 0.5)) for t in xrange(M)]
    edges = sample_edges(P, m)
    shape = (np.power(P0.shape[0], M), np.power(P0.shape[1], M))
    print '\tCreating sparse matrix..'
    A = sparse_matrix(edges, shape, symmetric)
    return A

def mosaicg_generator(A0, F0, M, noise=0.4):
    """ MOSAIC-G generator for mosaicked graphs,
    i.e., graphs with many binary attributes; e.g., social-affiliation networks
    Args:
        A0 (np.array)    : initiator matrix for adjacency
        F0 (np.array)    : initiator matrix for membership
        M (int) : number of recursive steps for MOSAIC-G
    """
    print 'MOSAIC-G GENERATOR'
    start = time.time()
    print '\tAdjacency matrix..'
    A = sample_graph(A0, M, noise)
    print '\tMembership matrix..'
    F = sample_graph(F0, M, noise, symmetric=False)
    end = time.time()
    print '\tGenerated mosaicked graph in', end-start, 'seconds'
    print '\tPickling graph for later at results/mosaic-g/..'
    scipy.sparse.save_npz('results/mosaic-g/A.npz', A, compressed=True)
    scipy.sparse.save_npz('results/mosaic-g/F.npz', F, compressed=True)
    return {'A': A, 'F': F}


#Python code for chapter 23 DSILT: Statistics

'''
-------------------------------------------------------------------------------
-----------------------Combinations and Permutations---------------------------
-------------------------------------------------------------------------------
'''

import math

def permutation(n, k):
    return math.factorial(n) / math.factorial(k)

def combination(n, k):
    return permutation(n, k) / math.factorial(k)

'''
-------------------------------------------------------------------------------
--------------------Bayes' Theorem and Bayesian Inference----------------------
-------------------------------------------------------------------------------
'''

import numpy as np
from scipy import stats
import matplotlib.pyplot as plt

#Simulate 1000 coin flips (Bernoulli trials) from a binomial dist
bernoulli_trials = np.random.binomial(n=1, p=0.5, size=1000)
print(np.mean(bernoulli_trials))

#PMF of Bernoulli dist gives probability of a positive hit for a single Bernoulli trial
print(stats.bernoulli.pmf(1, 0.5))

#Generate 1000 random binary numbers to simulate coin flipping data
d = np.random.randint(2, size=1000)

#For indpendent events, the probability of observing the data is the product of the PMF
print(np.product(stats.bernoulli.pmf(d, 0.5)))

#Look at the probability of the data on an x scale of 0-1
x = np.linspace(0, 1, 100)
prob = [np.product(stats.bernoulli.pmf(d, p)) for p in x]
print(prob)

#Plot distribution of a fair coin (prior)
fair_flips = bernoulli_flips = np.random.binomial(n=1, p=.5, size=1000)
p_fair = np.array([np.product(stats.bernoulli.pmf(fair_flips, p)) for p in x])
p_fair = p_fair / np.sum(p_fair)
plt.plot(x, p_fair)
plt.title('Prior Probability')
plt.show()

#Look at the probability of an unfair coin (sample)
d = np.random.binomial(n=1, p=0.8, size=1000)
prob = np.array([np.product(stats.bernoulli.pmf(d, p)) for p in x])
print(prob)

#Plot the posterior distribution after applying Bayes' Theorem
#This function stolen from http://dataconomy.com/2015/02/introduction-to-bayes-theorem-with-python/
def bayes(n_sample, n_prior=100, observed_p=0.8, prior_p=0.5):
    x = np.linspace(0, 1, 100)
    sample = np.random.binomial(n=1, p=observed_p, size=n_sample)
    observed_dist = np.array([np.product(stats.bernoulli.pmf(sample, p)) for p in x])
    prior_sample = np.random.binomial(n=1, p=prior_p, size=n_prior)
    prior = np.array([np.product(stats.bernoulli.pmf(prior_sample, p)) for p in x])
    prior = prior / np.sum(prior)
    posterior = [prior[i] * observed_dist[i] for i in range(prior.shape[0])]
    posterior = posterior / np.sum(posterior)

    fig, axes = plt.subplots(3, 1, sharex=True, figsize=(8,8))
    axes[0].plot(x, observed_dist)
    axes[0].set_title("Sampling Distribution")
    axes[1].plot(x, prior)
    axes[1].set_title("Prior Distribution")
    axes[2].plot(x, posterior)
    axes[2].set_title("Posterior Distribution")
    plt.tight_layout()
    plt.show()

    return posterior

bayes(100)
bayes(1000)

'''
-------------------------------------------------------------------------------
--------------------------------Markov Chains----------------------------------
-------------------------------------------------------------------------------
'''

'''
Example 1: Trump Speech Generator
'''

import numpy as np

#Trump speech generator
d = open('/home/dsilt/Desktop/dsilt-stats-code/23-24 Bayesian Statistics and Markov Models/trump_speeches.txt', encoding='utf8').read()
corpus = d.split()
#Function to generate all pairs of words
#Keys are all unique words and all of the words that appear after the key are stored as a list of values
def word_pairs(corpus):
    for i in range(len(corpus)-1):
        yield (corpus[i], corpus[i+1])
pairs = word_pairs(corpus)
words = {}
for w1, w2 in pairs:
    if w1 in words.keys():
        words[w1].append(w2)
    else:
        words[w1] = [w2]

#Choose a word to initialize the Markov chain and the length of the chain in number of words
first_word = np.random.choice(corpus)
nbr_words = 40
def markov_chain_text(initial_state, word_pair_dict, stop_time):
    state = initial_state
    chain = [state]
    for w in range(stop_time):
        state = np.random.choice(word_pair_dict[chain[-1]])
        chain.append(state)
    return ' '.join(chain)

print(markov_chain_text(first_word, words, nbr_words))

'''
Example 2: Markov Chain for voting in presidential elections
'''

#Model voting democrat vs republican in consecutive presidential elections
#Assume the following split of voting population for this elect (time t)
rep_1 = 0.55
dem_1 = 0.45
#Assume the following transition probabilities
rep_to_rep = 0.8
rep_to_dem = 0.2
dem_to_dem = 0.7
dem_to_rep = 0.3
#Calculate split of voting population at next election (t+1)
rep_2 = rep_1*rep_to_rep + dem_1*dem_to_rep
dem_2 = dem_1*dem_to_dem + rep_1*rep_to_dem
print('Republican vs Democrat Split for Next Election:', rep_2, 'vs', dem_2)
#Define initial state and transition matrix to make calculations simpler in the future
state_1 = np.array([rep_1, dem_1])
transm = np.array([[rep_to_rep, rep_to_dem], [dem_to_rep, dem_to_dem]])
#Define function to generalize the calculations above to any future time
def markov_chain(initial_state, transition_matrix, stop_time):
    state = initial_state
    chain = [state]
    for t in range(stop_time):
        state = np.round(np.dot(state, transm), 4)
        chain.append(state)
    return chain
print(markov_chain(state_1, transm, 10))

#Notice the proportion appears to converge around a rep to dem ratio of 60:40
#Steady state is when initial_state * transition_matrix = initial_state, meaning there are no more changes to state
#Adjust markov_chain to stop if a steady state has been reached
def markov_chain(initial_state, transition_matrix, stop_time):
    state = initial_state
    chain = [state]
    for t in range(stop_time):
        if (np.round(np.dot(state, transm), 4) == state).all():
            print('Steady state reached at time t =', t)
            break
        else:
            state = np.round(np.dot(state, transm), 4)
        chain.append(state)
    return chain
print(markov_chain(state_1, transm, 20))

'''
-------------------------------------------------------------------------------
-----------------------------Bayesian Regression-------------------------------
-------------------------------------------------------------------------------
'''

import pymc3 as pm
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

#Ignore all deprecation warnings from theano
import warnings
warnings.filterwarnings("ignore")

def glm_mcmc(df, formula, nbr_iters, likelihood=None, mcmc_sampler=None, target_accept=0.8, tune=500):
    '''
    This function is a wrapper for GLM Bayesian regression, using MCMC.  The
    default MCMC method is NUTS (No U-Turn Sampler), but others are available:
        Binary variables --> BinaryMetropolis
        Discrete variables --> Metropolis
        Continuous variables --> NUTS

    Parameters:
        df = pandas dataframe
        formula = regression formula as specified in R or Patsy
        nbr_iters = number of iterations to sample over in MCMC
        likelihood = the family of likelihood functions for the data (e.g. pm.glm.families.Normal())
        mcmc_sampler = the MCMC sampling method (e.g. pm.NUTS, pm.Metropolis), default NUTS
        target_accept = adapts the step size to make acceptance probability closer to target, higher target --> smaller steps, range(0,1)
        tune = number of tuning samples to use in each iteration, higher tune --> more probability space explored

    If acceptance probability is higher than the target, increase the tune
    parameters.  For more info about this, see:
        https://discourse.pymc.io/t/warning-when-nuts-probability-is-greater-than-acceptance-level/594

    If there are divergences after tuning, increase the target_accept
    parameter.  For more info about this, see:
        https://docs.pymc.io/notebooks/Diagnosing_biased_Inference_with_Divergences.html

    A great intro to GLM Bayesian regression with PYMC3 can be found here:
        https://docs.pymc.io/notebooks/GLM-linear.html
    '''
    bayesian_model = pm.Model()  #Initializes a Bayesian model container
    with bayesian_model:
        if likelihood is None:
            #Priors and observed sampling dist (likelihood) are automatically set
            pm.glm.GLM.from_formula(formula, df)
        else:
            pm.glm.GLM.from_formula(formula, df, family=likelihood)
        if mcmc_sampler is None:
            mcmc_sampler = pm.NUTS(target_accept=target_accept)
        else:
            mcmc_sampler = mcmc_sampler(target_accept=target_accept)
        #Perform inference to estimate the posterior, using nbr_iters MCMC sample posteriors
        trace = pm.sample(draws=nbr_iters, step=mcmc_sampler,
                          init='auto', n_init=200000,
                          cores=None, tune=tune, progressbar=True,
                          random_seed=14)
    return bayesian_model, trace

def patsy_formula(df, dependent_var, *excluded_cols):
    '''
    This function generates the R style formula for Patsy, omitting the
    dependent variable and any other specified variables from the right
    side of the equation.  This is useful because Patsy does not have
    a '.' operator.
    '''
    df_columns = list(df.columns.values)
    df_columns.remove(dependent_var)
    for col in excluded_cols:
        df_columns.remove(col)
    return dependent_var + ' ~ ' + ' + '.join(df_columns)

#GLM of mpg on disp
d = pd.read_csv('/home/dsilt/Desktop/dsilt-stats-code/23-24 Bayesian Statistics and Markov Models/mtcars.csv')
formula = patsy_formula(d, 'mpg')

#Bayesian regression
bayesian_reg, trace = glm_mcmc(d, formula, nbr_iters=2000)
print(pm.summary(trace))  #Regression results
#SD is the error term
pm.autocorrplot(trace)
plt.show()

#Plot the last half of the posterior estimates, as the first few traces are likely poor estimates of the parameters
#First few estimates are called "burn-in"
pm.traceplot(trace[1000:], lines={k: v['mean'] for k,v in pm.summary(trace[1000:]).iterrows()})
plt.show()
#The traces, produced to the right, should be stationary if the model converged
#The traces will show autocorrelation if there is any
#The distributions, produced to the left, can be used to get the mean of the parameter estimate, credible regions, and look at skew
#Distributions should look normal - if they do, and performance is not satisfactory, specifying normal priors might improve it

#Model PPC and RMSE for evaluation
ppc = pm.sample_ppc(trace[1000:], samples=500, model=bayesian_reg)
rmse = np.sqrt(np.sum((ppc['y'].mean(0).mean(0).T - d['mpg'])**2) / d.shape[0])

def get_r2(df, ppc, target):
    sse_model = np.sum((ppc['y'].mean(0).mean(0).T - df[[target]])**2)[0]
    sse_mean = np.sum((df[[target]] - df[target].mean())**2)[0]
    return 1-(sse_model / sse_mean)
print('R Squared:', get_r2(d, ppc, 'mpg'))

#Get the posterior predictions
traces = pm.trace_to_dataframe(trace)[['Intercept'] + list(d.drop(['mpg'], axis=1).columns)]
x = d.drop(['mpg'], axis=1)
x.insert(0, 'intercept', 1)
likelihoods = np.dot(x, traces.T)
likelihoods_sd = np.tile(pm.trace_to_dataframe(trace)[['sd']].T, (d.shape[0], 1))
likelihood = np.random.normal(likelihoods, likelihoods_sd)
print(likelihood.shape)  #Should have d.shape[0] rows and as many columns as MCMC samples

#Get credible intervals for the target variable
dfp = pd.DataFrame(np.percentile(likelihood,[2.5, 25, 50, 75, 97.5], axis=1).T,
                   columns=['0_25','25','50','75','97_5'])

#Plot target variable over index and the credible intervals from Bayesian regression
pal = sns.color_palette('Purples')
fig, ax = plt.subplots(1, 1, figsize=(7, 7))
ax.fill_between(d.index, dfp['0_25'], dfp['97_5'], alpha=0.7, color=pal[1], label='CR 95%')
ax.fill_between(d.index, dfp['25'], dfp['75'], alpha=0.5, color=pal[4], label='CR 50%')
ax.plot(d.index, dfp['50'], alpha=0.5, color=pal[5], label='Median')
plt.plot(d.index, d['mpg'], 'bo')
plt.legend()
plt.title('Bayesian Regression Predicted Credible Intervals vs Actual')
plt.show()

#Bayesian LASSO regression - this can be done by using a Laplacian prior
#First, look at Laplacian dist - the prior will force dist near 0 and only variables significantly different from 0 will escape
#Thus, by using Lapacian prior, only the variables with strong coefficients will stand out
from scipy.stats import norm, laplace

def plot_laplace_vs_normal(norm_sd=1., b=1.):

    dstrb = pd.DataFrame(index=np.linspace(-10, 10, 1000))
    dstrb['normal'] = norm.pdf(dstrb.index.values, loc=0, scale=norm_sd)
    b0 = max(b*.5,0)
    b2 = min(b*2,10)
    dstrb['laplace b={}'.format(b0)] = laplace.pdf(dstrb.index.values, loc=0, scale=b0)
    dstrb['laplace b={}'.format(b)] = laplace.pdf(dstrb.index.values, loc=0, scale=b)
    dstrb['laplace b={}'.format(b2)] = laplace.pdf(dstrb.index.values, loc=0, scale=b2)
    dstrb.plot(style=['--','-','-','-'], figsize=(12,4))
    plt.show()

plot_laplace_vs_normal()

def lasso_mcmc(df, formula, nbr_iters, likelihood=None, mcmc_sampler=None, target_accept=0.8, tune=500):
    '''
    This function is a wrapper for LASSO Bayesian regression, using MCMC.  The
    default MCMC method is Metropolis, but others are available:
        Binary variables --> BinaryMetropolis
        Discrete variables --> Metropolis
        Continuous variables --> NUTS (No U-Turn Sampler)

    Parameters:
        df = pandas dataframe
        formula = regression formula as specified in R or Patsy
        nbr_iters = number of iterations to sample over in MCMC
        likelihood = the family of likelihood functions for the data (e.g. pm.glm.families.Normal())
        mcmc_sampler = the MCMC sampling method (e.g. pm.NUTS, pm.Metropolis), default NUTS
        target_accept = adapts the step size to make acceptance probability closer to target, higher target --> smaller steps, range(0,1)
        tune = number of tuning samples to use in each iteration, higher tune --> more probability space explored

    If acceptance probability is higher than the target, increase the tune
    parameters.  For more info about this, see:
        https://discourse.pymc.io/t/warning-when-nuts-probability-is-greater-than-acceptance-level/594

    If there are divergences after tuning, increase the target_accept
    parameter.  For more info about this, see:
        https://docs.pymc.io/notebooks/Diagnosing_biased_Inference_with_Divergences.html

    A great intro to GLM Bayesian regression with PYMC3 can be found here:
        https://docs.pymc.io/notebooks/GLM-linear.html
    '''
    bayesian_model = pm.Model()  #Initializes a Bayesian model container
    with bayesian_model:

        priors = {"Intercept": pm.Laplace.dist(mu=0, b=0.1),
                  "Regressor": pm.Laplace.dist(mu=0, b=0.1)
                  }

        if likelihood is None:
            #Priors and observed sampling dist (likelihood) are automatically set
            pm.glm.GLM.from_formula(formula, df, priors=priors,
                                    family=pm.glm.families.Normal())
        else:
            pm.glm.GLM.from_formula(formula, df, priors=priors,
                                    family=likelihood)
        if mcmc_sampler is None:
            mcmc_sampler = pm.Metropolis(target_accept=target_accept)
        else:
            mcmc_sampler = mcmc_sampler(target_accept=target_accept)
        #Perform inference to estimate the posterior, using nbr_iters MCMC sample posteriors
        trace = pm.sample(draws=nbr_iters, step=mcmc_sampler,
                          init='auto', n_init=200000,
                          cores=None, tune=tune, progressbar=True,
                          random_seed=14)
    return bayesian_model, trace

lasso_reg, trace = lasso_mcmc(d, formula, nbr_iters=2000)
print(pm.summary(trace))
pm.traceplot(trace[1000:], lines={k: v['mean'] for k,v in pm.summary(trace[1000:]).iterrows()})
plt.show()
#So using Laplacian priors was a horrible idea

#Model PPC and RMSE for evaluation
ppc = pm.sample_ppc(trace[1000:], samples=500, model=lasso_reg)
rmse = np.sqrt(np.sum((ppc['y'].mean(0).mean(0).T - d['mpg'])**2) / d.shape[0])

print('R Squared:', get_r2(d, ppc, 'mpg'))  #R^2 shows the model does not fit the data at all

#Get the posterior predictions
traces = pm.trace_to_dataframe(trace)[['Intercept'] + list(d.drop(['mpg'], axis=1).columns)]
x = d.drop(['mpg'], axis=1)
x.insert(0, 'intercept', 1)
likelihoods = np.dot(x, traces.T)
likelihoods_sd = np.tile(pm.trace_to_dataframe(trace)[['sd']].T, (d.shape[0], 1))
likelihood = np.random.normal(likelihoods, likelihoods_sd)
print(likelihood.shape)  #Should have d.shape[0] rows and as many columns as MCMC samples

#Get credible intervals for the target variable
dfp = pd.DataFrame(np.percentile(likelihood,[2.5, 25, 50, 75, 97.5], axis=1).T,
                   columns=['0_25','25','50','75','97_5'])

#Plot target variable over index and the credible intervals from Bayesian regression
pal = sns.color_palette('Purples')
fig, ax = plt.subplots(1, 1, figsize=(7, 7))
ax.fill_between(d.index, dfp['0_25'], dfp['97_5'], alpha=0.7, color=pal[1], label='CR 95%')
ax.fill_between(d.index, dfp['25'], dfp['75'], alpha=0.5, color=pal[4], label='CR 50%')
ax.plot(d.index, dfp['50'], alpha=0.5, color=pal[5], label='Median')
plt.plot(d.index, d['mpg'], 'bo')
plt.legend()
plt.title('Bayesian Regression Predicted Credible Intervals vs Actual')
plt.show()

'''
-------------------------------------------------------------------------------
---------------------------------Naive Bayes-----------------------------------
-------------------------------------------------------------------------------
'''

from sklearn.naive_bayes import GaussianNB
import pandas as pd
from sklearn.metrics import confusion_matrix

d = pd.read_csv('/home/dsilt/Desktop/dsilt-stats-code/23-24 Bayesian Statistics and Markov Models/iris.csv')

#Train model with default hyperparameters
nb_model = GaussianNB()
nb_model.fit(d.drop(['Species'], axis=1), d['Species'])
preds = nb_model.predict(d.drop(['Species'], axis=1))
pred_probs = nb_model.predict_proba(d.drop(['Species'], axis=1))
print(confusion_matrix(d['Species'], preds))

23-24 Bayesian Statistics and Markov Models/23_bayesian_models.py

import numpy,from scipy,from theano,import pymc3


import numpy as np

class BinaryCrossEntropyLoss():
    '''
    Binary Cross Entropy Loss class that takes in real output and predictions which are probabilities and returns the loss.
    Based on the following article: https://quantdare.com/create-your-own-deep-learning-framework-using-numpy/
    '''
    def __call__(self, y, y_pred):
        self.last_y_pred = y_pred
        self.last_y = y

        assert y_pred.shape == y.shape

        n = y.shape[0]
        loss = np.nansum(- y * np.log(y_pred) - (1-y) * np.log(1-y_pred) ) / n
        self.last_loss = loss
        return self.last_loss

    def gradient(self):
        y = self.last_y
        y_pred = self.last_y_pred
        n = y.shape[0]
        self.dL_dy = (-(y/y_pred) + ((1-y)/(1-y_pred))) / n
        return self.dL_dy

class BCE(BinaryCrossEntropyLoss):

    def __init__(self):
        pass


def cone(D,age):
# outputs:
# topo -- topographic elevation of the cone [m]
# distance -- distance along the cone profile [m]
# coneslope -- mean slope of cone [deg]
# inputs:
# D -- landscape diffusivity [m^2/kyr]
# age -- age of the cone [kyr]

    import numpy as np

    # Topo and Grid Parameters
    conediam=876
    craterdiam=292
    nx=600
    dx=5
    coneslope=0.575

    # Create Initial Cone Topography
    icenter=round(nx/2)
    cone_radius=conediam/2
    crater_radius=craterdiam/2
    topo=1000+np.zeros(nx)
    dist=np.zeros(nx)
    distance=np.zeros(nx)
    for i in range(nx):
        distance[i]=i*dx
        dist[i]=abs(i*dx-icenter*dx);
        if dist[i]<=crater_radius:
            topo[i]=topo[i]+dist[i]*coneslope;
        if ((dist[i]>crater_radius) and (dist[i]<cone_radius)):
            topo[i]=1000+crater_radius*coneslope-(dist[i]-crater_radius)*coneslope;
        if dist[i]>cone_radius:
            topo[i]=1000+crater_radius*coneslope-(cone_radius-crater_radius)*coneslope;

    topo=topo-min(topo);

    # 1d Diffusion
    tend=age           # kyr

    toponew=np.zeros(nx)
    dt=0.1*dx*dx/D
    slope=np.zeros(nx)

    t=0
    while t<tend:

        for i in range(1,nx-1):
            toponew[i]=topo[i]+D*dt/(dx*dx)*(topo[i+1]-2*topo[i]+topo[i-1])

        toponew[0]=topo[0]+D*dt/(dx*dx)*(topo[1]-2*topo[0]+topo[0])
        toponew[nx-1]=topo[nx-1]+D*dt/(dx*dx)*(topo[nx-1]-2*topo[nx-1]+topo[nx-2])

        t+=dt
        topo=toponew[:]

    slope=np.gradient(topo,dx)
    slope=180/np.pi*np.arctan(abs(slope))

    maxtopo=max(topo)

    # Find array index associated with lowest elevation on the cone
    coneedge=-1
    i=0
    while coneedge==-1:
        if slope[i]>3:
            coneedge=i
        i+=1

    # Find array index associated with highest elevation on the cone
    maxind=-1;
    i=0;
    while maxind==-1:
        if ((topo[i+1]<topo[i]) or (i==nx-1)):
            maxind=i
        i+=1

    coneslope=(maxtopo-topo[coneedge])/(dx*(maxind-coneedge))
    coneslope=180/np.pi*np.arctan(coneslope)

    return (topo,distance,coneslope);


from numba_stats import qgaussian, t, norm
import numpy as np
import pytest
from scipy.integrate import quad

def q_sigma(nu, sigma):
    # https://en.wikipedia.org/wiki/Q-Gaussian_distribution
    # relation to Student's t-distribution

    # 1 / (2 sigma^2) = 1 / (3 - q)
    # sqrt((3 - q) / 2) = sigma

    q = (nu + 3.0) / (nu + 1.0)
    return q, sigma * np.sqrt(0.5 * (3.0 - q))

def test_qgaussian_pdf_vs_norm():
    x = np.linspace(-5, 5)

    expected = norm.pdf(x, 0.5, 1.2)
    got = qgaussian.pdf(x, 1, 0.5, 1.2)

    np.testing.assert_allclose(got, expected)

def test_qgaussian_cdf_vs_norm():
    x = np.linspace(-5, 5)

    expected = norm.cdf(x, 0.5, 1.2)
    got = qgaussian.cdf(x, 1, 0.5, 1.2)

    np.testing.assert_allclose(got, expected)

@pytest.mark.parametrize("nu", np.arange(1, 11))
def test_qgaussian_pdf_vs_t(nu):
    x = np.linspace(-5, 5)
    q, sigma = q_sigma(nu, 1.2)

    expected = t.pdf(x, nu, 0.1, 1.2)
    got = qgaussian.pdf(x, q, 0.1, sigma)

    np.testing.assert_allclose(got, expected)

@pytest.mark.parametrize("q", (1, 1.1, 1.2, 1.3, 1.4, 1.5, 1.9, 2.0, 2.5, 2.9))
def test_qgaussian_cdf(q):
    x = np.linspace(-5, 5, 10)

    expected = [quad(lambda y: qgaussian.pdf(y, q, 0.1, 1.2), 0, xi)[0] for xi in x]
    got = qgaussian.cdf(x, q, 0.1, 1.2) - qgaussian.cdf(0, q, 0.1, 1.2)

    np.testing.assert_allclose(got, expected)

@pytest.mark.parametrize("q", (1, 1.1, 1.2, 1.3, 1.4, 1.5, 1.9, 2.0, 2.5, 2.9))
def test_qgaussian_ppf(q):
    x = np.linspace(-5, 5, 10)

    def f(x):
        p = qgaussian.cdf(x, q, 2, 3)
        return qgaussian.ppf(p, q, 2, 3)

    expected = x
    got = f(x)

    np.testing.assert_allclose(got, expected)


'''
PyTorch has its own implementation of backward function for symmetric eigensolver https://github.com/pytorch/pytorch/blob/291746f11047361100102577ce7d1cfa1833be50/tools/autograd/templates/Functions.cpp#L1660
However, it assumes a triangular adjoint.
We reimplement it to return a symmetric adjoint
'''

import numpy as np
import torch

class EigenSolver(torch.autograd.Function):
    @staticmethod
    def forward(self, A):
        w, v = torch.symeig(A, eigenvectors=True)

        self.save_for_backward(w, v)
        return w, v

    @staticmethod
    def backward(self, dw, dv):
        w, v = self.saved_tensors
        dtype, device = w.dtype, w.device
        N = v.shape[0]

        F = w - w[:,None]
        # in case of degenerated eigenvalues, replace the following two lines with a safe inverse
        F.diagonal().fill_(np.inf);
        F = 1./F

        vt = v.t()
        vdv = vt@dv

        return v@(torch.diag(dw) + F*(vdv-vdv.t())/2) @vt

def test_eigs():
    M = 2
    torch.manual_seed(42)
    A = torch.rand(M, M, dtype=torch.float64)
    A = torch.nn.Parameter(A+A.t())
    assert(torch.autograd.gradcheck(DominantEigensolver.apply, A, eps=1e-6, atol=1e-4))
    print("Test Pass!")

if __name__=='__main__':
    test_eigs()


# Multiple Linear Regression

# Importing the libraries
import numpy as np
import pandas as pd

# Importing the dataset
dataset = pd.read_csv('Salary_Classification.csv')
temp = dataset.values
features = dataset.iloc[:, :-1].values
labels = dataset.iloc[:, -1].values

#Check if any NaN values in dataset
dataset.isnull().any(axis=0)

#check data types for each column
print dataset.dtypes

# Encoding categorical data
from sklearn.preprocessing import LabelEncoder
labelencoder = LabelEncoder()
features[:, 0] = labelencoder.fit_transform(features[:, 0])

from sklearn.preprocessing import OneHotEncoder
onehotencoder = OneHotEncoder(categorical_features = [0])
features = onehotencoder.fit_transform(features).toarray()

# Avoiding the Dummy Variable Trap
# dropping first column
features = features[:, 1:]

"""
With linear regression, or generalized linear models
estimated by maximum likelihood (or least squares)
you need to leave out one column.
Otherwise you will get a message about some
columns "left out because of singularities".

But if you estimate such models with regularization,
for example ridge, lasso er the elastic net,
then you should not leave out any columns.
The regularization takes care of the singularities,
and more important, the prediction obtained
may depend on which columns you leave out.
That will not happen when you do not use regularization.
"""

# Splitting the dataset into the Training set and Test set
from sklearn.model_selection import train_test_split
features_train, features_test, labels_train, labels_test = train_test_split(features, labels, test_size = 0.2, random_state = 0)

# Fitting Multiple Linear Regression to the Training set
from sklearn.linear_model import LinearRegression
regressor = LinearRegression()
regressor.fit(features_train, labels_train)

#To see the value of the intercept and slop calculated by the linear regression algorithm for our dataset, execute the following code.
print(regressor.intercept_)
print (regressor.coef_)

# Predicting the Test set results
Pred = regressor.predict(features_test)

import pandas as pd
import numpy as np
print (pd.DataFrame(Pred, labels_test))

# Prediction for a new values
# make this accorifng to the data csv
# Development is replaced by 1,0,0 to 0,0 to remove dummy trap

x = [0,0,1150,3,4]
x = np.array(x)
regressor.predict(x.reshape(1, -1))

le = labelencoder.transform(['Development'])
ohe = onehotencoder.transform(le.reshape(1,1)).toarray()
x = [ohe[0][1],ohe[0][2],1150,3,4]
x = np.array(x)
regressor.predict(x.reshape(1, -1))

# Getting Score for the Multi Linear Reg model
Score = regressor.score(features_train, labels_train)
Score = regressor.score(features_test, labels_test)

"""
Evaluating the Algorithm
The final step is to evaluate the performance of algorithm. We'll do this by finding the values for MAE, MSE and RMSE. Execute the following script:
"""
from sklearn import metrics
print('Mean Absolute Error:', metrics.mean_absolute_error(labels_test, Pred))
print('Mean Squared Error:', metrics.mean_squared_error(labels_test, Pred))
print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(labels_test, Pred)))

print np.mean(labels)
"""
You can see that the value of root mean squared error is 4379.8, which is lower than 10% of the mean value of the salaries (760038x10/100 = 7600). This means that our algorithm was not very accurate but can still make reasonably good predictions.

There are many factors that may have contributed to this inaccuracy, a few of which are listed here:

Need more data: Only one year worth of data isn't that much, whereas having multiple years worth could have helped us improve the accuracy quite a bit.
Bad assumptions: We made the assumption that this data has a linear relationship, but that might not be the case. Visualizing the data may help you determine that.
Poor features: The features we used may not have had a high enough correlation to the values we were trying to predict.
"""

"""
What is multiple regression?
//same as simple regression but multiple independent variables

//Assumptions of linear regression - do not blindly follow but check if the given problem is linear regression problem

// In above problem, profit is our dependent variables and others are independent variables

//but state (location) is a categorical variable and we need to handle this.
//For this you need to create dummy variables
//Dummy variables trap ( include one less dummy variables in predictor)
// Whenever you are building a model, always omit one dummy variables.
// So beware of dummy variable trap
//Show the image (dummy variable trap)
// Now lets start buidling a the model

// Some times you have to miss some of independent variable for model buidling (elimination) [selecting the right variables)
// All possible methods are given below
1. All in
2. Backward elimination
3. Forward selection
4. Bidirection elimination
5. Score comparison

"""
"""
About one hot encoding

Most of the machine learning algorithms such as linear regression, logistic regression, neural network, support vector machine works better with numerical features.

Quantitative features come with a numerical value and they can be directly used (Sometimes data preprocessing, normalization may have to use) as the input features of ML algorithms.

Ordinal features can be easily represented in numbers (Ex. First = 1, Second = 2, Third = 3 …). This is called Integer Encoding. Representing ordinal features using numbers makes sense because the dependency between each representation can be notated in a numerical way.

There are some algorithms that can directly deal with joint discrete distribution such as Markov chain / Naive Bayes / Bayesian network, tree based, etc. These algorithms can work with categorical data without any encoding; while we should encode the categorical features in a way to represent in a numerically to use as the input features for other ML algorithms. That means it’s better to change the categorical features to numerical most of the times
"""


"""
@file:          constrainedOptimization.py
@description:   answers to problems in exercises 7.3
                (Linear Algebra and its applications, David C. Lay 3ed)
@author:        Hiu-kong Dan
@date:          June 20, 2021
"""

import numpy as np
from numpy import linalg
from sympy import Matrix

def common(A):
    print(Matrix.eigenvects(Matrix(A)))
    print(Matrix.eigenvals(Matrix(A)))
    vecs = np.linalg.eig(A)
    print("unit eigenvectors:")
    for vec in vecs[1]:
        print(vec / np.linalg.norm(vec))
    print("eigenvalues: ", vecs[0])
    print("-"*20)

def problem14():
    A = np.array([[0, .5, 1.5, 15],
                  [.5, 0, 15, 1.5],
                  [1.5, 15, 0, .5],
                  [15, 1.5, .5, 0]])
    common(A)

def problem15():
    A = np.array([[0, 1.5, 2.5, 3.5],
                  [1.5, 0, 3.5, 2.5],
                  [2.5, 3.5, 0, 1.5],
                  [3.5, 2.5, 1.5, 0]])
    common(A)

def problem16():
    A = np.array([[4, -3, -5, -5],
                  [-3., 0, -3, -3],
                  [-5, -3, 0, -1],
                  [-5, -3, -1, 0]])
    common(A)

def problem17():
    A = np.array([[-6, -2, -2, -2],
                  [ -2, -10, 0, 0],
                  [ -2, 0, -13, 3],
                  [ -2, 0, 3, -13]])
    common(A)

if __name__ == '__main__':
    problem14()
    problem15()
    problem16()
    problem17()


"""Normalize features"""

import numpy as np

def normalize(features):
    """Normalize features.

    Normalizes input features X. Returns a normalized version of X where the mean value of
    each feature is 0 and deviation is close to 1.

    :param features: set of features.
    :return: normalized set of features.
    """

    # Copy original array to prevent it from changes.
    features_normalized = np.copy(features).astype(float)

    # Get average values for each feature (column) in X.
    features_mean = np.mean(features, 0)

    # Calculate the standard deviation for each feature.
    features_deviation = np.std(features, 0)

    # Subtract mean values from each feature (column) of every example (row)
    # to make all features be spread around zero.
    if features.shape[0] > 1:
        features_normalized -= features_mean

    # Normalize each feature values so that all features are close to [-1:1] boundaries.
    # Also prevent division by zero error.
    features_deviation[features_deviation == 0] = 1
    features_normalized /= features_deviation

    return features_normalized, features_mean, features_deviation


# from scratch
# https://github.com/SSQ/Coursera-Stanford-Greedy-Algorithms-Minimum-Spanning-Trees-and-Dynamic-Programming/blob/master/Programming%20Assignment%202/Cluster%201.ipynb
import numpy as np

file_path = "sample-data/clustering1.txt"
X = np.loadtxt(file_path, dtype=int)

clusters = {}
for x in range(1,501):
    clusters[x] = [x]

# sort the distance of nodes
sorted_X = np.argsort(X[:,2], axis=0)

k = 4

# d: dictionary, k: key
def get_key_by_value(d, k):
    for leader, nodes in d.items():
        if k in nodes:
            return leader

# c: dictionary. cluster , a: key. added value in dict, d: key. deleted value in dict
def update_cluster(c, a, d):
    c[a].extend(c[d])
    del c[d]

iteration = 0

print('whole_edges: ')
print(len(sorted_X))

# core algorithm
for i in sorted_X:
    min_edge = X[i,:]
    # print("minedge",min_edge)
    key1 = get_key_by_value(clusters, min_edge[0])
    key2 = get_key_by_value(clusters, min_edge[1])
    if key1 == key2:
        continue
    elif len(clusters[key1]) >= len(clusters[key2]):
        update_cluster(clusters, key1, key2)
    else:
        update_cluster(clusters, key2, key1)

    if len(clusters) == k:
        break

print(clusters)

clusters_keys = list(clusters.keys()) # [102, 384, 414, 462]
print("cluster keys", clusters_keys)

# initialize
spacing = []

distance=0
# set all spacing of k-clusters
for i in range(len(clusters_keys)-1):
    j = i+1
    for k in range(j, len(clusters_keys)):
        spacing.append([clusters_keys[i], clusters_keys[k], distance])
spacing = np.array(spacing)
print('spacing: ')
print(spacing)

def min_distance(X, clusters, cluster1, cluster2):
    minimum = X[:,2].max()
    for i in clusters[cluster1]:
        for j in clusters[cluster2]:
            if i < j:
                find_position = np.all([X[:,0]==i, X[:,1]==j], axis=0)
                find_edge = np.argwhere(find_position==True)
                actural_position = find_edge[0][0]
                distance = X[actural_position, 2]
                if minimum > distance:
                    minimum = distance
            else:
                find_position = np.all([X[:,0]==j, X[:,1]==i], axis=0)
                find_edge = np.argwhere(find_position==True)
                actural_position = find_edge[0][0]
                distance = X[actural_position, 2]
                if minimum > distance:
                    minimum = distance
    return minimum

for i in range(len(spacing)):
    spacing[i,2] = min_distance(X, clusters, spacing[i,0], spacing[i,1])

print('spacing: ')
print(spacing)
print('max distance clusters: ')
print(spacing[np.argmax(spacing[:,2]), :])
print('max distance: ')
print(np.amax(spacing[:,2]))