#implementation of Bayesian linear regression
import numpy as np
from scipy import stats

class BayesLinReg:

    def __init__(self, n_features, alpha, beta):
        self.n_features = n_features
        self.alpha = alpha
        self.beta = beta
        self.mean = np.zeros(n_features)
        self.cov_inv = np.identity(n_features) / alpha

    def learn(self, x, y):

        # Update the inverse covariance matrix (Bishop eq. 3.51)
        cov_inv = self.cov_inv + self.beta * np.outer(x, x)

        # Update the mean vector (Bishop eq. 3.50)
        cov = np.linalg.inv(cov_inv)
        mean = cov @ (self.cov_inv @ self.mean + self.beta * y * x)

        self.cov_inv = cov_inv
        self.mean = mean

        return self

    def predict(self, x):

        # Obtain the predictive mean (Bishop eq. 3.58)
        y_pred_mean = x @ self.mean

        # Obtain the predictive variance (Bishop eq. 3.59)
        w_cov = np.linalg.inv(self.cov_inv)
        y_pred_var = 1 / self.beta + x @ w_cov @ x.T

        return stats.norm(loc=y_pred_mean, scale=y_pred_var ** .5)

    @property
    def weights_dist(self):
        cov = np.linalg.inv(self.cov_inv)
        return stats.multivariate_normal(mean=self.mean, cov=cov)

#progressive validation to measure the performance of a model
from sklearn import datasets
from sklearn import metrics

X, y = datasets.load_boston(return_X_y=True)

model = BayesLinReg(n_features=X.shape[1], alpha=.3, beta=1)

y_pred = np.empty(len(y))

for i, (xi, yi) in enumerate(zip(X, y)):
    y_pred[i] = model.predict(xi).mean()
    model.learn(xi, yi)

print(metrics.mean_absolute_error(y, y_pred))

#In a Bayesian linear regression, the weights follow a distribution that quantifies their uncertainty.
#steps for producing a visualization of both distributions.

from mpl_toolkits.axes_grid1 import ImageGrid
import matplotlib.pyplot as plt
%matplotlib inline

np.random.seed(42)

# Pick some true parameters that the model has to find
weights = np.array([-.3, .5])

def sample(n):
    for _ in range(n):
        x = np.array([1, np.random.uniform(-1, 1)])
        y = np.dot(weights, x) + np.random.normal(0, .2)
        yield x, y

model = BayesLinReg(n_features=2, alpha=2, beta=25)

# The following 3 variables are just here for plotting purposes
N = 100
w = np.linspace(-1, 1, 100)
W = np.dstack(np.meshgrid(w, w))

n_samples = 5
fig = plt.figure(figsize=(7 * n_samples, 21))
grid = ImageGrid(
    fig, 111,  # similar to subplot(111)
    nrows_ncols=(n_samples, 3),  # creates a n_samplesx3 grid of axes
    axes_pad=.5  # pad between axes in inch.
)

# We'll store the features and targets for plotting purposes
xs = []
ys = []

def prettify_ax(ax):
    ax.set_xlim(-1, 1)
    ax.set_ylim(-1, 1)
    ax.set_xlabel('$w_1$')
    ax.set_ylabel('$w_2$')
    return ax

for i, (xi, yi) in enumerate(sample(n_samples)):

    pred_dist = model.predict(xi)

    # Prior weight distribution
    ax = prettify_ax(grid[3 * i])
    ax.set_title(f'Prior weight distribution #{i + 1}')
    ax.contourf(w, w, model.weights_dist.pdf(W), N, cmap='viridis')
    ax.scatter(*weights, color='red')  # true weights the model has to find

    # Update model
    model.learn(xi, yi)

    # Prior weight distribution
    ax = prettify_ax(grid[3 * i + 1])
    ax.set_title(f'Posterior weight distribution #{i + 1}')
    ax.contourf(w, w, model.weights_dist.pdf(W), N, cmap='viridis')
    ax.scatter(*weights, color='red')  # true weights the model has to find

    # Posterior target distribution
    xs.append(xi)
    ys.append(yi)
    posteriors = [model.predict(np.array([1, wi])) for wi in w]
    ax = prettify_ax(grid[3 * i + 2])
    ax.set_title(f'Posterior target distribution #{i + 1}')
    # Plot the old points and the new points
    ax.scatter([xi[1] for xi in xs[:-1]], ys[:-1])
    ax.scatter(xs[-1][1], ys[-1], marker='*')
    # Plot the predictive mean along with the predictive interval
    ax.plot(w, [p.mean() for p in posteriors], linestyle='--')
    cis = [p.interval(.95) for p in posteriors]
    ax.fill_between(
        x=w,
        y1=[ci[0] for ci in cis],
        y2=[ci[1] for ci in cis],
        alpha=.1
    )
    # Plot the true target distribution
    ax.plot(w, [np.dot(weights, [1, xi]) for xi in w], color='red')

# A nice property about Bayesian models is that they allow to quantify the uncertainty of predictions.
np.random.seed(42)

model = BayesLinReg(n_features=2, alpha=1, beta=25)
pct_in_ci = 0
pct_in_ci_hist = []
n = 5_000

for i, (xi, yi) in enumerate(sample(n)):

    ci = model.predict(xi).interval(.95)
    in_ci = ci[0] < yi < ci[1]
    pct_in_ci += (in_ci - pct_in_ci) / (i + 1)  # online update of an average
    pct_in_ci_hist.append(pct_in_ci)

    model.learn(xi, yi)

fig, ax = plt.subplots(figsize=(9, 6))
ax.plot(range(n), pct_in_ci_hist)
ax.axhline(y=.95, color='red', linestyle='--')
ax.set_title('Quality of the prediction interval along time')
ax.set_xlabel('# of observed samples')
ax.set_ylabel('% of predictions in 95% prediction interval')
ax.set_ylim(.9, 1)
ax.grid()


# A library for running Cholesky factorization and Cholesky incomplete
# factorization
# author : Etienne THIERY
import trunk.matgen
from numpy import *

def oldCompleteCholesky(M):
    ''' Computes the cholesky factorization of a matrix M, i.e the
    lower triangular matrix T such as M = T . T.transpose
    M should be a symmetric positive definite matrix
    runs in O(n^3/3) where M is a matrix of size nxn'''
    n = M.shape[0]
    T = zeros((n,n))
    col = 0
    while col < n:
        T[col][col] = M[col][col]
        row = 0
        while row < col:
            T[col][col] -= T[col][row]*T[col][row]
            row+=1
        T[col][col] = sqrt(T[col][col])
        row+=1
        while row < n:
            T[row][col] = M[row][col]
            k = 0
            while k < col:
                T[row,col] -= T[col,k]*T[row,k]
                k+=1
            T[row,col] /= T[col,col]
            row+=1
        col+=1

    return T

def completeCholesky(M):
    ''' Computes the cholesky factorization of a matrix M, i.e the
    lower triangular matrix T such as M = T . T.transpose
    M should be a symmetric positive definite matrix
    runs in O(n^3/3) where M is a matrix of size nxn'''
    n = M.shape[0]
    T = zeros((n,n))
    for col in range(n):
        T[col,col] = sqrt(M[col,col] - dot(T[col,:col], T[col,:col]))
        T[col+1:n,col] = (M[col+1:n,col] - dot(T[col+1:n,:col], (T[col,:col]).transpose())) / T[col,col]

    return T

def oldIncompleteCholesky(M):
    '''Only performs an incomplete cholesky factorization
    which result in only an approximation of the result, but is faster '''

    n = M.shape[0]
    T = zeros((n,n))
    col = 0
    while col < n:
        T[col][col] = M[col][col]
        row = 0
        if M[col][col] != 0:
            while row < col:
                T[col][col] -= T[col][row]*T[col][row]
                row+=1
            T[col][col] = sqrt(T[col][col])
            row+=1
        else:
            row = col+1

        while row < n:
            if M[row][col] != 0:
                T[row][col] = M[row][col]
                k = 0
                while k < col:
                    T[row,col] -= T[col,k]*T[row,k]
                    k+=1
                T[row,col] /= T[col,col]
            row+=1
        col+=1
    return T

def incompleteCholesky(M):
    '''Only performs an incomplete cholesky factorization
    which result in only an approximation of the result, but is faster '''
    n = M.shape[0]
    T = zeros((n,n))
    for col in range(n):
        T[col,col] = sqrt(M[col,col] - dot(T[col,:col], T[col,:col]))
        for row in range(col+1, n):
            if M[row,col] == 0:
                T[row,col] = 0
            else:
                T[row,col] = (M[row,col] - dot(T[row,:col], (T[col,:col]).transpose())) / T[col,col]
    return T


import itertools

from IMLearn.learners import UnivariateGaussian, MultivariateGaussian
import numpy as np
import plotly.graph_objects as go
import plotly.io as pio
pio.templates.default = "simple_white"

def test_univariate_gaussian():
    mu = 10
    sigma = 1
    # Question 1 - Draw samples and print fitted model
    m=1000 # number of samples
    samples = np.random.normal(mu,sigma,size=m)
    gaussian_param = UnivariateGaussian()
    gaussian_param.fit(samples)
    print(gaussian_param.mu_,gaussian_param.var_)

    # Question 2 - Empirically showing sample mean is consistent
    estimated_mean = []
    ms = np.arange(10,1010,10)
    for m in ms:
        X = np.random.normal(mu, sigma, size=m)
        estimated_mean.append(np.mean(X))
    go.Figure([go.Scatter(x=ms, y=estimated_mean, mode='markers+lines', name=r'$\widehat\mu$'),
               go.Scatter(x=ms, y=[mu]*len(ms), mode='lines', name=r'$\mu$')],
              layout=go.Layout(title=r"$\text{(5) Estimation of Expectation As Function Of Number Of Samples}$",
                               xaxis_title="$m\\text{ - number of samples}$",
                               yaxis_title="r$\hat\mu$",
                               height=300)).show()

    # # Question 3 - Plotting Empirical PDF of fitted model
    PDFs = gaussian_param.pdf(samples)
    go.Figure([go.Histogram(x=samples, opacity=0.75, bingroup=1, histnorm='probability density', marker_color="rgb(219,124,134)", name=r'$\hat\mu_1$')],
              layout=go.Layout(barmode='overlay',
                               title=r"$\text{(8) Mean estimator distribution}$",
                               xaxis_title="r$\hat\mu$",
                               yaxis_title="density",
                               height=300)).show()
    print(UnivariateGaussian.log_likelihood(mu,sigma,samples))

def test_multivariate_gaussian():
    m = 1000
    mean = np.array([0,0,4,0])
    sigma_matrix = np.array([1,0.2,0,0.5,0.2,2,0,0,0,0,1,0,0.5,0,0,1]).reshape(4,4)
    multi_gaussian_var = MultivariateGaussian()
    # Question 4 - Draw samples and print fitted model
    samples = np.random.multivariate_normal(mean,sigma_matrix,m)
    multi_gaussian_var.fit(samples)
    print(multi_gaussian_var.mu_)
    print( multi_gaussian_var.cov_)

    # Question 5 - Likelihood evaluation
    lin_space = np.linspace(-10,10,200)
    log_likelihoods=[]
    for f1 in lin_space:
        for f3 in lin_space:
            mu = np.array([f1,0,f3,0])
            samples = np.random.multivariate_normal(mu,sigma_matrix,m)
            log_likelihoods.append(MultivariateGaussian.log_likelihood(mu,sigma_matrix,samples))

    # Question 6 - Maximum likelihood

if __name__ == '__main__':
    np.random.seed(0)
    # test_univariate_gaussian()
    test_multivariate_gaussian()


#!/usr/bin/env python
# coding: utf-8

# # Hierarchical Clustering Lab
# In this notebook, we will be using sklearn to conduct hierarchical clustering on the [Iris dataset](https://archive.ics.uci.edu/ml/datasets/iris) which contains 4 dimensions/attributes and 150 samples. Each sample is labeled as one of the three type of Iris flowers.
#
# In this exercise, we'll ignore the labeling and cluster based on the attributes, then we'll compare the results of different hierarchical clustering techniques with the original labels to see which one does a better job in this scenario. We'll then proceed to visualize the resulting cluster hierarchies.
#
# ## 1. Importing the Iris dataset
#

# In[1]:

from sklearn import datasets

iris = datasets.load_iris()

# A look at the first 10 samples in the dataset

# In[2]:

iris.data[:10]

# ```iris.target``` contains the labels that indicate which type of Iris flower each sample is

# In[3]:

iris.target

# ## 2. Clustering
# Let's now use sklearn's ```AgglomerativeClustering``` to conduct the heirarchical clustering

# In[4]:

from sklearn.cluster import AgglomerativeClustering

# Hierarchical clustering
# Ward is the default linkage algorithm, so we'll start with that
ward = AgglomerativeClustering(n_clusters=3)
ward_pred = ward.fit_predict(iris.data)

# Let's also try complete and average linkages
#
# **Exercise**:
# * Conduct hierarchical clustering with complete linkage, store the predicted labels in the variable ```complete_pred```
# * Conduct hierarchical clustering with average linkage, store the predicted labels in the variable ```avg_pred```

# In[5]:

# Hierarchical clustering using complete linkage
# TODO: Create an instance of AgglomerativeClustering with the appropriate parameters
complete = AgglomerativeClustering(n_clusters=3, linkage="complete")
# Fit & predict
# TODO: Make AgglomerativeClustering fit the dataset and predict the cluster labels
complete_pred = complete.fit_predict(iris.data)

# Hierarchical clustering using average linkage
# TODO: Create an instance of AgglomerativeClustering with the appropriate parameters
avg = AgglomerativeClustering(n_clusters=3, linkage="average")
# Fit & predict
# TODO: Make AgglomerativeClustering fit the dataset and predict the cluster labels
avg_pred = avg.fit_predict(iris.data)

# To determine which clustering result better matches the original labels of the samples, we can use ```adjusted_rand_score``` which is an *external cluster validation index* which results in a score between -1 and 1, where 1 means two clusterings are identical of how they grouped the samples in a dataset (regardless of what label is assigned to each cluster).
#
# Cluster validation indices are discussed later in the course.

# In[6]:

from sklearn.metrics import adjusted_rand_score

ward_ar_score = adjusted_rand_score(iris.target, ward_pred)

# **Exercise**:
# * Calculate the Adjusted Rand score of the clusters resulting from complete linkage and average linkage

# In[7]:

# TODO: Calculated the adjusted Rand score for the complete linkage clustering labels
complete_ar_score = adjusted_rand_score(iris.target, complete_pred)

# TODO: Calculated the adjusted Rand score for the average linkage clustering labels
avg_ar_score = adjusted_rand_score(iris.target, avg_pred)

# Which algorithm results in the higher Adjusted Rand Score?

# In[8]:

print( "Scores: \nWard:", ward_ar_score,"\nComplete: ", complete_ar_score, "\nAverage: ", avg_ar_score)

# ## 3. The Effect of Normalization on Clustering
#
# Can we improve on this clustering result?
#
# Let's take another look at the dataset

# In[9]:

iris.data[:15]

# Looking at this, we can see that the forth column has smaller values than the rest of the columns, and so its variance counts for less in the clustering process (since clustering is based on distance). Let us [normalize](https://en.wikipedia.org/wiki/Feature_scaling) the dataset so that each dimension lies between 0 and 1, so they have equal weight in the clustering process.
#
# This is done by subtracting the minimum from each column then dividing the difference by the range.
#
# sklearn provides us with a useful utility called ```preprocessing.normalize()``` that can do that for us

# In[9]:

from sklearn import preprocessing

normalized_X = preprocessing.normalize(iris.data)
normalized_X[:10]

# Now all the columns are in the range between 0 and 1. Would clustering the dataset after this transformation lead to a better clustering? (one that better matches the original labels of the samples)

# In[10]:

ward = AgglomerativeClustering(n_clusters=3)
ward_pred = ward.fit_predict(normalized_X)

complete = AgglomerativeClustering(n_clusters=3, linkage="complete")
complete_pred = complete.fit_predict(normalized_X)

avg = AgglomerativeClustering(n_clusters=3, linkage="average")
avg_pred = avg.fit_predict(normalized_X)

ward_ar_score = adjusted_rand_score(iris.target, ward_pred)
complete_ar_score = adjusted_rand_score(iris.target, complete_pred)
avg_ar_score = adjusted_rand_score(iris.target, avg_pred)

print( "Scores: \nWard:", ward_ar_score,"\nComplete: ", complete_ar_score, "\nAverage: ", avg_ar_score)

# ## 4. Dendrogram visualization with scipy
#
# Let's visualize the highest scoring clustering result.
#
# To do that, we'll need to use Scipy's [```linkage```](https://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html) function to perform the clusteirng again so we can obtain the linkage matrix it will later use to visualize the hierarchy

# In[11]:

# Import scipy's linkage function to conduct the clustering
from scipy.cluster.hierarchy import linkage

# Specify the linkage type. Scipy accepts 'ward', 'complete', 'average', as well as other values
# Pick the one that resulted in the highest Adjusted Rand Score
linkage_type = 'ward'

linkage_matrix = linkage(normalized_X, linkage_type)

# Plot using scipy's [dendrogram](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.cluster.hierarchy.dendrogram.html) function

# In[13]:

from scipy.cluster.hierarchy import dendrogram
import matplotlib.pyplot as plt
plt.figure(figsize=(22,18))

# plot using 'dendrogram()'
dendrogram(linkage_matrix)

plt.show()

# ## 5. Visualization with Seaborn's ```clustermap```
#
# The [seaborn](http://seaborn.pydata.org/index.html) plotting library for python can plot a [clustermap](http://seaborn.pydata.org/generated/seaborn.clustermap.html), which is a detailed dendrogram which also visualizes the dataset in more detail. It conducts the clustering as well, so we only need to pass it the dataset and the linkage type we want, and it will use scipy internally to conduct the clustering

# In[ ]:

import seaborn as sns

sns.clustermap(normalized_X, figsize=(12,18), method=linkage_type, cmap='viridis')

# Expand figsize to a value like (18, 50) if you want the sample labels to be readable
# Draw back is that you'll need more scrolling to observe the dendrogram

plt.show()

# Looking at the colors of the dimensions can you observe how they differ between the three type of flowers? You should at least be able to notice how one is vastly different from the two others (in the top third of the image).

# In[ ]:


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
get_ipython().system('pip install pandas_datareader')
from pandas_datareader import data
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matploitlib', 'inline')

# In[8]:

test = data.DataReader(['TSLA', 'FB'], 'yahoo', start='2016/01/01', end='2021/04/30')
test.head(20)

# In[10]:

test = test['Adj Close']
test.head(20)

# In[11]:

tesla = test['TSLA'].pct_change().apply(lambda x: np.log(1+x))
tesla.head()

# In[12]:

var_tesla = tesla.var()
var_tesla

# In[13]:

facebook = test['FB'].pct_change().apply(lambda x: np.log(1+x))
facebook.head()

# In[14]:

var_facebook = facebook.var()
var_facebook

# In[15]:

tesla_volatility = np.sqrt(var_tesla * 250)
facebook_volatility = np.sqrt(var_facebook *250)
tesla_volatility , facebook_volatility

# In[16]:

test.pct_change().apply(lambda x: np.log(1+x)).std().apply(lambda x: x*np.sqrt(250)).plot(kind='bar')

# In[17]:

test1 = test.pct_change().apply(lambda x: np.log(1+x))
test1.head()

# In[18]:

test1['TSLA'].cov(test1['FB'])

# In[19]:

test1['TSLA'].corr(test1['FB'])

# In[20]:

test2 = test.pct_change().apply(lambda x: np.log(1+x))
test2.head()

# In[21]:

w = [0.2, 0.8]
e_r_ind = test2.mean()
e_r_ind

# In[22]:

e_r = (e_r_ind*w).sum()
e_r

# In[2]:

df = data.DataReader(['AAPL', 'NKE', 'GOOGL', 'AMZN'], 'yahoo', start='2016/01/01', end='2021/04/30')
df.head()

# In[3]:

df = df['Adj Close'] #Close pricing of the company's stock on the given day
df.head()

# # Covariance
#
# Covariance measures the directional relationship between the returns on two assets.
#
# A positive covariance means that returns of the two assets move together while a negative covariance means they move inversely. Risk and volatility can be reduced in a portfolio by pairing assets that have a negative covariance.
#
# ## cov_matrix
#
# Representing the Covariance of all the stocks using matrix for better understanding.
#
# ## Why are we taking log?
#
# This is beacuse the log of the returns is time additive.
#
# That is,
# If r13 is the returns for time between t3 and t1.
# r12 is the returns between t1 and t2 and
# r23 is the returns between t2 and t3
#
# Then, log(r13) = log(r12) + log(r23)
#
# For example:,
# If p1 = 100, p2 = 110 and p3 = 120,
# where p1 is price of stock in time 1
#
# Then:
#
# log(r12) = ln(p2/p1) = ln(110/100) = 9.53%
# log(r23) = ln(120/110) = 8.7% and
# log(r13) = log(r12) + log(r23) = 9.53 + 8.7 = 18.23%, which is same as ln(120/100).
#
# This means a log change of +0.1 today and then -0.1 tomorrow will give you the same value of stock as yesterday. This is not true if you simply compute percentage chang

# In[4]:

cov_matrix = df.pct_change().apply(lambda x: np.log(1+x)).cov()
cov_matrix

# # Correlation
#
# Correlation, in the finance and investment industries, is a statistic that measures the degree to which two securities move in relation to each other. Correlations are used in advanced portfolio management, computed as the correlation coefficient, which has a value that must fall between -1.0 and +1.0.
#
# We can think of correlation as a scaled version of covariance, where the values are restricted to lie between -1 and +1.
#
# A correlation of -1 means negative relation, i.e, if correlation between Asset A and Asset B is -1, if Asset A increases, Asset B decreases.
#
# A correlation of +1 means positive relation, i.e, if correlation between Asset A and Asset B is 1, if Asset A increases, Asset B increases.
#
# A correlation of 0 means no relation, i.e, if correlation between Asset A and Asset B is 0, they dont have any effect on each other.
#
# ## corr_matrix
#
# Correlation matrix

# In[5]:

corr_matrix = df.pct_change().apply(lambda x: np.log(1+x)).corr()
corr_matrix

# # Weights
#
# Let’s define an array of random weights for the purpose of calculation. These weights will represent the percentage allocation of investments between these two stocks. They must add up to 1.
#
# So, the problem of portfolio optimization is nothing but to find the optimal values of weights that maximizes expected returns while minimizing the risk (standard deviation).

# In[6]:

w = {'AAPL': 0.1, 'NKE': 0.2, 'GOOGL': 0.5, 'AMZN': 0.2}
port_var = cov_matrix.mul(w, axis=0).mul(w, axis=1).sum().sum()
port_var

# # Portfolio Expected Returns
#
# The mean of returns (given by change in prices of asset stock prices) give us the expected returns of that asset.
# The sum of all individual expected returns further multiplied by the weight of assets give us expected return for the portfolio.
#
# We use the resample() function to get yearly returns. The argument to function, ‘Y’, denotes yearly.
# If we dont perform resampling, we will get daily returns.

# In[7]:

ind_er = df.resample('Y').last().pct_change().mean()
ind_er

# In[8]:

w = [0.1, 0.2, 0.5, 0.2]
port_er = (w*ind_er).sum()
port_er

# # Plotting the Efficient Frontier
#
# Efficient frontier is a graph with ‘returns’ on the Y-axis and ‘volatility’ on the X-axis. It shows us the maximum return we can get for a set level of volatility, or conversely, the volatility that we need to accept for certain level of returns.

# In[9]:

#Volatility is given by annual standard deviation. It is multiplied by 250 because there are 250 trading days in a year.
ann_sd = df.pct_change().apply(lambda x: np.log(1+x)).std().apply(lambda x: x*np.sqrt(250))
ann_sd

# In[10]:

assets = pd.concat([ind_er, ann_sd], axis=1) # Creating a table for visualising returns and volatility of assets
assets.columns = ['Returns', 'Volatility']
assets

# # Analysis of the above Results
#
# Apple has the highest Volatility or we can say Risk, but also the highest return.

# Google has the lowest volatility but, the lowest porfit or return as well.
#
# Next, to plot the graph of efficient frontier, we need run a loop. In each iteration, the loop considers different weights for assets and calculates the return and volatility of that particular portfolio combination.
#
# We run this loop a 1000 times.
#
# To get random numbers for weights, we use the "np.random.random()" function. But, the sum of weights must be 1, so we divide those weights by their cumulative sum.

# In[11]:

p_ret = [] # Define an empty array for portfolio returns
p_vol = [] # Define an empty array for portfolio volatility
p_weights = [] # Define an empty array for asset weights

num_assets = len(df.columns)
num_portfolios = 10000

# In[12]:

for portfolio in range(num_portfolios):
    weights = np.random.random(num_assets)
    weights = weights/np.sum(weights)
    p_weights.append(weights)
    returns = np.dot(weights, ind_er) # Returns are the product of individual expected returns of asset and its
                                      # weights
    p_ret.append(returns)
    var = cov_matrix.mul(weights, axis=0).mul(weights, axis=1).sum().sum()# Portfolio Variance
    sd = np.sqrt(var) # Daily standard deviation
    ann_sd = sd*np.sqrt(250) # Annual standard deviation = volatility
    p_vol.append(ann_sd)

# In[13]:

data = {'Returns':p_ret, 'Volatility':p_vol}

for counter, symbol in enumerate(df.columns.tolist()):
    #print(counter, symbol)
    data[symbol+' weight'] = [w[counter] for w in p_weights]

# In[14]:

portfolios  = pd.DataFrame(data)
portfolios.head()

# You can see that there are a number of portfolios with different weights, returns and volatility. Plotting the returns and volatility from this dataframe will give us the efficient frontier for our portfolio.

# In[15]:

portfolios.plot.scatter(x='Volatility', y='Returns', marker='o', s=10, alpha=0.3, grid=True, figsize=[10,10])

# # Reading data form the Efficient Frontier
#
# Each point on the line (left edge) represents an optimal portfolio of stocks that maximises the returns for any given level of risk.
#
#
# The point (portfolios) in the interior are sub-optimal for a given risk level. For every interior point, there is another that offers higher returns for the same risk.
#
# On this graph, we can also see the combination of weights that will give you all possible combinations:
#
#

#
Minimum volatility (left most point)

#
Maximum returns (top most point)

#

#
# And everything in between.

# In[16]:

min_vol_port = portfolios.iloc[portfolios['Volatility'].idxmin()]
# idxmin() gives us the minimum value in the column specified.
min_vol_port

# The minimum volatility is in a portfolio where the weights of Apple, Nike, Google and Amazon are 11%, 38%, 31% and 19% respectively. This point can be plotted on the efficient frontier graph as shown:

# In[17]:

plt.subplots(figsize=[10,10])
plt.scatter(portfolios['Volatility'], portfolios['Returns'],marker='o', s=10, alpha=0.3)
plt.scatter(min_vol_port[1], min_vol_port[0], color='r', marker='*', s=500)

# The red star denotes the most efficient portfolio with minimum volatility.
#
# It is worthwhile to note that any point to the right of efficient frontier boundary is a sup-optimal portfolio.
#
# We found the portfolio with minimum volatility, but you will notice that the return on this portfolio is pretty low. Any sensible investor wants to maximize his return, even if it is a tradeoff with some level of risk.
#
# The question arises that how do we find this optimal risky portfolio and finally optimize our portfolio to the maximum?
#
# This is done by using a parameter called the Sharpe Ratio.
#
# The ratio is the average return earned in excess of the risk-free rate per unit of volatility or total risk. Volatility is a measure of the price fluctuations of an asset or portfolio.
#
#
# The risk-free rate of return is the return on an investment with zero risk, meaning it’s the return investors could expect for taking no risk.
#
# The optimal risky portfolio is the one with the highest Sharpe ratio.

# In[18]:

rf = 0.01 # risk factor
optimal_risky_port = portfolios.iloc[((portfolios['Returns']-rf)/portfolios['Volatility']).idxmax()]
optimal_risky_port

# You can notice that while the difference in risk between minimum volatility portfolio and optimal risky portfolio is just 4%, the difference in returns is a whopping 12%.
# We can plot this point too on the graph of efficient frontier.

# In[19]:

plt.subplots(figsize=(10, 10))
plt.scatter(portfolios['Volatility'], portfolios['Returns'],marker='o', s=10, alpha=0.3)
plt.scatter(min_vol_port[1], min_vol_port[0], color='r', marker='*', s=500)
plt.scatter(optimal_risky_port[1], optimal_risky_port[0], color='g', marker='*', s=500)

# The green star represents the Optimal Risky Portfolio.

# In[ ]:


import argparse
import logging
import os

import numpy as np
import pandas as pd
import scipy.stats as stats
import matplotlib.pyplot as plt

def load_data_MNTest(fl="./MCTestData.csv"):
    """
    Loads data stored in McNemarTest.csv
    :param fl: filename of csv file
    :return: labels, prediction1, prediction2
    """
    data = pd.read_csv(fl, header=None).to_numpy()
    labels = data[:, 0]
    prediction_1 = data[:, 1]
    prediction_2 = data[:, 2]
    return labels, prediction_1, prediction_2

def load_data_TMStTest(fl="./TMStTestData.csv"):
    """
    Loads data stored in fl
    :param fl: filename of csv file
    :return: y1, y2
    """
    data = np.loadtxt(fl, delimiter=",")
    y1 = data[:, 0]
    y2 = data[:, 1]
    return y1, y2

def load_data_FTest(fl="./FTestData.csv"):
    """
    Loads data stored in fl
    :param fl: filename of csv file
    :return: evaluations
    """
    errors = np.loadtxt(fl, delimiter=",")
    return errors

def McNemar_test(labels, prediction_1, prediction_2):
    """
    H0: model1 and model2 have the same performance
    H1: performances are not equal
    :param labels: the ground truth labels
    :param prediction_1: the prediction results from model 1
    :param prediction_2:  the prediction results from model 2
    :return: the test statistic chi2_Mc
    """
    m1 = (labels==prediction_1)
    m2 = (labels==prediction_2)
    B = np.sum(np.logical_and(m1, np.logical_not(m2)))
    C = np.sum(np.logical_and(m2, np.logical_not(m1)))
    chi2_Mc = (abs(B-C)-1)**2/(B+C)
    return chi2_Mc

def  TwoMatchedSamplest_test(y1, y2):
    """
    H0: model1 and model2 have the same performance
    H1: performances are not equal
    :param y1: runs of algorithm 1
    :param y2: runs of algorithm 2
    :return: the test statistic t-value
    """
    d = y1-y2
    n_test = len(y1)
    d_mean = np.mean(d)
    d_var  = np.sqrt(1.0/(n_test-1)*np.sum(np.square(d-d_mean)))
    t_value = np.sqrt(n_test)*d_mean/d_var
    return t_value

def Friedman_test(errors):
    """
    H0: all algorithms are equivalent in their performance and hence their average ranks
    H1: the average ranks for at least one algorithm is different
    :param errors: the error values of different algorithms on different datasets
    :return: chi2_F: the test statistic chi2_F value
    :return: FData_stats: the statistical data of the Friedan test data, you can add anything needed to facilitate
    solving the following post hoc problems
    """
    n, k = errors.shape

    # Rank matrix
    r = np.argsort(np.argsort(errors, axis=1), axis=1)

    rtotal_mean = np.mean(r)
    r_mean = np.mean(r, axis=0)
    ss_total = n*np.sum(np.square(r_mean-rtotal_mean))
    ss_error = 1.0/n/(k-1) * np.sum(np.square(r-rtotal_mean))

    chi2_F = ss_total/ss_error
    FData_stats = {'errors': errors, 'Rj': r_mean}
    return chi2_F, FData_stats

def Nemenyi_test(FData_stats):
    """
    Compares all pairs of algorithm to find best-performing algorithm after H0 of the Friedman-test was rejected
    :param FData_stats: the statistical data of the Friedan test data to be utilized in the post hoc problems
    :return: the test statisic Q value
    """
    r_mean = FData_stats['Rj']
    errors = FData_stats['errors']
    n, k = errors.shape
    Q_value = np.zeros((k, k), dtype=np.float16)
    inv = 1.0/np.sqrt(k*(k+1)/6/n)
    for i in range(k):
        for j in range(i+1,k):
            q = (r_mean[i]-r_mean[j])
            Q_value[i,j] = q*inv
    return Q_value

def box_plot(FData_stats):
    """
    :param FData_stats: the statistical data of the Friedan test data to be utilized in the post hoc problems
    """
    errors = FData_stats['errors']
    plt.boxplot(errors)

def main(args):
    # (a)
    labels, prediction_A, prediction_B = load_data_MNTest()
    chi2_Mc = McNemar_test(labels, prediction_A, prediction_B)

    # (b)
    y1, y2 = load_data_TMStTest()
    t_value = TwoMatchedSamplest_test(y1, y2)

    # (c)
    errors = load_data_FTest()
    chi2_F, FData_stats = Friedman_test(errors)

    # (d)
    Q_value = Nemenyi_test(FData_stats)

    # (e)
    box_plot(FData_stats)

if __name__ == '__main__':
    cmdline_parser = argparse.ArgumentParser('ex03')

    cmdline_parser.add_argument('-v', '--verbose', default='INFO', choices=['INFO', 'DEBUG'], help='verbosity')
    cmdline_parser.add_argument('--seed', default=12345, help='Which seed to use', required=False, type=int)
    args, unknowns = cmdline_parser.parse_known_args()
    np.random.seed(args.seed)
    log_lvl = logging.INFO if args.verbose == 'INFO' else logging.DEBUG
    logging.basicConfig(level=log_lvl)

    if unknowns:
        logging.warning('Found unknown arguments!')
        logging.warning(str(unknowns))
        logging.warning('These will be ignored')
    main(args)


# Reference Book: Python Data Science Handbook (page:(51-58))
# Date(12 April, 2019) Day-2, Time = 7:30 PM

import numpy as np
from scipy import special

np.random.seed(0)
# why use numpy??

def compute_reciprocals(values):
    output = np.empty(len(values))
    for i in range(len(values)):
        output[i] = 1.0 / values[i]
    return output

values = np.random.randint(1, 10, size=5)

# print(compute_reciprocals(values))
# 21 ms ± 195 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)

# @@@@@@@@@@@@@@@@@@@@@@@@ Introducing UFunc's @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
print(1.0/values)
'''
Vectorized operations in NumPy are implemented via ufuncs, whose main purpose is
to quickly execute repeated operations on values in NumPy arrays. Ufuncs are
extremely flexible—before we saw an operation between a scalar and an array, but we
can also operate between two arrays:
'''
a = np.arange(5)  # [0,1,2,3,4]
b = np.arange(1, 6)  # [1,2,3,4,5]
print(a / b)  # [0.         0.5        0.66666667 0.75       0.8       ]

# And ufunc operations are not limited to one-dimensional arrays—they can act on
# multidimensional arrays as well:
a = np.arange(9).reshape(3, 3)
a = 2 ** a
print(a)
'''
[[  1   2   4]
 [  8  16  32]
 [ 64 128 256]]
'''

# @@@@@@@@@@@@@@@@@@@@@@@@ Exploring UFunc's @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# Ufuncs exist in two flavors: unary ufuncs, which operate on a single input, and binary
# ufuncs, which operate on two inputs. We’ll see examples of both these types of func‐
# tions here.

# Array Arithmetic
npArray = np.arange(1, 5)  # [1,2,3,4]
print('\nnpArray     =', npArray)
print('npArray + 5 =', npArray + 5)
print('npArray - 5 =', npArray - 5)
print('npArray * 2 =', npArray * 2)
print('npArray / 2 =', npArray / 2)
print('npArray // 5 =', npArray // 2)
print('-npArray =', -npArray)
print('npArray ** 2 =', npArray ** 2)
print('npArray % 5 =', npArray % 2)
'''
npArray     = [1 2 3 4]
npArray + 5 = [6 7 8 9]
npArray - 5 = [-4 -3 -2 -1]
npArray * 2 = [2 4 6 8]
npArray / 2 = [0.5 1.  1.5 2. ]
npArray // 5 = [0 1 1 2]
-npArray = [-1 -2 -3 -4]
npArray ** 2 = [ 1  4  9 16]
npArray % 5 = [1 0 1 0]
'''
# All of these arithmetic operations are simply convenient wrappers around specific
# functions built into NumPy

print('\n', np.add(npArray, 2))  # [3 4 5 6]
print(np.subtract(npArray, 2))  # [-1  0  1  2]
print(np.negative(npArray))  # [-1 -2 -3 -4]
print(np.multiply(npArray, 3))  # [ 3  6  9 12]
print(np.divide(npArray, npArray))  # [1. 1. 1. 1.]
print(np.floor_divide(npArray, 2))  # [0 1 1 2]
print(np.power(npArray, 3))  # [ 1  8 27 64]
print(np.mod(npArray, npArray))  # [0 0 0 0]

# Absolute value
npArray = np.array([-2, -1, 0, 1, 2])
print('\n', npArray)
print(abs(npArray))  # abs is python built in function

# corresponding numpy ufunc is np.absolute()
# which is also available under the np.abs()

print(np.absolute(npArray))  # [2 1 0 1 2]
print(np.abs(npArray))  # [2 1 0 1 2]

# ufunc also handle complex data, in which the absolute value
# returns the magnitude:

npArray = np.array([3 - 4j, 4 - 3j, 2 + 0j, 0 + 1j])
print('\n', np.abs(npArray))  # [5. 5. 2. 1.]

# Trigonmetric functions
theta = np.linspace(0, np.pi, 3)

print('\n', theta)
print(np.sin(theta))  # [0.0000000e+00 1.0000000e+00 1.2246468e-16]
print(np.cos(theta))  # [ 1.000000e+00  6.123234e-17 -1.000000e+00]
print(np.tan(theta))  # [ 0.00000000e+00  1.63312394e+16 -1.22464680e-16]

# Inverse Trigonometric functions are also avaiable:
theta = np.array([-1, 0, 1])

print('\n', np.arcsin(theta))  # [-282.74333882    0.          282.74333882]
print(np.arccos(theta))  # [3.14159265 1.57079633 0.        ]
print(np.arctan(theta))  # [-0.78539816  0.          0.78539816]

#Exponents and logarithms
npArray = np.array([1, 2, 3])

print('\n', npArray)
print(np.exp(npArray))
print(np.exp2(npArray))
print(np.power(3, npArray))

# The inverse of the exponentials, the logarithms, are also available.
npArray = [1, 2, 4, 10]

print('\n', npArray)  # [1, 2, 4, 10]
print(np.log(npArray))  # [0.         0.69314718 1.38629436 2.30258509]
print(np.log2(npArray))  # [0.         1.         2.         3.32192809]
print(np.log10(npArray))  # [0.         0.30103    0.60205999 1.        ]

# There are also some specialized versions that are useful for maintaining precision
# with very small input:

npArray = [0, 0.001, 0.01, 0.1]
print("\nexp(npArray) - 1 =", np.expm1(npArray))
print("log(1 + npArray) =", np.log1p(npArray))
'''
exp(npArray) - 1 = [0.         0.0010005  0.01005017 0.10517092]
log(1 + npArray) = [0.         0.0009995  0.00995033 0.09531018]
'''

# Specialized ufuncs
# NumPy has many more ufuncs available, including hyperbolic trig functions, bitwise
# arithmetic, comparison operators, conversions from radians to degrees, rounding and
# remainders, and much more.
# Another excellent source for more specialized and obscure ufuncs is the submodule
# scipy.special . If you want to compute some obscure mathematical function on
# your data, chances are it is implemented in scipy.special . There are far too many
# functions to list them all, but the following snippet shows a couple that might come
# up in a statistics context:

# Gamma functions (generalized factorials) and related functions
npArray = np.array([1, 5, 10])

print("\ngamma(npArray)      =", special.gamma(npArray))
print("ln|gamma(npArray)| =", special.gammaln(npArray))
print("beta(npArray, 2)      =", special.beta(npArray, 2))
'''
amma(npArray)      = [1.0000e+00 2.4000e+01 3.6288e+05]
ln|gamma(npArray)| = [ 0.          3.17805383 12.80182748]
beta(npArray, 2)      = [0.5        0.03333333 0.00909091]
'''

# Error function (integral of Gaussian)
# its complement, and its inverse

npArray = np.array([0, 0.3, 0.7, 1.0])

print('\nerf(npArray) = ', special.erf(npArray))
print('erfc(npArray) = ', special.erfc(npArray))
print('erfinv(npArray) = ', special.erfinv(npArray))
'''
erf(npArray) =  [0.         0.32862676 0.67780119 0.84270079]
erfc(npArray) =  [1.         0.67137324 0.32219881 0.15729921]
erfinv(npArray) =  [0.         0.27246271 0.73286908        inf]
'''

# @@@@@@@@@@@@@@@@@@@@@@@@@@@@ Advanced Ufunc Features @@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# specifying output

# For large calculations, it is sometimes useful to be able to specify the array where the
# result of the calculation will be stored. Rather than creating a temporary array, you
# can use this to write computation results directly to the memory location where you’d
# like them to be. For all ufuncs, you can do this using the out argument of the
# function:

x = np.arange(5)
y = np.arange(5)
np.multiply(x, 10, out=y)
print('\n', y)

y = np.zeros(10)
np.power(2, x, out=y[::2])  # x[0,1,2,3,4]
print(y)  # [ 1.  0.  2.  0.  4.  0.  8.  0. 16.  0.]

#Aggregates
x = np.arange(1,6)
print('\n',np.add.reduce(x)) #15
print(np.multiply.reduce(x)) #120

#If we'd like to store all the intermediate results of the compution,
#we can instead use accumulate()
print(np.add.accumulate(x))  # [ 1  3  6 10 15]
print(np.multiply.accumulate(x))  # [  1   2   6  24 120]

#we also use dedicated numpy functions to compute the result
# use np.sum() instead of np.add.reduce()
# use np.prod() instead of np.multiply()
# use np.cumsum() instead of np.add.accumulate()
# use np.cumprod() instead of np.multiply.accumulate()

#Outer products
x = np.arange(1, 6)
print('\n',np.multiply.outer(x, x))
'''
[[ 1  2  3  4  5]
 [ 2  4  6  8 10]
 [ 3  6  9 12 15]
 [ 4  8 12 16 20]
 [ 5 10 15 20 25]]
'''

#for further readings https://docs.scipy.org/doc/numpy/reference/generated/


import numpy as np

class RunningMeanVar(object):
    r"""Estimates sample mean and variance by using `Chan's method`_.

    It supports for both scalar and multi-dimensional data, however, the input is
    expected to be batched. The first dimension is always treated as batch dimension.

    .. note::

        For better precision, we handle the data with `np.float64`.

    .. warning::

        To use estimated moments for standardization, remember to keep the precision `np.float64`
        and calculated as ..math:`\frac{x - \mu}{\sqrt{\sigma^2 + 10^{-8}}}`.

    Example:

        >>> f = RunningMeanVar(shape=())
        >>> f([1, 2])
        >>> f([3])
        >>> f([4])
        >>> f.mean
        2.499937501562461
        >>> f.var
        1.2501499923440393

    .. _Chan's method:
        https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Parallel_algorithm

    """
    def __init__(self, shape):
        self.shape = shape
        self.mean = np.zeros(shape, dtype=np.float64)
        self.var = np.ones(shape, dtype=np.float64)
        self.N = 1e-8  # numerical stability for variance term, and 1e-4 is for std

    def __call__(self, x):
        r"""Update the mean and variance given an additional batched data.

        Args:
            x (object): additional batched data.
        """
        x = np.asarray(x, dtype=np.float64)
        assert x.ndim == len(self.shape) + 1, f'expected {len(self.shape) + 1}, got {x.ndim}'

        batch_mean = x.mean(axis=0)
        batch_var = x.var(axis=0)
        batch_N = x.shape[0]

        new_N = self.N + batch_N
        delta = batch_mean - self.mean
        new_mean = self.mean + delta*(batch_N/new_N)
        M_A = self.N*self.var
        M_B = batch_N*batch_var
        M_X = M_A + M_B + (delta**2)*((self.N*batch_N)/new_N)
        new_var = M_X/new_N

        self.mean = new_mean
        self.var = new_var
        self.N = new_N

    @property
    def n(self):
        r"""Returns the total number of samples so far. """
        return int(self.N)


#
# Solve -u`` + k*u = f in (0, pi) with u(0) = u(pi) = 0   [1]
#

from __future__ import division
from sympy import Symbol
from lega.sine_basis import mass_matrix, stiffness_matrix, load_vector
from lega.sine_basis import sine_eval, sine_fft
import scipy.sparse.linalg as la
from math import pi as Pi
import numpy as np

def get_rhs(u, k):
    '''
    Verify that u satisfies boundary conditions and compute the right hand
    side f.
    '''
    x = Symbol('x')
    assert abs(u.subs(x, 0)) < 1E-15 and abs(u.subs(x, Pi)) < 1E-15
    # Right hand side if u is to be the solution
    f = -u.diff(x, 2) + k*u

    return f

def solve_helmholtz_1d(f, k, n):
    '''Solve the Helmoholtz problem by N Fourier sine polynomials.'''
    A = stiffness_matrix(n)
    M = mass_matrix(n)
    # The linear system of lhs of Helmoholtz is
    AA = (A + k*M)

    # Try to see how big of an error we make when computing rhs with fft
    # Integrated
    b = load_vector(f, n)
    # Try some frequency
    # F = sine_eval(8192, f)
    # bb = sine_fft(F)[:n]
    # print 2**n, '>>>', np.linalg.norm(b-bb)

    # The system is (A + k*M)*U = bb
    U = la.spsolve(AA, b)

    # Note that x is a vector of expansion coeffs of the solution w.r.t to
    # the sine basis
    return U

# -----------------------------------------------------------------------------

if __name__ == '__main__':
    from sympy import cos, pi, lambdify
    from lega.sine_basis import sine_function
    from sympy.plotting import plot
    from sympy.mpmath import quad
    from math import sqrt, log as ln
    import matplotlib.pyplot as plt

    # Setup
    x = Symbol('x')
    u = x*(x-pi)*cos(2*pi*x)
    k = 1
    f = get_rhs(u, k)

    n_max = 30

    n = 2
    tol = 1E-14
    converged = False

    ns = []
    errors = []
    while not converged:
        U = solve_helmholtz_1d(f, k, n)  # w.r.t to shen

        # Error using symobolic functions
        uh = sine_function(U)
        # Want L2 norm of the error
        e = u - uh
        error = sqrt(quad(lambdify(x, e**2), [0, Pi]))

        # Error by FFT
        Evec = sine_eval(f=e, N=2**16)
        e_k = sine_fft(Evec)
        # Use parseval
        error_ = sqrt(np.sum(e_k**2))

        if n != 2:
            ns.append(n)
            errors.append(error)
            rate = ln(error/error0)/ln(n0/n)
            print 'n=%d, |e|_2=%.4E(%.2f)  {e}_2=%.4E' % (n, error, rate, error_)

        converged = error < tol or n >= n_max
        error0, n0 = error, n
        n += 1

    plt.figure()
    plt.loglog(ns, errors)

    # Plot the final numerical one againt analytical
    p0 = plot(u, (x, 0, Pi), show=False)
    p1 = plot(uh, (x, 0, Pi), show=False)
    p1[0].line_color='red'
    p0.append(p1[0])
    p0.show()


import numpy as np
from numpy import exp, sqrt, cos, pi, sin

from base_function import BaseFunction

class Beale(BaseFunction):
    target_E = 0.
    target_coords = np.array([3., 0.5])
    xmin = np.array([-4.5,-4.5])
    xmax = np.array([4.5,4.5])
    def getEnergy(self, coords):
        x, y = coords
        return (1.5 - x + x*y)**2 + (2.25 - x + x * y**2)**2 + (2.625 - x + x * y**3)**2

    def getEnergyGradient(self, coords):
        E = self.getEnergy(coords)
        x, y = coords
        dx = 2. * (1.5 - x + x*y) * (-1. + y) + 2. * (2.25 - x + x * y**2) * (-1. + y**2) + 2. * (2.625 - x + x * y**3) * (-1. + y**3)
        dy = 2. * (1.5 - x + x*y) * (x) +       2. * (2.25 - x + x * y**2) * (2. * y * x) + 2. * (2.625 - x + x * y**3) * (3. * x * y**2)
        return E, np.array([dx, dy])

if __name__ == "__main__":
    f = Beale()
    f.test_potential(f.target_coords)
    print ""
    f.test_potential(f.get_random_configuration())
    f.test_potential(np.array([1.,1.]), print_grads=True)

    from base_function import makeplot2d
    v = 3.
    makeplot2d(f, nx=30)


import random
import numpy as np
import warnings

def sample_size_calculation(p: float, d: float = .05, z: float = 1.96, finite_population: bool = True,
                            population_size: int = None) -> int:
    """
        Sources:
            Daniel WW (1999). Biostatistics: A Foundation for Analysis in the Health Sciences. 7th edition.
                New York: John Wiley & Sons.
            Naing, L., T. Winn, and B. N. Rusli. 'Practical issues in calculating the sample size for prevalence studies
                Archives of orofacial Sciences 1 (2006): 9-14.

        Formula Without Finite Population
            n = Z^2P(1-P) / d^2
            where
                n: Sample Size
                Z^2: Z Statistics for a level of confidence
                P: Expected Prevalence
                d: Precision

        Formula With Finite Population
            n' = NZ^2P(1-P) / d^2(N-1) + Z^2P(1-P)
            where
                n': Sample size corrected for finite population
                N: Population size
                Z: Z Statistics for a level of confidence
                P: Expected Prevalence
                d: Precision

    !!: The finite population calculation should be used if n/N > .05. (Daniel, 1999).

    Setting d:
        Therefore, we recommend d as a half of P if P is below 0.1(10%)
        and if P is above 0.9 (90%), d can be {0.5(1-P)}. For example, if P is 0.04, investigators may use d=0.02,
        and if P is 0.98, we recommend d=0.01. (Naing, 2006)

    """

    # If the calculation is set to finite_population, the population size can't be None.
    if finite_population and not population_size:
        raise Exception('For finite population, set a population size.')

    # Without finite population
    if not finite_population:
        return int(np.round(((z ** 2) * p * (1 - p)) / (d ** 2), 0))

    # With finite population
    sample_size = np.round(
        (population_size * (z ** 2) * p * (1 - p)) / (((d ** 2) * (population_size - 1)) + ((z ** 2) * p * (1 - p))), 0)
    if sample_size / population_size < .05:
        warnings.warn('Computed sample size is less than 0.05 of population. Use infinite population calculation '
                      '(Daniel, 1999).', UserWarning)

    return int(sample_size)

def compute_confidence_intervals(x: np.array, z: float = 1.96) -> float:
    """
    Function to compute the confidence interval of the mean of a sample.

    Hazra, Avijit. "Using the confidence interval confidently." Journal of thoracic disease 9.10 (2017): 4125.
    Formula:
        CI = x̅ ± z × (std/√n)
        where
            CI: Confidence Interval
            x̅: Sample Mean
            z: Z Statistic for desired confidence interval
            std: Sample Standard Deviation
            n: Sample Size
    """

    return z * (x.std()/len(x)**.5)

if __name__ == '__main__':
    weighted_list = [1] * 2 + [0] * 98
    population = np.array([random.choice(weighted_list) for _ in range(1000)])
    computed_sample_size = sample_size_calculation(p=0.02, d=0.02, z=1.96, finite_population=True,
                                                   population_size=len(population))

    print(f'Population Size: {len(population)}')
    print(f'Computed Sample Size: {computed_sample_size}')

    # Selecting the sample
    sample = np.array(random.choices(population, k=computed_sample_size))
    confidence_interval = compute_confidence_intervals(sample)
    print('*'*50)
    print(f'True average: {population.mean()*100}')
    print(f'Sample Average: {sample.mean()*100} ± {confidence_interval*100}.')


import numpy
from scipy.linalg import cho_factor, LinAlgError, cho_solve

from assignment_2_3_4.config import Config
from assignment_2_3_4.optimizers import AbstractOptimizer, OptimizationStep
from assignment_2_3_4.oracles import AbstractOracle

class NewtonCholeskyOptimizer(AbstractOptimizer):
    name: str = "newton-cholesky"
    _default_tau = 1e-10

    def __init__(self, config: Config):
        super().__init__(config)
        self._tau = self._default_tau

    def reset_state(self):
        self._tau = self._default_tau

    def _get_direction(self, oracle: AbstractOracle, last_point: OptimizationStep) -> numpy.ndarray:
        is_tau_changed = False
        while True:
            try:
                _cho_factor = cho_factor(last_point.hessian)
                break
            except LinAlgError:
                diag_range = numpy.arange(last_point.hessian.shape[0])
                last_point.hessian[diag_range, diag_range] += self._tau
                self._tau *= 2
                is_tau_changed = True
        if is_tau_changed:
            self._tau /= 2
        return cho_solve(_cho_factor, -last_point.grad)

    def _aggregate_optimization_step(
        self,
        oracle: AbstractOracle,
        new_point: numpy.ndarray,
        passed_time: float,
    ) -> OptimizationStep:
        new_value, new_grad, new_hessian = oracle.fuse_value_grad_hessian(new_point)
        return OptimizationStep(new_point, new_value, new_grad, passed_time, oracle.n_calls, hessian=new_hessian)


import numpy as np
from numpy import linalg
from abc import abstractmethod
import pandas as pd
import math

pd.options.display.float_format = '{:,.8f}'.format
np.set_printoptions(suppress=True, precision=8)

TOR = pow(10.0, -5)
MAX_ITR = 30

class FixedPointMethod(object):

    def __init__(self):
        return

    @abstractmethod
    def f(self, x):
        return NotImplementedError('Implement f()!')

    @abstractmethod
    def run(self, x):
        return NotImplementedError('Implement run()!')

class FixedPoint(FixedPointMethod):

    def __init__(self):
        super(FixedPointMethod, self).__init__()

    def f(self, x):
        sol = np.zeros(len(x))
        sol[0] = math.cos(x[1] * x[2]) / 3.0 + 1.0 / 6.0
        sol[1] = math.sqrt(x[0] * x[0] + math.sin(x[2]) + 1.06) / 9.0 - 0.1
        sol[2] = -math.exp(-x[0] * x[1]) / 20.0 - (10.0 * math.pi - 3.0) / 60.0
        return sol

    def run(self, x):
        """
        given x_0 in R^3 as a starting point.

        :param x: x_0 as described
        :return: the minimizer x* of f
        """
        df = pd.DataFrame(columns=['x' + str(i + 1) for i in range(len(x))] + ['residual', 'actual-residual'])

        row = len(df)
        df.loc[row] = [xe for xe in x] + [np.nan, np.nan]

        while True:
            y = self.f(x)
            residual = linalg.norm(x - y, np.inf)
            x = y

            row = len(df)
            df.loc[row] = [ye for ye in y] + [residual, np.nan]
            if residual < TOR:
                break

        for i in range(len(df)):
            xk = np.array([df.loc[i][j] for j in range(len(x))])
            df.loc[i][4] = linalg.norm(x - xk, np.inf)

        print(df)

class FixedPointAcceleration(FixedPointMethod):

    def __init__(self):
        super(FixedPointMethod, self).__init__()

    def f(self, x):
        sol = np.zeros(len(x))
        sol[0] = math.cos(x[1] * x[2]) / 3.0 + 1.0 / 6.0
        sol[1] = math.sqrt(sol[0] * sol[0] + math.sin(x[2]) + 1.06) / 9.0 - 0.1
        sol[2] = -math.exp(-sol[0] * sol[1]) / 20.0 - (10.0 * math.pi - 3.0) / 60.0
        return sol

    def run(self, x):
        """
        given x_0 in R^3 as a starting point.

        :param x: x_0 as described
        :return: the minimizer x* of f
        """
        df = pd.DataFrame(columns=['x' + str(i + 1) for i in range(len(x))] + ['residual', 'actual-residual'])

        row = len(df)
        df.loc[row] = [xe for xe in x] + [np.nan, np.nan]
        while True:
            y = self.f(x)
            residual = linalg.norm(x - y, np.inf)
            x = y

            row = len(df)
            df.loc[row] = [ye for ye in y] + [residual, np.nan]
            if residual < TOR:
                break

        for i in range(len(df)):
            xk = np.array([df.loc[i][j] for j in range(len(x))])
            df.loc[i][4] = linalg.norm(x - xk, np.inf)

        print(df)

class FixedPoint1D(FixedPointMethod):

    def __init__(self):
        super(FixedPointMethod, self).__init__()

    def f(self, x):
        return math.cos(x)

    def g1(self, x):
        return np.array(x - pow(x, 3) - 4 * pow(x, 2) + 10)

    def g2(self, x):
        return np.array(math.sqrt(10 / x - 4 * x))

    def g3(self, x):
        return np.array(math.sqrt(10 - pow(x, 3)) / 2)

    def g4(self, x):
        return np.array(math.sqrt(10 / (4 + x)))

    def g5(self, x):
        return np.array(x - (pow(x, 3) + 4 * pow(x, 2) - 10) / (3 * pow(x, 2) + 8 * x))

    def run2(self, x0):
        df = pd.DataFrame(columns=['(sp) f(x)'])
        row = len(df)
        x = x0
        df.loc[row] = [x]
        for k in range(MAX_ITR):
            try:
                y = self.f(x)
            except ValueError:
                break
            residual = math.fabs(x - y)
            x = y

            row = len(df)
            df.loc[row] = [y]
            if residual < TOR or x > 1e9:
                break
        return df

    def run(self, x0):
        """
        given x_0 in R^3 as a starting point.

        :param x: x_0 as described
        :return: the minimizer x* of f
        """
        g = [self.g1, self.g2, self.g3, self.g4, self.g5]

        total_df = None
        for j in range(len(g)):
            df = pd.DataFrame(columns=['g' + str(j + 1) + '(x)'])
            row = len(df)
            x = x0
            df.loc[row] = [x]
            for k in range(MAX_ITR):
                try:
                    y = np.array(g[j](x))
                except ValueError:
                    break
                residual = math.fabs(x - y)
                x = y

                row = len(df)
                df.loc[row] = [y]
                if residual < TOR or x > 1e9:
                    break
            total_df = df if total_df is None else pd.concat([total_df, df], axis=1)

def main():
    x0 = np.array([0.1, 0.1, -0.1])
    x10 = 1.5
    # FixedPoint().run(x0)
    # FixedPointAcceleration().run(x0)
    # FixedPoint1D().run(x10)
    FixedPoint1D().run2(math.pi / 4)

    return

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-

from numpy import linspace, logspace, diff, zeros
from numpy import cos, sin, exp, log, pi
import matplotlib.pyplot as plt
from scipy.integrate import solve_ivp

####################################
# Implementación genérica de Taylor 2
# f(t,x): primera derivada de x respecto al tiempo
# g(t,x): segunda derivada de x respecto al tiempo
# x0: condición inicial
# t0, tf: tiempo inicial y final
# h: paso de integración
####################################
def taylor2(f, g, x0, t0, tf, h):
    N = int((tf - t0) / h)  # número de puntos
    t = linspace(t0, tf, N + 1)
    n = x0.shape[0]  # dimensión del problema
    x = zeros((n, N + 1))
    x[:, 0] = x0
    i = h * h / 2.0
    for k in range(N):
        x[:, k + 1] = x[:, k] + h * f(t[k], x[:, k]) + i * g(t[k], x[:, k])

    return t, x

####################################
# Implementación genérica de Heun
# f(t,x): derivada de x respecto al tiempo
# x0: condición inicial
# t0, tf: tiempo inicial y final
# h: paso de integración
####################################
def heun(f, x0, t0, tf, h):
    N = int((tf - t0) / h)  # número de puntos
    t = linspace(t0, tf, N + 1)
    n = x0.shape[0]  # dimensión del problema
    x = zeros((n, N + 1))
    x[:, 0] = x0
    for k in range(N):
        f1 = h * f(t[k], x[:, k])
        f2 = h * f(t[k] + h, x[:, k] + f1)
        x[:, k + 1] = x[:, k] + (f1 + f2) / 2.0

    return t, x

####################################
# Implementación genérica de Cauchy
# f(t,x): derivada de x respecto al tiempo
# x0: condición inicial
# t0, tf: tiempo inicial y final
# h: paso de integración
####################################
def cauchy(f, x0, t0, tf, h):
    N = int((tf - t0) / h)  # número de puntos
    t = linspace(t0, tf, N + 1)
    n = x0.shape[0]  # dimensión del problema
    x = zeros((n, N + 1))
    x[:, 0] = x0
    h2 = h / 2.0
    for k in range(N):
        x[:, k + 1] = x[:, k] + h * f(t[k] + h2, x[:, k] + h2 * f(t[k], x[:, k]))

    return t, x

########################
# EJEMPLO
########################
R = 1e3  # Valor de la resistencia
C = 1e-6  # Valor de la capacidad
w = 2.0 * pi * 1000  # frecuencia angular de la señal de entrada
A = 1.0  # amplitud de la señal de entrada
T = 5 * 2 * pi / w  # simulo cinco ciclos

####################################
# Solución
def xsol(t):
    x = -exp(-t / (R * C)) + cos(w * t) + w * R * C * sin(w * t)
    x = (A / (1 + (w * R * C) ** 2)) * x
    return x

####################################
# Derivada primera de x
def dx(t, x):
    return ((A * cos(w * t) - x) / (R * C))

####################################
# Derivada segunda de x
def d2x(t, x):
    return ((-A * w * sin(w * t) - ((A * cos(w * t) - x) / (R * C))) / (R * C))

####################################
# Plot ejemplo
def plotejemplo(h):
    x0 = zeros(1)
    t, xh = heun(dx, x0, 0, T, h)
    t, xc = cauchy(dx, x0, 0, T, h)
    t, xt = taylor2(dx, d2x, x0, 0, T, h)
    x = xsol(t)
    fig, ax = plt.subplots()
    ax.plot(t, x, label='Solución')
    ax.plot(t, xh[0, :], label='Heun')
    ax.plot(t, xc[0, :], label='Cauchy')
    ax.plot(t, xt[0, :], label='Taylor 2')
    ax.legend()
    plt.title('Ejemplo')
    fig, ax = plt.subplots()
    ax.plot(t, xh[0, :] - x, label='Heun')
    ax.plot(t, xc[0, :] - x, label='Cauchy')
    ax.plot(t, xt[0, :] - x, label='Taylor 2')
    ax.legend()
    plt.title('Error')

####################################
# Errores ejemplo
def errorejemplo():
    n = 5
    N = logspace(1, 5, n)
    h = T / N
    eh = zeros(n)
    ec = zeros(n)
    et = zeros(n)
    x0 = zeros(1)
    x = xsol(T)
    for k in range(n):
        t, xh = heun(dx, x0, 0, T, h[k])
        t, xc = cauchy(dx, x0, 0, T, h[k])
        t, xt = taylor2(dx, d2x, x0, 0, T, h[k])
        eh[k] = abs(xh[0, -1] - x)
        ec[k] = abs(xc[0, -1] - x)
        et[k] = abs(xt[0, -1] - x)
    fig, ax = plt.subplots()
    ax.loglog(h, eh, label='Heun')
    ax.loglog(h, ec, label='Cauchy')
    ax.loglog(h, et, label='Taylor 2')
    # ax.loglog(h, (h/h[0])**2*eh[0], 'b--')
    # ax.loglog(h, (h/h[0])**2*ec[0], 'r--')
    # ax.loglog(h, (h/h[0])**2*et[0], 'g--')
    ax.legend()
    plt.xlabel('h')
    plt.ylabel('error')
    plt.title('Error')
    print(diff(log(eh)) / diff(log(h)))
    print(diff(log(ec)) / diff(log(h)))
    print(diff(log(et)) / diff(log(h)))

####################################
# Estimación error ejemplo
def esterrorejemplo(h):
    x0 = zeros(1)
    x = xsol(T)
    t, xh1 = heun(dx, x0, 0, T, h)
    t, xc1 = cauchy(dx, x0, 0, T, h)
    t, xt1 = taylor2(dx, d2x, x0, 0, T, h)
    t, xh2 = heun(dx, x0, 0, T, h / 2)
    t, xc2 = cauchy(dx, x0, 0, T, h / 2)
    t, xt2 = taylor2(dx, d2x, x0, 0, T, h / 2)

    eh = abs(xh2[0, -1] - x)
    ec = abs(xc2[0, -1] - x)
    et = abs(xt2[0, -1] - x)
    eeh = abs(xh1[0, -1] - xh2[0, -1]) / 3.0
    eec = abs(xc1[0, -1] - xc2[0, -1]) / 3.0
    eet = abs(xt1[0, -1] - xt2[0, -1]) / 3.0
    print(abs(eh - eeh) / eh * 100)
    print(abs(ec - eec) / ec * 100)
    print(abs(et - eet) / et * 100)
    return eh, eeh, ec, eec, et, eet

####################################
# Comparación con RK45
def comp45(h):
    x0 = zeros(1)
    t, xh = heun(dx, x0, 0, T, h)
    # s45 = solve_ivp(dx,[0,T],x0,method='RK45', t_eval=None,
    #                 rtol = 1e-6, atol = 1e-8)
    # s45 = solve_ivp(dx,[0,T],x0,method='RK45', t_eval=None,
    #                 rtol = 1e-13, atol = 1e-14)
    s45 = solve_ivp(dx, [0, T], x0, method='RK45', t_eval=t,
                    rtol=1e-13, atol=1e-14)
    x = xsol(t)
    x1 = xsol(s45.t)

    fig, ax = plt.subplots()
    ax.plot(t, x, label='Solución')
    ax.plot(t, xh[0, :], label='Heun')
    ax.plot(s45.t, s45.y[0, :], label='RK45')
    ax.legend()
    plt.title('Ejemplo')
    fig, ax = plt.subplots()
    ax.plot(t[:-1], diff(t), label='Heun')
    ax.plot(s45.t[:-1], diff(s45.t), label='RK45')
    ax.legend()
    plt.title('Paso de integración')

    fig, ax = plt.subplots()
    ax.semilogy(t, abs(xh[0, :] - x), label='Heun')
    ax.semilogy(s45.t, abs(s45.y[0, :] - x1), label='RK45')
    ax.legend()
    plt.title('Error')

    print(s45.t.shape)
    print(t.shape)

####################################
# Plot ejemplo
def plotejemplo_ruku4(h, fruku4):
    x0 = zeros(1)
    t, xrk4 = fruku4(dx, x0, 0, T, h)

    t, xt = taylor2(dx, d2x, x0, 0, T, h)

    print("Calculating solution...")
    x = xsol(t)

    print("Plotting solution and RuKu 4.")
    fig, ax = plt.subplots()
    ax.plot(t, x, label='Solución')
    print(f"t: {t}\nxrk4: {xrk4}\nxrk4[0,:]: {xrk4[0:1]}")
    ax.plot(t, xrk4[:], label='RuKu 4')
    ax.legend()
    plt.title('Ejemplo')

    print("Plotting error.")
    fig, ax = plt.subplots()
    ax.plot(t, xrk4[:] - x, label='RuKu 4')
    # plt.axis("off")
    # ax.legend()
    # plt.title('Error')

    print("Showing plots.")
    plt.show()

# plotejemplo(T/10000)
# plt.close('all')
# errorejemplo()
# esterrorejemplo(T/1e4)
# comp45(T/1000)


from sympy import symbols, hessian, Matrix
from sympy.functions import exp

def gradient(f, v): return Matrix([f]).jacobian(v)

x, b1, b2, b3, b4, b5, b6, b7 = symbols('x,b1,b2,b3,b4,b5,b6,b7')

print('NIST Average Difficulty')
print('\nBennett5')
print('y = b1 * (b2+x)**(-1/b3)  +  e')

r = b1 * (b2+x)**(-1/b3)
v = [b1, b2, b3]

print('Jacobian:')
print(gradient(r, v))

print('Hessian:')
print(hessian(r, v))

print('\nBoxBOD')
print('b1*(1-exp(-b2*x))  +  e')

r = b1*(1-exp(-b2*x))
v = [b1, b2]

print('Jacobian:')
print(gradient(r, v))

print('Hessian:')
print(hessian(r, v))

print('\nEckerle4')
print('y = (b1/b2) * exp(-0.5*((x-b3)/b2)**2)  +  e')

r = (b1/b2) * exp(-0.5*((x-b3)/b2)**2)
v = [b1, b2, b3]

print('Jacobian:')
print(gradient(r, v))

print('Hessian:')
print(hessian(r, v))

print('\nMGH09')
print('y = b1*(x**2+x*b2) / (x**2+x*b3+b4)  +  e')

r = b1*(x**2+x*b2) / (x**2+x*b3+b4)
v = [b1, b2, b3, b4]

print('Jacobian:')
print(gradient(r, v))

print('Hessian:')
print(hessian(r, v))

print('\nMGH10')
print('y = b1 * exp(b2/(x+b3))  +  e')

r = b1 * exp(b2/(x+b3))
v = [b1, b2, b3]

print('Jacobian:')
print(gradient(r, v))

print('Hessian:')
print(hessian(r, v))

print('\nRat42')
print('y = b1 / (1+exp(b2-b3*x))  +  e')

r = b1 / (1+exp(b2-b3*x))
v = [b1, b2, b3]

print('Jacobian:')
print(gradient(r, v))

print('Hessian:')
print(hessian(r, v))

print('\nRat43')
print('y = b1 / ((1+exp(b2-b3*x))**(1/b4))  +  e')

r = b1 / ((1+exp(b2-b3*x))**(1/b4))
v = [b1, b2, b3, b4]

print('Jacobian:')
print(gradient(r, v))

print('Hessian:')
print(hessian(r, v))

print('\nThurber')
print('y = (b1 + b2*x + b3*x**2 + b4*x**3) / '
      '(1 + b5*x + b6*x**2 + b7*x**3)  +  e')

r = (b1 + b2*x + b3*x**2 + b4*x**3) / (1 + b5*x + b6*x**2 + b7*x**3)
v = [b1, b2, b3, b4, b5, b6, b7]

print('Jacobian:')
print(gradient(r, v))

print('Hessian:')
print(hessian(r, v))


#---------------------------
# organize imports
#---------------------------
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import random
import os
import warnings
from sklearn.datasets import load_boston
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import ElasticNet
from sklearn.tree import DecisionTreeRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.ensemble import AdaBoostRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.metrics import mean_squared_error

#---------------------------
# library specific options
#---------------------------
pd.options.display.float_format = '{:,.2f}'.format
sns.set(color_codes=True)
def warn(*args, **kwargs):
    pass
warnings.warn = warn
warnings.filterwarnings("ignore", category=FutureWarning)

#---------------------------
# analyze the dataset
#---------------------------
def analyze_dataset(dataset):
	print("[INFO] keys : {}".format(dataset.keys()))

	print("[INFO] features shape : {}".format(dataset.data.shape))
	print("[INFO] target shape   : {}".format(dataset.target.shape))

	print("[INFO] feature names")
	print(dataset.feature_names)

	print("[INFO] dataset summary")
	print(dataset.DESCR)

	df = pd.DataFrame(dataset.data)
	print("[INFO] df type : {}".format(type(df)))
	print("[INFO] df shape: {}".format(df.shape))
	print(df.head())

	df.columns = dataset.feature_names
	print(df.head())

	df["PRICE"] = dataset.target
	print(df.head())

	print("[INFO] dataset datatypes")
	print(df.dtypes)

	print("[INFO] dataset statistical summary")
	print(df.describe())

	# correlation between attributes
	print("PEARSON CORRELATION")
	print(df.corr(method="pearson"))
	sns.heatmap(df.corr(method="pearson"))
	plt.savefig("heatmap_pearson.png")
	plt.clf()
	plt.close()

	print("SPEARMAN CORRELATION")
	print(df.corr(method="spearman"))
	sns.heatmap(df.corr(method="spearman"))
	plt.savefig("heatmap_spearman.png")
	plt.clf()
	plt.close()

	print("KENDALL CORRELATION")
	print(df.corr(method="kendall"))
	sns.heatmap(df.corr(method="kendall"))
	plt.savefig("heatmap_kendall.png")
	plt.clf()
	plt.close()

	# show missing values
	print(pd.isnull(df).any())

	file_report = "boston_housing.txt"
	with open(file_report, "w") as f:
		f.write("Features shape : {}".format(df.drop("PRICE", axis=1).shape))
		f.write("\n")

		f.write("Target shape   : {}".format(df["PRICE"].shape))
		f.write("\n")

		f.write("\nColumn names")
		f.write("\n")
		f.write(str(df.columns))
		f.write("\n")

		f.write("\nStatistical summary")
		f.write("\n")
		f.write(str(df.describe()))
		f.write("\n")

		f.write("\nDatatypes")
		f.write("\n")
		f.write(str(df.dtypes))
		f.write("\n")

		f.write("\nPEARSON correlation")
		f.write("\n")
		f.write(str(df.corr(method="pearson")))
		f.write("\n")

		f.write("\nSPEARMAN correlation")
		f.write("\n")
		f.write(str(df.corr(method="spearman")))
		f.write("\n")

		f.write("\nKENDALL correlation")
		f.write("\n")
		f.write(str(df.corr(method="kendall")))

		f.write("\nMissing Values")
		f.write("\n")
		f.write(str(pd.isnull(df).any()))

	return df

#---------------------------
# visualize the dataset
#---------------------------
def visualize_dataset(df):
	colors = ["y", "b", "g", "r"]

	cols = list(df.columns.values)

	if not os.path.exists("plots/univariate/box"):
	    os.makedirs("plots/univariate/box")

	if not os.path.exists("plots/univariate/density"):
	    os.makedirs("plots/univariate/density")

	# draw a boxplot with vertical orientation
	for i, col in enumerate(cols):
		sns.boxplot(df[col], color=random.choice(colors), orient="v")
		plt.savefig("plots/univariate/box/box_" + str(i) + ".png")
		plt.clf()
		plt.close()

	# draw a histogram and fit a kernel density estimate (KDE)
	for i, col in enumerate(cols):
		sns.distplot(df[col], color=random.choice(colors))
		plt.savefig("plots/univariate/density/density_" + str(i) + ".png")
		plt.clf()
		plt.close()

	if not os.path.exists("plots/multivariate"):
	    os.makedirs("plots/multivariate")

	# bivariate plot between target and feature
	for i, col in enumerate(cols):
		if (i == len(cols) - 1):
			pass
		else:
			sns.jointplot(x=col, y="PRICE", data=df);
			plt.savefig("plots/multivariate/target_vs_" + str(i) + ".png")
			plt.clf()
			plt.close()

	# pairplot
	sns.pairplot(df)
	plt.savefig("plots/pairplot.png")
	plt.clf()
	plt.close()

#---------------------------
# train the model
#---------------------------
def train_model(df, dataset):
	X = df.drop("PRICE", axis=1)
	Y = df["PRICE"]
	print(X.shape)
	print(Y.shape)

	scaler = MinMaxScaler().fit(X)
	scaled_X = scaler.transform(X)

	seed      = 9
	test_size = 0.20

	X_train, X_test, Y_train, Y_test = train_test_split(scaled_X, Y, test_size = test_size, random_state = seed)

	print(X_train.shape)
	print(X_test.shape)
	print(Y_train.shape)
	print(Y_test.shape)

	# user variables to tune
	folds   = 10
	metric  = "neg_mean_squared_error"

	# hold different regression models in a single dictionary
	models = {}
	models["Linear"]        = LinearRegression()
	models["Lasso"]         = Lasso()
	models["ElasticNet"]    = ElasticNet()
	models["KNN"]           = KNeighborsRegressor()
	models["DecisionTree"]  = DecisionTreeRegressor()
	models["SVR"]           = SVR()
	models["AdaBoost"]      = AdaBoostRegressor()
	models["GradientBoost"] = GradientBoostingRegressor()
	models["RandomForest"]  = RandomForestRegressor()
	models["ExtraTrees"]    = ExtraTreesRegressor()

	# 10-fold cross validation for each model
	model_results = []
	model_names   = []
	for model_name in models:
		model   = models[model_name]
		k_fold  = KFold(n_splits=folds, random_state=seed)
		results = cross_val_score(model, X_train, Y_train, cv=k_fold, scoring=metric)

		model_results.append(results)
		model_names.append(model_name)
		print("{}: {}, {}".format(model_name, round(results.mean(), 3), round(results.std(), 3)))

	# box-whisker plot to compare regression models
	figure = plt.figure()
	figure.suptitle('Regression models comparison')
	axis = figure.add_subplot(111)
	plt.boxplot(model_results)
	axis.set_xticklabels(model_names, rotation = 45, ha="right")
	axis.set_ylabel("Mean Squared Error (MSE)")
	plt.margins(0.05, 0.1)
	plt.savefig("model_mse_scores.png")
	plt.clf()
	plt.close()

	# create and fit the best regression model
	best_model = GradientBoostingRegressor(random_state=seed)
	best_model.fit(X_train, Y_train)

	# make predictions using the model
	predictions = best_model.predict(X_test)
	print("[INFO] MSE : {}".format(round(mean_squared_error(Y_test, predictions), 3)))

	# plot between predictions and Y_test
	x_axis = np.array(range(0, predictions.shape[0]))
	plt.plot(x_axis, predictions, linestyle="--", marker="o", alpha=0.7, color='r', label="predictions")
	plt.plot(x_axis, Y_test, linestyle="--", marker="o", alpha=0.7, color='g', label="Y_test")
	plt.xlabel('Row number')
	plt.ylabel('PRICE')
	plt.title('Predictions vs Y_test')
	plt.legend(loc='lower right')
	plt.savefig("predictions_vs_ytest.png")
	plt.clf()
	plt.close()

	# plot model's feature importance
	feature_importance = best_model.feature_importances_
	feature_importance = 100.0 * (feature_importance / feature_importance.max())
	sorted_idx = np.argsort(feature_importance)
	pos = np.arange(sorted_idx.shape[0]) + .5
	plt.barh(pos, feature_importance[sorted_idx], align='center')
	plt.yticks(pos, dataset.feature_names[sorted_idx])
	plt.xlabel('Relative Importance')
	plt.title('Variable Importance')
	plt.savefig("feature_importance.png")
	plt.clf()
	plt.close()

#--------------------------
# MAIN FUNCTION
#--------------------------
if __name__ == '__main__':
	dataset = load_boston()

	df = analyze_dataset(dataset)
	visualize_dataset(df)
	train_model(df, dataset)


# # Polynomial Regression - python, sklearn
# We have a couple options we could use to solve [Polynomial Regression](https://en.wikipedia.org/wiki/Polynomial_regression) using [sklearn](http://scikit-learn.org). Below is one appraoch.
import numpy as np
import pandas as pd
from sklearn import linear_model
from sklearn.preprocessing import PolynomialFeatures

# ## Generate Data
# We'll generate sample sample data with known coefficients and randomly generated error. Specifically;
# $$ y = \beta_0 + \sum_i (\beta_i x_i) + \epsilon   \thinspace \thinspace \thinspace \thinspace \thinspace     \forall i \in {1..3}$$
# $$ \beta_0: 4 $$
# $$ \beta_1: 6 $$
# $$ \beta_2: 2 $$
# $$ \beta_3: -1 $$

x1 = np.array([[i,] for i in np.random.uniform(low=-2.0, high=4.0, size=10000)])
poly = PolynomialFeatures(degree=3)
X = poly.fit_transform(x1)
X = np.array([i[1:] for i in X])
calc_y  = lambda x: 4 + 6*x[0] + 2*x[1] - 1*x[2] + np.random.normal(0,4)
Y = np.apply_along_axis(calc_y, 1, X)

# ## Initialize & Run LinearRegression
# Notice we dropped our bias column above so we are going to include an intercept fit in our model below.

regr = linear_model.LinearRegression(fit_intercept=True)
regr.fit(X,Y)

dat = pd.DataFrame(data=np.dstack((np.append(np.array([regr.intercept_, ]), regr.coef_), [4,6,2,-1]))[0])
dat.columns = ['sklearn_coef', 'true_coef']
dat

# Plot Original Data Scatter Plot & Predict Function
x_range = np.array([[i,] for i in linspace(-2.0, 4.0, 100)])
x_values = np.array([i[1:] for i in poly.fit_transform(x_range)])
plt.scatter([i[0] for i in X], Y,  color='black')
plt.plot([i[0] for i in x_values], regr.predict(x_values), color='blue',linewidth=3)


import numpy as np

def alg(x1, y1, x2, y2, x3, y3):
    print("Method: 2nd degree 1")
    print("f(x)=ax²+bx+c")
    print("calculating...")

    # this is a linear matrix equation, or system of linear scalar equations.
    # we will fill this with a 2-dimensional array, with what we use to solve this.
    matrix = np.array(
        [
            [x1 ** 2, x1, 1],
            [x2 ** 2, x2, 1],
            [x3 ** 2, x3, 1]
        ])
    y = np.array([y1, y2, y3])

    # this numpy function solves this systems and we get an array
    solved = np.linalg.solve(matrix, y)

    # now we will set the values to the associated letter in the function
    a = solved[0]
    b = solved[1]
    c = solved[2]

    if np.allclose(np.dot(matrix, solved), y):
        # printing the solution
        print("done")
        print("f(x)=" + str(round(a, 3)) + "x²+" + str(round(b, 3)) + "x+" + str(round(c, 3)))
    else:
        print("Something went wrong.")
        print("This solution might be a bit inaccurate.")
        print("This is what we found:")
        print("f(x)=" + str(round(a, 3)) + "x²+" + str(round(b, 3)) + "x+" + str(round(c, 3)))


from functools import partial
from operator import ge, le
from typing import Callable, Dict, List, Tuple

import numpy as np
import zarr
from fixpoint import *
from numpy import linalg as LA
from utils import *

def cr(A, b, rtol=1.0e-8, etol=1.0e-8, max_it=25, x_0=None):
    if x_0 is None:
        x = np.zeros_like(b)
    else:
        x = x_0

    it = 0
    # Compute pseudoenergy
    E_old = 0.0
    E_new = quadratic_form(A, b, x)
    DeltaE = E_new - E_old
    # Compute residual and its norm
    r = b - np.einsum('ij,j->i', A, x)
    p = r
    rnorm = compute_residual_norm(A, b, x)
    print('Iteration #    Residual norm')
    # Report at start
    print('    {:4d}        {:.5E}       {:.5E}'.format(
        it, rnorm, abs(DeltaE)))
    while it < max_it:
        # Update solution vector
        Ap = np.einsum('ij,j->i', A, p)
        rtAr = np.einsum('i,ij,j', r, A, r)
        alpha = rtAr / np.einsum('i,i', Ap, Ap)
        x = x + alpha * p
        # Compute residual and its norm
        r = r - alpha * Ap
        rnorm = LA.norm(r)
        # Compute new pseudoenergy
        E_new = quadratic_form(A, b, x)
        DeltaE = E_new - E_old
        E_old = E_new

        # Update search direction
        beta = np.einsum('i,ij,j', r, A, r) / rtAr
        p = r + beta * p
        it += 1

        # Check convergence
        if rnorm < rtol and abs(DeltaE) < etol:
            # Print last statistics before breaking out
            print('    {:4d}        {:.5E}       {:.5E}'.format(
                it, rnorm, abs(DeltaE)))
            break

        # Report
        print('    {:4d}        {:.5E}       {:.5E}'.format(
            it, rnorm, abs(DeltaE)))
    else:
        raise RuntimeError(
            'Maximum number of iterations ({0:d}) exceeded, but residual norm {1:.5E} still greater than threshold {2:.5E}'.
            format(max_it, rnorm, rtol))
    return x

def cr_step(A, b, x, r, p):
    # Update solution vector
    Ap = np.einsum('ij,j->i', A, p)
    rtAr = np.einsum('i,ij,j', r, A, r)
    alpha = rtAr / np.einsum('i,i', Ap, Ap)
    x = x + alpha * p
    # Compute residual and its norm
    r = r - alpha * Ap
    # Update search direction
    beta = np.einsum('i,ij,j', r, A, r) / rtAr
    p = r + beta * p
    return x, r, p

def cr_stepper(A, b, iterate: Iterate):
    # Update vector and statistics
    x_new, r_new, p_new = cr_step(A, b, iterate['x'], iterate['r'], iterate['p'])
    E_new = quadratic_form(A, b, x_new)
    rnorm = LA.norm(r_new)
    xdiffnorm = LA.norm(x_new - iterate['x'])
    denergy = abs(E_new - iterate['E'])

    # In-place update of dictionary
    iterate['iteration counter'] += 1
    iterate['x'] = x_new
    iterate['r'] = r_new
    iterate['p'] = p_new
    iterate['E'] = E_new
    iterate['2-norm of residual'] = rnorm
    iterate['2-norm of error'] = xdiffnorm
    iterate['absolute pseudoenergy difference'] = denergy

def checkpointer(iterate: Iterate):
    zarr.save('data/cr.zarr', iterate['x'])

def main():
    print('Experiments with linear solvers')
    dim = 1000
    M = np.random.randn(dim, dim)
    # Make sure our matrix is SPD
    A = 0.5 * (M + M.transpose())
    A = A * A.transpose()
    A += dim * np.eye(dim)
    b = np.random.rand(dim)
    x_ref = LA.solve(A, b)

    # Plain CR
    x_cr = cr(A, b, rtol=1.0e-4, etol=1.0e-5, max_it=25)
    print('CR relative error to reference {:.5E}\n'.format(
        relative_error_to_reference(x_cr, x_ref)))

    # CR with iterator
    it_count = Stat(
        '# it.',
        '{:d}',
        kind='failure',
        criterion=Criterion(
            threshold=25,
            comparison=ge,
            message='Maximum number of iterations ({threshold:d}) exceeded'))

    rnorm = Stat(
        '||r||_2',
        '{:.5E}',
        kind='success',
        criterion=Criterion(
            threshold=1.0e-4,
            comparison=le,
            message='2-norm of residual below threshold {threshold:.1E}'))

    denergy = Stat(
        'abs(Delta E)',
        '{:.5E}',
        kind='success',
        criterion=Criterion(
            threshold=1.0e-5,
            comparison=le,
            message='Pseudoenergy variation below threshold {threshold:.1E}'))

    xdiffnorm = Stat(
        '||x_new - x_old||_2',
        '{:.5E}',
        kind='success',
        criterion=Criterion(
            threshold=1.0e-4,
            comparison=le,
            message='2-norm of error below threshold {threshold:.1E}'))

    energy = Stat('E', '{:.5E}', kind='report')

    stats = {
        '2-norm of residual': rnorm,
        'absolute pseudoenergy difference': denergy,
        '2-norm of error': xdiffnorm,
        'E': energy,
        'iteration counter': it_count
    }

    x_0 = np.zeros_like(b)
    guess = Iterate({
        'x': x_0,
        'r': compute_residual(A, b, x_0),
        'p': compute_residual(A, b, x_0),
        'E': quadratic_form(A, b, x_0),
        '2-norm of residual': compute_residual_norm(A, b, x_0),
        'absolute pseudoenergy difference': 0.0,
        '2-norm of error': 0.0
    })
    cr_loose = IterativeSolver(
        partial(cr_stepper, A, b), guess, stats, RuntimeError, checkpointer)

    for _ in cr_loose:
        pass

    print('\ncr_loose.niterations ', cr_loose.niterations)
    print('Conjugate residual relative error to reference {:.5E}\n'.format(
        relative_error_to_reference(cr_loose.iterate['x'], x_ref)))

if __name__ == '__main__':
    main()


#Plots for one-dimensional data -- HISTOGRAMS

import matplotlib.pyplot as plt
import numpy as np

x = np.random.normal(0, 1, 1000)
plt.hist(x, bins=20)
plt.show()
#Will display data as a normally distributed histogram
#Distribution is as a result of the method used to generate data
#Bars referred to as bins, and we can specify the number of bins
#Here, we specified 20 bins

#Two Seperate Plots

plt.subplot(1, 2, 1)
# 1 row, 2 columns and this is the first one
x = np.random.normal(0, 1, 1000)
plt.hist(x)

plt.subplot(1, 2, 2)
# 1 row, 2 columns and this is the second
x = np.random.uniform(-3, 3, 1000)
#Uniform distribution specified this time
plt.hist(x)

plt.show()

#Other plots and Ian's examples can be found here: https://github.com/ianmcloughlin/pyplot-examples


# The following are snap functions for finding a best approximated integer or rational number for a real number:

import numpy as np
from sympy import Rational

def bestApproximation(x,imax):
    # The input is a numpy parameter vector p.
    # The output is an integer specifying which parameter to change,
    # and a float specifying the new value.
    def float2contfrac(x,nmax):
        x = float(x)
        c = [np.floor(x)];
        y = x - np.floor(x)
        k = 0
        while np.abs(y)!=0 and k<nmax:
            y = 1 / float(y)
            i = np.floor(y)
            c.append(i)
            y = y - i
            k = k + 1
        return c

    def contfrac2frac(seq):
        ''' Convert the simple continued fraction in `seq`
            into a fraction, num / den
            '''
        num, den = 1, 0
        for u in reversed(seq):
            num, den = den + num*u, num
        return num, den

    def contFracRationalApproximations(c):
        return np.array(list(contfrac2frac(c[:i+1]) for i in range(len(c))))

    def contFracApproximations(c):
        q = contFracRationalApproximations(c)
        return q[:,0] / float(q[:,1])

    def truncateContFrac(q,imax):
        k = 0
        while k < len(q) and np.maximum(np.abs(q[k,0]), q[k,1]) <= imax:
            k = k + 1
        return q[:k]

    def pval(p):
        p = p.astype(float)
        return 1 - np.exp(-p ** 0.87 / 0.36)

    xsign = np.sign(x)
    q = truncateContFrac(contFracRationalApproximations(float2contfrac(abs(x),20)),imax)

    if len(q) > 0:
        p = np.abs(q[:,0] / q[:,1] - abs(x)).astype(float) * (1 + np.abs(q[:,0])) * q[:,1]
        p = pval(p)
        i = np.argmin(p)
        return (xsign * q[i,0] / float(q[i,1]), xsign* q[i,0], q[i,1], p[i])
    else:
        return (None, 0, 0, 1)

def integerSnap(p, top=1):
    p = np.array(p)
    metric = np.abs(p - np.round(p.astype(np.double)))
    chosen = np.argsort(metric)[:top]
    return dict(list(zip(chosen, np.round(p.astype(np.double))[chosen])))

def zeroSnap(p, top=1):
    p = np.array(p)
    metric = np.abs(p)
    chosen = np.argsort(metric)[:top]
    return dict(list(zip(chosen, np.zeros(len(chosen)))))

def rationalSnap(p, top=1):
    """Snap to nearest rational number using continued fraction."""
    p = np.array(p)
    snaps = np.array(list(bestApproximation(x,10) for x in p))
    chosen = np.argsort(snaps[:, 3])[:top]
    d = dict(list(zip(chosen, snaps[chosen, 1:3])))
    d = {k:  f"{val[0]}/{val[1]}" for k,val in d.items()}

    return d


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

##
## implementation of the MF-DFA algorithm as described in:
## http://mokslasplius.lt/rizikos-fizika/multifractality-time-series
##

import numpy as np
import matplotlib.pyplot as plt

def __fluctuations(profile,q,scale):
    # segment profile into equal chunks
    segments=int(len(profile) // scale)
    stridedProfile=np.lib.stride_tricks.as_strided(profile,shape=(segments,scale))
    # fit all chunks and evaluate fluctations
    xVals=np.arange(scale)
    fqs=np.zeros(segments)
    for i, segVals in enumerate(stridedProfile):
        coef=np.polyfit(xVals,segVals,1)
        fitVals=np.polyval(coef,xVals)
        fqs[i]=np.mean((segVals-fitVals)**2)
    return fqs

def MakeDfa(profile,q,scaleSample,showFqs=False):
    # sample fluctuations in given points
    fqs=np.zeros(len(scaleSample))
    for i, s in enumerate(scaleSample):
        if(q!=0):
            fqs[i]=np.mean(__fluctuations(profile,q,s)**(q/2))**(1/q)
        else:
            fqs[i]=np.exp(0.5*np.mean(np.log(__fluctuations(profile,q,s))))
    if(showFqs):
        plt.plot(scaleSample,fqs,label="q="+str(q))
    return fqs

def MakeMfDfa(series,qSample,scaleSample,showFqs=False):
    # obtain profile
    profile=np.cumsum(series-np.mean(series))
    logScaleSample=np.log10(scaleSample)
    hq=np.zeros(len(qSample))
    if(showFqs):
        plt.figure()
        plt.loglog()
        plt.xlabel('s')
        plt.ylabel('Fq(s)')
    for i, q in enumerate(qSample):
        logFqs=np.log10(MakeDfa(profile,q,scaleSample,showFqs=showFqs))
        hq[i]=np.polyfit(logScaleSample,logFqs,1)[0]
    if(showFqs):
        plt.show()
    return hq

def MakeSegMfDfa(series,qSample,scaleSample,segmentSize=None):
    if(segmentSize is None):
        return MakeMfDfa(series,qSample,scaleSample)
    hqs=()
    for i in np.arange(0,len(series)-segmentSize+1,segmentSize):
        hqs=hqs+(MakeMfDfa(series[i:i+segmentSize],qSample,scaleSample),)
    return np.mean(np.vstack(hqs),axis=0)


from sympy import *

def bisection(interval, function_expression, precision=0.01):
    variable = var("x")
    function = sympify(function_expression)
    if function.subs(variable, interval[0]) * function.subs(variable, interval[1]) >= 0:
        return "this function has no roots in the given interval."
    initiation_point = interval[0]
    while interval[1] - interval[0] >= precision:
        initiation_point = (interval[0] + interval[1]) / 2
        if function.subs(variable, initiation_point) == 0.0:
            break
        if function.subs(variable, initiation_point) * function.subs(variable, interval[0]) < 0:
            interval[1] = initiation_point
        else:
            interval[0] = initiation_point
    return "The value of the root is : {}".format(initiation_point)

given_interval = eval(input("please enter and interval like [a, b] where a and b are numbers: "))
given_function = input("please enter a function like 2 * x + 3 * sin(x)")
print(bisection(given_interval, given_function))


import numpy as np
from time import time

class GradientDescent:
    __slots__ = ["learning_rate", "weights", "heights", "beta", "data_x", "cost"]

    def __init__(self, data_x, data_y, learning_rate=0.00001):
        try:
            self.data_x = data_x.values
        except AttributeError:
            self.data_x = data_x
        self.weights = np.append(np.ones((self.data_x.shape[0], 1)),
                                 self.data_x,
                                 axis=1)
        try:
            self.heights = data_y.values
        except AttributeError:
            self.heights = data_y
        self.beta = np.append(np.ones((1, 1)) * self.heights.sum() / len(self.heights),
                              np.zeros((self.data_x.shape[1], 1)),
                              axis=0)
        self.learning_rate = learning_rate
        self.cost = []

    def __derivatives_sum_of_squared_residuals(self):
        prediction = np.dot(self.weights, self.beta)
        pred_gap = prediction - self.heights
        self.cost.append((pred_gap ** 2).sum())
        derivative = 2 * np.dot(self.weights.T,  pred_gap)
        self.beta = self.beta - (derivative * self.learning_rate)
        return derivative[0]

    @staticmethod
    def r2_score(y_true, y_pred):
        first_ssr = ((y_true - (y_true.sum() / len(y_true))) ** 2).sum()
        last_ssr = ((y_true - y_pred) ** 2).sum()
        return (first_ssr - last_ssr) / first_ssr

    def optimizer(self, number_of_steps=False):
        time_start = time()
        if number_of_steps:
            i = 1
            while i <= number_of_steps:
                self.__derivatives_sum_of_squared_residuals()
                if self.beta[0][0] != self.beta[0][0]:
                    raise Exception("Learning rate should be chosen lower!")
                print("\r", f"Iteration:{i}", end="")
                i += 1
        else:
            i = 0
            while 1:
                i += 1
                intercept_derivative = self.__derivatives_sum_of_squared_residuals()
                if self.beta[0][0] != self.beta[0][0]:
                    raise Exception("Learning rate should be chosen lower!")
                print("\r", f"Intercept Slope:{intercept_derivative}", end="")
                if i < 2:
                    continue
                if 0.0001 > abs(intercept_derivative):
                    break
        time_stop = time()
        benchmark = round(time_stop - time_start, 2)
        print("\nCompleted in {} seconds".format(benchmark))
        model_output_predict = np.dot(self.weights, self.beta)
        r2_score = self.r2_score(self.heights, model_output_predict)
        print("R-Squared:%{}".format(100 * r2_score))

    def test(self, data_x, data_y):
        try:
            x = data_x.values
        except AttributeError:
            x = data_x
        x_test = np.append(np.ones((x.shape[0], 1)),
                           x,
                           axis=1)
        try:
            height = data_y.values
        except AttributeError:
            height = data_y

        test_data_predict = np.dot(x_test, self.beta)
        r2_score = self.r2_score(height, test_data_predict)
        print("Test Score: %{}".format(100 * r2_score))

    def get_parameters(self):
        return self.beta[0][0], self.beta[1:]

    def predict(self, x):
        input_x = np.append(np.ones((x.shape[0], 1)), x, axis=1)
        return np.dot(input_x, self.beta)

class LinReg:
    __slots__ = ["x_train", "x", "y", "theta_init", "theta"]

    def __init__(self, data_x, data_y):
        try:
            self.x_train = data_x.values
        except AttributeError:
            self.x_train = data_x
        self.x = np.append(np.ones((self.x_train.shape[0], 1)),
                           self.x_train,
                           axis=1)
        try:
            self.y = data_y.values
        except AttributeError:
            self.y = data_y
        self.theta_init = np.append(np.ones((1, 1)) * (self.y.sum() / len(self.y)),
                                    np.ones((self.x_train.shape[1], 1)) * 0,
                                    axis=0)
        self.theta = np.empty((len(self.x), 0))

    def __r2_inside(self, x, y):
        first_ssr = ((y - np.dot(x, self.theta_init)) ** 2).sum()
        last_ssr = ((y - np.dot(x, self.theta)) ** 2).sum()
        return (first_ssr - last_ssr) / first_ssr

    def train(self):
        start_time = time()
        self.theta = np.linalg.lstsq(self.x, self.y, rcond=None)[0]
        end_time = time()
        print(f"Completed in {round(end_time - start_time, 2)} seconds.")
        print(f"Training R2-Score: % {self.__r2_inside(self.x, self.y) * 100}")
        print(f"Intercept: {self.theta[0][0]}, Coefficients: {self.theta[1:].reshape(1, len(self.theta) - 1)}")

    def test(self, test_x, test_y):
        if self.theta.size != 0:
            try:
                x_test = test_x.values
            except AttributeError:
                x_test = test_x
            x = np.append(np.ones((x_test.shape[0], 1)),
                          x_test,
                          axis=1)
            try:
                y = test_y.values
            except AttributeError:
                y = test_y
            self.theta_init[0] = (y.sum() / len(y))
            print(f"Testing R2-Score: % {self.__r2_inside(x, y) * 100}")
        else:
            raise Exception("Model not trained!")

    def predict(self, x):
        if self.theta.size != 0:
            input_x = np.append(np.ones((x.shape[0], 1)), x, axis=1)
            return np.dot(input_x, self.theta)
        else:
            raise Exception("Model not trained!")

    @staticmethod
    def r2_score(y_true, y_predict):
        first_ssr = ((y_true - (y_true.sum() / len(y_true))) ** 2).sum()
        last_ssr = ((y_true - y_predict) ** 2).sum()
        return (first_ssr - last_ssr) / first_ssr

    @property
    def intercept(self):
        if self.theta is not None:
            return self.theta[0][0]
        else:
            raise Exception("Model not trained!")

    @property
    def coefficients(self):
        if self.theta is not None:
            return self.theta[1:].reshape(1, len(self.theta) - 1)
        else:
            raise Exception("Model not trained!")

class LogReg:
    __slots__ = ["x", "y", "learning_rate", "iteration", "weights"]

    def __init__(self, x, y, learning_rate=0.01, iteration=1000):
        self.x = x
        self.y = y
        self.learning_rate = learning_rate
        self.iteration = iteration
        self.weights = np.append(np.full((1, 1), 0.0), np.full((self.x.shape[0], 1), 0.01), axis=0)
        self.x = np.append(np.ones((1, self.x.shape[1])), self.x, axis=0)

    @staticmethod
    def __sigmoid(z):
        return 1 / (1 + np.exp(-z))

    def __forward_propagation(self):
        return np.dot(self.weights.T, self.x)

    def __backward_propagation(self, y):
        return np.dot(self.x, (y - self.y).T) / self.x.shape[1]

    def __inner_predict(self, x):
        z = self.__sigmoid(np.dot(self.weights.T, x))
        prediction = np.zeros((1, x.shape[1]))
        for i in range(z.shape[1]):
            prediction[0, i] = 0 if z[:, i] <= 0.5 else 1
        return prediction

    @staticmethod
    def accuracy(y_true, y_pred):
        return 100 - np.average(np.abs(y_pred - y_true)) * 100

    def train(self):
        cost_list = []
        for i in range(self.iteration):
            out = self.__sigmoid(self.__forward_propagation())
            loss = -self.y * np.log(out) - (1 - out) * np.log(1 - out)
            cost = loss.sum() / self.x.shape[1]
            cost_list.append(cost)
            derivatives = self.__backward_propagation(out)
            self.weights = self.weights - self.learning_rate * derivatives
            print("\r", f"Iteration:{i}", end="")
        print(f"\n%{self.accuracy(self.y, self.__inner_predict(self.x))}")

    def predict(self, x):
        x = np.append(np.ones((1, x.shape[1])), x, axis=0)
        return self.__inner_predict(x)

class KNN:
    __slot__ = ["inputs", "outputs", "k_neighbors", "metric_func", "model_type"]

    def __init__(self, x, y, k_neighbors=5, metric="euclidean", model="classification"):
        self.inputs = x
        self.outputs = y
        self.k = k_neighbors
        self.model_type = model
        metrics_list = {"euclidean": self.__euclidean,
                        "minkowski": self.__minkowski,
                        "manhattan": self.__manhattan,
                        "hamming": self.__hamming}

        self.metric_func = metrics_list[metric]
        self.distances = dict()

    @staticmethod
    def __euclidean(x, y):
        return np.sqrt(np.abs(x - y)).sum()

    @staticmethod
    def __minkowski(x, y):
        pass

    @staticmethod
    def __manhattan(x, y):
        pass

    @staticmethod
    def __hamming(x, y):
        pass

    @staticmethod
    def __most_frequent(different_classes):
        counter = 0
        most_freq = different_classes[0]

        for element in different_classes:
            element_freq = different_classes.count(element)
            if element_freq > counter:
                counter = element_freq
                most_freq = element
        return most_freq

    def predict(self, x_test):
        for index, sample in enumerate(self.inputs):
            dist = self.metric_func(x_test, sample)
            self.distances[dist] = index
        list_ = list(self.distances)
        list_.sort()
        first_k_elements = list_[:self.k]
        if self.model_type == "classification":
            different_classes = []
            for element in first_k_elements:
                different_classes.append(self.outputs[self.distances[element]])
            return self.__most_frequent(different_classes)[0]

        if self.model_type == "regression":
            sum_ = 0
            for element in first_k_elements:
                sum_ += self.outputs[self.distances[element]]
            return (sum_ / self.k)[0]


import numpy as np

class Normalizer:

    def __init__(self):
        self.meanList = None
        self.stdList = None

    def Normalization(self , datas):
        if datas.ndim == 1 :
            self.meanList = np.mean( datas )
            self.stdList = np.std( datas )
        else :
            self.meanList = np.mean( datas , axis = 1)
            self.stdList = np.std( datas , axis = 1 )
        return (( datas.T - self.meanList ) / self.stdList).T

    def DeNormalization(self, datas):
        return (( datas.T * self.stdList + self.meanList ) ).T

if __name__ == "__main__":

    data = np.array([[1,2,3],[2,4,5]])
    mean = np.mean(data ,axis = 1)
    std = np.std(data ,axis = 1)
    print()

    k = np.array([10,10,10])
    k2 = np.array([100,100])

    res1 = data + k
    res2 = data.T + k2
    res3 = res2.T

    nm = Normalizer()
    norm = nm.Normalization(data)
    denorm = nm.DeNormalization(norm)

    print()