import numpy as np
import matplotlib.pyplot as mpl

# -------------------------------------|Declaração de funções|-------------------------------------#
def g0(x):
    return 1

def g1(x):
    return x

def g2(x):
    return x ** 2

def g3(x):
    return x ** 3

def produto_interno(a, b, x, n, tipo):

    res = 0
    if tipo == "f" or tipo == "F":
        for i in range(n):
            res += a(x[i]) * b(x[i])

    elif tipo == "v" or tipo == "V":
        for i in range(n):
            res += a(x[i]) * b[i]

    return res

def g_de_x(gx, x, coeficientes, a, b):

    for i in range(int(a * 200), int(b * 200 + 1)):
        i /= 200

        x.append(i)
        gx[0].append(coeficientes[0][0] + coeficientes[0][1] * i)
        gx[1].append(coeficientes[1][0] + coeficientes[1][1] * i + coeficientes[1][2] * i ** 2)
        gx[2].append(
            coeficientes[2][0] + coeficientes[2][1] * i + coeficientes[2][2] * i ** 2 + coeficientes[2][3] * i ** 3
        )

def erro(x, y, coeficientes):

    v = np.zeros((3))

    for i in range(len(x)):

        v[0] += abs(y[i] - (coeficientes[0][0] + coeficientes[0][1] * x[i])) ** 2
        v[1] += abs(y[i] - (coeficientes[1][0] + coeficientes[1][1] * x[i] + coeficientes[1][2] * (x[i] ** 2))) ** 2
        v[2] += (
            abs(
                y[i]
                - (
                    coeficientes[2][0]
                    + coeficientes[2][1] * x[i]
                    + coeficientes[2][2] * (x[i] ** 2)
                    + coeficientes[2][3] * (x[i] ** 3)
                )
            )
            ** 2
        )

    return v

def grafico4(x, y, xg, g):
    mpl.figure()
    mpl.plot(xg, g[0], label="Aproximação com grau 1", alpha=0.7, color="#eda405")
    mpl.plot(xg, g[1], label="Aproximação com grau 2", alpha=0.7, color="#e30000")
    mpl.plot(xg, g[2], label="Aproximação com grau 3", alpha=0.7, color="#9900ff")
    mpl.plot(x, y, "bo", label="Dados originais", alpha=0.5)
    mpl.xlabel("x")
    mpl.ylabel("y/g1(x)/g2(x)/g3(x)")
    mpl.title("Minimos Quadrados")
    mpl.legend()
    mpl.grid(True)

# -------------------------------------------------------------------------------------------------#

# -----------------------------------|Dados a serem aproximados|-----------------------------------#
x = [1.0, 1.1, 1.3, 1.5, 1.9, 2.1]
y = [1.84, 1.96, 2.21, 2.45, 2.94, 3.18]
# -------------------------------------------------------------------------------------------------#

# Variável para controlar o plotamento de gráficos
graficos = True

# ------------------------------------|Declaração de variáveis|------------------------------------#
gx = [[], [], []]
xg = []
g = [g0, g1, g2, g3]

coeficientes = [[], [], []]

minq_a = [[], [], []]
minq_a[0] = np.zeros((2, 2))
minq_a[1] = np.zeros((3, 3))
minq_a[2] = np.zeros((4, 4))

minq_b = [[], [], []]
minq_b[0] = np.zeros((2))
minq_b[1] = np.zeros((3))
minq_b[2] = np.zeros((4))
# -------------------------------------------------------------------------------------------------#

# --------------------------|Calculando as matrizes de produtos internos|--------------------------#
for i in range(3):

    for j in range(2 + i):

        for k in range(2 + i):

            minq_a[i][j][k] = produto_interno(g[j], g[k], x, len(x), "f")

        minq_b[i][j] = produto_interno(g[j], y, x, len(x), "v")

    # Resolvendo o sistema linear atual
    coeficientes[i] = np.linalg.solve(minq_a[i], minq_b[i])
# -------------------------------------------------------------------------------------------------#

# Calculando o erro cometido pelos polinomios através da variancia entre a função e os dados
v = erro(x, y, coeficientes)

# ----------------------|Printando os polinomios com seus respectivos erros|-----------------------#
print("g1(x) = %f +%fx" % (coeficientes[0][0], coeficientes[0][1]))
print("Erro =", v[0], end="\n\n")

print("g2(x) = %f +%fx %fx^2" % (coeficientes[1][0], coeficientes[1][1], coeficientes[1][2]))
print("Erro =", v[1], end="\n\n")

print("g3(x) = %f +%fx +%fx^2 %fx^3" % (coeficientes[2][0], coeficientes[2][1], coeficientes[2][2], coeficientes[2][3]))
print("Erro =", v[2], end="\n\n")
# -------------------------------------------------------------------------------------------------#

# --------------------------------------|Plotando os gráficos|-------------------------------------#
if graficos == True:

    g_de_x(gx, xg, coeficientes, 0.95, 2.15)
    grafico4(x, y, xg, gx)

    gx = [[], [], []]
    xg = []

    g_de_x(gx, xg, coeficientes, -10, 10)
    grafico4(x, y, xg, gx)

    mpl.show()
# -------------------------------------------------------------------------------------------------#

Lista-implementacao-3/1 - Minimos quadrados/minimos quadrados.py


import pandas as pd
import numpy as np
import pandas_datareader as pdr

def Exercise_1():
    symbol = "SPY"
    start_date = "2016-01-01"
    end_date = "2018-08-20"
    data = pdr.get_data_stooq(symbol, start=start_date, end=end_date)
    data.sort_index(inplace=True)
    return data

def Exercise_2(data):
    filepath = "data.csv"
    data.to_csv(filepath)
    return

def Exercise_3():
    filepath = "data.csv"
    data2 = pd.read_csv(filepath, parse_dates=True, index_col=0)
    return data2

def Exercise_4(data):
    close_mean = data.Close.rolling(20).mean()
    data['MA'] = close_mean
    return data

def Exercise_5(data):
    std_dev = data.Close.rolling(20).std()
    data['BB_Up'] = data.Close + std_dev * 2
    data['BB_Dn'] = data.Close - std_dev * 2
    return data

def Exercise_6(data):
    # note you can also use: rets = data.Close.pct_change()
    rets = (data.Close - data.Close.shift(1)) / data.Close.shift(1)
    data['Returns'] = rets
    return data

def Exercise_7(data):
    # generate volume-weighted returns and store in our data
    data['VR'] = data.Returns * data.Volume

    # create the custom function that does steps b-d.
    def vmomentum(col_data: pd.Series):
        c_up = 0
        c_dn = 0
        best_up = 0
        best_dn = 0

        for x in col_data:
            if x > 0:
                c_up += x
                best_up = c_up if c_up > best_up else best_up
                best_dn = c_dn if c_dn < best_dn else best_dn
                c_dn = 0
            elif x < 0:
                c_dn += x  # adds a negative value
                best_dn = c_dn if c_dn < best_dn else best_dn
                best_up = c_up if c_up > best_up else best_up
                c_up = 0

        spread = best_up + best_dn
        total = np.abs(col_data).sum()
        ratio = spread / total
        return ratio

    # apply the custom function over a 20 day window for our data
    data['VMom'] = data.VR.rolling(20).apply(vmomentum, raw=False)

    return data

def unified_solution():
    data = Exercise_1()  # request data
    Exercise_2(data)     # store data to disk

    data = Exercise_3()  # load data from disk
    data = Exercise_4(data)  # append the moving average
    data = Exercise_5(data)  # append the Bollinger bands
    data = Exercise_6(data)  # append the returns
    data = Exercise_7(data)  # append our momentum indicator
    return data


def cmap(mapCen, mapData):
    mapResults = []
    numFeatures = len(mapData[0])
    ##for each data piece
    for dataToMap in mapData:
        eucDists = []
        ##check Square Euclidean Distance to each centroid
        for cen in mapCen:
            SED = 0
            for feature in range(numFeatures):
                dif = cen[feature] - dataToMap[feature]
                SED = SED + (dif * dif)
            eucDists.append(SED)
        ##add the index of the lowest distance (which will be the cetroid ID) to the results
        mapResults.append(eucDists.index(min(eucDists)))
    return(mapResults)

def featureAverage(fData):
    numFData = len(fData)
    sumFData = 0
    for fDataEle in fData:
        sumFData = sumFData + fDataEle
    meanFData = sumFData/numFData
    return(meanFData)

def sortDataToCluster(dataToSort, dataIndicies, k):
    clusteredData = []
    ##set up empty results array
    for cluster in range(k):
        clusteredData.append([])
    ##dataIndicies[d] returns the centroid ID of the data stored in dataToSrt[d]
    for d in range(len(dataToSort)):
        clusteredData[dataIndicies[d]].append(dataToSort[d])
    return(clusteredData)

import numpy as np
import time

##read in data from text file, split into lines and turned into floats
f = open("data.txt","r")
data = [[float(num) for num in line.split()] for line in f.readlines()]
f.close()

numdata = len(data) ##length of dataset
features = len(data[0]) ##length of a single data piece

k = 15
centroidID = np.random.permutation(numdata)
centroids = np.random.rand(k,features)
for i in range(k):
    centroids[i] = data[i]

start_time = time.time_ns()

results = cmap(centroids,data)

end_time = time.time_ns()

clusteredData = sortDataToCluster(data,results,k)
##clusteredData[class][data][feature]

##setup empty results
results = []

##for each cluster
for i in range(k):
    ##get number of Data in Cluster (numDiC)
    numDiC = len(clusteredData[i])
    ##for each feature in current cluster
    for j in range(features):
        featureList = []
        ##collect each value of chosen feature from data in cluster
        for m in range(numDiC):
            featureList.append(clusteredData[i][m][j])
        ##if no data in cluster, use centroid instead
        if (len(featureList)) == 0:
            featureList.append(centroids[i][j])
        ##send off to be calculated
        results.append(featureAverage(featureList))

##rebuild centroids from results
newCentroids = centroids
for i in range(k):
    for j in range(features):
        newCentroids[i][j] = results[(features*i) + j]
##check to see if centroids have changed
compare = centroids == newCentroids
if compare.all():
    ##if so stop classifying
    #break
    print('Finished')
else:
    centroids = newCentroids
    ##if not keep classifying

print("time taken: %s" % ((end_time - start_time)/1000000000))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Nov  2 16:19:21 2020

Copyright 2020 by Hadrien Montanelli.
"""
# Standard library imports:
import numpy as np

def acf(series):
    '''Compute the sample autocorrelation function of a series.

    Input
    -----
    series : numpy.ndarray
        A time series as a Tx1 array for T data points.

    Output
    ------
    output : numpy.ndarray
        The Tx1 array of sample autocorrelation.

    Example
    -------
        import numpy as np
        from learnpy.timeseries import autocorr

        x_t = np.linspace(-1, 1, 100) + 5e-1*np.random.randn(100)
        output = autocorr(x_t)
        print(output)

    See also the 'example_autocorr' file.
    '''
    # Get the number of data points:
    T = len(series)

    # Compute the sample mean and sample variance:
    sample_mean = 1/T*sum(series)
    sample_var = 1/T*sum((series - sample_mean)**2)

    # Compute the sample autocorrelation function:
    sample_acf = np.zeros(T)
    sample_acf[0] = 1
    for h in range(1, T):
        sample_acf[h] = 1/T*sum((series[h:T] - sample_mean)
                                * (series[:T-h] - sample_mean))
        sample_acf[h] /= sample_var

    return sample_acf


import numpy as np
import pywt
import cv2
import matplotlib.pyplot as plt
import Assignment_1.add_snp_noise as snp

img = cv2.imread('image_3.png', 0)
print(img.shape)
noise_img = snp.add_snp(img, 10)

# reference : https://docs.opencv.org/3.4/d5/db5/tutorial_laplace_operator.html
ddepth = cv2.CV_16S
kernel_size = 3
denoised = cv2.GaussianBlur(img, (3, 3), 8)
dst = cv2.Laplacian(denoised, ddepth, kernel_size)
abs_dst = cv2.convertScaleAbs(dst)

temp = cv2.add(img, noise_img)
final = cv2.add(temp, abs_dst)

coeffs = pywt.dwt2(final, 'haar')

cA, (cH, cV, cD) = coeffs
d = 1, (2, 3, 4)
print(d)

thres = 3 * np.std(cA)
a = coeffs[0]
# a = pywt.threshold(coeffs[0], thres, 'soft')
b = pywt.threshold(coeffs[1][0], thres, 'soft', np.mean(cH))
c = pywt.threshold(coeffs[1][1], thres, 'soft', np.mean(cV))
d = pywt.threshold(coeffs[1][2], thres, 'soft', np.mean(cD))

coeff = a, (b, c, d)

# plt.imshow(a, cmap='gray')
# plt.show()

# print("#", len(coeffs[1][1]))
#
fig = plt.figure()

ax1 = fig.add_subplot(221)
ax1.imshow(cA, cmap='gray')
ax1.set_title('LL')

ax2 = fig.add_subplot(222)
ax2.imshow(cH, cmap='gray')
ax2.set_title('LH')

ax3 = fig.add_subplot(223)
ax3.imshow(cV, cmap='gray')
ax3.set_title('HL')

ax4 = fig.add_subplot(224)
ax4.imshow(cD, cmap='gray')
ax4.set_title('HH')

plt.tight_layout()
fig = plt.gcf()
plt.show()
#
# coeffs_2 = pywt.dwt2(cA, 'haar')
#
# cA_2, (cH_2, cV_2, cD_2) = coeffs_2

fig = plt.figure()

# ax1 = fig.add_subplot(221)
# ax1.imshow(cA_2, cmap='gray')
# ax1.set_title('LL')
#
# ax2 = fig.add_subplot(222)
# ax2.imshow(cH_2, cmap='gray')
# ax2.set_title('LH')
#
# ax3 = fig.add_subplot(223)
# ax3.imshow(cV_2, cmap='gray')
# ax3.set_title('HL')
#
# ax4 = fig.add_subplot(224)
# ax4.imshow(cD_2, cmap='gray')
# ax4.set_title('HH')

res = pywt.idwt2(coeff, 'haar')

ax1 = fig.add_subplot(121)
ax1.imshow(final, cmap='gray')
ax1.set_title('Noisy image')

ax2 = fig.add_subplot(122)
ax2.imshow(res, cmap='gray')
ax2.set_title('Result')
#
plt.tight_layout()
fig = plt.gcf()
plt.show()


import numpy as np
import pandas as pd
import statsmodels.api as sm
import seaborn as sns
sns.set()
from scipy import stats
stats.chisqprob = lambda chisq, df: stats.chi2.sf(chisq, df)

raw_data = pd.read_csv('2.02. Binary predictors.csv')
data = raw_data.copy()
data['Admitted'] = data['Admitted'].map({'Yes': 1, 'No': 0})
data['Gender'] = data['Gender'].map({'Female': 1, 'Male': 0})

y = data['Admitted']
x1 = data[['SAT', 'Gender']]

# ## Regression
x = sm.add_constant(x1)
reg_log = sm.Logit(y, x)
results_log = reg_log.fit()
results_log.summary()

np.exp(1.9449)
# Given the same SAT score, a female is 7 times more likely to get admitted than a male.

# ## Training the Model
np.set_printoptions(formatter={'float': lambda x: "{0:0.2f}".format(x)})
# Used this formatting to show a more logical result.
results_log.predict()
# These are the Predicted Values of my model
np.array(data['Admitted'])
# While these are the actual values of the data.

results_log.pred_table()
cm_df = pd.DataFrame(results_log.pred_table())
cm_df.columns = ['Predicted 0', 'Predicted 1']
cm_df = cm_df.rename(index={0: 'Actual 0', 1: 'Actual 1'})
# I then compare the Predicted Values of the model I trained and the Actual Values using the Confusion Matrix.
cm_df
cm = np.array(cm_df)
accuracy_train = (cm[0, 0]+cm[1, 1])/cm.sum()
accuracy_train
# The accuracy of my model using the training data is 94.6%

# ## Testing the Model and assessing its accuracy
test = pd.read_csv('2.03. Test dataset.csv')
# In this exercise the data was split into 90-10 beforehand.
test['Admitted'] = test['Admitted'].map({'Yes': 1, 'No': 0})
test['Gender'] = test['Gender'].map({'Female': 1, 'Male': 0})
test

x
# This is the Regression variable I defined earlier that contains independent variables and their constant.

test_actual = test['Admitted']
test_data = test.drop(['Admitted'], axis=1)
test_data = sm.add_constant(test_data)
# Decided to drop 'Admitted' because Order is important. The coefficients of the Regression expect it.

def confusion_matrix(data, actual_values, model):
        pred_values = model.predict(data)
        bins = np.array([0, 0.5, 1])
        cm = np.histogram2d(actual_values, pred_values, bins=bins)[0]
        accuracy = (cm[0, 0]+cm[1, 1])/cm.sum()
        return cm, accuracy

cm = confusion_matrix(test_data, test_actual, results_log)
cm
# Shows the Confusion Matrix and the Accuracy of my model.
# The accuracy is 89.47%
cm_df = pd.DataFrame(cm[0])
cm_df.columns = ['Predicted 0', 'Predicted 1']
cm_df = cm_df.rename(index={0: 'Actual 0', 1: 'Actual 1'})
cm_df
# Here I made a more presentable Confusion Matrix.

print('Missclassification rate: '+str((1+1)/19))
# The opposite of accuracy is the Misclassification Rate.
# My model's Misclassification Rate is 10.53&.

import numpy,from scipy,import statsmodels


#from DreamDiff import DreamDiff as ad
from DreamDiff.DreamDiff import DreamDiff as ad
import numpy as np
import matplotlib.pyplot as plt

class Optimize:
    '''
    Class containing a suite of root-finding, optimization, and
    animation tools for Newton's method, gradient descent, and
    Nesterov's accelerated gradient descent.
    '''

    def newtons_method(f, x0, epsilon=0.000001, max_iters=500):
        '''
        Implements Newton's root-finding method for scalar functions.

        INPUTS
        ======
        f: str, required
            Input function on which to perform Newton's method.
        x0: int or float, required
            Starting x-value at which to initialize the algorithm.
        epsilon: int or float, optional
            Solution accuracy threshold.
        max_iters: int, optional
            The maximum number of times to run the algorithm.

        RETURNS
        =======
        If a root is found, the algorithm returns its x-value, a list
        containing the previous points at which the derivative was
        evaluated, a list of the function's values at those points, and
        a list of the function's derivatives at those points in a 4-tuple.
        If no root is found or a zero derivative is reacahed, None is
        returned.
        '''
        # Initialize xn at the starting point
        xn = x0

        # Create a DreamDiff object to access private methods
        x = ad(1.0)

        # Parse the input function string
        f_parsed = x._parse_input(f)

        # Initialize empty lists to hold results
        xn_history = []
        yn_history = []
        der_history = []

        # Run the algorithm at most 'max_iters' times
        for i in range(max_iters):

            # Calculate values, derivatives, and store in lists
            xn_history.append(xn)
            new_f = x._evaluate_function(f_parsed, xn)
            yn = new_f.val[0]
            der = new_f.der[0]
            yn_history.append(yn)
            der_history.append(der)

            # If threshold is met, terminate the algorithm and return results
            if abs(yn) < epsilon:
                print('Found solution after {} iterations.'.format(i))
                print('Solution is: {}'.format(xn))
                return (xn, xn_history, yn_history, der_history)

            # If a zero derivative is reached, terminate the algorithm and
            # return None
            if der == 0:
                print('Reached zero derivative. No solution found.')
                return None

            # Apply the Newton's method update to xn
            xn = xn - yn/der

        # If no solution is found within 'max_iters', tell the user
        print('No solution found after max iterations.')

    def animate_newtons(f, x0, epsilon=0.000001, max_iters=500, runtime=20):
        '''
        Creates an animation of Newton's root-finding method for scalar
        functions by plotting the function and showing the tangent lines
        at which the gradient was evaluated.

        INPUTS
        ======
        f: str, required
            Input function on which to perform Newton's method.
        x0: int or float, required
            Starting x-value at which to initialize the algorithm.
        epsilon: int or float, optional
            Solution accuracy threshold.
        max_iters: int, optional
            The maximum number of times to run the algorithm.
        runtime: int, optional
            The length of time to run the animation.

        RETURNS
        =======
        A matplotlib animataion showing how Newton's method located
        a root of the input function if a root was found, otherwise
        None.
        '''
        # Get the root and history of x-values, y-values, and derivatives
        # for the input function using newtons_method
        results = Optimize.newtons_method(f, x0, epsilon, max_iters)

        # Check if newtons_method returned None
        if results == None:
            print("Newton's method did not find a root.")
            return None
        else:
            root, x_vals, y_vals, grad_vals = results[0], results[1], results[2], results[3]

            # Create a DreamDiff object to access private methods
            x = ad(1.0)

            # Create a list of x-vals searched
            t_vals = np.arange(min(x_vals)-2, max(x_vals)+2, 0.1)

            # Parse the input function string
            f_parsed = x._parse_input(f)

            # Calculate the values of the input function in the range of x-vals searched
            f_vals = [x._evaluate_function(f_parsed, t).val[0] for t in t_vals]

            # Initialize the animation
            fig = plt.figure()
            fig.canvas.draw()
            fig.suptitle("Newton's Method for f(x)="+f)

            # Run the animation for the duration of 'runtime'
            while runtime > 0:

                # Plot the animation, where each frame shows the function and
                # tangent line at which the gradient was evaluated
                for i in range(len(x_vals)):
                    grad = grad_vals[i]
                    intercept = y_vals[i] - grad*x_vals[i]
                    tangent_y = grad*t_vals + intercept
                    plt.cla()
                    plt.grid(True)
                    plt.plot(t_vals, f_vals, label='Function')
                    plt.plot(t_vals, tangent_y, '-r', label='Tangent')
                    plt.xlim(t_vals[0], t_vals[-1])
                    plt.ylim(min(f_vals), max(f_vals))
                    plt.xlabel('x')
                    plt.ylabel('f(x)')
                    plt.legend()
                    plt.pause(0.2)
                    runtime -= 1
            return None

    def grad_descent(f, x0, epsilon=0.000001, max_iters=500, eta=0.1):
        '''
        Implements the gradient descent optimization algorithm.

        INPUTS
        ======
        f: str, required
            Input function on which to perform gradient descent.
        x0: int or float, required
            Starting x-value at which to initialize the algorithm.
        epsilon: int or float, optional
            Solution accuracy threshold.
        max_iters: int, optional
            The maximum number of times to run the algorithm.
        eta: int or float, optional
            The learning rate, which controls the algorithm step size.

        RETURNS
        =======
        If a minimum is found, the algorithm returns its x-value, a list
        containing the previous points at which the derivative was
        evaluated, a list of the function's values at those points, and
        a list of the function's derivatives at those points in a 4-tuple.
        If no minimum is found, None is returned.
        '''
        # Initialize xn at the starting point
        xn = x0

        # Create a DreamDiff object to access private methods
        x = ad(1.0)

        # Parse the input function string
        f_parsed = x._parse_input(f)

        # Initialize empty lists to hold results
        xn_history = []
        yn_history = []
        der_history = []

        # Run the algorithm at most 'max_iters' times
        for i in range(max_iters):

            # Calculate values, derivatives, and store in lists
            xn_history.append(xn)
            new_f = x._evaluate_function(f_parsed, xn)
            yn = new_f.val[0]
            der = new_f.der[0]
            yn_history.append(yn)
            der_history.append(der)

            # Apply the gradient descent update (scaled by learning rate)
            new_xn = xn - eta*der

            # If threshold is met, terminate the algorithm and return results
            if abs(new_xn - xn) < epsilon:
                print('Found solution after {} iterations.'.format(i))
                print('Solution is: {}'.format(new_xn))
                return (new_xn, xn_history, yn_history, der_history)

            # Update xn
            xn = new_xn

        # If no solution is found within 'max_iters', tell the user and return None
        print('No solution found after max iterations.')
        return None

    def nesterov_grad_descent(f, x0, epsilon, max_iters, eta):
        '''
        Implements Nesterov's accelerated gradient descent
        optimization algorithm, which uses a momentum parameter 't'.

        INPUTS
        ======
        f: str, required
            Input function on which to perform gradient descent.
        x0: int or float, required
            Starting x-value at which to initialize the algorithm.
        epsilon: int or float, optional
            Solution accuracy threshold.
        max_iters: int, optional
            The maximum number of times to run the algorithm.
        eta: int or float, optional
            The learning rate, which controls the algorithm step size.

        RETURNS
        =======
        If a minimum is found, the algorithm returns its x-value, a list
        containing the previous points at which the derivative was
        evaluated, a list of the function's values at those points, and
        a list of the function's derivatives at those points in a 4-tuple.
        If no minimum is found, None is returned.
        '''
        # Initialize xn at the starting point
        xn = x0

        # Create a DreamDiff object to access private methods
        x = ad(1.0)

        # Initialize the momentum coefficient
        t = 1.0

        # Initialize the y-value
        y = x0

        # Parse the input function string
        f_parsed = x._parse_input(f)

        # Initialize empty lists to hold results
        xn_history = []
        yn_history = []
        der_history = []

        # Run the algorithm at most 'max_iters' times
        for i in range(max_iters):

            # Calculate values, derivatives, and store in lists
            xn_history.append(xn)
            new_f = x._evaluate_function(f_parsed, y)
            yn = new_f.val[0]
            der = new_f.der[0]
            yn_history.append(yn)
            der_history.append(der)

            # Apply Nesterov's momentum update
            new_t = 0.5*(1 + np.sqrt(1 + 4*t**2))

            # Calculate the new gradient (scaled by learning rate)
            new_xn = y - eta*der
            new_y = new_xn + (t - 1.0)/(new_t)*(new_xn - xn)

            # If threshold is met, terminate the algorithm and return results
            if abs(new_xn - xn) < epsilon:
                print('Found solution after {} iterations.'.format(i))
                print('Solution is: {}'.format(new_xn))
                return (new_xn, xn_history, yn_history, der_history)

            # Reset conditions
            if np.dot(y - new_xn, new_xn - xn) > 0:
                new_y = new_xn
                new_t = 1

            # Update xn, y, t
            xn = new_xn
            y = new_y
            t = new_t

        # If no solution is found within 'max_iters', tell the user and return None
        print('No solution found after max iterations.')
        return None

    def animate_grad_desc(f, x0, epsilon=0.000001, max_iters=500, eta=0.1, method='grad', runtime=20):
        '''
        Creates an animation of the gradient descent method for scalar
        functions by plotting the function and showing the tangent lines
        at which the gradient was evaluated.

        INPUTS
        ======
        f: str, required
            Input function on which to perform gradient descent.
        x0: int or float, required
            Starting x-value at which to initialize the algorithm.
        epsilon: int or float, optional
            Solution accuracy threshold.
        max_iters: int, optional
            The maximum number of times to run the algorithm.
        eta: int or float, optional
            The learning rate, which controls the algorithm step size.
        method: str, optional
            The gradient descent method to use: input 'grad' to use
            standard gradient descent and 'nesterov' to use
            Nesterov's accelerated gradient descent.
        runtime: int, optional
            The length of time to run the animation.

        RETURNS
        =======
        A matplotlib animataion showing how gradient descent located
        a minimum of the input function, depending on the descent method
        used.
        '''
        # Get the minimum and history of x-values, y-values, and derivatives
        # for the input function using grad_descent
        if method == 'grad':
            results = Optimize.grad_descent(f, x0, epsilon, max_iters, eta)
        elif method == 'nesterov':
            results = Optimize.nesterov_grad_descent(f, x0, epsilon, max_iters, eta)
        else:
            raise Exception(f'invalid gradient descent method: {method}')

        # Check if grad_descent returned None
        if results == None:
            print('Gradient descent did not find a minimum.')
            return None
        else:
            minimum, x_vals, y_vals, grad_vals = results[0], results[1], results[2], results[3]

            # Create a DreamDiff object to access private methods
            x = ad(1.0)

            # Create a list of x-vals searched
            t_vals = np.arange(min(x_vals)-2, max(x_vals)+2, 0.1)

            # Parse the input function string
            f_parsed = x._parse_input(f)

            # Calculate the values of the input function in the range of x-vals searched
            f_vals = [x._evaluate_function(f_parsed, t).val[0] for t in t_vals]

            # Initialize the animation
            fig = plt.figure()
            fig.canvas.draw()
            fig.suptitle("Gradient Descent for f(x)="+f)

            # Run the animation for the duration of 'runtime'
            while runtime > 0:

                # Plot the animation, where each frame shows the function and
                # tangent line at which the gradient was evaluated
                for i in range(len(x_vals)):
                    grad = grad_vals[i]
                    intercept = y_vals[i] - grad*x_vals[i]
                    tangent_y = grad*t_vals + intercept
                    plt.cla()
                    plt.grid(True)
                    plt.plot(t_vals, f_vals, label='Function')
                    plt.plot(t_vals, tangent_y, '-r', label='Tangent')
                    plt.xlim(t_vals[0], t_vals[-1])
                    plt.ylim(min(f_vals), max(f_vals))
                    plt.xlabel('x')
                    plt.ylabel('f(x)')
                    plt.legend()
                    plt.pause(0.2)
                    runtime -= 1
            return None

    def plot_coeffs(coeffs, x_array, y_array):
        '''
        Plots coefficients along with original points

        INPUTS
        ======
        coeffs: array of length 3n, required
            The x values for user's given points, must be same length as y_array

        x_array: array of numeric values of length n+1, required
            The x values for user's given points, must be same length as y_array
        y_array: array of numeric values of length n+1, required
            The y values for user's given points, must be same length as x_array

        RETURNS
        =======
        True if plotted, nothing otherwise.
        '''
        # check x and y for all points
        if not (len(x_array) == len(y_array)):
            raise Exception('Number of x values for data points do not match number of y values')

        num_eq = int(len(coeffs)/3)

        # plot each equation
        for i in range(num_eq):
            # define quadratic function for plotting
            def f(x, coeffs, i):
                return coeffs[i*3]*(x**2) + coeffs[(i*3)+1]*x + coeffs[(i*3)+2]

            temp_x = []
            # plot individual quadratic function
            temp_arange = np.arange(start = x_array[i], stop=x_array[i+1], step=0.01)
            for j in temp_arange:
                temp_x.append(f(j, coeffs, i))
            plt.plot(temp_arange, temp_x)

        # plot points
        plt.plot(x_array, y_array, 'ro')
        plt.title('Quadratic Splines')
        plt.show()
        return True

    def quadratic_spline(x_array, y_array, plot = False):
        '''
        Runs Quadratic Spline interpolation on a set of points.

        INPUTS
        ======
        x_array: array of numeric values, required
            The x values for user's given points, must be same length as y_array
        y_array: array of numeric values, required
            The x values for user's given points, must be same length as x_array
        plot: bool, optional, default = False
            If plot is True, plots the piecewise quadratic splines using matplotlib.pyplot

        RETURNS
        =======
        Coeffs: list of floats
            The list of coefficients to the quadratic spline equations in list as [a_1, b_1, c_1,... a_n, b_n, c_n]

        Prints piecewise function using coefficients.

        If plot==True, displays quadratic splines using matplotlib.pyplot
        '''

        # ensure all points have x and y values
        if not (len(x_array) == len(y_array)):
            raise Exception('number of x values for data points do not match number of y values')

        # ensure at least 3 data points for quadratic splines
        if len(x_array)<3:
            raise Exception('Too few data points for quadratic spline interpolation')

        n = len(x_array) - 1
        sys_of_equations = np.zeros((3*n, 3*n))

        # define AutoDiff objects a,b,c
        def a(ad_val):
            return ad(ad_val**2)
        def b(ad_val):
            return ad(ad_val)
        def c(ad_val):
            return ad(1)

        # define 3n linear equations
        for i in range(n):
            # Equations to connect quadratic between point i, i+1
            sys_of_equations[3*i][(3*i)] = a(x_array[i]).val
            sys_of_equations[3*i][((3*i)+1)] = b(x_array[i]).val
            sys_of_equations[3*i][((3*i)+2)] = c(x_array[i]).val
            sys_of_equations[(3*i)+ 1][(3*i)] = a(x_array[i+1]).val
            sys_of_equations[(3*i)+ 1][((3*i)+1)] = b(x_array[i+1]).val
            sys_of_equations[(3*i)+ 1][((3*i)+2)] = c(x_array[i+1]).val

            if i!=n-1:
                # Constraints so derivative matches at all interior points
                sys_of_equations[(3*i)+ 2][(3*i)] = a(x_array[i+1]).der
                sys_of_equations[(3*i)+ 2][((3*i)+1)] = b(x_array[i+1]).der
                sys_of_equations[(3*i)+ 2][((3*i)+2)] = c(x_array[i+1]).der
                sys_of_equations[(3*i)+ 2][(3*(i+1))] = -(a(x_array[i+1]).der)
                sys_of_equations[(3*i)+ 2][((3*(i+1))+1)] = -(b(x_array[i+1]).der)
                sys_of_equations[(3*i)+ 2][((3*(i+1))+2)] = -(c(x_array[i+1]).der)

        # Last of 3n equations
        sys_of_equations[3*n -1][0] = 1

        # construct y array for linear equations
        y = np.zeros(3*n)
        for i in range(n):
            y[(3*i)] = y_array[i]
            y[(3*i)+1] = y_array[i+1]
            y[(3*i)+2] = 0

        # solve for coeffs
        coeffs_all = np.linalg.lstsq(sys_of_equations, y, rcond=None)
        coeffs = coeffs_all[0]

        # truncate coeffs
        for i in range(len(coeffs)):
            coeffs[i] = float(f"{coeffs[i]:.4f}")

        print("Quadratic Spline Equations: ")
        for index in range(n):
            print("S_{index}(x) = {a}*x^2 + {b}*x + {c} for x ∈ [{i_0}, {i_1}]".format(
                index=index, a=coeffs[index*3], b=coeffs[index*3+1], c=coeffs[index*3+2], i_0=x_array[index], i_1=x_array[index+1]
                ))
        # plot
        if plot == True:
            spline_plot = Optimize.plot_coeffs(coeffs, x_array, y_array)

        return coeffs


# В программе реализована простая линейная регрессия
# -*- coding: utf-8 -*-
"""
Created on 28 April, 2019
@author jswanglp

requirements:
    matplotlib==2.0.2
    numpy==1.15.4
    tensorflow==1.12.0
    scikit_learn==0.23.1

"""

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt

if __name__ == '__main__':

    # Настройка параметров
    tf.app.flags.DEFINE_float('learning_rate', 3e-2, 'learning rate, default is 3e-2.')
    tf.app.flags.DEFINE_integer('num', 32, 'number of samples, default is 32.')
    tf.app.flags.DEFINE_integer('num_epochs', 50, 'number of epochs, default is 50.')
    FLAGS = tf.app.flags.FLAGS

    graph = tf.Graph()
    with graph.as_default():

        with tf.name_scope('Input'):
            x_input = tf.placeholder(tf.float32, shape=[None,], name='x_input')
            y_input = tf.placeholder(tf.float32, shape=[None,], name='y_input')
        with tf.name_scope('w_and_b'):
            w = tf.Variable(2.0, name='weight')
            b = tf.Variable(1.0, name='biases')
            y = tf.add(tf.multiply(x_input, w), b)
        with tf.name_scope('Train'):
            loss_op = tf.reduce_mean(tf.pow(y_input - y, 2))
            train_op = tf.train.AdamOptimizer(FLAGS.learning_rate).minimize(loss_op)

        gradients_node = tf.gradients(loss_op, w)

        # Настройка сессии
        sess = tf.Session()
        with sess.as_default():
            sess.run(tf.global_variables_initializer())

        # Генерация наборов обучающих данных
            x_pure = np.random.randint(-10, 100, FLAGS.num)
            x_train = x_pure + np.random.randn(FLAGS.num) / 10  # шум в направлении х
            y_train = 3 * x_pure + 2 + np.random.randn(FLAGS.num) / 10  # шум в направлении y
            Gradients = []
            Loss = []
            for i in range(FLAGS.num_epochs):
                _, gradients, loss = sess.run([train_op, gradients_node, loss_op],
                                              feed_dict={x_input: x_train,
                                                        y_input: y_train})
                print_list = [i+1, loss, gradients[0]]
                print("epoch: {0[0]} \t loss: {0[1]:.2f} \t gradients: {0[2]:.2f}".format(print_list))
                Gradients.append(gradients)
                Loss.append(loss)
        sess.close()

    fig = plt.figure(1, (14, 6))
    AX = [fig.add_subplot(i) for i in range(121,123)]
    name = ['Loss', 'Gradients']
    color = ['r', 'b']
    data = [Loss, Gradients]
    for na, ax, co, d in zip(name, AX, color, data):
        ax.plot(np.linspace(0, FLAGS.num_epochs, FLAGS.num_epochs), d, co, label=na)
        ax.set_title(na)
        ax.legend()

    plt.show()


"""Implementations of sequence alignment and optimal BST algorithms.

These algorithms provide examples of dynamic programming.

For now, I did not implement the reconstruction functions because the idea
is so similar to the algorithms in Chapter 16.
"""

import numpy as np

def read_seq_file(filename):
    """Reads data from sequence alignment test file.

    Args:
        filename (str): The file containing the edge list.

    Returns:
        str: The first sequence of characters.
        str: The second sequence of characters.
        int: The cost per gap in a sequence.
        int: The cost per mismatch in a sequence.
    """
    with open(filename, 'r') as f:
        next(f) # Skip first line
        cost_gap, cost_mismatch = next(f).strip().split()
        cost_gap, cost_mismatch = int(cost_gap), int(cost_mismatch)
        seq_x = next(f).strip()
        seq_y = next(f).strip()
        return seq_x, seq_y, cost_gap, cost_mismatch

def nw_score(seq_x, seq_y, cost_gap, cost_mismatch):
    """Returns the Needleman-Wunsch score for two sequences.

    The NW score is based on an optimal alignment, allowing gaps.

    I'm using a numpy array but avoiding vectorization to keep things minimal.

    Args:
        seq_x (str): The first sequence.
        seq_y (str): The second sequence.
        cost_gap (int): Cost per gap.
        cost_mismatch (int): Cost per mismatch.

    Returns:
        int: The NW score.
    """
    # Initialize array for holding solutions
    cache = np.zeros((len(seq_x)+1, len(seq_y)+1))
    # Base case #1 - seq_y is empty
    for i in range(len(seq_x) + 1):
        cache[i,0] = i * cost_gap
    # Base case #2 - seq_x is empty
    for j in range(len(seq_y) + 1):
        cache[0,j] = j * cost_gap
    # Systematically solve all sub-problems
    for i in range(1, len(seq_x) + 1):
        for j in range(1, len(seq_y) + 1):
            if seq_x[i-1] == seq_y[j-1]:
                match_cost = 0
            else:
                match_cost = cost_mismatch
            cost_1 = cache[i-1,j-1] + match_cost
            cost_2 = cache[i-1, j] + cost_gap
            cost_3 = cache[i, j-1] + cost_gap
            cache[i,j] = min(cost_1, cost_2, cost_3)
    return cache[len(seq_x), len(seq_y)]

def read_bst_file(filename):
    """Reads data from sequence alignment test file.

    Args:
        filename (str): The file containing the edge list.

    Returns:
        list: The frequencies for each key.
    """
    with open(filename, 'r') as f:
        next(f) # Skip first line
        freq = [int(x) for x in next(f).strip().split(',')]
        return freq

def opt_bst(freq):
    """Returns the search cost for an optimal BST.

    This algorithm finds an optimal BST, provided key frequencies.

    The costs are not normalized, so search cost is not average length.

    Args:
        freq (list): The frequencies for each key.

    Returns:
        int: The search cost for the optimal BST.
    """
    # Initialize array for holding solutions
    n = len(freq)
    cache = np.zeros((n+1, n+1))
    # Base case
    for i in range(n + 1):
        cache[i,i] = 0
    # Systematically solve all subproblems
    for s in range(n):
        for i in range(1, n + 1 - s):
            pk = sum(freq[i-1:i+s])
            min_cost = float('inf')
            for r in range(i, i + s + 1):
                min_cost = min(min_cost, cache[i-1,r-1] + cache[r,i+s])
            cache[i-1][i+s] = pk + min_cost
    return cache[0][n]

if __name__ == '__main__':
    import time

    nw_tests = [('problem17.8nw.txt', 224)]
    bst_opt_tests = [('problem17.8optbst.txt', 2780)]

    # Run NW correctness and timing tests
    print('Starting NW correctness tests...')
    for test_case in nw_tests:
        nw_inputs = read_seq_file(test_case[0])
        start = time.time()
        score = nw_score(*nw_inputs)
        end = round(time.time() - start, 3)
        assert score == test_case[1], f'Failed correctness for {test_case[0]}.'
        print(f'Passed correctness for {test_case[0]} in {end} seconds.')
    print('All NW correctness tests passed.')

    # Run NW correctness and timing tests
    print('Starting optimal BST correctness tests...')
    for test_case in bst_opt_tests:
        freq = read_bst_file(test_case[0])
        start = time.time()
        cost = opt_bst(freq)
        end = round(time.time() - start, 3)
        assert cost == test_case[1], f'Failed correctness for {test_case[0]}.'
        print(f'Passed correctness for {test_case[0]} in {end} seconds.')
    print('All optimal BST correctness tests passed.')


# Originally from https://cvw.cac.cornell.edu/python/exercise
# Modified by the cloudmesh team
from __future__ import print_function, division
"""
An estimate of the numerical value of pi via Monte Carlo integration.
Computation is distributed across processors via MPI.
"""

import numpy as np
from mpi4py import MPI
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import sys
from numba import jit
from cloudmesh.common.StopWatch import StopWatch

StopWatch.start("Overall time")
@jit(nopython=True)
def throw_darts(n):
    """
    returns an array of n uniformly random (x,y) pairs lying within the
    square that circumscribes the unit circle centered at the origin,
    i.e., the square with corners at (-1,-1), (-1,1), (1,1), (1,-1)
    """
    darts = 2*np.random.random((n,2)) - 1
    return darts

def in_unit_circle(p):
    """
    returns a boolean array, whose elements are True if the corresponding
    point in the array p is within the unit circle centered at the origin,
    and False otherwise -- hint: use np.linalg.norm to find the length of a vector
    """
    return np.linalg.norm(p,axis=-1)<=1.0

def estimate_pi(n, block=100000):
    """
    returns an estimate of pi by drawing n random numbers in the square
    [[-1,1], [-1,1]] and calculating what fraction land within the unit circle;
    in this version, draw random numbers in blocks of the specified size,
    and keep a running total of the number of points within the unit circle;
    by throwing darts in blocks, we are spared from having to allocate
    very large arrays (and perhaps running out of memory), but still can get
    good performance by processing large arrays of random numbers
    """
    total_number = 0
    i = 0
    while i < n:
        if n-i < block:
            block = n-i
        darts = throw_darts(block)
        number_in_circle = np.sum(in_unit_circle(darts))
        total_number += number_in_circle
        i += block
    return (4.*total_number)/n

def estimate_pi_in_parallel(comm, N):
    """
    on each of the available processes,
    calculate an estimate of pi by drawing N random numbers;
    the manager process will assemble all of the estimates
    produced by all workers, and compute the mean and
    standard deviation across the independent runs
    """

    if rank == 0:
        data = [N for i in range(size)]
    else:
        data = None
    data = comm.scatter(data, root=0)
    #
    pi_est = estimate_pi(N)
    #
    pi_estimates = comm.gather(pi_est, root=0)
    if rank == 0:
        return pi_estimates

def estimate_pi_statistics(comm, Ndarts, Nruns_per_worker):
    results = []
    for i in range(Nruns_per_worker):
        result = estimate_pi_in_parallel(comm, Ndarts)
        if rank == 0:
            results.append(result)
    if rank == 0:
        pi_est_mean = np.mean(results)
        pi_est_std  = np.std(results)
        return pi_est_mean, pi_est_std

if __name__ == '__main__':
    """
    for N from 4**5 to 4**14 (integer powers of 4),
    compute mean and standard deviation of estimates of pi
    by throwing N darts multiple times (Nruns_total times,
    distributed across workers)
    """
    comm = MPI.COMM_WORLD
    rank = comm.Get_rank()
    size = comm.Get_size()
    if rank == 0:
        print("MPI size = {}".format(size))
        sys.stdout.flush()
    Nruns_total = 64
    Nruns_per_worker = Nruns_total // size
    #
    estimates = []
    for log4N in range(5,15):
        N = int(4**log4N)
        result = estimate_pi_statistics(comm, N, Nruns_per_worker)
        if rank == 0:
            pi_est_mean, pi_est_std = result
            estimates.append((N, pi_est_mean, pi_est_std))
            print(N, pi_est_mean, pi_est_std)
            sys.stdout.flush()
    if rank == 0:
        estimates = np.array(estimates)
        plt.figure()
        plt.errorbar(np.log2(estimates[:,0]), estimates[:,1], yerr=estimates[:,2])
        plt.ylabel('estimate of pi')
        plt.xlabel('log2(number of darts N)')
        plt.savefig('pi_vs_log2_N.png')
        plt.figure()
        plt.ylabel('log2(standard deviation)')
        plt.xlabel('log2(number of darts N)')
        plt.plot(np.log2(estimates[:,0]), np.log2(estimates[:,2]))
        plt.savefig('log2_std_vs_log2_N.png')
    MPI.Finalize()

StopWatch.stop("Overall time")
StopWatch.benchmark()