import numpy as np
import cv2
import os
import argparse
from skimage.exposure import rescale_intensity
from scipy.stats import multivariate_normal
import matplotlib as mpl
mpl.use('TkAgg')
from matplotlib import pyplot as plt

parser = argparse.ArgumentParser('Geneate dataset BlurMicroscopy')
parser.add_argument('--output_fold', dest='output_fold', help='output directory', type=str, default='./dataset/')
parser.add_argument('--num_imgs', dest='num_imgs', help='number of images',type=int, default=3000)
parser.add_argument('--image_size', dest='image_size', help='image_size = H = W', type=int, default=256)
parser.add_argument('--std_r', dest='std_r', help='std of additive Gaussion white noise', type=float, default=5.)
parser.add_argument('--phase', dest='phase', help='test or train', type=str, default='train')
args = parser.parse_args()

def Gaussian_2D(m=0, sigma=1.):
    """
    :param M: sample num
    :param m: mean
    :param sigma: std
    :return: Gaussian distribution
    """
    mean = np.zeros(2) + m
    cov = np.eye(2) * sigma
    Gaussian = multivariate_normal(mean=mean, cov=cov)
    return Gaussian

def generate_bean(bean_size=10, sigma=0.2, M=50, is_plot=False):
    """
    :param bean_size: bean_size
    :param sigma: std
    :param M: sample num
    :param is_plot: bool plot images
    :return: image of a bean
    """
    Gaussian = Gaussian_2D(m=0, sigma=sigma)
    X, Y = np.meshgrid(np.linspace(-0.5, 0.5, M), np.linspace(-0.5, 0.5, M))
    d = np.dstack([X, Y])
    Z = np.zeros((M, M))
    for i in range(len(d)):
        for j in range(len(d[0])):
            x, y = d[i][j]
            if x ** 2 + y ** 2 <= 0.5*0.5-0.01:
                Z[i][j] = Gaussian.pdf((x, y))

    Z = Z.reshape(M, M)
    max_Z = np.max(Z)
    img_Z = np.uint8(np.asarray(Z)/max_Z*255)
    indensity = np.random.uniform(low=0.5, high=1.0)
    img_Z = indensity * img_Z

    if bean_size < M:
        bean = cv2.resize(img_Z, (bean_size, bean_size), interpolation=cv2.INTER_CUBIC)
    else:
        bean = img_Z

    if is_plot:
        cv2.imwrite("bean_size10.png", bean)
        cv2.imwrite("bean_size50.png", img_Z)

        plt.figure(0)
        plt.imshow(img_Z, cmap='gray', vmin=0, vmax=255)
        plt.figure(1)
        plt.imshow(bean, cmap='gray', vmin=0, vmax=255)

        plt.show()
    return bean

def generate_sharp_img(image_size=256, bean_size=10, bean_min=3, bean_max=10, is_plot=False):
    """
    Generate a sharp image with beans
    :param image_size: image_H = image_W = image_size
    :param bean_size: diameter
    :param bean_min: min num of beans
    :param bean_max: man num of beans
    :param is_plot: bool plot images
    :return: a sharp image with beans
    """
    bean = generate_bean(bean_size=bean_size, sigma=0.2, M=50)
    bean_num = np.random.randint(low=bean_min, high=bean_max)
    bean_locs = np.random.randint(low=bean_size/2+1, high=image_size-bean_size/2-1, size=(bean_num, 2))

    background = np.zeros((image_size, image_size))

    for bean_loc in bean_locs:
        bean_loc_x, bean_loc_y = bean_loc
        left = bean_loc_y - bean_size/2 if 0 < bean_loc_y - bean_size/2 < image_size else 0
        right = bean_loc_y + bean_size/2 if 0 < bean_loc_x - bean_size/2 < image_size else image_size
        up = bean_loc_x - bean_size/2 if 0 < bean_loc_x - bean_size/2 < image_size else 0
        down = bean_loc_x + bean_size/2 if 0 < bean_loc_x - bean_size/2 < image_size else image_size

        background[up:down, left:right] += bean

    if is_plot:
        # cv2.imwrite("sample_img.png", background)
        plt.figure()
        plt.imshow(background, cmap='gray', vmin=0, vmax=255)
        plt.show()
    return background

def convolve(image, kernel):
    """
    2D convolution
    :param image:
    :param kernel:
    :return: convolved image
    """
    (iH, iW) = image.shape[:2]
    (kH, kW) = kernel.shape[:2]

    pad = (kW - 1) // 2
    image = cv2.copyMakeBorder(image, pad, pad, pad, pad, cv2.BORDER_REPLICATE)
    output = np.zeros((iH, iW), dtype="float32")

    for y in np.arange(pad, iH + pad):
        for x in np.arange(pad, iW + pad):
            roi = image[y - pad:y + pad + 1, x - pad:x + pad + 1]
            k = (roi * kernel).sum()
            output[y - pad, x - pad] = k
        # rescale the output image to be in the range [0, 255]
    output = rescale_intensity(output, in_range=(0, 255))
    output = (output*255).astype("uint8")
    return output

def kernel_fit(loc):
    """
    Estimated psf of laser
    :param loc: (x, y)
    :return: z
    """
    x, y = loc
    x, y = x * 50, y * 50
    z = np.exp(-np.log(2) * (x * x + y * y) / (160.5586 * 160.5586)) * 255
    return z

def get_kernel(is_plot=False):
    """
    Compute cropped blur kernel
    :param is_plot: bool
    :return: blur kernel
    """
    M = 61
    X, Y = np.meshgrid(np.linspace(-30, 31, M), np.linspace(-30, 31, M))
    d = np.dstack([X, Y])
    Z = np.zeros((M, M))
    for i in range(len(d)):
        for j in range(len(d[0])):
            x, y = d[i][j]
            Z[i][j] = kernel_fit((x, y))

    Z = Z.reshape(M, M)
    img_Z = np.uint8(np.asarray(Z))
    crop_size = 22
    crop_Z = img_Z[crop_size:M-crop_size, crop_size:M-crop_size]
    kernel = crop_Z / np.float(np.sum(crop_Z))
    if is_plot:
        print(crop_Z.shape)
        print(crop_Z)
        # psf = cv2.imread("psf.png", 0)
        # plt.figure()
        # plt.imshow(psf, cmap='gray', vmin=0, vmax=255)
        plt.figure()
        plt.imshow(img_Z, cmap='gray', vmin=0, vmax=255)
        plt.figure()
        plt.imshow(crop_Z, cmap='gray', vmin=0, vmax=255)
        plt.show()
    return kernel

def generate_dataset(name_folder, num_imgs, image_size=256, std_r=5, bean_size=10, bean_min=3, bean_max=10):
    is_plot = False
    kernel = get_kernel()

    for i in range(num_imgs):
        name_prefix = '%04d' % (i+1)
        name_blur = name_folder + name_prefix + "_blur.png"
        name_sharp = name_folder + name_prefix + "_sharp.png"

        sharp = generate_sharp_img(image_size=image_size, bean_size=bean_size,
                                   bean_min=bean_min, bean_max=bean_max, is_plot=False)
        blurry = cv2.filter2D(sharp, -1, kernel)  # convolve(sharp, kernel)
        noise_img = np.random.normal(loc=0, scale=std_r, size=blurry.shape)
        blurry_noisy = blurry + noise_img

        cv2.imwrite(name_sharp, sharp)
        cv2.imwrite(name_blur, blurry_noisy)

        if is_plot:
            # cv2.imwrite("sharp_sample.png", sharp)
            # cv2.imwrite("blurry_sample.png", blurry)
            # cv2.imwrite("blurry_noisy_sample.png", blurry_noisy)

            plt.figure()
            plt.imshow(sharp, cmap='gray', vmin=0, vmax=255)
            plt.figure()
            plt.imshow(blurry, cmap='gray', vmin=0, vmax=255)
            plt.figure()
            plt.imshow(blurry_noisy, cmap='gray', vmin=0, vmax=255)
            plt.show()
    return

if __name__ == "__main__":
    """
    Example:
        python generate_dataset.py --phase train --num_imgs 5
    """
    args = parser.parse_args()

    img_output_fold = args.output_fold + args.phase + '/'  # Output Folder

    if not os.path.isdir(img_output_fold):
        os.makedirs(img_output_fold)

    generate_dataset(img_output_fold, args.num_imgs, image_size=args.image_size, std_r=args.std_r)

    # bean = generate_bean(bean_size=3, sigma=0.2, M=50)
    # cv2.imwrite("bean_size3.png", bean)


import numpy as np

class LinearRegression:

    def __init__(self , learning_rate = 0.01 , n_itres = 1000 , norm = False ):
        self.learning_rate = learning_rate
        self.n_itres       = n_itres
        self.weights       = None
        self.bias          = None
        self.norm          = norm

    def fit(self , X , y):
        if self.norm == True:
            self.NormalEquation(X , y)
        else:
            self.GradinetDescent(X , y)

    def NormalEquation(self , X , y):
        n        = len(X)
        X        = np.concatenate((np.ones((n , 1)) , X) , axis = 1)
        XT       = X.transpose()
        XTX      = np.dot(XT , X)
        XTXinv   = np.linalg.inv(XTX)
        XTXinvXT = np.dot(XTXinv , XT)
        self.B   = np.dot(XTXinvXT , y)

    def GradinetDescent(self , X , y):
        self.weights = np.zeros(n_feats)
        self.bias    = 0

        for _ in range(self.n_itres):
            y_predicted = np.dot(X , self.weights) + self.bias
            dw = (1 / n_samples) * np.dot(X.T , (y_predicted - y))
            db = (1 / n_samples) * np.sum(y_predicted - y)
            self.weights = self.weights - self.learning_rate * dw
            self.bias    = self.bias - self.learning_rate * db

    def _predictNorm(self , x_test):
        x_test = np.concatenate(([1] , x_test ))
        return np.dot(x_test , self.B)

    def _predictGD(self , X):
        y_predicted = np.dot(X , self.weights) + self.bias
        return y_predicted

    def predict(self , x_test):
        if self.norm == True:
            _predictNorm(x_test)
        else:
            _predictGD(x_test)


"""
This implements a heuristic for the TSP, rather than an exact algorithm, and as
a result will be able to handle much larger problem sizes.  Here is a data file
describing a TSP instance
(original source: http://www.math.uwaterloo.ca/tsp/world/bm33708.tsp).

The first line indicates the number of cities. Each city is a point in the
plane, and each subsequent line indicates the x- and y-coordinates of a single
city.

The distance between two cities is defined as the Euclidean distance --- that
is, two cities at locations (x,y)(x,y) and (z,w)(z,w) have distance
sqrt[(x-z)^2 + (y-w)^2] between them.

1.  Start the tour at the first city.
2.  Repeatedly visit the closest city that the tour hasn't visited yet.
    In case of a tie, go to the closest city with the lowest index.
    For example, if both the third and fifth cities have the same distance from
    the first city (and are closer than any other city), then the tour should
    begin by going from the first city to the third city.
3.  Once every city has been visited exactly once, return to the first city to
    complete the tour.
"""

import numpy as np

def greedy_tsp(coords):
    """This function is an implementation of greedy algorithm for travelling
    sales man problem

    Examples
    >>> coords = np.array([
    >>>    [1, 1, 2],
    >>>    [2, 5, 8],
    >>>    [3, 1, 3],
    >>>    [4, 1, 4],
    >>>    [5, 2, 4]
    >>> ])
    >>> mindist, path = greedy_tsp(coords)
    >>> print(mindist)
    15.21110255092798
    >>> print(path)
    [1, 3, 4, 5, 2, 1]

    """
    mindist = 0
    origin_coord = coords[0, 1:]
    start = 0
    path = [coords[0, 0]]
    while coords.shape[0] > 1:
        start_coord = coords[start, 1:]
        coords = np.delete(coords, start, 0)
        squared_dists = ((coords[:, 1:] - start_coord) ** 2).sum(axis=1)
        start = np.argmin(squared_dists)
        mindist += np.sqrt(squared_dists[start])
        path.append(coords[start, 0])
    end_coord = coords[start, 1:]
    mindist += np.sqrt(((end_coord - origin_coord) ** 2).sum())
    path.append(path[0])
    return mindist, path

if __name__ == "__main__":
    coords = np.loadtxt("nn.txt", skiprows=1)
    mindist, _ = greedy_tsp(coords)
    print(mindist)


import numpy as np
import math

# Create random input and output data
x = np.linspace(-math.pi, math.pi, 2000)
y = np.sin(x)

# Randomly initialize weights
a = np.random.randn()
b = np.random.randn()
c = np.random.randn()
d = np.random.randn()

learning_rate = 1e-6
for t in range(5000):
    # Forward pass, compute predicted y
    # y = a + b * x + c * x^2 + d * x^3
    y_pred = a + b * x + c * (x**2) + d * (x**3)

    # Compute and print loss
    loss = np.square(y_pred - y).sum()
    if t % 100 == 99:
        print("Epoch: {} Loss: {}.".format(t, loss))

    # Backprop to compute gradients of a, b, c, d with respect to loss
    grad_y_pred = 2.0 * (y_pred - y)
    grad_a = grad_y_pred.sum()
    grad_b = (grad_y_pred * x).sum()
    grad_c = (grad_y_pred * x**2).sum()
    grad_d = (grad_y_pred * x**3).sum()

    # Update weights
    a -= learning_rate * grad_a
    b -= learning_rate * grad_b
    c -= learning_rate * grad_c
    d -= learning_rate * grad_d

print("Result: f(x) = {} + {} * x + {} * x**2 + {} * x**3".format(a, b, c, d))

Learning PyTorch/Learning PyTorch with Example/polynomial_numpy.py


#!/usr/bin/env python

#   mp_fft.py
#   2018-08-25  Markku-Juhani O. Saarinen <mjos@iki.fi>

#   high-precision convolutions with FFT. Original adopted via apfft from
#   https://jakevdp.github.io/blog/2013/08/28/understanding-the-fft/

import numpy as np
import mpmath as mp

mp_real = np.vectorize(mp.re)

# FFT and inverse FFT using mpmath

def mp_fft(v, inv_fft=False):

    n = v.shape[0]
    if n == 1:
        return v

    if np.log2(n) % 1 > 0:
        raise ValueError("size of b must be a power of 2")

    # inverse op changes this sign, and scaling at the end
    s_pi = mp.pi * mp.mpc(1j)
    if not inv_fft:
        s_pi = -s_pi

    m = np.array([ [mp.mpc(1), mp.mpc(1)], [mp.mpc(1), mp.mpc(-1)] ])
    m = m.dot(v.reshape((2, -1)))

    # build-up each level of the recursive calculation all at once
    while m.shape[0] < n:
        m_even = m[:, :m.shape[1] / 2]
        m_odd = m[:, m.shape[1] / 2:]
        l = m.shape[0]
        w = mp.exp(s_pi / mp.mpf(l));
        a = np.empty(l, dtype=mp.mpc)
        x = mp.mpc(1)
        for i in range(l - 1):
            a[i] = x;
            x *= w
        a[l - 1] = x
        a = a[:, None]
        m = np.vstack([m_even + a * m_odd,
                       m_even - a * m_odd])

    # scale the result if inverse
    if inv_fft:
        m /= mp.mpf(n)

    return m.ravel()

# convolution on arrays with FFT

def mp_conv(x, y):
    xy = mp_fft(x, False) * mp_fft(y, False)
    xy = mp_fft(xy, True)
    return mp_real(xy)

# -- test code --

# equivalent exhaustive method for testing

def mp_conv2(x, y):
    l = x.shape[0]
    r = np.array([mp.mpf(0)] * l)
    for i in range(l):
        for j in range(l):
            r[(i + j) % l] += x[i] * y[j]
    return mp_real(r)

# a vector of random numbers, scaled to sum up to 1

def mp_rand_dist(l):
    v = np.array([mp.rand() for i in range(l)])
    return v / v.sum()

# returns a very small number if fft works

def test_fft():
    n = 2**10
    a = mp_rand_dist(n)
    b = mp_rand_dist(n)
    c = mp_conv(a, b)
    d = mp_conv2(a, b)
    eps = 0
    for i in range(n):
        eps = max(eps, mp.fabs(c[i] - d[i]))
    return eps

# run the test
#mp.mp.dps = 100
#print test_fft()


# 使用sympy.diff求导定义函数  ，求解其导数
from sympy import *

init_printing(use_unicode=True)
x = symbols("x")
f = log(x)
# 一阶导数
pprint(diff(f, x))
# 二阶导数可以传入第三个参数，表示阶数
pprint(diff(f, x, 2))

print("--------------------------------")

f2 = tan(x)
pprint(diff(f2, x))


from numpy import ndarray, power

def mean_sum_squares(error_array: ndarray) -> float:
    """
    Returns the expected value of sum of squares.

    Parameters
    ----------
    error_array:
     N-by-m array where N is the number of samples and m is the number of output.

    Returns
    -------
    mean_sum_squares:
     The expected value of sum of squares.
    """
    assert error_array.ndim >= 2

    return power(error_array, 2.0).sum(axis=1).mean()


import numpy as np
from metrics import *

class SimpleLogisticRegression:

    def __init__(self):
        self.betas = None

    def predict_proba(self, X, intercept=True):
        X = np.c_[np.ones(X.shape[0]), X] if intercept else X

        try:
            z = np.dot(X, self.betas)
        except TypeError as e:
            raise Exception('Model is not fitted. Call fit before predict').with_traceback(e.__traceback__)
        return 1 / (1 + np.exp(-z))

    def predict(self, X, threshold = 0.5):
        return np.array([1 if e > threshold else 0 for e in self.predict_proba(X)])

    def __gradient_descent(self, X, y, learning_rate, steps, seed, verbosity, cost_func='cross_entropy', **kwargs):
        np.random.seed(seed)

        minimize = {
            'cross_entropy': cross_entropy
        }

        self.betas = np.random.rand(X.shape[1])

        for i in range(steps):
            try:
                predicted = self.predict_proba(X, intercept=False)
            except TypeError as e:
                raise Exception('Model is not fitted. Call fit before predict').with_traceback(e.__traceback__)

            try:
                error = minimize.get(cost_func, lambda: 'Invalid')(predicted, y)
            except ValueError as e:
                raise Exception('X and y have different lenght')

            if verbosity != 0 and i % verbosity == 0 :
                print('Step: {} --- Error: {}'.format(i, error))

            error = predicted - y
            gradient = np.dot(X.T, error)/(len(X))
            self.betas -= learning_rate*gradient

        self.is_fitted = True

    def fit(self, X, y, mode='gradient_descent', learning_rate=0.01, steps=10000, seed=0, verbosity=0, cost_func='cross_entropy'):
        modes = {
            'gradient_descent': self.__gradient_descent
        }

        try:
            X = np.c_[np.ones(X.shape[0]), X]
        except AttributeError as e:
            raise Exception('X must a be a numpy array').with_traceback(e.__traceback__)

        try:
            modes.get(mode, lambda: 'Invalid')(X=X,
                                                y=y,
                                                learning_rate=learning_rate,
                                                steps=steps,
                                                seed=seed,
                                                verbosity=verbosity,
                                                cost_func=cost_func)
        except TypeError as e:
            Exception('Invalid parameter type').with_traceback(e.__traceback__)
        except Exception as e:
            raise e


# Задача № 11
'''Для вычисления несобственного интеграла модернизуруем метод Симпсона'''

import math
import numpy as np
import matplotlib.pyplot as plt

def function_graph(function):
    '''Функция, которая рисует графики'''
    fig = plt.subplots()
    x = np.linspace(1, 2)
    plt.plot(x, function(x))
    plt.show()

def simpson(f, x1, x2, n, e):
    '''Функция вычисления интеграла,
    где x1,x2 - границы интегрирования,
    f - целевая функция, n - разбиение,
    e - шаг, на который отходим от границ интеграирования '''
    a=x1+e
    b=x2
    h=(b-a)/n
    k=0.0
    x=a + h
    for i in range(1,(n//2) + 1):
        k += 4*f(x)
        x += 2*h
    x = a + 2*h
    for i in range(1,n//2):
        k += 2*f(x)
        x += 2*h
    return (h/3)*(f(a)+f(b)+k)

def function(x):
    return math.log(x)/(1+x**2)

print(simpson(function, 0.0, 1.0, 10000000, 0.000000001))

#print(function(10))
#function_graph(function)


import numpy as np
import matplotlib.pyplot as plt

from O5 import prepare_data_wine

# 1. Standardize data

X_train, X_test, y_train, y_test = prepare_data_wine(standardize=True,
                                                     split=True)

# 2. Compute mean vector for each class

np.set_printoptions(precision=4)
mean_vectors = []
for label in np.sort(np.unique(y_train)):
    mean_vectors.append(np.mean(X_train[y_train == label], axis=0))
    print('MV %s: %s\n' % (label, mean_vectors[label - 1]))

# 3.1 Create within-class scatter matrix

d = 13  # number of features
S_W = np.zeros((d, d))
for label, mv in zip(range(1, 4), mean_vectors):
    class_scatter = np.zeros((d, d))

    for row in X_train[y_train == label]:
        row, mv = row.reshape(d, 1), mv.reshape(d, 1)
        class_scatter += (row - mv).dot((row - mv).T)
        S_W += class_scatter

print(f"Within-class scatter matrix: {S_W.shape[0]} x {S_W.shape[1]}")

# The assumption that we are making when we are computing the scatter matrices
# is that the class labels in the training dataset are uniformly distributed.
# However, if we print the number of class labels, we see that this assumption
# is violated:
print(f'Class label distribution: {np.bincount(y_train)[1:]}')

# Thus, we want to scale the individual scatter matrices, S(i), before we sum
# them up as scatter matrix S(W). When we divide the scatter matrices by the
# number of class-examples, nn ii , we can see that computing the scatter
# matrix is in fact the same as
# computing the covariance matrix, Σ(i) —the covariance matrix is a normalized
# version of the scatter matrix:
# Σ(i) = 1/n(i) * S(i) =1/n(i) * ∑(x−m(i))(x-m(i))^T

# Compute scaled within-class scatter matrix:

d = 13
S_W = np.zeros((d, d))
for label, mv in zip(range(1, 4), mean_vectors):
    class_scatter = np.cov(X_train[y_train == label].T)
    S_W += class_scatter
print(f"Within-class scatter matrix: {S_W.shape[0]} x {S_W.shape[1]}")

# 3.2 Compute between-class scatter matrix:

mean_overall = np.mean(X_train, axis=0)
d = 13  # Number of features
S_B = np.zeros((d, d))

for i, mean_vector in enumerate(mean_vectors):
    n = X_train[y_train == i + 1, :].shape[0]
    mean_vector = mean_vector.reshape(d, 1)  # Make column vector
    mean_overall = mean_overall.reshape(d, 1)
    S_B += n * (mean_vector - mean_overall).dot((mean_vector - mean_overall).T)

print(f'Between-class scatter matrix: {S_B.shape[0]}x{S_B.shape[1]}')

# 4 Compute the eigenvectors and corresponding eigenvalues of the matrx,
# S(w, -1) * S(b)
eigen_values, eigen_vectors = np.linalg.eig(np.linalg.inv(S_W).dot(S_B))

# 5 Sort the eigenvalues in descending order
eigen_pairs = [(np.abs(eigen_values[i]), eigen_vectors[:, i])
               for i in range(len(eigen_values))]
eigen_pairs = sorted(eigen_pairs, key=lambda k: k[0], reverse=True)
print(f'Eigenvalues in descending order: \n')
for eigen_value in eigen_pairs:
    print(eigen_value[0])

# In LDA, the number of linear discriminants is at most c−1, where c is the
# number of class labels, since the in-between scatter matrix, S(B), is the
# sum of c matrices with rank one or less. We can indeed see that we only have
# two nonzero eigenvalues (the eigenvalues 3-13 are not exactly zero, but this
# is due to the floating-point arithmetic in NumPy).

tot = sum(eigen_values.real)
discr = [(i / tot) for i in sorted(eigen_values.real, reverse=True)]
cum_discr = np.cumsum(discr)

plt.bar(range(1, 14), discr, alpha=0.5, align='center',
        label='Individual "discriminability"')
plt.step(range(1, 14), cum_discr, where='mid',
         label='Cumulative "discriminablity"')
plt.ylabel('"Discriminability" ratio')
plt.xlabel('Linear Discriminants')
plt.ylim([-0.1, 1.1])
plt.legend(loc='best')
plt.tight_layout()
plt.show()

# 6 Choose the k eigenvectors that correspond to the k largest eigenvalues
# to construct a d * k - dimensional transformation matrix, W; the
# eigenvectors are the columns of this matrix.

w = np.hstack((eigen_pairs[0][1][:, np.newaxis].real,
               eigen_pairs[1][1][:, np.newaxis].real))
print('Matrix W: \n', w)

# 7 Project the examples onto the new feature space using the transformation
# matrix, W.

X_train_lda = X_train.dot(w)
colors = ['r', 'b', 'g']
markers = ['s', 'x', 'o']

for l, c, m in zip(np.unique(y_train), colors, markers):
    plt.scatter(X_train_lda[y_train == l, 0],
                X_train_lda[y_train == l, 1] * (-1),
                c=c, label=l, marker=m)

plt.xlabel('LD 1')
plt.ylabel('LD 2')
plt.legend(loc='lower right')
plt.tight_layout()
plt.show()


import scipy.stats as spss
import pandas as pd

# Data - continuous distributions
df_uniform = spss.uniform.rvs(size=10000,loc=10,scale=20)
df_normal = spss.norm.rvs(size=10000,loc=10,scale=100)
df_std_norm = spss.norm.rvs(size=10000,loc=0,scale=1)
df_gamma = spss.gamma.rvs(a=5,size=10000)
df_exponential = spss.expon.rvs(scale=1,loc=0,size=10000)
df_beta = spss.beta.rvs(5, 2, loc=0, scale=1, size=10000) #, random_state=None)
# discrete distributions
df_binomial = spss.binom.rvs(n=10,p=0.8,size=10000)
df_poisson = spss.poisson.rvs(mu=3,size=10000)
df_bernoulli = spss.bernoulli.rvs(size=10000,p=0.6)

# Plot
from plotly.subplots import make_subplots
import plotly.graph_objects as go
fig = make_subplots(rows=3, cols=3, # shared_xaxes=True,
     vertical_spacing=0.2, horizontal_spacing=0.1,
    specs=[[{"type": "xy"}, {"type": "xy"}, {"type": "xy"}],
           [{"type": "xy"}, {"type": "xy"}, {"type": "xy"}],
           [{"type": "xy"}, {"type": "xy"}, {"type": "xy"}]])
    #subplot_titles=("Plot 1", "Plot 2"))

fig.update_layout(template="plotly_dark")

fig.add_trace(go.Histogram(x=df_uniform, showlegend=False),
              row=1, col=1,)
fig.add_trace(go.Histogram(x=df_normal, showlegend=False),
              row=1, col=2,)
fig.add_trace(go.Histogram(x=df_std_norm, showlegend=False),
              row=1, col=3,)
fig.add_trace(go.Histogram(x=df_gamma, showlegend=False),
              row=2, col=1,)
fig.add_trace(go.Histogram(x=df_exponential, showlegend=False),
              row=2, col=2,)
fig.add_trace(go.Histogram(x=df_beta, showlegend=False),
              row=2, col=3,)
fig.add_trace(go.Histogram(x=df_binomial, showlegend=False),
              row=3, col=1,)
fig.add_trace(go.Histogram(x=df_poisson, showlegend=False),
              row=3, col=2,)
fig.add_trace(go.Histogram(x=df_bernoulli, showlegend=False),
              row=3, col=3,)

# Title
offset_top=0.12
fig.add_annotation(dict(xref='paper',yref='paper',x=0.5,y=offset_top+1.22,xanchor='center',yanchor='top',
  font=dict(family='Arial',size=24,color='grey'),showarrow=False,
  text="Statistical Distributions"))
# Subtitle
fig.add_annotation(dict(xref='paper',yref='paper',x=0.5,y=offset_top+1.095,xanchor='center',yanchor='top',
  font=dict(family='Arial',size=16,color='grey'),showarrow=False,
  text="Arrangement of values of a variable showing their frequency of occurrence"))
# Footer
offset_bottom=0.05
fig.add_annotation(dict(xref='paper',yref='paper',x=0.5,y=-(offset_bottom+0.02),xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='#30DayChartChallenge - statistics - 2021/04/09'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.5,y=-(offset_bottom+0.075),xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Datasets created using scipy.stats'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.5,y=-(offset_bottom+0.15),xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='twitter.com/vivekparasharr | github.com/vivekparasharr | vivekparasharr.medium.com'))

# Annotations
offset_chart_titles = 1.12
fig.add_annotation(dict(xref='paper',yref='paper',x=0.13,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Uniform Distribution'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.48,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Normal Distribution'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.84,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Standard Normal Distribution'))
offset_chart_titles = 0.72
fig.add_annotation(dict(xref='paper',yref='paper',x=0.13,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Gamma Distribution'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.48,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Exponential Distribution'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.84,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Beta Distribution'))
offset_chart_titles = 0.32
fig.add_annotation(dict(xref='paper',yref='paper',x=0.13,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Binomial Distribution'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.48,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Poisson Distribution'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.84,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=12, color='grey'),showarrow=False,
  text='Bernoulli Normal Distribution'))

# Annotations for formulas
offset_chart_titles = 1.07
fig.add_annotation(dict(xref='paper',yref='paper',x=0.13,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=10, color='grey'),showarrow=False,
  text='uniform.rvs(size=10000,loc=10,scale=20)'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.48,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=10, color='grey'),showarrow=False,
  text='norm.rvs(size=10000,loc=10,scale=100)'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.84,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=10, color='grey'),showarrow=False,
  text='norm.rvs(size=10000,loc=0,scale=1)'))
offset_chart_titles = 0.67
fig.add_annotation(dict(xref='paper',yref='paper',x=0.13,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=10, color='grey'),showarrow=False,
  text='gamma.rvs(a=5,size=10000)'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.48,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=10, color='grey'),showarrow=False,
  text='expon.rvs(scale=1,loc=0,size=10000)'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.84,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=10, color='grey'),showarrow=False,
  text='beta.rvs(5, 2, loc=0, scale=1, size=10000)'))
offset_chart_titles = 0.27
fig.add_annotation(dict(xref='paper',yref='paper',x=0.13,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=10, color='grey'),showarrow=False,
  text='binom.rvs(n=10,p=0.8,size=10000)'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.48,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=10, color='grey'),showarrow=False,
  text='poisson.rvs(mu=3,size=10000)'))
fig.add_annotation(dict(xref='paper',yref='paper',x=0.84,y=offset_chart_titles,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=10, color='grey'),showarrow=False,
  text='bernoulli.rvs(size=10000,p=0.6)'))

# Annotations for type of distribution
fig.add_annotation(dict(xref='paper',yref='paper',x=-0.09,y=1.06,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=16, color='grey'),showarrow=False,
  text='Continuous', textangle=270))
fig.add_annotation(dict(xref='paper',yref='paper',x=-0.09,y=0.68,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=16, color='grey'),showarrow=False,
  text='Continuous', textangle=270))
fig.add_annotation(dict(xref='paper',yref='paper',x=-0.09,y=0.25,xanchor='center',yanchor='top',
  font=dict(family='Arial', size=16, color='grey'),showarrow=False,
  text='Discrete', textangle=270))

fig.show()


"""
Statistical tests.
"""
import numpy as np

from split_data import split_by_label

def anova_test(y: np.ndarray, x: np.ndarray,
               ind_col_anova: list = None,
               label_b: int = -1) -> tuple:
    """Anova test on features for features selection.

    Args:
        y (np.ndarray): input data features.
        x (np.ndarray): input data.
        ind_col_anova (list) : index of x columns.
        label_b (int, optional): label of "b" event. Defaults to -1.

    Returns:
        tuple : (ind_F, F) with F f_value of features sorted,
        ind_F index of features.
    """
    tX_a, tX_b = split_by_label(y, x, label_b)
    mean_labels = [np.mean(tX_a, 0), np.mean(tX_b, 0)]
    var_between_groups = np.var(mean_labels, 0)
    var_in_groups = np.var(tX_a, 0) + np.var(tX_b, 0)
    f = var_between_groups / var_in_groups
    F = np.argsort(f)
    if ind_col_anova is not None:
        for i in range(0, len(F)):
            F[i] = ind_col_anova[F[i]]
    return F, np.sort(f)


import numpy as np
from scipy.stats import gaussian_kde
import matplotlib
import matplotlib.pyplot as plt

# class defining the posterior distribution for the coin flip example
class target_coin_flip():
    def __init__(self, NN, NH):
        self.NN = NN # number of tosses
        self.NH = NH # number of heads
    def evaluate(self, H):
        # samples from the posterior
        pH=1 if (H>=0 and H<=1) else 0.0
        return np.power(H,self.NH)*np.power(1-H,self.NN-self.NH)*pH

#class defining the logarithmic representation of the posterior
class target_log_coin_flip():
    # TODO : use the "target_coin_flip" class as a starting point
    #        and edit appropriately to convert to logarithmic scale
    def __init__(self, num_tosses, num_heads):
        self.num_heads = num_heads
        self.num_tails = num_tosses - num_heads

    def evaluate(self, H):
        if 0.0 < H < 1.0:
            return self.num_heads * np.log(H) + self.num_tails * np.log(1 - H)
        else:
            return -np.inf

def MCMC(target, starting_sample, num_iters=1e6, burnin=1e4):
    print("Running MCMC")
    # Markov Chain Monte Carlo (MCMC) - Metropolis
    current_sample = starting_sample
    # proposal distribution is a Gaussian with mean 0.0 and std 0.1
    proposal_mean = 0.0
    proposal_sigma = 0.1
    samples = []
    samples.append(starting_sample)
    for iter_ in range(int(num_iters)):
        # GENERATION BASED ON PROPOSAL
        next_sample_candidate = current_sample + proposal_mean + np.random.randn()*proposal_sigma
        # ACCEPTANCE PROBABILITY - Metropolis
        acceptance_prob = np.min([1.0, target.evaluate(next_sample_candidate)/(target.evaluate(current_sample))])
        # ACCEPT OR REJECT with uniform probability
        temp = np.random.rand()
        if temp <= acceptance_prob:
            # ACCEPTING THE CANDIDATE SAMPLE
            samples.append(next_sample_candidate)
            current_sample = next_sample_candidate
        else:
            samples.append(current_sample)
    if len(samples)>burnin:
        # keep only samples after burn-in iterations
        samples=samples[int(burnin):]
    else:
        raise ValueError("Number of samples {:} smaller than burnin period {:}.".format(len(samples), burnin))
    return samples

def MCMCLOG(logtarget, starting_sample, num_iters=1e6, burnin=1e4):
    print("Running MCMCMHLOG")
    # Markov Chain Monte Carlo (MCMC) - Metropolis Hastings
    # TODO: start from the "MCMC" function above and edit whenever
    #       necessary to convert to logarithmic scale
    current_sample = starting_sample
    # proposal distribution is a Gaussian with mean 0.0 and std 0.1
    proposal_mean = 0.0
    proposal_sigma = 0.1
    samples = [starting_sample]
    for _ in range(int(num_iters)):
        # GENERATION BASED ON PROPOSAL
        next_sample_candidate = current_sample + proposal_mean + np.random.randn()*proposal_sigma
        # ACCEPTANCE PROBABILITY - Metropolis
        acceptance_prob = np.min([0.0, logtarget.evaluate(next_sample_candidate) - logtarget.evaluate(current_sample)])
        # ACCEPT OR REJECT with uniform probability
        temp = np.log(np.random.rand())
        if temp <= acceptance_prob:
            # ACCEPTING THE CANDIDATE SAMPLE
            samples.append(next_sample_candidate)
            current_sample = next_sample_candidate
        else:
            samples.append(current_sample)
    if len(samples)>burnin:
        # keep only samples after burn-in iterations
        samples=samples[int(burnin):]
    else:
        raise ValueError("Number of samples {:} smaller than burnin period {:}.".format(len(samples), burnin))
    return samples

if __name__ == "__main__":

    #IN NN TOSSES, NH TIMES HEAD (NN>=NH)
    NN = 3000 # NN = 300 / 3000 tosses
    NH = 1500 # NH = 150 / 1500 heads
    burnin = 1e2 # number of burn-in iterations
    num_iters = 1e5 # number of total MCMC iterations
    starting_sample = 0.5 # starting point for MCMC algorithm

    target = target_coin_flip(NN,NH)
    samples_mcmc = MCMC(target, starting_sample, num_iters, burnin)
    # plot histogram with samples drawn from posterior distribution with the MCMC algorithm
    plt.hist(samples_mcmc, density=True, facecolor='g', alpha=0.7, bins=10, label="MCMC") # if you're using python>3, use density=True instead of normed=True
    plt.xlim([0,1])
    plt.legend()

    target = target_log_coin_flip(NN,NH)
    samples_mcmclog = MCMCLOG(target, starting_sample, num_iters, burnin)
    plt.hist(samples_mcmclog, density=True, facecolor='b', alpha=0.4, bins=10, label="MCMC-LOG")
    plt.xlim([0,1])
    plt.legend()
    plt.show()


import numpy as np
from itertools import accumulate

def time_series_sim(k,ga,diff,delta_t,N,G):
    """ returns a time series that is the solution of a Langevin equation describing a Brownian particle in a harmonic potential
    :param k: Spring constant
    :param ga: friction coefficient
    :param diff: Diffusion coefficient
    :param delta_t: time step
    :param N: number of samples that are returned
    :param G: ratio of simulated data points and returned samples (100 means that only every 100th point is used)
    :return:
    """
    # differential equation x_i = x_(i-1) - k/gamma*x_(i-1) + sqrt(2*D*delta_t)*w_i
    def next_point(x, y):
        amplitude = np.sqrt(2 * diff * delta_t / float(G))
        return x - k / ga * x * delta_t / float(G) + amplitude * y

    # random force
    w=np.random.normal(0,1,N*G)
    xx = np.fromiter(accumulate(w, next_point), np.float)

    return xx[::G]  # only use every G point

def time_series(A,D,delta_t,N):
    """ returns a time series that is the solution of a Langevin equation describing a Brownian particle in a harmonic potential
    :param A: mean square amplitude - oscillator strength
    :param D: Diffusion coefficient
    :param delta_t: time step
    :param N: number of samples that are returned
    :return:
    """
    # using Smolukowski solution for simulation

    #first point
    x=[np.random.normal(0,np.sqrt(A))]
    stddev = np.sqrt(A * (1.0 - np.exp(-2.0 * D / A * delta_t)))

    for i in range(N-1):
        x.append(np.random.normal(x[-1]*np.exp(-D/A*delta_t),stddev))

    return np.array(x)

def time_series2(A,gamma,delta_t,N):
    """ returns a time series that is the solution of a Langevin equation describing a Brownian particle in a harmonic potential
    :param A: mean square amplitude - oscillator strength
    :param gamma: friction coefficient
    :param delta_t: time step
    :param N: number of samples that are returned
    :return:
    """
    # using Smolukowski solution for simulation

    #first point
    x=[np.random.normal(0,np.sqrt(A))]
    stddev = np.sqrt(A * (1.0 - np.exp(-2.0 * gamma * delta_t)))

    for i in range(N-1):
        x.append(np.random.normal(x[-1]*np.exp(-gamma*delta_t),stddev))

    return np.array(x)

# differential equation x_i = x_(i-1) - k/gamma*x_(i-1) + sqrt(2*D*delta_t)*w_i
# using 4th-order Runge-Kutta
def next_point_RK4(x,y):
    k0=-a*x
    k1=-a*(x+k0/2.0)
    k2=-a*(x+k1/2.0)
    k3=-a*(x+k2)
    return x + (k0+2*k1+2*k2+k3)/6 + ampl*y


#
# Description: example of a bootstrapping and forward curve generation
# script, this can be used to build a set of curves for different currencies
# TODO: include some spline smoothing to the zero curve, from first principles!
#

from sympy.solvers import solve
from sympy import Symbol, abs, Real
x = Symbol('x', real=True)

import pylab as pylab

def g(yieldCurve, zeroRates,n, verbose):
    '''
        generates recursively the zero curve
        expressions eval('(0.06/1.05)+(1.06/(1+x)**2)-1')
        solves these expressions to get the new rate
        for that period

    '''
    if len(zeroRates) >= len(yieldCurve):
        print("\n\n\t+zero curve boot strapped [%d iterations]" % (n))
        return
    else:
        legn = ''
        for i in range(0,len(zeroRates),1):
            if i == 0:
                legn = '%2.6f/(1+%2.6f)**%d'%(yieldCurve[n], zeroRates[i],i+1)
            else:
                legn = legn + ' +%2.6f/(1+%2.6f)**%d'%(yieldCurve[n], zeroRates[i],i+1)
        legn = legn + '+ (1+%2.6f)/(1+x)**%d-1'%(yieldCurve[n], n+1)
        # solve the expression for this iteration
        if verbose:
            print("-[%d] %s" % (n, legn.strip()))
        rate1 = solve(eval(legn), x)
        # Abs here since some solutions can be complex
        rate1 = min([Real(abs(r)) for r in rate1])
        if verbose:
            print("-[%d] solution %2.6f" % (n, float(rate1)))
        # stuff the new rate in the results, will be
        # used by the next iteration
        zeroRates.append(rate1)
        g(yieldCurve, zeroRates,n+1, verbose)

verbose = True
tenors = [.1,.25,0.5,1,2,3,5,7,10,20,30]
#
# money market, futures, swap rates
#
yieldCurve = [0.07,	0.09,	0.15,	0.21,	0.37,	0.57,	1.13,	1.70,	2.31,	3.08	,3.41]

#yieldCurve = [0.05, 0.06, 0.07, 0.08 ,0.085 ,0.0857 ,0.0901,0.0915,0.0925,0.0926,0.0934,0.0937]
zeroRates = [yieldCurve[0]] # TODO: check that this is the correct rate

print("\n\n\tAlexander Baker, March 2012\n\tYield Curve Bootstrapper\n\tAlexander Baker\n\n")

# kick off the recursive code
g(yieldCurve, zeroRates, 1, verbose)
print("\tZeroRate Array",zeroRates)

pylab.plot(tenors,yieldCurve)
pylab.plot(tenors,zeroRates)
pylab.show()


from aerosandbox.numpy import sin, cos, linalg
from aerosandbox.numpy.array import array
import numpy as _onp
from typing import Union, List

def rotation_matrix_2D(
        angle,
        as_array: bool = True,
):
    """
    Gives the 2D rotation matrix associated with a counterclockwise rotation about an angle.
    Args:
        angle: Angle by which to rotate. Given in radians.
        as_array: Determines whether to return an array-like or just a simple list of lists.

    Returns: The 2D rotation matrix

    """
    s = sin(angle)
    c = cos(angle)
    rot = [
        [c, -s],
        [s, c]
    ]
    if as_array:
        return array(rot)
    else:
        return rot

def rotation_matrix_3D(
        angle: Union[float, _onp.ndarray],
        axis: Union[_onp.ndarray, List, str],
        as_array: bool = True,
        axis_already_normalized: bool = False
):
    """
    Yields the rotation matrix that corresponds to a rotation by a specified amount about a given axis.

    An implementation of https://en.wikipedia.org/wiki/Rotation_matrix#Rotation_matrix_from_axis_and_angle

    Args:

        angle: The angle to rotate by. [radians]
        Direction of rotation corresponds to the right-hand rule.
        Can be vectorized.

        axis: The axis to rotate about. [ndarray]
        Can be vectorized; be sure axis[0] yields all the x-components, etc.

        as_array: boolean, returns a 3x3 array-like if True, and a list-of-lists otherwise.

            If you are intending to use this function vectorized, it is recommended you flag this False. (Or test before
            proceeding.)

        axis_already_normalized: boolean, skips axis normalization for speed if you flag this true.

    Returns:
        The rotation matrix, with type according to the parameter `as_array`.
    """
    s = sin(angle)
    c = cos(angle)

    if isinstance(axis, str):
        if axis.lower() == "x":
            rot = [
                [1, 0, 0],
                [0, c, -s],
                [0, s, c]
            ]
        elif axis.lower() == "y":
            rot = [
                [c, 0, s],
                [0, 1, 0],
                [-s, 0, c]
            ]
        elif axis.lower() == "z":
            rot = [
                [c, -s, 0],
                [s, c, 0],
                [0, 0, 1]
            ]
        else:
            raise ValueError("If `axis` is a string, it must be `x`, `y`, or `z`.")
    else:
        ux = axis[0]
        uy = axis[1]
        uz = axis[2]

        if not axis_already_normalized:
            norm = (ux ** 2 + uy ** 2 + uz ** 2) ** 0.5
            ux = ux / norm
            uy = uy / norm
            uz = uz / norm

        rot = [
            [c + ux ** 2 * (1 - c), ux * uy * (1 - c) - uz * s, ux * uz * (1 - c) + uy * s],
            [uy * ux * (1 - c) + uz * s, c + uy ** 2 * (1 - c), uy * uz * (1 - c) - ux * s],
            [uz * ux * (1 - c) - uy * s, uz * uy * (1 - c) + ux * s, c + uz ** 2 * (1 - c)]
        ]

    if as_array:
        return array(rot)
    else:
        return rot

def rotation_matrix_from_euler_angles(
        roll_angle: Union[float, _onp.ndarray] = 0,
        pitch_angle: Union[float, _onp.ndarray] = 0,
        yaw_angle: Union[float, _onp.ndarray] = 0,
        as_array: bool = True
):
    """
    Yields the rotation matrix that corresponds to a given Euler angle rotation.

    Note: This uses the standard (yaw, pitch, roll) Euler angle rotation, where:
    * First, a rotation about x is applied (roll)
    * Second, a rotation about y is applied (pitch)
    * Third, a rotation about z is applied (yaw)

    In other words: R = R_z(yaw) @ R_y(pitch) @ R_x(roll).

    Note: To use this, pre-multiply your vector to go from body axes to earth axes.
        Example:
            >>> vector_earth = rotation_matrix_from_euler_angles(np.pi / 4, np.pi / 4, np.pi / 4) @ vector_body

    See notes:
    http://planning.cs.uiuc.edu/node102.html

    Args:
        roll_angle: The roll angle, which is a rotation about the x-axis. [radians]
        pitch_angle: The pitch angle, which is a rotation about the y-axis. [radians]
        yaw_angle: The yaw angle, which is a rotation about the z-axis. [radians]
        as_array:

    Returns:

    """
    sa = sin(yaw_angle)
    ca = cos(yaw_angle)
    sb = sin(pitch_angle)
    cb = cos(pitch_angle)
    sc = sin(roll_angle)
    cc = cos(roll_angle)

    rot = [
        [ca * cb, ca * sb * sc - sa * cc, ca * sb * cc + sa * sc],
        [sa * cb, sa * sb * sc + ca * cc, sa * sb * cc - ca * sc],
        [-sb, cb * sc, cb * cc]
    ]

    if as_array:
        return array(rot)
    else:
        return rot

def is_valid_rotation_matrix(
        a: _onp.ndarray,
        tol=1e-9
) -> bool:
    """
    Returns a boolean of whether the given matrix satisfies the properties of a rotation matrix.

    Specifically, tests for:
        * Volume-preserving
        * Handedness of output reference frame
        * Orthogonality of output reference frame

    Args:
        a: The array-like to be tested
        tol: A tolerance to use for truthiness; accounts for floating-point error.

    Returns: A boolean of whether the array-like is a valid rotation matrix.

    """

    def approx_equal(x, y):
        return (x > y - tol) and (x < y + tol)

    det = linalg.det(a)
    is_volume_preserving_and_right_handed = approx_equal(det, 1)

    eye_approx = a.T @ a
    eye = _onp.eye(a.shape[0])
    is_orthogonality_preserving = True
    for i in range(eye.shape[0]):
        for j in range(eye.shape[1]):
            if not approx_equal(eye_approx[i, j], eye[i, j]):
                is_orthogonality_preserving = False

    return (
            is_volume_preserving_and_right_handed and
            is_orthogonality_preserving
    )


import numpy as np
import matplotlib.pyplot as plt
import csv
import math
from pprint import pprint

# Read data from csv file
data = []
with open('UsaFuelPassenger.csv') as f:
    data = [list(map(float, i)) for i in csv.reader(f)]
    # random.shuffle(data)

# Make train sets
x_train = np.array(list(map(lambda x: x[0], data[:])), dtype='float')
y_train = np.array(list(map(lambda y: y[1], data[:])), dtype='float')

# Make validation sets
x_val = np.array(list(map(lambda x: x[0], data[8:])), dtype='float')
y_val = np.array(list(map(lambda y: y[1], data[8:])), dtype='float')

plt.scatter(x_train, y_train, c='blue')
plt.scatter(x_val, y_val, c='red')
data.sort()  # uncommend it , to see graph with connected points.
plt.plot(list(map(lambda x: x[0], data)), list(map(lambda x: x[1], data)), c='black')
plt.show()

# In[82]:

# Actual polynomial output value
def polynomial_output(x, a):
    return np.sum([param[0] * x ** i for i, param in enumerate(a)], axis=0)

# Design matrix
def dm(x_train, M):
    return np.array([[x ** j for j in range(M + 1)] for x in x_train], dtype='float')

# Mean squared error
def MSE(x, y, a):
    return np.mean(np.square(y - polynomial_output(x, a)))

# Draw model
def plot_model(a, x_train, y_train, x_val, y_val, M=None, error=None, lambda_=None):
    x = np.linspace(0, max(x_train) + 1, 100)
    y = polynomial_output(x, a)

    plt.scatter(x_train, y_train, c='blue', label='Training data')
    plt.scatter(x_val, y_val, c='red', label='Validation data')
    plt.plot(x, y, c='red', label=f'λ: {lambda_}')

    plt.xlim(right=max(x_train) + 1, left=0)
    plt.ylim(top=max(y_train) + 20, bottom=0)

    if error:
        plt.title(f'Error = {error}')
    plt.suptitle(f'M = {M}')
    plt.legend()
    plt.show()

# Model selection
M_range = range(8)
lambdas = [0.005, 0.03, 0.4]

# First we need to find the power of polynomial
# by choosing the best model with the least training error.
y = y_train[:, None]
models = []

for M in M_range:
    X = dm(x_train, M)
    f = X.transpose() @ X
    s = np.linalg.inv(f)
    th = s @ X.transpose()
    a = th @ y  # https://en.wikipedia.org/wiki/Polynomial_regression#Matrix_form_and_calculation_of_estimates
    # a = np.linalg.inv(X.transpose() @ X) @ X.transpose() @ y  # https://en.wikipedia.org/wiki/Polynomial_regression#Matrix_form_and_calculation_of_estimates
    val_error = MSE(x_val, y_val, a)
    models.append([a, val_error, M])
    plot_model(a, x_train, y_train, x_val, y_val, M=M, error=val_error)

model = min(models, key=lambda x: x[1])

print('Best fitted model to training data:')
plot_model(model[0], x_train, y_train, x_val, y_val, M=model[2], error=model[1])

# Then we can avoid overfitting with regularization.
regs = []
for regularization_lambda in lambdas:
    X = dm(x_train, model[2])
    eps = regularization_lambda * np.eye(model[2] + 1)
    a = np.linalg.inv(X.transpose() @ X + eps) @ X.transpose() @ y
    regs.append([a, MSE(x_train, y_train, a), MSE(x_val, y_val, a), regularization_lambda, model[2]])
    plt.show()

# Plot best model
print('Best model:')
lambda_ = min(regs, key=lambda x: x[2])
plot_model(lambda_[0], x_train, y_train, x_val, y_val, M=lambda_[4], error=lambda_[2], lambda_=lambda_[3])
print(lambda_[0])

# In[85]:

def get_prediction(x, a):
    y = polynomial_output(x, a)
    if y < 0:
        y = 0
    return y

x = 2
print(get_prediction(x, lambda_[0]))


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
from neldermead import NelderMead
from tests import utils
from scipy.optimize import minimize
import math

def rastrigin(x):
    x = x.reshape(-1)
    dim = len(x)
    return 10.0 * dim + np.sum([x[i] ** 2 - 10.0 * math.cos(2.0 * math.pi * x[i]) for i in range(dim)])

def test_run_d10_rastrigin():
    print("test_run_d10:")
    dim = 10
    iteration_number = 1000
    centroid = np.zeros([dim, 1])
    gamma = np.sqrt(dim)
    simplex = utils.get_regular_simplex(dim, centroid, gamma)

    nm = NelderMead(dim, rastrigin, simplex)
    x_best, f_best = nm.optimize(iteration_number)
    print("f_best:{}".format(f_best))

    params = {'maxiter': iteration_number,
              'initial_simplex': simplex.T,
              'xatol': 1e-20,
              'fatol': 1e-20
              }
    scipy_nm_f_best = minimize(rastrigin, x0=np.zeros(dim), method='Nelder-Mead', options=params).fun

    print("scipy_f_best:{}".format(scipy_nm_f_best))

    assert abs(f_best - scipy_nm_f_best) < 1e-5


"""
Implementation of the bigClAM algorithm.

Throughout the code, we will use tho following variables

  * F refers to the membership preference matrix. It's in [NUM_PERSONS, NUM_COMMUNITIES]
   so index (p,c) indicates the preference of person p for community c.
  * A refers to the adjency matrix, also named friend matrix or edge set. It's in [NUM_PERSONS, NUM_PERSONS]
    so index (i,j) indicates is 1 when person i and person j are friends.
"""

import numpy as np
import pickle
from util.generate_data import Datagen, gen_json,gen_json2
from util.pre_data import preData
import json

def sigm(x):
    return np.divide(np.exp(-1.*x),1.-np.exp(-1.*x))

def log_likelihood(F, A):
    """implements equation 2 of
    https://cs.stanford.edu/people/jure/pubs/bigclam-wsdm13.pdf"""
    A_soft = F.dot(F.T)

    # Next two lines are multiplied with the adjacency matrix, A
    # A is a {0,1} matrix, so we zero out all elements not contributing to the sum
    FIRST_PART = A*np.log(1.-np.exp(-1.*A_soft))
    sum_edges = np.sum(FIRST_PART)
    SECOND_PART = (1-A)*A_soft
    sum_nedges = np.sum(SECOND_PART)

    log_likeli = sum_edges - sum_nedges
    return log_likeli

def gradient(F, A, i):
    """Implements equation 3 of
    https://cs.stanford.edu/people/jure/pubs/bigclam-wsdm13.pdf

      * i indicates the row under consideration

    The many forloops in this function can be optimized, but for
    educational purposes we write them out clearly
    """
    N, C = F.shape

    neighbours = np.where(A[i])
    nneighbours = np.where(1-A[i])

    sum_neigh = np.zeros((C,))
    for nb in neighbours[0]:
        dotproduct = F[nb].dot(F[i])
        sum_neigh += F[nb]*sigm(dotproduct)

    sum_nneigh = np.zeros((C,))
    #Speed up this computation using eq.4
    for nnb in nneighbours[0]:
        sum_nneigh += F[nnb]

    grad = sum_neigh - sum_nneigh
    return grad

def train(A, C, iterations = 100):
    # initialize an F
    N = A.shape[0]
    F = np.random.rand(N,C)

    for n in range(iterations):
        for person in range(N):
            grad = gradient(F, A, person)

            F[person] += 0.005*grad

            F[person] = np.maximum(0.001, F[person]) # F should be nonnegative
        ll = log_likelihood(F, A)
        print('At step %5i/%5i ll is %5.3f'%(n, iterations, ll))
    return F

if __name__ == "__main__":

    # p2c = pickle.load(open('data/p2c.pl','rb'))
    ##generate data
    # datagen = Datagen(100, [.3, .3, .2, .2],[.2, .3, .3, .2] , .1).gen_assignments().gen_adjacency()
    # p2c = datagen.person2comm
    # adj = datagen.adj
    adj = preData()
    F = train(adj, 37)
    F_argmax = np.argmax(F,1)
    # data = gen_json(adj, p2c, F_argmax)
    data=gen_json2(adj,F_argmax)
    # with open('../data/data.json','w') as f:
    with open('ui/assets/data.json','w') as f:
        json.dump(data,f, indent=4)

    for i,row in enumerate(F):
        print(row)
        # print(p2c[i])


from collections import Counter
from typing import Dict, Sequence, List, Tuple, Hashable, Optional

from sympy import laguerre, assoc_laguerre, prod, factorial, Rational, Poly
from sympy.abc import x

from ccc.errors import ConstraintNotImplementedError

class PermutationCounter:
    """
    Count constrained permutations of a sequence.

    """

    def __init__(
        self,
        sequence: Sequence[Hashable],
        constraints: Optional[List[Tuple]] = None,
        same_distinct: bool = False,
    ) -> None:

        self.sequence: Sequence[Hashable] = sequence
        self.length: int = len(sequence)
        self.frequencies: Counter = Counter(sequence)

        self.constraints: List[Tuple] = constraints
        self.polynomials: Dict[Hashable, Poly] = {}

        self.same_distinct: bool = same_distinct
        self._correction_factor: int = 1

        if not constraints:
            return

        if len(constraints) == 1:
            op, *args = constraints[0]
            getattr(self, "impose_constraint_" + op)(*args)

        else:
            raise ConstraintNotImplementedError(
                "Imposing multiple constraints is not supported for permuations"
            )

    def impose_constraint_derangement(self) -> None:
        """
        Permutation has no items in their original position.

        Uses the Laguerre polynomial approach detailed in
        "Derangements and Laguerre polynomials", S. Even & J. Gillis,
        Mathematical Proceedings of the Cambridge Philosophical Society,
        Volume 79, Issue 1 January 1976 , pp. 135-143.

        """
        for item, count in self.frequencies.items():
            self.polynomials[item] = laguerre(count, x)

        if self.same_distinct:
            self._correction_factor = prod(factorial(freq) for freq in self.frequencies.values())

    def impose_constraint_no_adjacent(self) -> None:
        """
        Permutation has no consecutive items equal to each other.

        Uses the generalised Laguerre polynomial approach detailed in
        "Counting words with Laguerre series", J. Taylor, The Electronic
        Journal of Combinatorics (E-JC), Volume 21, Issue 2 (2014).

        (https://www.combinatorics.org/ojs/index.php/eljc/article/view/v21i2p1)

        """
        for item, count in self.frequencies.items():
            self.polynomials[item] = assoc_laguerre(count, -1, x)

        if self.same_distinct:
            self._correction_factor = prod(factorial(freq) for freq in self.frequencies.values())

    def count_unconstrained_permutations(self) -> int:
        """
        Number of permutations where no constraint is specified.

        If a, b, ..., n are the counts of each item in the sequence,
        this is equal to:

            len(sequence)! / a! / b! / ... /n!

        If instances of the same symbol are considered distinct, then
        the number of permutations is equal to:

            len(sequence)!

        """
        if self.same_distinct:
            return factorial(self.length)

        return factorial(self.length) / prod(factorial(freq) for freq in self.frequencies.values())

    def probability(self) -> Rational:
        """
        Probability that a permutation of the sequence meets the
        specified contraints.
        """
        return self.count() / self.count_unconstrained_permutations()

    def count(self) -> int:
        """
        Number of permutations of the sequence that meets the
        specified contraints.
        """
        if not self.constraints:
            return self.count_unconstrained_permutations()

        terms = prod(self.polynomials.values()).apart().as_ordered_terms()
        return abs(sum(eval_gamma(t) for t in terms)) * self._correction_factor

def eval_gamma(term):
    """
    Evaluate the term k*x**n as k*n!
    """
    coeff, exp = term.as_coeff_exponent(x)
    return coeff * factorial(exp)


import numpy as np
from scipy.spatial.distance import squareform
from scipy.cluster.hierarchy import linkage

def seriation(Z,N,cur_index):
    '''
        input:
            - Z is a hierarchical tree (dendrogram)
            - N is the number of points given to the clustering process
            - cur_index is the position in the tree for the recursive traversal
        output:
            - order implied by the hierarchical tree Z

        seriation computes the order implied by a hierarchical tree (dendrogram)
    '''
    if cur_index < N:
        return [cur_index]
    else:
        left = int(Z[cur_index-N,0])
        right = int(Z[cur_index-N,1])
        return (seriation(Z,N,left) + seriation(Z,N,right))

def compute_serial_matrix(dist_mat,method="ward"):
    '''
        input:
            - dist_mat is a distance matrix
            - method = ["ward","single","average","complete"]
        output:
            - seriated_dist is the input dist_mat,
              but with re-ordered rows and columns
              according to the seriation, i.e. the
              order implied by the hierarchical tree
            - res_order is the order implied by
              the hierarhical tree
            - res_linkage is the hierarhical tree (dendrogram)

        compute_serial_matrix transforms a distance matrix into
        a sorted distance matrix according to the order implied
        by the hierarchical tree (dendrogram)
    '''
    N = len(dist_mat)
    flat_dist_mat = squareform(dist_mat)
    res_linkage = linkage(flat_dist_mat, method=method, optimal_ordering=True)
    res_order = seriation(res_linkage, N, N + N-2)
    seriated_dist = np.zeros((N,N))
    a,b = np.triu_indices(N,k=1)
    seriated_dist[a,b] = dist_mat[ [res_order[i] for i in a], [res_order[j] for j in b]]
    seriated_dist[b,a] = seriated_dist[a,b]

    return seriated_dist, res_order, res_linkage


# -*- coding: utf-8 -*-
"""
Created on Mon Feb 14 11:40:09 2022

@author: pendl
"""

import cv2
import numpy as np
import matplotlib.pyplot as plt

def computeSVD(mat):

    A=np.array(mat)
    m , n =A.shape
    #print(A.shape)
    AAt=np.dot(A,A.T)
    AtA=np.dot(A.T,A)

    w, U = np.linalg.eig(AAt)
    idx = np.flip(np.argsort(w))
    eval_AAt = w[idx]
    U = U[:, idx]

    p = min(m,n)
    sigma=np.zeros((m,n))
    for i in range(p):
        sigma[i,i]= np.abs(np.sqrt(eval_AAt[i]))

    w_2, V = np.linalg.eig(AtA)
    idx = np.flip(np.argsort(w_2))
    eval_AtA = w_2[idx]
    V= V[:,idx]
    return U,sigma,V

def computeHomography(ps1, ps2):

    if (len(ps1) < 4) or (len(ps2) < 4):
        print("Need atleast four points to compute SVD.")
        return 0

    x = ps1[:, 0]
    y = ps1[:, 1]
    xp = ps2[:, 0]
    yp = ps2[:,1]

    nrows= 8

    A = []
    for i in range(int(nrows/2)):
        row1 = np.array([-x[i], -y[i], -1, 0, 0, 0, x[i]*xp[i], y[i]*xp[i], xp[i]])
        A.append(row1)
        row2 = np.array([0, 0, 0, -x[i], -y[i], -1, x[i]*yp[i], y[i]*yp[i], yp[i]])
        A.append(row2)

    A = np.array(A)
    print("Computing Homography matrix for ")
    print(A)
    U, E, V = computeSVD(A)
    H_vertical = V[:, V.shape[1] - 1]
    H = H_vertical.reshape([3,3])
    H = H / H[2,2]
    return H

pointset1 = np.array([[5, 5], [150, 5], [150, 150], [5, 150]])
pointset2 = np.array([[100, 100], [200, 80], [220, 80], [100, 200]])

computeHomography(pointset1, pointset2)


# pylint: disable=missing-function-docstring, missing-module-docstring/
from pyccel.decorators import types

#==============================================================================

@types( int )
def sum_natural_numbers( n ):
    x = 0
    for i in range( 1, n+1 ):
        x += i
    return x

# ...
@types( int )
def factorial( n ):
    x = 1
    for i in range( 2, n+1 ):
        x *= i
    return x

# ...
@types( int )
def fibonacci( n ):
    x = 0
    y = 1
    for i in range( n ): # pylint: disable=unused-variable
        z = x+y
        x = y
        y = z
    return x

# ...
@types( int )
def double_loop( n ):
    x = 0
    for i in range( 3, 10 ): # pylint: disable=unused-variable
        x += 1
        y  = n*x
        for j in range( 4, 15 ): # pylint: disable=unused-variable
            z = x-y
    return z

# ...
@types( 'int[:,:](order=C)' )
def double_loop_on_2d_array_C( z ):

    from numpy import shape

    m, n = shape( z )

    for i in range( m ):
        for j in range( n ):
            z[i,j] = i-j

# ...
@types( 'int[:,:](order=F)' )
def double_loop_on_2d_array_F( z ):

    from numpy import shape

    m, n = shape( z )

    for i in range( m ):
        for j in range( n ):
            z[i,j] = i-j

# ...
@types( 'int[:,:](order=C)' )
def product_loop_on_2d_array_C( z ):

    from numpy     import shape
    from itertools import product

    m, n = shape( z )

    x = [i for i in range(m)]
    y = [j for j in range(n)]

    for i,j in product( x, y ):
        z[i,j] = i-j

# ...
@types( 'int[:,:](order=F)' )
def product_loop_on_2d_array_F( z ):

    from numpy     import shape
    from itertools import product

    m, n = shape( z )

    x = [i for i in range(m)]
    y = [j for j in range(n)]

    for i,j in product( x, y ):
        z[i,j] = i-j

# ...
@types( 'int[:]' )
def map_on_1d_array( z ):

    @types( int )
    def f( x ):
        return x+5

    res = 0
    for v in map( f, z ):
        res *= v

    return res

# ...
@types( 'int[:]' )
def enumerate_on_1d_array( z ):

    res = 0
    for i,v in enumerate( z ):
        res += v*i

    return res

# ...
@types( int )
def zip_prod( m ):

    x = [  i for i in range(m)]
    y = [2*j for j in range(m)]

    res = 0
    for i1,i2 in zip( x, y ):
        res += i1*i2

    return res

# ...
@types( 'real[:], real[:]' )
def product_loop_on_real_array( z, out ):

    from numpy     import shape

    n, = shape( z )

    for i in range(n):
        out[i] = z[i]**2

# ...
@types('int,int,int')
def fizzbuzz_search_with_breaks( fizz, buzz, max_val ):
    for i in range(1,max_val+1):
        if i%fizz == 0 and i%buzz == 0:
            break
    return i

# ...
@types('int,int,int')
def fizzbuzz_sum_with_continue( fizz, buzz, max_val ):
    fizzbuzz_sum = 0
    for i in range(1,max_val+1):
        if i%fizz != 0:
            continue
        if i%buzz != 0:
            continue
        fizzbuzz_sum += i
    return fizzbuzz_sum

# ...
@types(int)
def fibonacci_while(n):
    x = 0
    y = 1
    i = 1
    while i <= n:
        z = x+y
        x = y
        y = z
        i = i + 1
    return x

# ...
@types(int)
def sum_nat_numbers_while(n):
    x = 0
    i = 0
    while i <= n:
        x += i
        i = i + 1
    return x

# ...
@types(int,int)
def double_while_sum(n, m):
    x = 0
    y = 0
    i = 0
    while x <= n:
        while y <= m:
            i += y
            y = y + 1
        i += x
        x = x + 1
    return i

# ...
@types( int )
def factorial_while( n ):
    x = 1
    i = 1
    while i <= n:
        x = i * x
        i = i + 1
    return x

@types( int )
def while_not_0( n ):
    while n:
        n -= 1
    return n


import torch
import numpy as np

def my_softmax(x):
    """
    softmax原理：
    输入任意实数x，输出[0, 1]代表概率，且概率之和为1,
    本质上是在做0-1概率分布的归一化
    这里利用了 非负实数集合/sum 的值域为[0,1]来做计算
    这样不仅能限定值域为[0, 1]，还能让输出值之和为1,
    而sigmoid只能做到前者

    为什么softmax需要exp？softmax和max的关系？
    要判断x属于哪个类别，首先能想到的是x属于哪个类的概率最高，我就认为它属于哪个类，概率可以用[0,1]表示，
    所以我遍历一下概率向量，直接把最大的概率置为1，其他置为0即可，这是很直接的max想法，但max函数是不可
    导的，不能求梯度做反向传播来学习。怎么办呢？我们还是再回顾一下目标
    1. 希望总的概率和为1
    2. 希望只有一个概率为1，其他都为0
    3. 函数要可导

    第一个目标很好实现，做一个归一化即可，即把x变成非负实数，再除以sum，即可得到。
    比较简单的让x平移使得最小值变为0，满足非负实数的条件，再除以sum： (x - min(x)) / sum((x - min(x)))
    假如x是[0.3, 0.8, 0.9]，那么计算后得到[0, 0.456, 0.544],
    问题来了，0.456和0.544比较接近，我们能不能让0.544变得更接近1， 而0.456更接近0？
    也就是说，我们怎么能让微小的变化变得更大？我们应该马上就能想到指数函数可以做到！
    这样，用极限的思维来看，最大的概率会扩大得比其他概率大得多，其他概率相当于无穷小，
    再除以sum，最大的概率就会接近1，其他相对无穷小的概率就会接近0。
    而用e做底数的指数函数，在结合交叉熵做反向传播时，很方便算导数，因此使用了exp，
    这样一来，我们达到了上面3个目标，使得max值尽量接近1，而且是连续可导的，相比max
    会很平滑，所以是soft的
    """

    # 由于对任意实数x取指数函数很容易导致类型上溢，
    # 而且softmax(x) 和 softmax(x + c)是等价的（softmax主要是用x向量内各个值的相对差来做计算，所以整体平移相对差不变，结果也不变）
    # 因此先整体减去一个最大值做预处理，让x变成(-∞, 0]
    x = x - np.max(x)

    # 指数函数可以把x变成正实数，此时值域变成了(0, 1]，和不为1
    e = np.exp(x)
    # 除以sum后结果还是(0, 1]，但是和为1
    # 虽然概率永远不可能为0，但是可以无限接近0，我们认为这样就够了
    # 这也是exp的小弊端
    return e / np.sum(e)

def my_softmax_grad(y, loss_value):
    # softmax要分n类（这里我们分10类）的求导有两种情况
    # 第一种情况：
    #    假如有一个yj，代表x分到第j类的概率，那么yj对yj的偏导等于yj - yj ** 2
    # 第二种情况：
    #    假如有一个yj，代表x分到第j类的概率，那么除了第j个y值，其他n-1个y值，对这个yj的偏导等于 0 -yi * yj    (i从0到n-1，i等于j除外)

    # n个y值，生成对角矩阵，第j行第j列，代表x被分到第j类的概率yj
    diag_matrix = np.diag(y)
    # np.outer用来生成双层for循环，生成n*n的矩阵，矩阵的i,j处位置代表yi * yj的结果
    multi_matrix = np.outer(y, y)
    # 对角矩阵 - 相乘矩阵 = 偏导矩阵，在对角部分等于yj - yj * yj, 在非对角部分等于0 - yi * yj
    partial_matrix = diag_matrix - multi_matrix

    # partial_matrix的第j行，代表x的分类概率向量（y，n个值）中每一个分量（概率）对x被分在第j类的偏导
    # loss_value有n个值，代表x被分为这n类的误差,所以-loss_value * partial_matrix[j]等于
    # x被分到第j类的梯度
    return np.dot(partial_matrix, loss_value)

if __name__ == '__main__':
    class_num = 10  # 分10类
    input = np.random.rand(class_num)  # 1x10 vector
    loss = np.random.rand(class_num)

    print("torch的softmax:")
    print(torch.softmax(torch.tensor(input), dim=0).numpy())
    print("自己写的softmax:")
    print(my_softmax(input))

    t = torch.tensor(input, requires_grad=True)
    s = torch.softmax(t, dim=0)
    s.backward(torch.tensor(loss))
    print("torch的softmax grad:")
    print(t.grad.numpy())
    print("自己写的softmax grad:")
    print(my_softmax_grad(my_softmax(input), loss))


import numpy as np
from data import io

'''
flatten(imageset)
    - converts an image to a column vector
'''
def flatten(imageset):
    flat = imageset.reshape(imageset.shape[ 0 ], -1).T
    return flat

'''
weight(U, dataset, shi)
    - calculate weight of each train sample
    - using first k pca
'''
def weight(U, dataset, shi):
    return np.dot(U.T, dataset - shi)

'''
pca(k, trainset):
    - returns the first k pca
'''
def pca(k, trainset):
    shi = np.asmatrix(trainset.mean(axis = 1)).T
    phi = trainset - shi
    covariance = np.dot(phi, phi.T)
    eig_vals, eig_vecs = np.linalg.eig(covariance)

    i = eig_vals.argsort()[-k:][::-1]
    eig_vals = eig_vals[i]
    component = eig_vecs[:, i]

    return component, shi

'''
train_pca(n, k, image_size, directory)
    - calculate pca, weight
'''
def train_pca(n, k, image_size, directory):
    trainset = io.load(n, image_size, directory)
    print("trainset shape:", trainset.shape)

    trainset = flatten(trainset)
    print("trainset shape after flattening:", trainset.shape)

    U, shi = pca(k, trainset)
    print("Principle components:", U.shape)
    print("Average Column Vector:", shi.shape)

    train_weight = weight(U, trainset, shi)
    print("train weight data shape:", train_weight.shape)

    return U, shi, train_weight

'''
test_pca(n, image_size, directory, U, shi, train_weight)
    - maps the testset with pca
    - calculates test weights
'''
def test_pca(n, image_size, directory, U, shi, train_weight):
    testset = io.load(n, image_size, directory)
    print("testset shape:", testset.shape)

    testset = flatten(testset)
    print("testset shape after flattening::", testset.shape)

    test_weight = weight(U, testset, shi)
    return test_weight


import pdb
import numpy as np
import matplotlib.pyplot as plt
import math

######################################################################
#
# Learning algorithms and helpers
#
######################################################################

# Ordinary least squares regression
# X is an n x d matrix
# Y is an n x 1 matrix
# returns weights: d x 1 matrix
def ols(X, y):
    return (X.T * X).I * X.T * y

# "Ridge" regression with lambda = l; otherwise like ols
# Penalizes the offset weight, which is sometimes undesirable
def olsr(X, y, l = 0):
    d = X.shape[1]
    return (X.T * X + l * np.identity(d)).I * X.T * y

# Given an order for a polynomial feature space, d
# return a *function* that maps a single value into a vector of d+1 features
def polynomialFeatures(d):
    def p(xi):
        return np.matrix([xi[0,0]**i for i in range(d+1)])
    return p

# Given X, an array of training examples, and b, a bandwidth,
# return a *function* that maps a single value into a vector of features
def RBFs(X, b):
    def g(x1, x2, b):
        return np.exp(-((np.linalg.norm(x1 - x2) / b)**2))
    return lambda xi: np.matrix([g(xi, x, b) for x in X] + [1.0])

# Given a function, such as the one generated by polynomialFeatures,
# which takes a d-dimensional vector and returns a D-dimensional
# vector, and an n x d matrix X, return an n x D matrix of feature
# vectors
def applyFeatureFun(phi, X):
    return np.vstack([phi(X[i,:]) for i in range(X.shape[0])])

# Given a D x 1 weight vector and a "feature function" that takes a
# d-dimensional vector into a D x 1 vector, return a **function** that
# takes a d-dimensional vector into a scalar regression value, but
# mapping the input into the high-dimensional feature space and then
# taking the dot product with the weights.
def makeRegressor(w, phi):
    def r(x):
        return (phi(x)*w)[0,0]
    return r

# Given an n x d matrix X of input values and an n x 1 matrix y of
# target values, and a function that maps a row of X into a
# prediction, return the root mean squared error of the predictor
# applied to the X values.
def rmse(X, y, predictor):
    return np.sqrt(sse(X, y, predictor)/len(y))

# As for rmse, but just return the sum squared error.
def sse(X, y, predictor):
    p = np.matrix([predictor(X[i,:]) for i in range(X.shape[0])]).T
    assert p.shape == y.shape
    return np.sum(np.square(p - y))

# Generic gradient descent:
# - f is a function mapping a vector of parameters to a score
#     (not strictly necessary)
# - df is a function mapping a vector of parameters to the gradient at
#     that point
# - x0 is an initial parameter vector
# - step_size is the step size
#
#
# - terminates after max_iter iterations, or when parameters have not changed
#    by more than eps from one iteration to the next
#   in every iteration updates the current param vector by the gradient
# Returns: the final parameter vector, a list of scores (one per iteration)
#   and a list of parameter vectors (one per iteration)
def gd(f, df, x0, step_size = .01, max_iter = 1000, eps = .00001):
    diff = 1
    iterCount = 0
    fs = []
    xs = []

    fs.append(f(x0))
    xs.append(x0)
    while (diff > eps and iterCount < max_iter):
        x = x0 - step_size * df(x0)
        diff = np.linalg.norm(x - x0)
        fs.append(f(x))
        xs.append(x0)

        iterCount += 1
        x0 = x
    # your code here

    return x, fs, xs

# Linear regression using gradient descent.
# X and y are as for ordinary least squares;
# w0 is the initial weight vector
# max_iter is as for gd
# l currently unused, but can be used to do ridge regression
# Returns: final weight vector, a list of scores, list of weight vectors

def gdLinReg(X, y, l = 0, step_size = 0.01, w0 = None, max_iter = 1000):
    # w is d by 1; X is n by d
    # return result is d by 1
    n = float(len(y))
    def df(w):
        return 2 * np.sum(np.multiply(X*w - y, X), axis = 0).T + l * w
    def f(w):
        return np.sum(np.square(X*w - y)) + l * float(w.T * w)

    if w0 is None: w0 = np.matrix(np.ones(X.shape[1])).T
    return gd(f, df, w0, step_size = step_size, max_iter = max_iter)

# Stochastic gradient descent
# X is an n x d matrix of input examples
# y is an n x 1 matrix of outputs (we could combine X and Y if we wanted to)
# f is a function taking a parameter vector, a 1 x d input example,
#     and a 1 x 1 output value, and returning an error; we want to
#     minimize this error in expectation over the whole data set
# df has the same inputs as f, but returns a gradient vector of the same
#     dimension as w0
# if earlyTermination is true, then terminate if params move by less
# than epsilon; otherwise, terminate after max_iter iterations.

# Returns last parameter vector, list of scores, list of param
# vectors, the iteration on which termination happened, and a Boolean
# indicating whether termination was early or not.

def sgd(X, y, f, df, w0, step_size = .01, max_iter = 20000, eps = .0000001,
            earlyTermination = False):
    n = y.shape[0]
    prev_w = w0
    fs = []; ws = [prev_w]
    for i in range(max_iter):
        j = np.random.randint(n)
        Xj = X[j]; yj = y[j]
        w = prev_w - step_size * df(prev_w, Xj, yj)
        if earlyTermination and np.all(abs(w - prev_w) < eps):
            return w, fs, ws, i, False
        fs.append(f(prev_w, Xj, yj)); ws.append(w)
        prev_w = w
    return w, fs, ws, i, True

# Linear regression computed by stochastic gradient descent.
# Parameters are as for gdLinReg.

# Returns last parameter vector, list of scores, list of param
# vectors, the iteration on which termination happened, and a Boolean
# indicating whether termination was early or not.

def sgdLinReg(X, y, l = 0, step_size = 0.01, w0 = None, max_iter = 20000):
    def df(w, Xj, yj):
        return 2 * ((Xj * w - yj) * Xj).T

    def f(w, Xj, yj):
        return np.square(Xj * w - yj)

    if w0 is None: w0 = np.matrix(np.ones(X.shape[1])).T
    return sgd(X, y, f, df, w0, step_size = step_size, max_iter = max_iter)

######################################################################
#
# Data
#
######################################################################

# Uses data set from Chapter 1 of Bishop, stored in file
# "curvefitting.txt".  They are 10 points drawn from (x, sin(2 pi x))
# with noise added (but I'm not sure how much.)

# If random is not False, it should be an integer, and instead of
# returning data from the file, we will generate a new random data set
# of that size, with 0 mean, 0.2 stdev Gaussian noise.

# if addOnes is true, return: n x 1 matrix X, n x 2 matrix F (with
# column of 1's added) and n x 1 matrix Y.

def getCurveData(addOnes = False, random = False):
    if random:
        X = np.matrix([[i / float(random)] for i in range(random + 1)])
        noise = np.random.normal(scale = 0.2, size = (random+1, 1))
        y = np.matrix([[np.sin(2 * np.pi * X[i,0])] for i in range(X.shape[0])]) + noise
    else:
        data = np.loadtxt('curvefitting.txt')
        X, y = np.matrix(data[0]).T, np.matrix(data[1]).T
    if addOnes:
        F = np.append(np.ones_like(X), X, 1)
        return X, F, y
    else:
        return X, y

# Get the Blog data

def getBlogData(n):
    dir = '/home/binder/entwurf6/teaching/2017/ai_2017/wek1/code/'
    X = np.matrix(np.genfromtxt(dir+'x_train.csv',
                                    delimiter = ','))
    if n == None:
        n = X.shape[0]
    else:
        X = X[:n]

    y = np.matrix(np.genfromtxt(dir+'y_train.csv',
                                    delimiter = ',')).T[:n]
    XTest = np.matrix(np.genfromtxt(dir+'x_test.csv',
                                    delimiter = ','))
    yTest = np.matrix(np.genfromtxt(dir+'y_test.csv',
                                    delimiter = ',')).T
    return X, y, XTest, yTest

######################################################################
#
# Plotting stuff
#
######################################################################

def tidyPlot(xmin, xmax, ymin, ymax, center = False, title = None,
                 xlabel = None, ylabel = None):
    plt.ion()
    plt.figure(facecolor="white")
    ax = plt.subplot()
    if center:
        ax.spines['left'].set_position('zero')
        ax.spines['right'].set_color('none')
        ax.spines['bottom'].set_position('zero')
        ax.spines['top'].set_color('none')
        ax.spines['left'].set_smart_bounds(True)
        ax.spines['bottom'].set_smart_bounds(True)
        ax.xaxis.set_ticks_position('bottom')
        ax.yaxis.set_ticks_position('left')
    else:
        ax.spines["top"].set_visible(False)
        ax.spines["right"].set_visible(False)
        ax.get_xaxis().tick_bottom()
        ax.get_yaxis().tick_left()
    eps = .05
    plt.xlim(xmin-eps, xmax+eps)
    plt.ylim(ymin-eps, ymax+eps)
    if title: ax.set_title(title)
    if xlabel: ax.set_xlabel(xlabel)
    if ylabel: ax.set_ylabel(ylabel)
    return ax

def plotData(ax, x, y, style = 'ro', label = None):
    if style is None:
        ax.plot(x, y, label = label)
    else:
        ax.plot(x, y, style, label = label)
    plt.show()

# w is a (1 x 2) matrix
def plotLine(ax, w, xmin, xmax, nPts = 100):
    b = float(w[0])
    m = float(w[1])
    plotFun(ax, lambda x: m*x + b, xmin, xmax, nPts)

def plotFun(ax, f, xmin, xmax, nPts = 100, label = None):
    x = np.linspace(xmin, xmax, nPts)
    y = np.vstack([f(np.matrix([[xi]])) for xi in x])
    ax.plot(x, y, label = label)
    plt.show()

def smooth(n, vals):
    # Run a box filter of size n
    x = sum(vals[0:n])
    result = [x]
    for i in range(n, len(vals)):
        x = x - vals[i-n] + vals[i]
        result.append(x)
    return result

######################################################################
#
# Tests
#
######################################################################

######################################################################
#
# Ordinary least squares in 2D (one constant input dimension)

def t1():
    X, F, y = getCurveData(True)

    w = ols(F, y)

    print ('w', w.T)
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    ax = tidyPlot(xmin, xmax, ymin, ymax, xlabel = 'x', ylabel = 'y')
    plotData(ax, X, y)
    plotLine(ax, w, xmin, xmax)

######################################################################
#
# Ordinary least squares in polynomial feature spaces
# Plots predictors

def t2(ds = range(1, 10)):
    X, y = getCurveData()
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')

    plotData(ax, X, y)

    for d in ds:
        phi = polynomialFeatures(d)
        phiD = applyFeatureFun(phi, X)
        w = ols(phiD, y)
        predictor = makeRegressor(w, phi)

        plotFun(ax, predictor, xmin, xmax, label = str(d))
        print ('Order', d, 'Training RMSE', rmse(X, y, predictor))
        print ('     w', w.T)
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

######################################################################
#
# Ordinary least squares in polynomial feature spaces
# Plots train and test error versus order of polynomial basis

def t3(ds = range(1, 10)):
    X, y = getCurveData()
    XTest, yTest = getCurveData(random = 10)
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')
    plotData(ax, X, y)
    plotData(ax, XTest, yTest, style = 'go')
    trainErr = []
    testErr = []

    for d in ds:
        phi = polynomialFeatures(d)
        phiD = applyFeatureFun(phi, X)
        w = ols(phiD, y)
        predictor = makeRegressor(w, phi)
        plotFun(ax, predictor, xmin, xmax)
        trainErr.append(rmse(X, y, predictor))
        testErr.append(rmse(XTest, yTest, predictor))
        print ('Order', d, 'Training RMSE', trainErr[-1])
        print ('        Test RMSE', testErr[-1])

    ax = tidyPlot(0, len(ds), 0, max(max(testErr), max(trainErr)), center =True,
                      xlabel = 'Polynomial order', ylabel = 'RMSE')
    ax.plot(ds, trainErr, label = 'train err')
    ax.plot(ds, testErr, label = 'test err')
    ax.legend()

######################################################################
#
# Ordinary least squares in RBF feature spaces
# Plots train and test error versus bandwidth

# LPK: these are good b values (.01, .05, .1, .5, 1, 2, 4)

def tRBF(bs = []):
    X, y = getCurveData()
    XTest, yTest = getCurveData(random = 10)
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')
    plotData(ax, X, y)
    trainErr = []
    testErr = []

    for b in bs:
        phi = RBFs(X, b)
        phiD = applyFeatureFun(phi, X)
        w = olsr(phiD, y, 1e-5)
        predictor = makeRegressor(w, phi)
        plotFun(ax, predictor, xmin, xmax, label = str(b))
        trainErr.append(rmse(X, y, predictor))
        testErr.append(rmse(XTest, yTest, predictor))
        print ('Bandwidth', b, 'Training RMSE', trainErr[-1])
        print ('        Test RMSE', testErr[-1])
        print ('    w', w.T)
    ax.legend(loc = 'best')

    if len(bs) > 1:
        ax = tidyPlot(min(bs), max(bs), 0, max(max(testErr), max(trainErr)),
                        xlabel = 'Bandwidth', ylabel = 'RMSE')
        ax.plot(bs, trainErr, label = 'train err')
        ax.plot(bs, testErr, label = 'test err')
        ax.legend(loc = 'best')

######################################################################
#
# Ordinary least squares in polynomial feature spaces
# Plots train and test error versus size of training set

def t4(trainSizes = (10, 15, 20, 30, 50, 100), showData = False):
    XTest, yTest = getCurveData(random = 100)
    xmin, xmax = float(np.min(XTest)), float(np.max(XTest))
    ymin, ymax = float(np.min(yTest)), float(np.max(yTest))
    trainErr = []
    testErr = []
    phi = polynomialFeatures(9)

    if not showData:
        ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y',
                          title = 'Predictors for different train sizes')

    for ts in trainSizes:
        X, y = getCurveData(random = ts-1)
        if showData:
            ax = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y',
                          title = 'Train size = ' + str(ts))
            plotData(ax, X, y)
        phiD = applyFeatureFun(phi, X)
        w = ols(phiD, y)
        predictor = makeRegressor(w, phi)
        plotFun(ax, predictor, xmin, xmax, label=str(ts))
        trainErr.append(rmse(X, y, predictor))
        testErr.append(rmse(XTest, yTest, predictor))
        print ('Train size', ts, 'Training RMSE', trainErr[-1])
        print ('        Test RMSE', testErr[-1])
        print ('     w', w.T)
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))
    if len(trainSizes) > 1:
        ax = tidyPlot(0, max(trainSizes), 0, max(testErr), center = True,
                      xlabel = 'Training set size', ylabel = 'RMSE')
        ax.plot(trainSizes, trainErr, label = 'train err')
        ax.plot(trainSizes, testErr, label = 'test err')
        ax.legend(loc="upper right")

######################################################################
#
# Ridge regression in polynomial feature spaces
# Plot train and test error versus ridge parameter lambda

def t5(logLambdaValues = (-50, -40, -30, -20, -15, -10, -1, 0, 1, 10)):
    XTest, yTest = getCurveData(random = 100)
    xmin, xmax = float(np.min(XTest)), float(np.max(XTest))
    ymin, ymax = float(np.min(yTest)), float(np.max(yTest))
    trainErr = []
    testErr = []
    phi = polynomialFeatures(9)

    ax = tidyPlot(xmin, xmax, ymin-1, ymax+1,
                      xlabel = 'x', ylabel = 'y',
                          title = 'Predictors for different lambda values')
    X, y = getCurveData()
    phiD = applyFeatureFun(phi, X)
    plotData(ax, X, y)
    for llv in logLambdaValues:
        w = olsr(phiD, y, np.exp(llv))
        predictor = makeRegressor(w, phi)
        plotFun(ax, predictor, xmin, xmax, label=str(llv))
        trainErr.append(rmse(X, y, predictor))
        testErr.append(rmse(XTest, yTest, predictor))
        print ('Log lambda', llv, 'Training RMSE', trainErr[-1])
        print ('        Test RMSE', testErr[-1])
        print (w.T)
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

    if len(logLambdaValues) > 1:
        ax = tidyPlot(min(logLambdaValues), max(logLambdaValues),
                        0, max(testErr), center = True,
                        xlabel = 'Log lambda', ylabel = 'RMSE')
        ax.plot(logLambdaValues, trainErr, label = 'train err')
        ax.plot(logLambdaValues, testErr, label = 'test err')
        ax.legend(loc="best")

######################################################################
#
# Batch gradient descent in 2D feature space
# Plots error as a function of iteration number for different step sizes
# Individual plots of trajectory of w during optimization

def t6(learning_rates = (.0001, .001, .01, .05, .07, .075, .1)):
    X, F, y = getCurveData(True)
    ax = tidyPlot(0, 1000, 2, 8,
                      xlabel = 'iteration', ylabel = 'err')
    for lr in learning_rates:
        w, scores, vals = gdLinReg(F, y, step_size = lr)

        plotData(ax, range(len(scores)), scores, style = None,
                     label = str(lr))
        print ('lr', lr, 'w', w.T, 'err', scores[-1])
        dmin = min(-1.5, np.min(vals)); dmax = max(1.5, np.max(vals))
        nax = tidyPlot(dmin, dmax, dmin, dmax,
                       xlabel = 'w0', ylabel = 'w1',
                       title = 'step size = '+str(lr), center = True)
        plotData(nax, [float(xd) for (xd, yd) in vals],
                      [float(yd) for (xd, yd) in vals],
                      style = 'bo-')
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

######################################################################
#
# Stochastic  gradient descent in 2D feature space
# Plots error as a function of iteration number for different step sizes
# Individual plots of trajectory of w during optimization

def t7(learning_rates = (.0001, .001, .01, .1, .5, 1)):
    X, F, y = getCurveData(True)
    ax = tidyPlot(0, 1000, 0, 100,
                      xlabel = 'iteration', ylabel = 'err')
    for lr in learning_rates:
        w, scores, vals, iters, failed = sgdLinReg(F, y, step_size = lr)

        smoothScores = smooth(len(y)*4, [float(s) for s in scores])
        plotData(ax, range(len(smoothScores)), smoothScores,
                     style = None, label = str(lr))
        print ('lr', lr, 'w', w.T, 'err', smoothScores[-1])
        dmin = min(-1.5, np.min(vals)); dmax = max(1.5, np.max(vals))
        nax = tidyPlot(dmin, dmax, dmin, dmax,
                       xlabel = 'w0', ylabel = 'w1',
                       title = 'step size = '+str(lr), center = True)
        plotData(nax, [float(xd) for (xd, yd) in vals],
                      [float(yd) for (xd, yd) in vals],
                      style = 'bo-')
    ax.legend(loc="best")

######################################################################
#
# Batch gradient descent in polynomial feature space
# Plots train and test error as a function of iteration number

def t8(order = 9):
    goodw = np.matrix(\
      [[  3.49512436e-01,   2.32326298e+02,  -5.32086773e+03,   4.85596769e+04,
         -2.31598783e+05,   6.39931981e+05,  -1.06162004e+06,   1.04222515e+06,
         -5.57590377e+05,   1.25180846e+05]]).T
    zeros = np.matrix(np.zeros([order+1, 1]))
    medw = np.matrix(\
      [[  3.0e-01,   2.0e+02,  -5.0e+03,   5.0e+04,
         -2.0e+05,   6.0e+05,  -1.0e+06,   1.0e+06,
         -5.0e+05,   1.0e+05]]).T

    X, y = getCurveData()
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    XTest, yTest = getCurveData(random = 100)

    phi = polynomialFeatures(order)
    phiD = applyFeatureFun(phi, X)
    lr = 0.005
    w, scores, vals = gdLinReg(phiD, y, step_size = .01, w0 = zeros,
                                   max_iter = 10000)
    print ('w', w.T, 's', scores[-1], 'iter', len(scores))
    iVals = []; trainVals = []; testVals = []
    axPred = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')
    for i in range(0, len(vals), 100):
        predictor = makeRegressor(vals[i], phi)
        plotFun(axPred, predictor, xmin, xmax, label = 'iters = '+str(i))
        iVals.append(i)
        trainVals.append(rmse(X, y, predictor))
        testVals.append(rmse(XTest, yTest, predictor))
    plotData(axPred, X, y)
    axPred.legend(loc="upper left", bbox_to_anchor=(1,1))

    ax = tidyPlot(0, np.max(iVals), 0, 1, #np.max(testVals),
                   xlabel = 'iteration', ylabel = 'rmse')
    plotData(ax, iVals, trainVals, label = 'training error', style = 'bo-')
    plotData(ax, iVals, testVals, label = 'testing error', style = 'ro-')
    ax.legend(loc="best")

def t9(order = 9):
    zeros = np.matrix(np.zeros([order+1, 1]))
    goodw = np.matrix(\
      [[  3.49512436e-01,   2.32326298e+02,  -5.32086773e+03,   4.85596769e+04,
         -2.31598783e+05,   6.39931981e+05,  -1.06162004e+06,   1.04222515e+06,
         -5.57590377e+05,   1.25180846e+05]]).T
    medw = np.matrix(\
      [[  3.0e-01,   2.0e+02,  -5.0e+03,   5.0e+04,
         -2.0e+05,   6.0e+05,  -1.0e+06,   1.0e+06,
         -5.0e+05,   1.0e+05]]).T
    randw = np.matrix(np.random.randn(order+1,1))

    X, y = getCurveData()
    xmin, xmax = float(min(X)), float(max(X))
    ymin, ymax = float(min(y)), float(max(y))
    XTest, yTest = getCurveData(random = 100)

    phi = polynomialFeatures(order)
    phiD = applyFeatureFun(phi, X)
    lr = 0.05
    w, scores, vals, iters, failed = sgdLinReg(phiD, y, step_size = lr,
                                                   max_iter = 100000,
                                                   w0 = randw)
    print ('w', w.T, 's', scores[-1], 'iter', len(scores))
    iVals = []; trainVals = []; testVals = []
    axPred = tidyPlot(xmin, xmax, ymin-1, ymax+1, xlabel = 'x', ylabel = 'y')
    for i in range(0, len(vals), 5000):
        predictor = makeRegressor(vals[i], phi)
        plotFun(axPred, predictor, xmin, xmax, label = 'iters = '+str(i))
        iVals.append(i)
        trainVals.append(rmse(X, y, predictor))
        testVals.append(rmse(XTest, yTest, predictor))
    plotData(axPred, X, y)

    ax = tidyPlot(0, np.max(iVals), 0, 1, #np.max(testVals),
                   xlabel = 'iteration', ylabel = 'rmse')
    plotData(ax, iVals, trainVals, label = 'training error', style = 'bo-')
    plotData(ax, iVals, testVals, label = 'testing error', style = 'ro-')
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

# Blog feedback data, with analytic ridge regression
def t10(n = None,
        logLambdaValues = [-14, -12, -10, -8, -6, -4, -2, 0, 2, 4, 5, 6, 7,
                               8, 9, 10, 12, 14, 16, 18, 20]):
    X, y, XTest, yTest = getBlogData(n = n)
    print (X.shape, y.shape)
    iVals = []; trainVals = []; testVals = []
    for llv in logLambdaValues:
        w = olsr(X, y, np.exp(llv))
        predictor = lambda x: x * w
        iVals.append(llv)
        trainVals.append(rmse(X, y, predictor))
        testVals.append(rmse(XTest, yTest, predictor))
        print (llv, trainVals[-1], testVals[-1])

    ax = tidyPlot(np.min(iVals), np.max(iVals), 28, 40, #np.max(testVals),
                   xlabel = 'Log lambda', ylabel = 'rmse',
                   title = 'Training data size = '+str(n))
    plotData(ax, iVals, trainVals, label = 'training error', style = 'bo-')
    plotData(ax, iVals, testVals, label = 'testing error', style = 'ro-')
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

# Blog feedback data, with gradient regression.
# Add ridge!

# lambda = 80; rmse = 31.8; mse = 1024
# lambda = ? ; mse = 1015   (seems untrustworthy)
# log lambda = 2.8
# lambda 100, MSE 894

# lr = .000000005 good for whole data set (or divide by 5)

def t11(lr = .000000005, n = None):
    X, y, XTest, yTest = getBlogData(n = n)
    print (X.shape, y.shape)
    iVals = []; trainVals = []; testVals = []
    #w, scores, vals = gdLinReg(X, y, step_size = lr, max_iter = 20000)
    w, scores, vals, iters, failed = \
                            sgdLinReg(X, y,
                                          step_size = lr, max_iter = 2000000)
    print ('s', scores[-1], 'iter', len(scores))
    iVals = []; trainVals = []; testVals = []
    for i in range(0, len(vals), 50000):
        predictor = lambda x: float(x * vals[i])
        iVals.append(i)
        trainVals.append(rmse(X, y, predictor))
        testVals.append(rmse(XTest, yTest, predictor))

    print (trainVals)
    print (testVals)

    ax = tidyPlot(0, np.max(iVals), 0, 400, #np.max(testVals),
                   xlabel = 'iteration', ylabel = 'rmse')
    plotData(ax, iVals, trainVals, label = 'training error', style = 'bo-')
    plotData(ax, iVals, testVals, label = 'testing error', style = 'ro-')
    ax.legend(loc="upper left", bbox_to_anchor=(1,1))

def tGD(step_sizes = []):
    def f(x):
        return (2 * x + 3)**2
    def df(x):
        return 2 * 2 * (2 * x + 3)
    x0 = 0
    for ss in step_sizes:
        x, fs, xs = gd(f, df, x0, step_size = ss)
        print ('ss', ss, 'x', x)
        nax = tidyPlot(-4, 1, 0, 15,
                       xlabel = 'x', ylabel = 'f(x)',
                       title = 'step size = '+str(ss), center = True)
        plotFun(nax, f, -4, 1)
        plotData(nax, xs, fs, style = 'ro-')

def tGD2(x0 = 0):
    def f(x):
        return (x - 2) * (x - 3) * (x + 3) * (x + 1)
    def df(x):
        return 9 - (22 * x) - (3 * x**2) + (4 * x**3)
    ss = 0.01
    x, fs, xs = gd(f, df, x0, step_size = ss)
    nax = tidyPlot(-4, 4, -25, 25,
                       xlabel = 'x', ylabel = 'f(x)',
                       title = 'step size = '+str(ss), center = True)
    plotFun(nax, f, -4, 4)
    plotData(nax, xs, fs, style = 'ro-')

if __name__ == '__main__':
    print ('Loaded learn.py')
    # tGD2()
    stepSizes = [0.01]
    tGD(stepSizes)
    plt.show(block=True)