import numpy as np
import pandas as pd
from numpy.random import RandomState
from sklearn.cluster import KMeans as kmeans

def sse_l2(x, y):
    return np.sum((x - y) ** 2)

def wss(X, labels, centroids):
    """
    Calculate the WSS or cohesion, which is the within SSE
    of distances from each sample to its centroid
    """
    k = centroids.shape[0]
    wss = 0
    for i in range(k):
        cluster = X[labels == i]
        centroid = centroids[i]
        wss += sse_l2(cluster, centroid)

    return wss

def bss(X, labels, centroids):
    """
    Calculate the BSS or separation, which is the weighted SSE
    between each cluster centroid and the global centroid
    """
    k = centroids.shape[0]
    global_mean = X.mean(axis=0)
    cluster_sizes = np.array([len(X[labels == i]) for i in range(k)])

    dists = np.sum((centroids - global_mean) ** 2, axis=1)
    bss = np.sum(dists * cluster_sizes)
    return bss

def _sanitize(X):
    return X.values if isinstance(X, pd.DataFrame) else X

class KMeans:

    def __init__(self, k, tol=1e-7, max_iter=100, n_init=10, random_state=None):
        self.k = k
        self.tol = tol
        self.max_iter = max_iter
        self.n_init = n_init
        self.centroids_ = None

        if random_state is not None:
            random_state = np.int64(random_state)
        self.random_state = RandomState(seed=random_state)

    def fit(self, X):
        """
        Run the algorithm n_init times and keep
        the one with best cohesion
        """
        results = []
        for _ in range(self.n_init):
            res = self._fit(X)
            results.append(res)

        # evaluate results, keep the best
        best_result = self.eval_clusterings(results)
        (self.labels_,
         self.centroids_,
         self.n_iter_,
         self.cohesion_,
         self.separation_) = best_result
        return self

    def _fit(self, X):
        """
        Run main k-means algorithm to classify the data into
        k clusters
        """
        X = _sanitize(X)

        if self.centroids_ is None:
            # select k initial centroids randomly
            centroids_i = set()

            while len(centroids_i) != self.k:
                # ensure we generate a set of k unique centroids
                centroids_i = {self.random_state.randint(X.shape[0]) for _ in range(self.k)}

            centroids = X[list(centroids_i)]
        else:
            centroids = self.centroids_

        curr_iter = 0
        variance_diff = self.tol * 2
        variances = np.zeros(self.k)

        while curr_iter < self.max_iter and np.any(variance_diff > self.tol):
            # assign each point the closest centroid
            labels = self._assign(X, centroids)
            self._fix_empty(X, labels, centroids)

            # calculate new centroids
            for i in range(self.k):
                cluster = X[labels == i]
                new_centroid = np.mean(cluster, axis=0)
                centroids[i] = new_centroid

            # calculate difference in variance from previous clustering
            variances_old = variances
            variances = np.empty_like(variances_old)
            for i in range(self.k):
                cluster = X[labels == i]
                var = np.var(cluster, axis=0)
                variances[i] = np.linalg.norm(var, 2)

            variance_diff = variances - variances_old

            curr_iter += 1

        cohesion = wss(X, labels, centroids)
        separation = bss(X, labels, centroids)

        return labels, centroids, curr_iter, cohesion, separation

    def fit_predict(self, X):
        return self.fit(X).labels_

    def predict(self, X):
        if self.centroids_ is None:
            raise ValueError('This model is not fitted yet.')

        X = _sanitize(X)
        labels = self._assign(X, self.centroids_)
        return labels

    def _assign(self, X, centroids):
        """
        Assign each sample to the cluster of the closest centroid
        """
        # calculate distances to centroids
        centroid_dists = np.ndarray((X.shape[0], self.k))
        for i in range(self.k):
            # here einsum = sum squared factors
            diff = X - centroids[i]
            centroid_dists[:, i] = np.sqrt(np.einsum('ij,ij->i', diff, diff))

        return np.argmin(centroid_dists, axis=1)

    def _fix_empty(self, X, labels, centroids):
        """
        Fill/fix empty clusters if there are any by taking a random
        sample from the cluster with highest SSE and using it as a new
        centroid for the empty cluster, while ensuring that centroid
        was not already in use.
        """
        counts = np.array([np.count_nonzero(labels == i) for i in range(self.k)])

        # while there is an empty cluster
        while np.any(counts == 0):
            # index of empty cluster
            empty_i = np.argmin(counts)

            # compute SSE
            cluster_sses = []
            for i in range(self.k):
                if i != empty_i:
                    cluster = X[labels == i]
                    sse = sse_l2(cluster, cluster.mean())
                    cluster_sses.append(sse)
                else:
                    cluster_sses.append(0)

            # choose as new centroid a random point
            # from the cluster with highest SSE
            highest_i = np.argmax(cluster_sses)

            random_i = self.random_state.randint(counts[highest_i])
            new_centroid = X[labels == highest_i][random_i]

            # if it is already used as centroid, look for another one
            already_used = np.any([np.array_equal(new_centroid, c) for c in centroids])
            while already_used:
                random_i = self.random_state.randint(counts[highest_i])
                new_centroid = X[labels == highest_i][random_i]
                already_used = np.any([np.array_equal(new_centroid, c) for c in centroids])

            # reassign centroid
            labels[np.where(labels == highest_i)[0][random_i]] = empty_i
            centroids[empty_i] = new_centroid

            counts = np.array([np.count_nonzero(labels == i) for i in range(self.k)])

    def eval_clusterings(self, clusterings):
        """
        Custom criterion to evaluate clusterings and select the best
        """
        sse = np.array([[c[3], c[4]] for c in clusterings])

        intra_min = np.percentile(sse[:, 0], 10)
        intra_max = np.percentile(sse[:, 0], 90)
        intra_slope = 1 / (intra_min - intra_max)
        intra_b = -intra_max*intra_slope

        inter_min = np.percentile(sse[:, 1], 10)
        inter_max = np.percentile(sse[:, 1], 90)
        inter_slope = 1 / (inter_max - inter_min)
        inter_b = -inter_min*inter_slope

        intra_score = np.vectorize(lambda x: 0.75*(intra_slope*x + intra_b) if x > intra_min else 0.75*1)
        inter_score = np.vectorize(lambda x: 0.25*(inter_slope*x + inter_b) if x < inter_max else 0.25*1)

        sse = intra_score(sse[:, 0].clip(min=0)) + inter_score(sse[:, 1].clip(min=0))
        best_clustering = clusterings[np.argmax(sse)]

        return best_clustering

class BisectingKMeans:

    def __init__(self, k, tol=1e-7, max_iter=100, n_init=10,
                 criterion='heterogeneity', random_state=None):
        self.k = k
        self.tol = tol
        self.max_iter = max_iter
        self.n_init = n_init
        self.random_state = random_state

        if criterion == 'heterogeneity':
            self.eval_func = lambda x: sse_l2(x, x.mean())
        elif criterion == 'size':
            self.eval_func = len
        else:
            raise ValueError('Unknown criterion {}. Possible values'
                        ': \'heterogeneity\', \'size\''.format(criterion))

    def fit(self, X):
        X = _sanitize(X)

        labels = np.zeros(X.shape[0], dtype=np.int)
        clusters = [(self.eval_func(X), 0)]
        for i in range(1, self.k):
            target_label = self._pick(clusters)
            cluster_mask = labels == target_label

            bicluster = KMeans(2, self.tol, self.max_iter, self.n_init,
                self.random_state).fit_predict(X[cluster_mask])

            # mark one half of the new partition with a new label
            # we can keep the old label for the other half
            new_cluster_mask = np.where(cluster_mask)[0][bicluster == 0]
            labels[new_cluster_mask] = i

            self._insert(target_label, i, clusters, labels, X)

        centroids = np.array([X[labels == i].mean(axis=0) for i in range(self.k)])

        self.labels_ = labels
        self.cohesion_ = wss(X, labels, centroids)
        self.separation_ = bss(X, labels, centroids)
        return self

    def fit_predict(self, X):
        return self.fit(X).labels_

    def _insert(self, label_a, label_b, clusters, labels, X):
        # compute score according to criterion
        cluster_a = X[labels == label_a]
        cluster_b = X[labels == label_b]
        score_a = self.eval_func(cluster_a)
        score_b = self.eval_func(cluster_b)

        # add to list
        clusters.append((score_a, label_a))
        clusters.append((score_b, label_b))

    def _pick(self, clusters):
        # pick and remove the cluster with the highest value
        i = np.argmax([c[0] for c in clusters])
        return clusters.pop(i)[1]

class KMeanspp(KMeans):

    def _fit(self, X):
        if self.centroids_ is None:
            # run improved init only when not fitted already
            X = _sanitize(X)

            # select first random centroid
            centroids = np.ndarray((self.k, X.shape[1]))
            rand_i = self.random_state.randint(X.shape[0])
            centroids[0] = X[rand_i]

            dists = np.ndarray((X.shape[0], self.k))
            for i in range(1, self.k):
                # compute squared distances to all centroids
                for j in range(i):
                    diff = X - centroids[j]
                    dists[:, j] = np.einsum('ij,ij->i', diff, diff)

                # pick as centroid the point with largest minimum distance
                # to any centroid
                min_dists = np.min(dists[:, :i], axis=1)
                farthest_point_i = np.argmax(min_dists)
                centroids[i] = X[farthest_point_i]

            self.centroids_ = centroids
            res = super()._fit(X)
            self.centroids_ = None  # reset attribute since it is set by self.fit
            return res

        else:
            return super()._fit(X)

class FuzzyCMeans:

    def __init__(self, c, m=2, eps=1e-7, max_iter=100, n_init=10, random_state=None):
        self.c = c
        self.m = m
        self.eps = eps
        self.max_iter = max_iter
        self.n_init = n_init
        self.V_ = None

        if random_state is not None:
            random_state = np.int64(random_state)
        self.random_state = RandomState(seed=random_state)

    def fit(self, X):
        """
        Run the algorithm n_iter times and keep
        the one with best cohesion
        """
        results = []
        for _ in range(self.n_init):
            res = self._fit(X)
            results.append(res)

        # evaluate results, keep the best
        best_i = np.argmin([res[4] for res in results])
        (self.labels_,
         self.U_,
         self.V_,
         self.n_iter_,
         self.perfindex_) = results[best_i]
        return self

    def _fit(self, X):
        """
        Run main fuzzy c-means algorithm to classify the data into
        c labels or clusters
        """
        X = _sanitize(X)

        if self.V_ is None:
            # select c initial centroids randomly
            centroids_i = set()
            while len(centroids_i) != self.c:
                # ensure we generate a set of c unique centroids
                centroids_i = {self.random_state.randint(X.shape[0]) for _ in range(self.c)}

            V = X[list(centroids_i)] * 1.001    # avoid divisions by 0
        else:
            V = self.V_

        curr_iter = 0
        V_diff = self.eps * 2

        while curr_iter < self.max_iter and np.any(V_diff > self.eps):
            # compute U
            U = self._compute_U(X, V)

            # compute V
            V_old = V
            V = self._compute_V(X, U)

            V_diff = np.linalg.norm(V - V_old, 2, axis=1)
            curr_iter += 1

        labels = np.argmax(U, axis=0)
        perfindex = self._performance_index(X, U, V)

        return labels, U, V, curr_iter, perfindex

    def fit_predict(self, X):
        return self.fit(X).labels_

    def predict(self, X):
        if self.V_ is None:
            raise ValueError('This model is not fitted yet.')

        X = _sanitize(X)

        V = self.V_
        U = self._compute_U(X, V)
        return np.argmax(U, axis=0)

    def _compute_U(self, X, V):
        # precompute distances to centroids
        dists = np.empty((self.c, X.shape[0]))
        for i in range(self.c):
            diff = X - V[i]
            dists[i, :] = np.sqrt(np.einsum('ij,ij->i', diff, diff))

        # init with 1s, it will always be at least 1
        # since we always divide the distance to one cluster
        # by itelf for every cluster
        U = np.ones_like(dists)
        exp = 2 / (self.m - 1)

        # main formula broadcasted
        for i in range(self.c):
            for j in range(self.c):
                if i != j:  # thats why we init with 1s
                    U[i] += (dists[i] / dists[j]) ** exp

        return U ** -1.0

    def _compute_V(self, X, U):
        V = np.empty((self.c, X.shape[1]))
        Um = U ** self.m

        # main formula broadcasted
        for i in range(self.c):
            Um_col = Um[i].reshape(-1, 1)
            V[i] = np.sum(Um_col * X, axis=0) / np.sum(Um_col)

        return V

    def _performance_index(self, X, U, V):
        # precompute errors
        inner_errors = np.empty((self.c, X.shape[0]))
        for i in range(self.c):
            # here einsum = sum of squared factors
            diff = X - V[i]
            inner_errors[i, :] = np.einsum('ij,ij->i', diff, diff)

        global_mean = X.mean(axis=0)
        diff = V - global_mean
        outer_errors = np.einsum('ij,ij->i', diff, diff).reshape(-1, 1)

        # main formula
        perfindex = np.sum(U * (inner_errors - outer_errors))
        return perfindex


import pymc3 as pm
import numpy as np
import pandas as pd
from theano import shared
import scipy.stats as stats
import matplotlib.pyplot as plt
import arviz as az

az.style.use('arviz-darkgrid')

## Simple linear regression
np.random.seed(1)
N = 100
alpha_real = 2.5
beta_real = 0.9
eps_real = np.random.normal(0, 0.5, size=N)

x = np.random.normal(10, 1, N)
y_real = alpha_real + beta_real * x
y = y_real + eps_real

# we can center the data
#x = x - x.mean()
# or standardize the data
#x = (x - x.mean())/x.std()
#y = (y - y.mean())/y.std()

# In[4]:

_, ax = plt.subplots(1, 2, figsize=(8, 4))
ax[0].plot(x, y, 'C0.')
ax[0].set_xlabel('x')
ax[0].set_ylabel('y', rotation=0)
ax[0].plot(x, y_real, 'k')
az.plot_kde(y, ax=ax[1])
ax[1].set_xlabel('y')
plt.tight_layout()
plt.savefig('B11197_03_02.png', dpi=300)

# In[5]:

with pm.Model() as model_g:
    α = pm.Normal('α', mu=0, sd=10)
    β = pm.Normal('β', mu=0, sd=1)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = pm.Deterministic('μ', α + β * x)
    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_g = pm.sample(2000, tune=1000)

# In[6]:

az.plot_trace(trace_g, var_names=['α', 'β', 'ϵ'])
plt.savefig('B11197_03_03.png', dpi=300)

# ### Modyfing the data before running the models

# In[7]:

az.plot_pair(trace_g, var_names=['α', 'β'], plot_kwargs={'alpha': 0.1})
plt.savefig('B11197_03_04.png', dpi=300)

# ### interpreting the posterior

# In[8]:

plt.plot(x, y, 'C0.')

alpha_m = trace_g['α'].mean()
beta_m = trace_g['β'].mean()

draws = range(0, len(trace_g['α']), 10)
plt.plot(x, trace_g['α'][draws] + trace_g['β'][draws]
         * x[:, np.newaxis], c='gray', alpha=0.5)

plt.plot(x, alpha_m + beta_m * x, c='k',
         label=f'y = {alpha_m:.2f} + {beta_m:.2f} * x')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.legend()
plt.savefig('B11197_03_05.png', dpi=300)

# In[9]:

plt.plot(x, alpha_m + beta_m * x, c='k',
         label=f'y = {alpha_m:.2f} + {beta_m:.2f} * x')

sig = az.plot_hpd(x, trace_g['μ'], credible_interval=0.98, color='k')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.legend()
plt.savefig('B11197_03_06.png', dpi=300)

# In[10]:

ppc = pm.sample_posterior_predictive(trace_g, samples=2000, model=model_g)

# In[11]:

plt.plot(x, y, 'b.')
plt.plot(x, alpha_m + beta_m * x, c='k',
         label=f'y = {alpha_m:.2f} + {beta_m:.2f} * x')

az.plot_hpd(x, ppc['y_pred'], credible_interval=0.5, color='gray')
az.plot_hpd(x, ppc['y_pred'], color='gray')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.savefig('B11197_03_07.png', dpi=300)

# In[12]:

az.r2_score(y, ppc['y_pred'])

# ### The multivariate normal distribution
# Actually the bivariate

# In[13]:

sigma_x1 = 1
sigmas_x2 = [1, 2]
rhos = [-0.90, -0.5, 0, 0.5, 0.90]

k, l = np.mgrid[-5:5:.1, -5:5:.1]
pos = np.empty(k.shape + (2,))
pos[:, :, 0] = k
pos[:, :, 1] = l

f, ax = plt.subplots(len(sigmas_x2), len(rhos),
                     sharex=True, sharey=True, figsize=(12, 6),
                     constrained_layout=True)
for i in range(2):
    for j in range(5):
        sigma_x2 = sigmas_x2[i]
        rho = rhos[j]
        cov = [[sigma_x1**2, sigma_x1*sigma_x2*rho],
               [sigma_x1*sigma_x2*rho, sigma_x2**2]]
        rv = stats.multivariate_normal([0, 0], cov)
        ax[i, j].contour(k, l, rv.pdf(pos))
        ax[i, j].set_xlim(-8, 8)
        ax[i, j].set_ylim(-8, 8)
        ax[i, j].set_yticks([-5, 0, 5])
        ax[i, j].plot(0, 0,
                      label=f'$\\sigma_{{x2}}$ = {sigma_x2:3.2f}\n$\\rho$ = {rho:3.2f}', alpha=0)
        ax[i, j].legend()
f.text(0.5, -0.05, 'x_1', ha='center', fontsize=18)
f.text(-0.05, 0.5, 'x_2', va='center', fontsize=18, rotation=0)
plt.savefig('B11197_03_08.png', dpi=300)

# In[14]:

data = np.stack((x, y)).T
with pm.Model() as pearson_model:

    μ = pm.Normal('μ', mu=data.mean(0), sd=10, shape=2)

    σ_1 = pm.HalfNormal('σ_1', 10)
    σ_2 = pm.HalfNormal('σ_2', 10)
    ρ = pm.Uniform('ρ', -1., 1.)
    r2 = pm.Deterministic('r2', ρ**2)

    cov = pm.math.stack(([σ_1**2, σ_1*σ_2*ρ],
                         [σ_1*σ_2*ρ, σ_2**2]))

    y_pred = pm.MvNormal('y_pred', mu=μ, cov=cov, observed=data)

    trace_p = pm.sample(1000)

# In[15]:

az.plot_trace(trace_p, var_names=['r2'])
plt.savefig('B11197_03_9.png', dpi=300)

# In[16]:

az.summary(trace_p, var_names=['r2'])

# ## Robust linear regression

# In[17]:

ans = pd.read_csv('../data/anscombe.csv')
x_3 = ans[ans.group == 'III']['x'].values
y_3 = ans[ans.group == 'III']['y'].values
x_3 = x_3 - x_3.mean()

# In[18]:

_, ax = plt.subplots(1, 2, figsize=(10, 5))
beta_c, alpha_c = stats.linregress(x_3, y_3)[:2]
ax[0].plot(x_3, (alpha_c + beta_c * x_3), 'k',
           label=f'y ={alpha_c:.2f} + {beta_c:.2f} * x')
ax[0].plot(x_3, y_3, 'C0o')
ax[0].set_xlabel('x')
ax[0].set_ylabel('y', rotation=0)
ax[0].legend(loc=0)
az.plot_kde(y_3, ax=ax[1], rug=True)
ax[1].set_xlabel('y')
ax[1].set_yticks([])
plt.tight_layout()
plt.savefig('B11197_03_10.png', dpi=300)

# In[19]:

with pm.Model() as model_t:
    α = pm.Normal('α', mu=y_3.mean(), sd=1)
    β = pm.Normal('β', mu=0, sd=1)
    ϵ = pm.HalfNormal('ϵ', 5)
    ν_ = pm.Exponential('ν_', 1/29)
    ν = pm.Deterministic('ν', ν_ + 1)

    y_pred = pm.StudentT('y_pred', mu=α + β * x_3,
                         sd=ϵ, nu=ν, observed=y_3)

    trace_t = pm.sample(2000)

# In[20]:

varnames = ['α', 'β', 'ϵ', 'ν']
az.plot_trace(trace_t, var_names=varnames);

# In[21]:

beta_c, alpha_c = stats.linregress(x_3, y_3)[:2]

plt.plot(x_3, (alpha_c + beta_c * x_3), 'k', label='non-robust', alpha=0.5)
plt.plot(x_3, y_3, 'C0o')
alpha_m = trace_t['α'].mean()
beta_m = trace_t['β'].mean()
plt.plot(x_3, alpha_m + beta_m * x_3, c='k', label='robust')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.legend(loc=2)
plt.tight_layout()

plt.savefig('B11197_03_11.png', dpi=300)

# In[22]:

az.summary(trace_t, var_names=varnames)

# In[23]:

ppc = pm.sample_posterior_predictive(trace_t, samples=200, model=model_t, random_seed=2)

# In[24]:

data_ppc = az.from_pymc3(trace=trace_t, posterior_predictive=ppc)
ax = az.plot_ppc(data_ppc, figsize=(12, 6), mean=True)
plt.xlim(0, 12)
plt.savefig('B11197_03_12.png', dpi=300)

# ## Hierarchical linear regression

# In[25]:
N = 20
M = 8
idx = np.repeat(range(M-1), N)
idx = np.append(idx, 7)
np.random.seed(314)

alpha_real = np.random.normal(2.5, 0.5, size=M)
beta_real = np.random.beta(6, 1, size=M)
eps_real = np.random.normal(0, 0.5, size=len(idx))

y_m = np.zeros(len(idx))
x_m = np.random.normal(10, 1, len(idx))
y_m = alpha_real[idx] + beta_real[idx] * x_m + eps_real

_, ax = plt.subplots(2, 4, figsize=(10, 5), sharex=True, sharey=True)
ax = np.ravel(ax)
j, k = 0, N
for i in range(M):
    ax[i].scatter(x_m[j:k], y_m[j:k])
    ax[i].set_xlabel(f'x_{i}')
    ax[i].set_ylabel(f'y_{i}', rotation=0, labelpad=15)
    ax[i].set_xlim(6, 15)
    ax[i].set_ylim(7, 17)
    j += N
    k += N
plt.tight_layout()
plt.savefig('B11197_03_13.png', dpi=300)

# In[26]:

x_centered = x_m - x_m.mean()

# In[27]:

with pm.Model() as unpooled_model:
    α_tmp = pm.Normal('α_tmp', mu=0, sd=10, shape=M)
    β = pm.Normal('β', mu=0, sd=10, shape=M)
    ϵ = pm.HalfCauchy('ϵ', 5)
    ν = pm.Exponential('ν', 1/30)

    y_pred = pm.StudentT('y_pred', mu=α_tmp[idx] + β[idx] * x_centered,
                         sd=ϵ, nu=ν, observed=y_m)

    α = pm.Deterministic('α', α_tmp - β * x_m.mean())

    trace_up = pm.sample(2000)

# In[28]:

az.plot_forest(trace_up, var_names=['α', 'β'], combined=True)
plt.savefig('B11197_03_14.png', dpi=300)

#
#  <img src='B11197_03_15.png' width="700">

# In[29]:

with pm.Model() as hierarchical_model:
    # hyper-priors
    α_μ_tmp = pm.Normal('α_μ_tmp', mu=0, sd=10)
    α_σ_tmp = pm.HalfNormal('α_σ_tmp', 10)
    β_μ = pm.Normal('β_μ', mu=0, sd=10)
    β_σ = pm.HalfNormal('β_σ', sd=10)

    # priors
    α_tmp = pm.Normal('α_tmp', mu=α_μ_tmp, sd=α_σ_tmp, shape=M)
    β = pm.Normal('β', mu=β_μ, sd=β_σ, shape=M)
    ϵ = pm.HalfCauchy('ϵ', 5)
    ν = pm.Exponential('ν', 1/30)

    y_pred = pm.StudentT('y_pred', mu=α_tmp[idx] + β[idx] * x_centered,
                         sd=ϵ, nu=ν, observed=y_m)

    α = pm.Deterministic('α', α_tmp - β * x_m.mean())
    α_μ = pm.Deterministic('α_μ', α_μ_tmp - β_μ * x_m.mean())
    α_σ = pm.Deterministic('α_sd', α_σ_tmp - β_μ * x_m.mean())

    trace_hm = pm.sample(1000)

# In[30]:

az.plot_forest(trace_hm, var_names=['α', 'β'], combined=True)
plt.savefig('B11197_03_16.png', dpi=300)

# In[31]:

_, ax = plt.subplots(2, 4, figsize=(10, 5), sharex=True, sharey=True,
                     constrained_layout=True)
ax = np.ravel(ax)
j, k = 0, N
x_range = np.linspace(x_m.min(), x_m.max(), 10)
for i in range(M):
    ax[i].scatter(x_m[j:k], y_m[j:k])
    ax[i].set_xlabel(f'x_{i}')
    ax[i].set_ylabel(f'y_{i}', labelpad=17, rotation=0)
    alpha_m = trace_hm['α'][:, i].mean()
    beta_m = trace_hm['β'][:, i].mean()
    ax[i].plot(x_range, alpha_m + beta_m * x_range, c='k',
               label=f'y = {alpha_m:.2f} + {beta_m:.2f} * x')
    plt.xlim(x_m.min()-1, x_m.max()+1)
    plt.ylim(y_m.min()-1, y_m.max()+1)
    j += N
    k += N
plt.savefig('B11197_03_17.png', dpi=300)

# ## Polynomial regression

# In[32]:

x_2 = ans[ans.group == 'II']['x'].values
y_2 = ans[ans.group == 'II']['y'].values
x_2 = x_2 - x_2.mean()

plt.scatter(x_2, y_2)
plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.savefig('B11197_03_18.png', dpi=300)

# In[33]:

with pm.Model() as model_poly:
    α = pm.Normal('α', mu=y_2.mean(), sd=1)
    β1 = pm.Normal('β1', mu=0, sd=1)
    β2 = pm.Normal('β2', mu=0, sd=1)
    ϵ = pm.HalfCauchy('ϵ', 5)

    mu = α + β1 * x_2 + β2 * x_2**2

    y_pred = pm.Normal('y_pred', mu=mu, sd=ϵ, observed=y_2)

    trace_poly = pm.sample(2000)

# In[34]:

#az.plot_trace(trace_poly);

# In[35]:

#az.summary(trace_poly)

# In[36]:

x_p = np.linspace(-6, 6)
y_p = trace_poly['α'].mean() + trace_poly['β1'].mean() *     x_p + trace_poly['β2'].mean() * x_p**2
plt.scatter(x_2, y_2)
plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.plot(x_p, y_p, c='C1')
plt.savefig('B11197_03_19.png', dpi=300)

# ## Multiple Linear regression

# In[37]:

np.random.seed(314)
N = 100
alpha_real = 2.5
beta_real = [0.9, 1.5]
eps_real = np.random.normal(0, 0.5, size=N)

X = np.array([np.random.normal(i, j, N) for i, j in zip([10, 2], [1, 1.5])]).T
X_mean = X.mean(axis=0, keepdims=True)
X_centered = X - X_mean
y = alpha_real + np.dot(X, beta_real) + eps_real

# In[38]:

def scatter_plot(x, y):
    plt.figure(figsize=(10, 10))
    for idx, x_i in enumerate(x.T):
        plt.subplot(2, 2, idx+1)
        plt.scatter(x_i, y)
        plt.xlabel(f'x_{idx+1}')
        plt.ylabel(f'y', rotation=0)

    plt.subplot(2, 2, idx+2)
    plt.scatter(x[:, 0], x[:, 1])
    plt.xlabel(f'x_{idx}')
    plt.ylabel(f'x_{idx+1}', rotation=0)

scatter_plot(X_centered, y)
plt.savefig('B11197_03_20.png', dpi=300)

# In[39]:

with pm.Model() as model_mlr:
    α_tmp = pm.Normal('α_tmp', mu=0, sd=10)
    β = pm.Normal('β', mu=0, sd=1, shape=2)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α_tmp + pm.math.dot(X_centered, β)

    α = pm.Deterministic('α', α_tmp - pm.math.dot(X_mean, β))

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_mlr = pm.sample(2000)

# In[40]:

varnames = ['α', 'β', 'ϵ']
#az.plot_trace(trace_mlr, var_names=varnames);

# In[41]:

az.summary(trace_mlr, var_names=varnames)

# ## Confounding variables and redundant variables

# In[42]:

np.random.seed(42)
N = 100
x_1 = np.random.normal(size=N)
x_2 = x_1 + np.random.normal(size=N, scale=1)
#x_2 = x_1 + np.random.normal(size=N, scale=0.01)
y = x_1 + np.random.normal(size=N)
X = np.vstack((x_1, x_2)).T

# In[43]:

scatter_plot(X, y)
plt.savefig('B11197_03_21.png', dpi=300)

# In[44]:

with pm.Model() as m_x1x2:
    α = pm.Normal('α', mu=0, sd=10)
    β1 = pm.Normal('β1', mu=0, sd=10)
    β2 = pm.Normal('β2', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β1 * X[:, 0] + β2 * X[:, 1]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x1x2 = pm.sample(2000)

with pm.Model() as m_x1:
    α = pm.Normal('α', mu=0, sd=10)
    β1 = pm.Normal('β1', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β1 * X[:, 0]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x1 = pm.sample(2000)

with pm.Model() as m_x2:
    α = pm.Normal('α', mu=0, sd=10)
    β2 = pm.Normal('β2', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β2 * X[:, 1]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x2 = pm.sample(2000)

# In[45]:

az.plot_forest([trace_x1x2, trace_x1, trace_x2],
               model_names=['m_x1x2', 'm_x1', 'm_x2'],
               var_names=['β1', 'β2'],
               combined=False, colors='cycle', figsize=(8, 3))
plt.savefig('B11197_03_22.png', dpi=300)

# In[46]:

# just repeating the code from a couple of cells before, but with a lower value of `scale`.
np.random.seed(42)
N = 100
x_1 = np.random.normal(size=N)
x_2 = x_1 + np.random.normal(size=N, scale=0.01)
y = x_1 + np.random.normal(size=N)
X = np.vstack((x_1, x_2)).T

# In[47]:

scatter_plot(X, y)
plt.savefig('B11197_03_23.png', dpi=300)

# In[48]:

with pm.Model() as model_red:
    α = pm.Normal('α', mu=0, sd=10)
    β = pm.Normal('β', mu=0, sd=10, shape=2)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + pm.math.dot(X, β)

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_red = pm.sample(2000)

# In[49]:

az.plot_forest(trace_red, var_names=['β'], combined=True, figsize=(8, 2))
plt.savefig('B11197_03_24.png', dpi=300)

# In[50]:

az.plot_pair(trace_red, var_names=['β'])
plt.savefig('B11197_03_25.png', dpi=300)

# ## Masking effect variables

# In[51]:

np.random.seed(42)
N = 126
r = 0.8
x_1 = np.random.normal(size=N)
x_2 = np.random.normal(x_1, scale=(1 - r ** 2) ** 0.5)
y = np.random.normal(x_1 - x_2)
X = np.vstack((x_1, x_2)).T

# In[52]:

scatter_plot(X, y)
plt.savefig('B11197_03_26.png', dpi=300, figsize=(5.5, 5.5))

# In[53]:

with pm.Model() as m_x1x2:
    α = pm.Normal('α', mu=0, sd=10)
    β1 = pm.Normal('β1', mu=0, sd=10)
    β2 = pm.Normal('β2', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β1 * X[:, 0] + β2 * X[:, 1]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x1x2 = pm.sample(1000)

with pm.Model() as m_x1:
    α = pm.Normal('α', mu=0, sd=10)
    β1 = pm.Normal('β1', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β1 * X[:, 0]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x1 = pm.sample(1000)

with pm.Model() as m_x2:
    α = pm.Normal('α', mu=0, sd=10)
    β2 = pm.Normal('β2', mu=0, sd=10)
    ϵ = pm.HalfCauchy('ϵ', 5)

    μ = α + β2 * X[:, 1]

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=y)

    trace_x2 = pm.sample(1000)

# In[54]:

az.plot_forest([trace_x1x2, trace_x1, trace_x2],
               model_names=['m_x1x2', 'm_x1', 'm_x2'],
               var_names=['β1', 'β2'],
               combined=True, colors='cycle', figsize=(8, 3))
plt.savefig('B11197_03_27.png', dpi=300, bbox_inches='tight')

# ## Variable variance

# In[55]:

data = pd.read_csv('../data/babies.csv')
data.plot.scatter('Month', 'Lenght')
plt.savefig('B11197_03_28.png', dpi=300)

# In[56]:

with pm.Model() as model_vv:
    α = pm.Normal('α', sd=10)
    β = pm.Normal('β', sd=10)
    γ = pm.HalfNormal('γ', sd=10)
    δ = pm.HalfNormal('δ', sd=10)

    x_shared = shared(data.Month.values * 1.)

    μ = pm.Deterministic('μ', α + β * x_shared**0.5)
    ϵ = pm.Deterministic('ϵ', γ + δ * x_shared)

    y_pred = pm.Normal('y_pred', mu=μ, sd=ϵ, observed=data.Lenght)

    trace_vv = pm.sample(1000, tune=1000)

# In[57]:

plt.plot(data.Month, data.Lenght, 'C0.', alpha=0.1)

μ_m = trace_vv['μ'].mean(0)
ϵ_m = trace_vv['ϵ'].mean(0)

plt.plot(data.Month, μ_m, c='k')
plt.fill_between(data.Month, μ_m + 1 * ϵ_m, μ_m -
                 1 * ϵ_m, alpha=0.6, color='C1')
plt.fill_between(data.Month, μ_m + 2 * ϵ_m, μ_m -
                 2 * ϵ_m, alpha=0.4, color='C1')

plt.xlabel('x')
plt.ylabel('y', rotation=0)
plt.savefig('B11197_03_29.png', dpi=300)

# In[58]:

x_shared.set_value([0.5])
ppc = pm.sample_posterior_predictive(trace_vv, 2000, model=model_vv)
y_ppc = ppc['y_pred'][:, 0]

# In[59]:

ref = 53
density, l, u = az._fast_kde(y_ppc)
x_ = np.linspace(l, u, 200)
plt.plot(x_, density)
percentile = int(sum(y_ppc <= ref) / len(y_ppc) * 100)
plt.fill_between(x_[x_ < ref], density[x_ < ref],
                 label='percentile = {:2d}'.format(percentile))
plt.xlabel('length')
plt.yticks([])
plt.legend()
plt.savefig('B11197_03_30.png', dpi=300)

# In[ ]:

x_4 = ans[ans.group == 'IV']['x'].values
y_4 = ans[ans.group == 'IV']['y'].values

with pm.Model() as model_t2:
    α = pm.Normal('α', mu=0, sd=100)
    β = pm.Normal('β', mu=0, sd=1)
    ϵ = pm.HalfCauchy('ϵ', 5)
    ν = pm.Exponential('ν', 1/30)
    #ν = pm.Gamma('ν', mu=20, sd=15)
    #ν = pm.Gamma('ν', 2, 0.1)

    y_pred = pm.StudentT('y_pred', mu=α + β * x_4, sd=ϵ, nu=ν, observed=y_4)
    trace_t2 = pm.sample(2000)

import numpy,import scipy,from theano,import pymc3


#!/usr/bin/env python
# coding: utf-8

# DO NOT EDIT
# Autogenerated from the notebook copula.ipynb.
# Edit the notebook and then sync the output with this file.
#
# flake8: noqa
# DO NOT EDIT

# # Copula - Multivariate joint distribution

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from scipy import stats

sns.set_style("darkgrid")
sns.mpl.rc("figure", figsize=(8, 8))

# When modeling a system, there are often cases where multiple parameters
# are involved. Each of these parameters could be described with a given
# Probability Density Function (PDF). If would like to be able to generate a
# new set of parameter values, we need to be able to sample from these
# distributions-also called marginals. There are mainly two cases: *(i)*
# PDFs are independent; *(ii)* there is a dependency. One way to model the
# dependency it to use a **copula**.

# ## Sampling from a copula
#
# Let's use a bi-variate example and assume first that we have a prior and
# know how to model the dependence between our 2 variables.
#
# In this case, we are using the Gumbel copula and fix its hyperparameter
# `theta=2`. We can visualize it's 2-dimensional PDF.

from statsmodels.distributions.copula.api import (CopulaDistribution,
                                                  GumbelCopula,
                                                  IndependenceCopula)

copula = GumbelCopula(theta=2)
_ = copula.plot_pdf()  # returns a matplotlib figure

# And we can sample the PDF.

sample = copula.rvs(10000)
h = sns.jointplot(x=sample[:, 0], y=sample[:, 1], kind="hex")
_ = h.set_axis_labels("X1", "X2", fontsize=16)

# Let's come back to our 2 variables for a second. In this case we
# consider them to be gamma and normally distributed. If they would be
# independent from each other, we could sample from each PDF individually.
# Here we use a convenient class to do the same operation.
#
# ### Reproducibility
#
# Generating reproducible random values from copulas required explicitly
# setting the `seed` argument.
# `seed` accepts either an initialized NumPy `Generator` or `RandomState`,
# or any argument acceptable
# to `np.random.default_rng`, e.g., an integer or a sequence of integers.
# This example uses an
# integer.
#
# The singleton `RandomState` that is directly exposed in the `np.random`
# distributions is
# not used, and setting `np.random.seed` has no effect on the values
# generated.

marginals = [stats.gamma(2), stats.norm]
joint_dist = CopulaDistribution(copula=IndependenceCopula(),
                                marginals=marginals)
sample = joint_dist.rvs(512, random_state=20210801)
h = sns.jointplot(x=sample[:, 0], y=sample[:, 1], kind="scatter")
_ = h.set_axis_labels("X1", "X2", fontsize=16)

# Now, above we have expressed the dependency between our variables using
# a copula, we can use this copula to sample a new set of observation with
# the same convenient class.

joint_dist = CopulaDistribution(copula, marginals)
# Use an initialized Generator object
rng = np.random.default_rng([2, 0, 2, 1, 0, 8, 0, 1])
sample = joint_dist.rvs(512, random_state=rng)
h = sns.jointplot(x=sample[:, 0], y=sample[:, 1], kind="scatter")
_ = h.set_axis_labels("X1", "X2", fontsize=16)

# There are two things to note here. *(i)* as in the independent case, the
# marginals are correctly showing a gamma and normal distribution; *(ii)*
# the dependence is visible between the two variables.

# ## Estimating copula parameters
#
# Now, imagine we already have experimental data and we know that there is
# a dependency that can be expressed using a Gumbel copula. But we don't
# know what is the hyperparameter value for our copula. In this case, we can
# estimate the value.
#
# We are going to use the sample we just generated as we already know the
# value of the hyperparameter we should get: `theta=2`.

copula = GumbelCopula()
theta = copula.fit_corr_param(sample)
print(theta)

# We can see that the estimated hyperparameter value is close to the value
# set previously.

import numpy,from scipy,from statsmodels


## Visualize Arakawa C-grid numbering
import numpy as np
import matplotlib.pyplot as plt

nx = 3
ny = 3

Lx = nx
Ly = ny

dx = Lx/nx
dy = Ly/ny

# grid vectors for T-points
x_T = np.arange(dx/2.,Lx,dx)
y_T = np.arange(dy/2.,Ly,dy)

# grid vectors for u-points
x_u = x_T[:-1] + dx/2.
y_u = y_T

#grid vectors for v-points
x_v = x_T
y_v = y_T[:-1] + dy/2.

# grid vectors for q-points
x_q = np.arange(0,Lx+dx/2.,dx)
y_q = np.arange(0,Ly+dy/2.,dy)

## meshgrid

xx_T,yy_T = np.meshgrid(x_T,y_T)
xx_u,yy_u = np.meshgrid(x_u,y_u)
xx_v,yy_v = np.meshgrid(x_v,y_v)
xx_q,yy_q = np.meshgrid(x_q,y_q)

xx_T = xx_T.flatten()
xx_u = xx_u.flatten()
xx_v = xx_v.flatten()
xx_q = xx_q.flatten()

yy_T = yy_T.flatten()
yy_u = yy_u.flatten()
yy_v = yy_v.flatten()
yy_q = yy_q.flatten()

fig,ax = plt.subplots(1,1)

ax.plot(xx_T,yy_T,'r.',ms=10,mec='k')
ax.plot(xx_u,yy_u,'g.',ms=10,mec='k')
ax.plot(xx_v,yy_v,'b.',ms=10,mec='k')
ax.plot(xx_q,yy_q,'k.',ms=10)

ax.plot(x_v,[0]*nx,'b.',ms=10,alpha=.6)
ax.plot(x_v,[Ly]*nx,'b.',ms=10,alpha=.6)
ax.plot([0]*ny,y_u,'g.',ms=10,alpha=.6)
ax.plot([Lx]*ny,y_u,'g.',ms=10,alpha=.6)

plt.grid()

for xx,yy,vv in zip([xx_T,xx_u,xx_v,xx_q],[yy_T,yy_u,yy_v,yy_q],['T','u','v','q']):
    for i in range(len(xx)):
        ax.text(xx[i]+0.03,yy[i]+0.03,r'$'+vv+'_{%i}$' %(i+1),fontsize=15)

ax.set_xlabel(r'$x$')
ax.set_ylabel(r'$y$')

ax.set_xticks(range(Lx+1))
ax.set_yticks(range(Ly+1))

ax.set_xlim(-0.1,nx+0.1)
ax.set_ylim(-0.1,ny+0.1)

ax.set_frame_on(False)

dxstring = [(r'$%i\Delta x$' % i) if i > 1 else r'$\Delta x$' for i in range(1,nx)]
dystring = [(r'$%i\Delta y$' % i) if i > 1 else r'$\Delta y$' for i in range(1,ny)]

ax.set_xticklabels([r'$0$']+dxstring+[r'$L_x$'])
ax.set_yticklabels([r'$0$']+dystring+[r'$L_y$'])
plt.tight_layout()
plt.show()


### Start initialization
import sys
import numpy as np
import matplotlib.pyplot as plt
### End initialization
### Start Alt2: Streamlines with streamplot ###
w = 3
Y, X = np.mgrid[-w:w:100j, -w:w:100j]
R = np.sqrt(X**2+Y**2)
U = -Y / R**2 # Note that -Y/R = -sin(phi)
V = X / R**2  # Note that X/R = cos(phi)

fig = plt.figure()
ax = fig.add_subplot(111)

strm = ax.streamplot(X, Y, U, V, linewidth=2)

ax.set_title(r'Streamplot, vortex field ' +\
        r'$\vec{F} = \frac{J}{2\pi\rho}\hat\varphi$')
ax.set_xlabel('$x$')
ax.set_ylabel('$y$')

ax.set_aspect('equal')

plt.savefig('streamlines-vortex.png')
### End Alt2

### Start Alt1: Streamlines as 2d curves ###
def curve(rho0,x):
    """Fieldline for vortex streamlines in Cartesian coordinates.
    Assume z=0 => Fieldline in xy-plane.
    We also avoid the singularity at x=y=0."""
    if max(abs(x)) > rho0:
        print('Fieldline for rho_0=%3.1f is only defined for |x| <= %3.1f' %(rho0,rho0))
        return None
    return np.sqrt(rho0**2 - x**2)

fig = plt.figure()
ax = fig.add_subplot(111)

# Grid of x points
nx = 64

# Streamline constant
for rho0 in np.linspace(1,3,5):

    # Grid of x points
    x = np.linspace(-rho0,rho0, nx)

    yp = curve(rho0,x)
    ym = -curve(rho0,x)

    # Plot the streamlines with an appropriate colormap and arrow style
    ax.plot(x, yp, 'b-', x, ym, 'b-', linewidth=2)
    ax.annotate("", xy=(max(x), 0.1), xytext=(max(x), 0), arrowprops=dict(arrowstyle="->",color='b'))

ax.set_title(r'Streamplot, vortex field $\vec{F} = \frac{J}{2\pi\rho}\hat\varphi$')
ax.set_xlabel('$x$')
ax.set_ylabel('$y$')

ax.set_aspect('equal')
### End Alt1

#plt.show()
plt.savefig('streamlines-vortex-curves.png')


%matplotlib inline
import tqdm
import numpy as np
import matplotlib.pyplot as plt
from IPython import display # ?
display.set_matplotlib_formats('svg') # ?

import torch
from torch import Tensor
from torchvision import transforms, datasets

data_transform = transforms.Compose([transforms.Grayscale(), transforms.ToTensor(), transforms.Normalize(mean=[0], std=[1])])

mnist_train = datasets.MNIST(root='./data', train=True, download=True, transform=data_transform)
mnist_test = datasets.MNIST(root='./data', train=False, download=True, transform=data_transform)

xcount = torch.ones((10, 28, 28), dtype=torch.float32)
ycount = torch.ones((10), dtype=torch.float32)
##* Note: by inititalizing as ones, this conduct Laplace smoothing also.

for data, label in mnist_train:
    y = int(label)
    ycount[y] += 1
    xcount[y,:,:] += data.squeeze() # data.shape=(1,28,28); xcount[y,:,:].shape=(28,28)

py = ycount / ycount.sum()
px = xcount / ycount.reshape((10, 1, 1)) # ycount.shape=(10), xcount.shape=(10,28,28), non-broadcastable
##* Note this py, px are also the MLE.

## hotmap count for each class
fig,ax = plt.subplots(1,10,figsize=(10, 10))
for i in range(10):
    ax[i].imshow(xcount[i,:,:].numpy(), cmap='hot')
    ax[i].axes.get_xaxis().set_visible(False)
    ax[i].axes.get_yaxis().set_visible(False)
plt.show()
print('Class probabilities', py)

## give prediction/recognition (as discrimnant task)
data, label = mnist_test[0]
# recall that we classify y as to the class maximize the posterior prob.
# so we need to compute:
# 1) conditional liklihood. p(x|y). log-lik is computed to avoid overflow.
# 2) p(y): this is governed by the py we trained.
logpx = torch.log(px)
logpxneg = torch.log(1-px)
logpy = torch.log(py)

condloglik = data * logpx + (1-data) * logpxneg
condloglik = condloglik.sum((1,2))
bayespost = condloglik + logpy # log[p(x|y)p(y)] i.e. log[p(y|x)]
bayespost -= torch.max(bayespost) # Normalizing to prevent overflow or underflow
post = torch.exp(bayespost)

post /= post.sum()
#* normalizing the post to make it a distribution.
#* note that: \sum_y {p(x|y)p(y)}=p(x); so this gives posterior distribution
torch.argmax(post) # map prediction
# wrap this as function
del(logpx, logpy, logpxneg, condloglik, bayespost, post)

def bayespost(data, px, py):
    logpx = torch.log(px)
    logpxneg = torch.log(1-px)
    logpy = torch.log(py)

    logpost = (data*logpx + (1-data)*logpxneg).sum((1,2)) + logpy
    logpost -= logpost.max()
    post = logpost.exp()

    post /= post.sum()
    return post

post = bayespost(data, px, py)
post.argmax().item()

# now lets try 10 images for testing
fig, ax = plt.subplots(2, 10, figsize=(10, 3))
cnt = 0
for data, label in mnist_test:
    post = bayespost(data, px, py)
    # pic
    ax[1, cnt].bar(range(10), post) # draw the posterior
    ax[1, cnt].axes.get_yaxis().set_visible(False)
    ax[0, cnt].imshow(data.reshape((28,28)), cmap='hot')
    ax[0, cnt].axes.get_xaxis().set_visible(False)
    ax[0, cnt].axes.get_yaxis().set_visible(False)
    cnt += 1

# now calculate the testing error
cnt = 0
err = 0
for data, label in mnist_test:
    cnt += 1
    post = bayespost(data,px,py)

    if (post[int(label)] < post.max()): err += 1

print(f'Naive Bayes has error rate = {err/cnt}')


# Principal component analysis is a method that rotates the dataset in a way such that
# the rotated features are statistically uncorrelated. This rotation is often followed by
# selecting only a subset of the new features, according to how important they are for
# explaining the data.

import matplotlib.pyplot as plt

import mglearn

import numpy as np

from sklearn.datasets import load_breast_cancer
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA

cancer = load_breast_cancer()

def visualize_cancer_dataset_histograms():
    # One of the most common applications of PCA is visualizing high-dimensional datasets.
    fig, axes = plt.subplots(15, 2, figsize=(10, 20))
    malignant = cancer.data[cancer.target == 0]
    benign = cancer.data[cancer.target == 1]

    ax = axes.ravel()

    # Create a histogram for each of the features, counting how often a data point appears with a feature in a certain range (called a bin).
    for i in range(30):
        _, bins = np.histogram(cancer.data[:, i], bins=50)
        ax[i].hist(malignant[:, i], bins=bins, alpha=.5)
        ax[i].hist(benign[:, i], bins=bins, alpha=.5)
        ax[i].set_title(cancer.feature_names[i])
        ax[i].set_yticks(())
    ax[0].set_xlabel("Feature magnitude")
    ax[0].set_ylabel("Frequency")
    ax[0].legend(["malignant", "benign"], loc="best")
    plt.show()

# these plots dont show us anything about the interactions between variables and how these relate to the classes.
# visualize_cancer_dataset_histograms()

def visualize_cancer_dataset_pca():
    # Before applying PCA, scale the data so that each feature has unit variance.
    scaler = StandardScaler()
    scaler.fit(cancer.data)
    X_scaled = scaler.transform(cancer.data)

    pca = PCA(n_components=2)  # keep the first 2 principal components of the data
    pca.fit(X_scaled)

    X_pca = pca.transform(X_scaled)  # transform data onto the first two principal components.

    print("Original shape: {}".format(str(X_scaled.shape)))  # (569, 30)
    print("Reduced shape: {}".format(str(X_pca.shape)))  # (569, 2)

    print("PCA component shape: {}".format(pca.components_.shape))  # (2, 30)

    # plot first vs second principal component
    plt.figure(figsize=(8, 8))
    mglearn.discrete_scatter(X_pca[:, 0], X_pca[:, 1], cancer.target)
    plt.legend(cancer.target_names, loc="best")
    plt.gca().set_aspect("equal")
    plt.xlabel("First principal component")
    plt.ylabel("Second principal component")
    plt.show()

    # Visualize PCA coefficients
    plt.matshow(pca.components_, cmap='viridis')
    plt.yticks([0, 1], ["First component", "Second component"])
    plt.colorbar()
    plt.xticks(range(len(cancer.feature_names)), cancer.feature_names, rotation=60, ha='left')
    plt.xlabel("Feature")
    plt.ylabel("Principal components")
    plt.show()

visualize_cancer_dataset_pca()


# ！/usr/bin/python
# -*- coding: utf-8 -*-
# Created by PJZero on 17-4-22

import numpy as np

def gen_line_data(sample_num=100):
    """
    y = 3*x1 + 4*x2
    :return:
    """
    x1 = np.linspace(0, 9, sample_num)
    x2 = np.linspace(4, 13, sample_num)
    x = np.concatenate(([x1], [x2]), axis=0).T
    y = np.dot(x, np.array([3, 4]).T)  # y 列向量
    return x, y

def line_regression_mat(samples, labels):
    """
    线性回归：具体请参照西瓜书
    :param samples:
    :param labels:
    :return:
    """
    assert isinstance(samples, np.ndarray)
    assert isinstance(labels, np.ndarray)
    return np.dot(np.linalg.inv(np.dot(samples.T, samples)),
                  np.dot(samples.T, labels))

if __name__ == '__main__':
    x, y = gen_line_data()
    weight = line_regression_mat(x, y)
    print(weight)  # [ 3.  4.]


import pandas as pd
import numpy as np
from scipy import stats
from numpy.linalg import inv
import matplotlib.pylab as plt
import os

os.chdir('e:/MIT4/6.439/pset1')
# 1.2
df_gamma = pd.read_csv('data/gamma-ray.csv')
df_gamma.head()
lam = np.sum(df_gamma['count'])/np.sum(df_gamma['seconds'])
lam

df_gamma['count'] = pd.to_numeric(df_gamma['count'], downcast='signed')
df_gamma['count'].dtype

df_gamma['x!'] = df_gamma['count'].apply(np.math.factorial)

# 1.4
df_golub = pd.read_csv('data/golub_data/golub.csv', index_col=0)
df_cl = pd.read_csv('data/golub_data/golub_cl.csv' ,index_col=0)
df_names = pd.read_csv('data/golub_data/golub_gnames.csv', index_col=0)

df_golub.columns = list(range(1,39))
# 3051 genes of 18 patients
df_golub
df_cl
df_cl
(1-df_cl)['x'].sum()
# How many genes are associated with the different tumor types (meaning that their expression level differs between the two tumor types) using (i) the uncorrected p-values, (ii)the Holm-Bonferroni correction, and (iii) the Benjamini-Hochberg correction?

# split the data into ALL and AML
col_ALL = (df_cl.loc[df_golub.columns] == 0).transpose().values.tolist()[0]
col_AML = (df_cl.loc[df_golub.columns] == 1).transpose().values.tolist()[0]
df_ALL = df_golub[df_golub.columns[col_ALL]]
df_AML = df_golub[df_golub.columns[col_AML]]
df_ALL
# hypothesis testing
from scipy.stats import ttest_ind

# test each gene
t_stat, p_value = [],[]
for i in range(df_golub.shape[0]):
    # t, p =  ttest_ind( df_ALL.iloc[i], df_AML.iloc[i])
    t, p =  ttest_ind( df_ALL.iloc[i], df_AML.iloc[i], equal_var=False )
    t_stat.append(t)
    p_value.append(p)

df_welch_ttest = pd.DataFrame(index = range(1, df_golub.shape[0]+1))
df_welch_ttest['t_stat'] = t_stat
df_welch_ttest['p_value'] = p_value

df_welch_ttest
df_welch_ttest['significant_uncorrected'] = df_welch_ttest['p_value']<0.05
pvals = df_welch_ttest['p_value']

# bonferroni correction
def holm_bonferroni(pvals, alpha=0.05):
    m, pvals = len(pvals), np.asarray(pvals)
    ind = np.argsort(pvals)
    test = [p > alpha/(m+1-k) for k, p in enumerate(pvals[ind])]
    significant = np.zeros(np.shape(pvals), dtype='bool')
    significant[ind[0:m-np.sum(test)]] = True
    return significant

# Benjamini-Hochberg procedure
def BH(pvals, q=0.05):
    m = len(pvals)
    significant = np.zeros(m, dtype='bool')
    sort_ind = np.argsort(pvals).astype(int)+1 # sort the p-values

    for i in range(1,m+1): #i = the individual p-value’s rank
         if pvals[sort_ind[i]] < (i)*q/m:
            significant[sort_ind[i]-1] = True # record the significant index
    return significant

significant_pvals = holm_bonferroni(pvals, alpha=0.05)
df_welch_ttest['significant_pvals'] = significant_pvals
significant_pvals_BH = BH(pvals, q=0.05)
df_welch_ttest['significant_pvals_BH'] = significant_pvals_BH
df_welch_ttest

df_welch_ttest.sum()

# 1.6
syn_x = pd.read_csv('data/syn_X.csv',header=None)
syn_y = pd.read_csv('data/syn_Y.csv',header=None)

syn_x.columns = ['x1','x2']
syn_x['x0'] = 1

syn_x = syn_x[['x0','x1','x2']]
X = syn_x.values
Y = syn_y.values
X.shape
#beta = (X'X)^-1X'Y
a = np.matmul(inv(np.matmul(np.transpose(X),X)),np.transpose(X))

beta = np.matmul(a,Y)

beta

def gradientDescent(X, Y, beta_0, alpha, t):
    m, n = X.shape # m is number of cases, n is the number of variables
    cost = pd.DataFrame(np.zeros([t,2]))
    cost.columns = ['step','cost']
    beta = beta_0

    for i in range(t):
        # vectorized gradient: X'*(Y-X*beta)
        res = Y- np.matmul(X, beta)
        beta = beta + 2 * alpha * (1/m) * np.matmul(np.transpose(X), res)
        # calculate the cost base on current beta
        cost['step'][i] = i
        cost['cost'][i] = calCost(X, Y, beta)

    cost.plot(kind = 'scatter', x = 'step',y = 'cost')
    return beta, cost

def calCost(X, Y, beta):
    m, n = X.shape
    # vectorized cost: (X*beta - Y)'(X*beta - Y)
    residual = Y- np.matmul(X, beta)
    return (1/(2*m))*np.matmul(np.transpose(residual), residual)

beta_0 = np.matrix('0.5; 0.5; 0.5')
alpha = 0.1
t = 50
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

beta_0 = np.matrix('0.5; 0.5; 0.5')
alpha = 0.01
t = 50
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

beta_0 = np.matrix('0.5; 0.5; 0.5')
alpha = 0.05
t = 50
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

beta_0 = np.matrix('0.5; 0.5; 0.5')
alpha = 0.8
t = 50
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

beta_0 = np.random.rand(3,1)
alpha = 0.05
t = 100
beta, cost = gradientDescent(X, Y, beta_0,alpha, t )

df_mort_0 =  pd.read_csv('data/mortality.csv')
df_mort = df_mort_0

# check the scatterplot

df_mort.head()
# check the correlation matrix
import seaborn as sns

corr = df_mort.iloc[:,2:].corr()
corrplot = sns.heatmap(corr,xticklabels=corr.columns,yticklabels=corr.columns, linewidths=.01,cmap="YlGnBu")
fig = corrplot.get_figure()
fig.savefig('corrplot.png')

df_mort.columns
df_mort = df_mort.drop(['HC'], axis = 1) # drop the intercorrelated

sns.set(style="ticks")
pairplot = sns.pairplot(df_mort.iloc[:,1:], diag_kind="kde",markers="+",plot_kws=dict(s=50, edgecolor="b", linewidth=1),diag_kws=dict(shade=True))

pairplot.savefig('pairplot1.png')
# log-transformation
df_mort[['SO2','NOx','Pop']] = np.log(df_mort[['SO2','NOx','Pop']])
df_morthead()
# normalize the Data
data = df_mort.iloc[:,1:]
data = (data - data.mean())/(data.max() - data.min())
df_mort.iloc[:,1:] = data

df_mort.plot(kind = 'bar', x = 'City', y = 'Mortality',fontsize = 5)
plt.savefig('city.pdf')
df_mort[df_mort['Mortality'] == df_mort['Mortality'].max()]
df_mort[df_mort['Mortality'] == df_mort['Mortality'].min()]

# GD on raw data
Y_r = pd.DataFrame(df_mort_0['Mortality']).values
X_r = df_mort_0.iloc[:,2:].values
Y_r.shape
m, n = X_r.shape

beta_0 = np.random.rand(n,1)
beta_0

alpha = 0.00001
t = 1000
beta, cost = gradientDescent(X_r, Y_r, beta_0, alpha, t )
cost

Y= pd.DataFrame(df_mort['Mortality']).values
X = df_mort.iloc[:,2:].values
m, n = X.shape
beta_0 = np.random.rand(n,1)
alpha = 0.05
t = 2000
beta, cost = gradientDescent(X, Y, beta_0, alpha, t )
beta

#plot the residual
import scipy.stats as stats
import pylab

residuals = np.transpose(Y - np.matmul(X, beta))
res_list = sorted(residuals[0].tolist())
def q_q_plot(data):
    norm=np.random.normal(0,2,len(data))
    norm.sort()

    plt.plot(norm,data,"o")
    z = np.polyfit(norm,data, 1)
    p = np.poly1d(z)
    plt.plot(norm,p(norm),"k--", linewidth=2)
    plt.title("Normal Q-Q plot", size=20)
    plt.xlabel("Theoretical quantiles", size=18)
    plt.ylabel("Expreimental quantiles", size=18)
    plt.tick_params(labelsize=16)
    plt.savefig('qqplot.png')
    plt.show()

q_q_plot(res_list)

#

import numpy,from numpy,import scipy,from scipy


import numpy as np

class CrossEntropyLoss():
    """
    Numerically stable Cross Entropy Loss (Logistic Loss) function.

    Reference: https://web.stanford.edu/~jurafsky/slp3/5.pdf
    Page 7, Formula 5.12

    Parameters
    ----------
    prob: array_like
        Predicted probabilities.
    y: array_like
        Actual labels.
    epsilon: scalar
        If any of probabilities values are 0 or 1, loss will be undefined, so epsilon is added inside the logs.

    Returns
    -------
    loss: scalar
        Cross Entropy Loss value of given predicted probabilities and actual labels.
    """
    def __call__(self, prob, y, epsilon=1e-15):
        loss = ( -( y * np.log(prob + epsilon) + (1 - y) * np.log(1 - prob + epsilon) )).mean()
        return loss

class Accuracy():
    """
    Calculates mean accuracy of predictions on a given data.

    Parameters
    ----------
    y_pred: array_like
        Predicted labels.
    y_true: array_like
        Actual labels.

    Returns
    -------
    mean_accuracy: scalar
        Mean accuracy of predictions on a given data.
    """
    def __call__(self, y_pred, y_true):
        accuracy = (y_pred == y_true).mean()
        return accuracy

class MeanSquaredError():
    """
    Calculates mean squared error of predictions on a given data.

    Parameters
    ----------
    y_pred: array_like
        Predicted values.
    y_true: array_like
        Actual values.

    Returns
    -------
    error: scalar
        Mean squared error of predictions on a given data.
    """
    def __call__(self, y_pred, y_true):
        error = np.abs(y_pred - y_true).mean()
        return error

class RSquared():
    """
    Calculate R^2.

    Parameters
    ----------
    y_pred: array_like
        Predicted values.
    y_true: array_like
        Actual values.

    Returns
    -------
    r_squared: scalar
        Calculated R^2.
    """
    def __call__(self, y_true, y_pred):

        # Calculate Pearson product-moment correlation coefficient
        correlation_matrix = np.corrcoef(y_true, y_pred)
        correlation = correlation_matrix[0,1]

        # Calculate R^2
        r_squared = correlation**2

        return r_squared


"""
    Mamy dwie macierze Am×n i Bn×k. Należy napisać funkcję obliczającą ich iloczyn.

    1) Za pomocą iteracji po elementach (potrójna pętla).
    2) Korzystając z faktu, że każdy element macierzy  Cij jest
       iloczynem skalarnym  i-tego rzędu macierzy A oraz
       j-tego rzędu macierzy B (podwójna pętla)
    3) Używając funkcji:  np.dot lub np.tensordot (bez pętli)
"""

import numpy as np

MESSAGE_REGARDING_DIFFERENT_DIMENSIONS = 'The number of columns ' \
                                         'in the first matrix ' \
                                         'should be the same as ' \
                                         'the number of rows in the second matrix.'

def C1(A, B):

    # Wymiary m, n, k

    # m - liczka wierszy macierzy A
    # n - liczba kolumn macierzy A, liczba wierszy macierzy B
    # k - liczba kolumn macierzy B

    m, n_1 = A.shape
    n_2, k = B.shape

    # Jeżeli liczba kolumn macierzy A jest różna od liczby wierszy macierzy B
    # wyrzuć wyjątek
    if n_1 != n_2:
        raise ValueError(MESSAGE_REGARDING_DIFFERENT_DIMENSIONS)

    # Macierz C będzie miał wymiary (m,k)
    C = np.zeros((m, k))

    for i in range(m):

        for j in range(k):

            # Element Cij to iloczyn skalarny i-tego wiersza macierzy A i j-tej kolumny macierzy B

            scalar_product = 0

            # Zarówno każdy wiersz macierzy A jak i każda kolumna macierzy B zawiera n elementów
            for l in range(n_1):
                scalar_product += A[i][l] * B.T[j][l]

            C[i][j] = scalar_product

    return C

def C2(A, B):

    # Wymiary m, n, k

    # m - liczka wierszy macierzy A
    # n - liczba kolumn macierzy A, liczba wierszy macierzy B
    # k - liczba kolumn macierzy B

    m, n_1 = A.shape
    n_2, k = B.shape

    # Jeżeli liczba kolumn macierzy A jest różna od liczby wierszy macierzy B
    # wyrzuć wyjątek
    if n_1 != n_2:
        raise ValueError(MESSAGE_REGARDING_DIFFERENT_DIMENSIONS)

    # Macierz C będzie miał wymiary (m,k)
    C = np.zeros((m, k))

    for i in range(m):

        for j in range(k):
            # Element Cij to iloczyn skalarny i-tego wiersza macierzy A i j-tej kolumny macierzy B
            # aby go obliczyć używamy funkcji wbudowanej inner
            scalar_product = np.inner(A[i], B.T[j])
            C[i][j] = scalar_product

    return C

def C3(A, B):

    # n - liczba kolumn macierzy A, liczba wierszy macierzy B
    n_1 = A.shape[1]
    n_2 = B.shape[0]

    # Jeżeli liczba kolumn macierzy A jest różna od liczby wierszy macierzy B
    # wyrzuć wyjątek
    if n_1 != n_2:
        raise ValueError(MESSAGE_REGARDING_DIFFERENT_DIMENSIONS)

    C = np.dot(A, B)
    return C

# Tests
m = np.arange(9).reshape(3,3)
assert np.prod(C1(np.eye(3),m) == m)
assert np.prod(C1(m,m) == C3(m,m))
assert np.prod(C2(m,m) == C3(m,m))

try:
    C1(np.arange(9).reshape(3,3),np.ones((2,3)))
except (BaseException, Exception) as error:
    if type(error) != ValueError or \
            str(error) != MESSAGE_REGARDING_DIFFERENT_DIMENSIONS:
        raise AssertionError("Second dimension of matrix "
                             "should be the same as dimension of vector!.")

try:
    C2(np.arange(9).reshape(3,3),np.ones((2,3)))
except (BaseException, Exception) as error:
    if type(error) != ValueError or \
            str(error) != MESSAGE_REGARDING_DIFFERENT_DIMENSIONS:
        raise AssertionError("Second dimension of matrix "
                             "should be the same as dimension of vector!.")

try:
    C3(np.arange(9).reshape(3, 3), np.ones((2, 3)))
except (BaseException, Exception) as error:
    if type(error) != ValueError or \
            str(error) != MESSAGE_REGARDING_DIFFERENT_DIMENSIONS:
        raise AssertionError("Second dimension of matrix "
                             "should be the same as dimension of vector!.")

print("All tests were successful.")


"""Principal Component Analysis."""

import numpy as np

from typing import Optional

from alchina.exceptions import InvalidInput, NotFitted
from alchina.utils import features_reshape

class PCA(object):
    """Principal Component Analysis algorithm."""

    def __init__(self, n_components: Optional[int] = None):
        self.n_components = n_components

        self._covariance_matrix = None
        self._U_reduced = None
        self._mean = None

    def fit(self, X):
        """Train the model."""
        X = features_reshape(X)
        max_dimension = min(X.shape)

        if self.n_components is None:
            self.n_components = max_dimension
        elif self.n_components > max_dimension:
            raise InvalidInput(f"n_components must be lesser than {max_dimension}")

        self._covariance_matrix = np.cov(X.T)
        self._mean = np.mean(X, axis=0)

        U, _, _ = np.linalg.svd(self._covariance_matrix)
        self._U_reduced = U[:, : self.n_components]

    def transform(self, X):
        """Transform the input."""
        if self._U_reduced is None or self._mean is None:
            raise NotFitted("the model must be fitted before usage")

        return (X - self._mean).dot(self._U_reduced)

    def fit_transform(self, X):
        """Fit the model and transform the input."""
        self.fit(X)
        return self.transform(X)

    def score_samples(self, X):
        """Compute the log-likelihood of all samples."""
        if self._U_reduced is None or self._mean is None:
            raise NotFitted("the model must be fitted before usage")

        X = features_reshape(X)

        n_features = X.shape[1]
        precision = np.linalg.inv(self._covariance_matrix)
        residuals = X - self._mean

        return -(1 / 2) * (
            -np.log(np.linalg.det(precision))
            + np.sum((residuals * np.dot(residuals, precision)), axis=1)
            + n_features * np.log(2 * np.pi)
        )

    def score(self, X):
        """Compute the mean of log-likelihood of all samples."""
        return np.mean(self.score_samples(X))


'''
One-versus-all regularised logistic regression

Bence Mélykúti
23/02/2018
'''

import numpy as np
import csv
import itertools
import matplotlib.pyplot as plt
from sklearn import linear_model
from sklearn.preprocessing import PolynomialFeatures
import scipy.io # to open Matlab's .mat files
import time

# own implementation, not really recommended or necessary:
def reglogreg_manual(X, y, num_labels, lambda0):
#    all_theta = np.zeros((num_labels, 1+X.shape[1]))
    data_prediction = np.zeros((X.shape[0], num_labels))

    for i in range(num_labels):
        lreg = linear_model.LogisticRegression(C=1/lambda0, fit_intercept=True, intercept_scaling=100)
        y_i = np.zeros_like(y)
        y_i[y==i] = 1
        y_i[y!=i] = 0

        lreg.fit(X, y_i) # there is no first column with all 1s, but it is taken care of by fit_intercept=True
#        all_theta[i, 0] = lreg.intercept_
#        all_theta[i, 1:] = np.ravel(lreg.coef_)

        data_prediction[:,i] = np.ravel(lreg.predict_proba(X)[:,1])

    pred = np.argmax(data_prediction, axis=1)
    return pred, data_prediction#, all_theta

# built-in version; benchmarking shows that it's not much faster
def reglogreg(X, y, lambda0):
    lreg = linear_model.LogisticRegression(C=1/lambda0, fit_intercept=True, intercept_scaling=100, multi_class='ovr') # ovr = one-vs-rest
    lreg.fit(X,y)
    pred = lreg.predict(X)
    data_prediction = lreg.predict_proba(X)
    trainacc = lreg.score(X,y)
    return pred, data_prediction, trainacc

mat = scipy.io.loadmat('../machine-learning-ex3/ex3/ex3data1.mat')
mat['y'][mat['y'][:]==10]=0 # this sets character identifier 10 to 0
#mat['y'] = mat['y'] % 10 # this is another way of setting 10 to 0
num_labels=10 # number of classes

X = mat['X']
y = mat['y']

# Although PolynomialFeatures could cope with it, we avoid using polynomial features as we have many (400), not only 2 variates in each sample.
#degree=6
#X=mapFeature(X[:,0], X[:,1], degree)

# Regularised logistic regression

# regularisation parameter
lambda0 = 1 # 1e-2 is a better value; set to tiny value (e.g. 1e-10) to achieve no regularisation

start_time = time.time()

# Option 1
#pred, data_prediction = reglogreg_manual(X, y, num_labels, lambda0) # takes 3.04 sec with lambda0=1, 8 sec with lambda0=1e-2

# Option 2
# https://stackoverflow.com/questions/843277/how-do-i-check-if-a-variable-exists#843293
if 'trainacc' in globals():
    del trainacc
pred, data_prediction, trainacc = reglogreg(X, y, lambda0) # takes 2.99 sec with lambda0=1, 7.7 sec with lambda0=1e-2
print('Time elapsed: {:.2f} sec'.format(time.time() - start_time))

print('{0} data points out of {1} correctly classified.'.format(np.sum(np.ravel(y)==pred),X.shape[0]))
print('Train Accuracy:')

if 'trainacc' in globals():
    print(trainacc * 100, 'percent')
#else: # This is an alternative calculation.
print(np.mean(np.ravel(y)==pred) * 100, 'percent')
print('Expected accuracy (with lambda = 1): 94.9 (approx)')


import numpy as np
from math import atan2, asin

def Euler2Quaternion(phi, theta, psi):
    cth = np.cos(theta/2)
    cph = np.cos(phi/2)
    cps = np.cos(psi/2)

    sth = np.sin(theta/2)
    sph = np.sin(phi/2)
    sps = np.sin(psi/2)

    e0 = cps*cth*cph+sps*sth*sph
    e1 = cps*cth*sph-sps*sth*cph
    e2 = cps*sth*cph+sps*cth*sph
    e3 = sps*cth*cph+cps*sth*sph
    e = np.array([e0,e1,e2,e3])
    return e

def Quaternion2Euler(e):
    e0 = e.item(0)
    e1 = e.item(1)
    e2 = e.item(2)
    e3 = e.item(3)

    phi = atan2(2*(e0*e1 + e2*e3),(e0**2+e3**2-e1**2-e2**2))
    theta = asin(2*(e0*e2-e1*e3))
    psi = atan2(2*(e0*e3 + e1*e2),(e0**2+e1**2-e2**2-e3**2))
    return [phi,theta,psi]

def Quaternion2Rotation(e):
    e0 = e.item(0)
    e1 = e.item(1)
    e2 = e.item(2)
    e3 = e.item(3)

    R = np.array([[e0**2 + e1**2 - e2**2 - e3**2, 2*(e1*e2 - e0*e3), 2*(e1*e3 + e0*e2)],
                  [2*(e1*e2 + e0*e3), e0**2 - e1**2 + e2**2 - e3**2, 2*(e2*e3 - e0*e1)],
                  [2*(e1*e3 - e0*e2), 2*(e2*e3 + e0*e1), e0**2 - e1**2 - e2**2 + e3**2]
                  ])
    return R


# %%
'''
# General Relativity
'''

# %%
import numpy as np
import matplotlib.pyplot as plt
from functools import partial
import ipywidgets

def gamma(b):
    return 1/np.sqrt(1-b**2)

# %%
'''
### Hw1.3

How does an isotropically emitting gamma source look like from the point of view of a moving observer?
'''

# %%
@np.vectorize
def theta(theta_prime,beta):
    """
    Returns the angle of the radiation,
    emitted by the moving observer O' at an angle theta_prime,
    as seen by the stationary observer O,
    if the relative speed between O and O' is beta*c.
    """

    s = np.sin(theta_prime)/(gamma(beta)*(1 + np.cos(theta_prime)*beta))
    c = (np.cos(theta_prime) + beta)/(1 + np.cos(theta_prime)*beta)
    if s >= 0:
        return np.arccos(c)
    else:
        return 2*np.pi - np.arccos(c)

# %%
# varying beta

theta_prime = np.pi/2

betas = np.arange(0,1,0.01)

thetas = [theta(theta_prime=theta_prime,beta=b) for b in betas]
thetas = np.array(thetas)

plt.figure()
plt.plot(betas,thetas)
plt.xlabel('beta')
plt.ylabel('theta')
plt.show()

# %%
# varying theta_prime

beta = 0.9

theta_primes = np.arange(0,2*np.pi,2*np.pi/100)

thetas = [theta(theta_prime=t,beta=beta) for t in theta_primes]
thetas = np.array(thetas)

plt.figure()
plt.plot(theta_primes*180/np.pi,thetas*180/np.pi)
plt.xlabel('theta\'')
plt.ylabel('theta')
plt.show()

# %%
# multiple rays

n_rays = 16

theta_primes = np.arange(0,2*np.pi,2*np.pi/n_rays)

def plot_rays(beta):

    thetas = [theta(theta_prime=t,beta=beta) for t in theta_primes]
    thetas = np.array(thetas)

    plt.figure()
    for t in thetas:
        plt.plot([0,np.cos(t)],[0,np.sin(t)])
    plt.show()

ipywidgets.interact(plot_rays, beta = (0,0.99,0.01))

# %%
# Intensity

n_rays = 16
amp_factor = 1000

theta_primes = np.arange(0,np.pi,np.pi/(amp_factor*n_rays))
theta_scan = np.arange(0,np.pi+np.pi/n_rays,np.pi/n_rays)

def plot_intensity(beta):

    thetas = [theta(theta_prime=t,beta=beta) for t in theta_primes]
    thetas = np.array(thetas)

    intensity = np.zeros_like(theta_scan)

    j = 0
    last_i = 0
    for i,t in enumerate(thetas):
        if j >= len(intensity):
                print('error')
                break
        if t > theta_scan[j]:
            #print(i)
            intensity[j] = i - last_i
            last_i = i
            j += 1
    intensity[-1] = n_rays*amp_factor - last_i
    2
    intensity /= 1.*n_rays*amp_factor

    plt.figure()
    plt.plot(theta_scan[1:]*180/np.pi,intensity[1:])
    plt.xlabel('theta')
    plt.ylabel('normalized intensity')
    plt.title('angular spectrum')
    plt.show()

#plot_intensity(0.3)

ipywidgets.interact(plot_intensity, beta = (0,0.99,0.01))

# %%
# accounting for 3-dimensionality

n_rays = 16
amp_factor = 1000

theta_primes = np.arange(0,np.pi,np.pi/(amp_factor*n_rays))
theta_scan = np.arange(0,np.pi+np.pi/n_rays,np.pi/n_rays)

def plot_intensity(beta):

    thetas = [theta(theta_prime=t,beta=beta) for t in theta_primes]
    thetas = np.array(thetas)

    intensity = np.zeros_like(theta_scan)

    j = 0
    for i,t in enumerate(thetas):
        if j >= len(intensity):
                print('error')
                break
        intensity[j] += (0.00001 + np.sin(theta_primes[i]))/(0.0001 + np.sin(theta_scan[j]))
        if t > theta_scan[j]:
            #print(i)
            j += 1

    s = sum(intensity)
    print(intensity,s)
    intensity /= 1.*s

    plt.figure()
    plt.semilogy(theta_scan[1:]*180/np.pi,intensity[1:])
    plt.xlabel('theta')
    plt.ylabel('normalized intensity')
    plt.title('angular spectrum')
    plt.show()

# plot_intensity(0.)

ipywidgets.interact(plot_intensity, beta = (0,0.99,0.01))

# %%
# just to clarify how (arc)cos and (arc)sin work

theta_primes = np.arange(0,2*np.pi,2*np.pi/100)

plt.figure()
plt.plot(theta_primes,np.arcsin(np.sin(theta_primes)), label = 'sin')
plt.plot(theta_primes,np.arccos(np.cos(theta_primes)), label = 'cos')
plt.legend()
plt.show()

# %%
# Analytical solution to the intensity problem:
# assume in its own frame the source emits energy isotropically:
# what is the angular distribution seen by the observer?
# It will be cylindrically symmetric of course: what is the distribution wrt theta?

import sympy as sp
sp.init_printing()

v = sp.symbols('v', real=True)
gamma_sp = 1 / sp.sqrt(1-v**2)

theta_prime = sp.symbols('theta_prime', real=True)

# angle in the observer frame O
theta_orig = sp.atan(1/gamma_sp/(1/sp.tan(theta_prime) + v/sp.sin(theta_prime)))

# distribution means the derivative of theta in O wrt theta in O'
distribution = sp.diff(theta_orig, theta_prime)
# taking account of the size of the spherical portion.
# the absolute value is needed because of some issues with the arccosine and arccsine
# which are difficult to deal with in sympy
distribution_3d = sp.Abs(distribution * sp.sin(theta_orig) / sp.sin(theta_prime))

# %%
def plot_distribution(relative_velocity):

    # take the generic distributions and insert the actual velocity
    new_distribution = distribution.subs(v, relative_velocity)
    new_distribution_3d = distribution_3d.subs(v, relative_velocity)

    x = sp.symbols('x')

    # we make functions from the sympy relations, the inverse is
    # inserted because we want to plot d (theta_prime) / d (theta)
    f = sp.lambdify(x, 1/new_distribution.subs(theta_prime, x), 'numpy')
    f_3d = sp.lambdify(x, 1/new_distribution_3d.subs(theta_prime, x), 'numpy')

    theta_primes = np.linspace(1e-10, np.pi-1e-10)
    thetas = theta(theta_primes, relative_velocity)

    plt.ylabel("dtheta_prime/dtheta")
    plt.xlabel("theta")
    plt.semilogy(thetas, f(theta_primes), label="2d")
    plt.semilogy(thetas, f_3d(theta_primes), label="3d")
    plt.legend()
    plt.suptitle(f"relative velocity = {relative_velocity:.4f}")
    plt.show()

def log_plot(log_velocity):
    plot_distribution(1-10**(-log_velocity))

# ipywidgets.interact(plot_distribution, relative_velocity=(0,0.99,0.01), continuous_update=False)
ipywidgets.interact(log_plot, log_velocity=(0,2.5,0.05), continuous_update=False)

# %%


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import helper

def null_vector():
  # MISSING: Create a vector of size 10 and of type np.float32 full of zeros.
  return np.array([])

def chess():
  # MISSING: Create a 8x8 matrix and fill it with a checkerboard pattern
  # starting with 1 in the upper left and finishing with 0.
  # The matrix type should be np.int32.
  return np.array([])

def polar(a):
  # MISSING: "a" is a Nx2 matrix representing cartesian coordinates,
  # convert them to polar coordinates.
  # This function should return a Nx2 matrix where:
  # - the first column is the radius.
  #  -the second column is the angle.
  return np.array([])

def cap(a, b):
  # MISSING: Cap all elements of "a" at "b" (i.e., max(a_{i,j}) <= b).
  # BONUS: Make sure to do so without modifying the input arguments.
  return np.array([])

def moving_average(a):
  # MISSING: Compute the moving average of the elements of "a" over a
  # window of size 3.
  return np.array([])

def main():
  helper.check_solution(null_vector, [0] * 10)
  helper.check_solution(chess, [[1, 0, 1, 0, 1, 0, 1, 0],
                                [0, 1, 0, 1, 0, 1, 0, 1],
                                [1, 0, 1, 0, 1, 0, 1, 0],
                                [0, 1, 0, 1, 0, 1, 0, 1],
                                [1, 0, 1, 0, 1, 0, 1, 0],
                                [0, 1, 0, 1, 0, 1, 0, 1],
                                [1, 0, 1, 0, 1, 0, 1, 0],
                                [0, 1, 0, 1, 0, 1, 0, 1]])
  helper.check_solution(polar, [[0.70710678118, np.pi / 4.],
                                [0.70710678118 * 2, np.pi / 4.]],
                        np.array([[.5, .5], [1., 1.]], np.float32),
                        approx=True)
  helper.check_solution(cap, [1, 2, 2], np.array([1, 2, 3]), 2)
  helper.check_solution(cap, [2, 1, 2], np.array([3., 1., 3.]), 2)
  helper.check_solution(moving_average, [2, 3], np.array([1., 2., 3., 4.]))
  helper.check_solution(moving_average, [2], np.array([2., 2., 2.]))

if __name__ == '__main__':
  main()


"""
Supplemental functions for NumPy
"""
import numpy as np

def norm1d(array):
    shifted = array - (array.min())
    return shifted / shifted.max()

def scale1d(array, min=0, max=1):
    scaled = max * norm1d(array)
    return scaled + min

class PolynomialLinearModel:
    "A convenience class for creating polynomial linear models."
    def __init__(self, xs, ys, degree):
        (self.xs, self.ys) = (xs, ys)
        self.degree = degree
        self.y_mean = self.get_y_mean()
        (self.results, self.model, self.ys_predicted,
         self.ss_tot, self.ss_reg, self.ss_res,
         self.r_squared) = (None, None, None, None, None, None, None)
        (self.coeffs, self.residuals, self.rank,
         self.singular_values, self.rcond) = (None, None, None, None, None)
        self.polyfit()

    def polyfit(self):
        (self.coeffs, self.residuals, self.rank,
        self.singular_values, self.rcond) = np.polyfit(
            self.xs, self.ys, self.degree, full=True)
        self.model = np.poly1d(self.coeffs)
        self.ys_predicted = self.model(self.xs)
        self.ss_tot = self.get_ss_tot()
        self.ss_reg = self.get_ss_reg()
        self.ss_res = self.get_ss_res()
        self.r_squared = self.get_r_squared()
        self.results = {
            "coeffs": self.coeffs.tolist(),
            "residuals": self.residuals.tolist(),
            "rank": self.rank,
            "singular-values": self.singular_values.tolist(),
            "rcond": float(self.rcond),
            "y-mean": float(self.y_mean),
            "ss-tot": float(self.ss_tot),
            "ss-reg": float(self.ss_reg),
            "ss-res": float(self.ss_res),
            "r-squared": float(self.r_squared)}

    def predict(self, xs):
        return self.model(xs)

    def get_y_mean(self):
        return self.ys.sum() / self.ys.size

    def get_ss_tot(self):
        return ((self.ys - self.get_y_mean()) ** 2).sum()

    def get_ss_reg(self):
        return ((self.ys_predicted - self.get_y_mean()) ** 2).sum()

    def get_ss_res(self):
        return ((self.ys - self.ys_predicted) ** 2).sum()

    def get_r_squared(self):
        return 1 - self.get_ss_res() / self.get_ss_tot()

    def __str__(self):
        return str(self.results)

    def __repr__(self):
        return self.__str__()


# -*- coding: utf-8 -*-
import numpy as np

def build_householder(a):
    """Return factors of householder matrix (I - tau * vv.T).

    This algorithm refers to the folloing article:
        G.H Golub and C.F Van Loan,
        "Matrix Computations 4th Edition", 2013.

    Arguments:
        a (array_like): A vector.

    Returns:
        v (numpy.ndarray): Householder vector.
            This vector is modified so that the 1st element is 1.
            See the reference for more details.
        tau (float): A coefficient of householder matrix.
        diagonal (float): A diagonal value which is result of householder
            transformation.
    """
    v = np.array(a, dtype=np.float)
    if v.ndim != 1:
        raise ValueError("vector must be 1 dimensional array")
    # build v and tau
    diagonal = np.linalg.norm(v)
    if v[0] >= 0:
        v[0] += diagonal
        diagonal = -diagonal
    else:
        v[0] -= diagonal
    v /= v[0]
    tau = 2 / np.inner(v, v)

    return v, tau, diagonal

def triangularize(A):
    """Return result of applying householder transformation to a matrix.

    This algorithm refers to the folloing article:
        G.H Golub and C.F Van Loan,
        "Matrix Computations 4th Edition", 2013.

    Arguments:
        A (array_like): A matrix.

    Returns:
        A (numpy.ndarray): A upper trianglar matrix.
            This lower trianglar submatrix is overwritten by unit
            householder vectors. See the reference for more details.
    """
    A = np.array(A, dtype=np.float)
    if A.ndim != 2:
        raise ValueError("matrix must be 2 dimensional array")
    # triangularization
    m, n = A.shape
    mn = n if m > n else m - 1
    for j in range(mn):
        v, tau, A[j, j] = build_householder(A[j:, j])
        A[j + 1:, j] = v[1:]
        A[j:, j + 1:] -= np.outer(tau * v, np.dot(v, A[j:, j + 1:]))

    return A

def householder(A):
    """Return factors of QR factorization using householder transformation.

    This algorithm refers to the folloing article:
        G.H Golub and C.F Van Loan,
        "Matrix Computations 4th Edition", 2013.

    Arguments:
        A (array_like): A matrix.

    Returns:
        Q (numpy.ndarray): A orthogonal matrix.
        R (numpy.ndarray): A upper trianglar matrix.
    """
    R = np.array(A, dtype=np.float)
    if R.ndim != 2:
        raise ValueError("matrix must be 2 dimensional array")
    # initialization
    m, n = R.shape
    Q = np.identity(m)
    # factorization
    mn = n if m > n else m - 1
    for j in range(mn):
        v, tau, R[j, j] = build_householder(R[j:, j])
        R[j + 1:, j] = 0
        R[j:, j + 1:] -= np.outer(tau * v, np.dot(v, R[j:, j + 1:]))
        Q[:, j:] -= np.outer(tau * np.dot(Q[:, j:], v), v)

    return Q, R

def build_wy(A):
    """Return factors of WY representaion (I - WY.T).

    This algorithm refers to the folloing article:
        G.H Golub and C.F Van Loan,
        "Matrix Computations 4th Edition", 2013..

    Arguments:
        A (array_like): A matrix.

    Returns:
        W (numpy.ndarray): W matrix of WY representation.
        Y (numpy.ndarray): A unit lower trianglar matrix.
            This upper trianglar submatrix is overwritten by
            the result of applying tiangulation to A.
    """
    Y = np.array(A, dtype=np.float)
    if Y.ndim != 2:
        raise ValueError("matrix must be 2 dimensional array")
    # initialization
    m, n = Y.shape
    mn = n if m > n else m - 1
    v, tau, Y[0, 0] = build_householder(Y[:, 0])
    Y[1:, 0] = v[1:]
    Y[:, 1:] -= np.outer(tau * v, np.dot(v, Y[:, 1:]))
    W = (2 / np.inner(v, v) * v).reshape(-1, 1)
    # build W and Y
    for j in range(1, mn):
        v, tau, Y[j, j] = build_householder(Y[j:, j])
        Y[j + 1:, j] = v[1:]
        Y[j:, j + 1:] -= np.outer(tau * v, np.dot(v, Y[j:, j + 1:]))
        v = np.concatenate([np.zeros(j), v], axis=0)
        w = tau * (v - np.dot(W, np.dot(Y[j:, :j].T, v[j:])))
        W = np.concatenate([W, w.reshape((-1, 1))], axis=1)

    return W, Y

def wy(A, panels=64):
    """Return factors of QR factorization using WY representaion (I - WY.T).

    This algorithm refers to the folloing article:
        G.H Golub and C.F Van Loan,
        "Matrix Computations 4th Edition", 2013.

    Arguments:
        A (array_like): A matrix.
        panels (int, optional): A panel size (column size).

    Returns:
        Q (numpy.ndarray): A orthogonal matrix.
        R (numpy.ndarray): A upper trianglar matrix.
    """
    R = np.array(A, dtype=np.float)
    if R.ndim != 2:
        raise ValueError("matrix must be 2 dimensional array")
    # initialization
    m, n = R.shape
    Q = np.identity(m)
    # block householder QR factorization
    j = 0
    mn = n if m > n else m - 1
    while j < mn:
        b = min(j + panels, mn)
        W, R[j:, j:b] = build_wy(R[j:, j:b])
        Y = np.tril(R[j:, j:b], k=-1) + np.eye(m - j, b - j)
        Q[:, j:] -= np.dot(np.dot(Q[:, j:], W), Y.T)
        R[j:, b:] -= np.dot(Y, np.dot(W.T, R[j:, b:]))
        R[j:, j:b] = np.triu(R[j:, j:b])
        j = b

    return Q, np.triu(R)

def build_compact_wy(A):
    """Return factors of Compact-WY representaion (I - YTY.T).

    This algorithm refers to the folloing article:
        Robert Schreiber and Charles Van Loan,
        "A Storage-Efficient WY Representation for Products of Householder
        Transformations", 1989.

    Arguments:
        A (array_like): A matrix.

    Returns:
        Y (numpy.ndarray): A unit lower trianglar matrix.
            This upper trianglar submatrix is overwritten by
            the result of applying tiangulation to A.
        T (numpy.ndarray): A upper triangular matrix.
    """
    Y = np.array(A, dtype=np.float)
    if Y.ndim != 2:
        raise ValueError("matrix must be 2 dimensional array")
    # initialization
    m, n = Y.shape
    mn = n if m > n else m - 1
    v, tau, Y[0, 0] = build_householder(Y[:, 0])
    Y[1:, 0] = v[1:]
    Y[:, 1:] -= np.outer(tau * v, np.dot(v, Y[:, 1:]))
    T = np.array([[2 / np.inner(v, v)]])
    # build Y and T
    for j in range(1, mn):
        v, tau, Y[j, j] = build_householder(Y[j:, j])
        Y[j + 1:, j] = v[1:]
        Y[j:, j + 1:] -= np.outer(tau * v, np.dot(v, Y[j:, j + 1:]))
        v = np.concatenate([np.zeros(j), v], axis=0)
        t = (-tau * np.dot(T, np.dot(Y[j:, :j].T, v[j:]))).reshape(-1, 1)
        T = np.block([[T, t], [np.zeros((1, j)), np.array([[tau]])]])

    return Y, T

def compact_wy(A, panels=64):
    """Return factors of QR factorization using Compact-WY
    representaion (I - YTY.T).

    This algorithm refers to the folloing article:
        Robert Schreiber and Charles Van Loan,
        "A Storage-Efficient WY Representation for Products of Householder
        Transformations", 1989.

    Arguments:
        A (array_like): A matrix.
        panels (int, optional): A panel size (column size).

    Returns:
        Q (numpy.ndarray): A orthogonal matrix.
        R (numpy.ndarray): A upper trianglar matrix.
    """
    R = np.array(A, dtype=np.float)
    if R.ndim != 2:
        raise ValueError("matrix must be 2 dimensional array")
    # initialization
    m, n = R.shape
    Q = np.identity(m)
    # block householder QR factorization
    j = 0
    mn = n if m > n else m - 1
    while j < mn:
        b = min(j + panels, mn)
        R[j:, j:b], T = build_compact_wy(R[j:, j:b])
        Y = np.tril(R[j:, j:b], k=-1) + np.eye(m - j, b - j)
        Q[:, j:] -= np.dot(np.dot(Q[:, j:], Y), np.dot(T, Y.T))
        R[j:, b:] -= np.dot(np.dot(Y, T.T), np.dot(Y.T, R[j:, b:]))
        R[j:, j:b] = np.triu(R[j:, j:b])
        j = b

    return Q, R


"""
    gradient descent method1：
    gradient methods with fixed step size
"""

import numpy as np

def main():

    x = np.random.rand(100)     # creat 100 numbers from 0 to 1.
    y = 10 + 5 * x              # creat a linear function

    print(len(x), len(y))
    epsilon = 0.01   # set threshold to stop loop

    cnt = 0
    alpha = 0.001   # set the step size of gradient update
    theta0 = 0      # x0=1, according to theta0
    theta1 = 0      # x1=xi

    error1 = 0
    error0 = 0

    while True:
        sum0 = 0
        sum1 = 0
        cnt = cnt + 1
        for i in range(0, len(x)):     # BGD method
            diff = y[i] - (theta0 + theta1 * x[i])  # gradient function

            sum0 = sum0 + diff          # theta0 derivative
            sum1 = sum1 + diff*x[i]     # theta1 derivative

        theta0 = theta0 + alpha * sum0/len(x)   # gradient update
        theta1 = theta1 + alpha * sum1/len(x)   # gradient update

        error1 = 0
        for i in range(0, len(x)):
            error1 = error1 + (y[i] - (theta0 + theta1 * x[i]))**2  # loss function

        if abs(error1) < epsilon:   # get a whole minimum numer
            print('error1-0:', abs(error1))
            break
        # if abs(error1-error0) < epsilon:   # get a relative minimum numer
        #     print('error1-0:', abs(error1))
        #     break

        else:
            error0 = error1     # get a relative minimum number
            print('error1:', error1)

        print('theta0:', theta0, 'theta1:', theta1)

if __name__ == '__main__':
    main()

gradient descent/gradient descent method1.py


import numpy as np

def B_mat_RM(x: list, y: list, xg:float, yg:float):
    """Computes strain-displacement matrix (B matrix) for bending moments
    and shear forces.

        Arg:
            x:     X coordinates of the element  [list]
            y:     Y coordinates of the element  [list]
            xg:    Gauss point - local X coordinate [float]
            yg:    Gauss point - local X coordinate [float]
        Returns:
            b_mat_b:    strain-displacement matrix for bending moments
            b_mat_s:    strain-displacement matrix for shear forces
            area:       area of the element
    """

    N = np.zeros((3, 1))
    N[0] = 1.0 - xg - yg
    N[1] = xg
    N[2] = yg

    loc_dxN = np.zeros((3, 1))
    loc_dxN[0] = -1.0
    loc_dxN[1] = 1.0
    loc_dxN[2] = 0.0

    loc_dyN = np.zeros((3, 1))
    loc_dyN[0] = -1.0
    loc_dyN[1] = 0.0
    loc_dyN[2] = 1.0

    jac_m = np.zeros((2, 2))
    jac_m[0, 0] = x[0]*loc_dxN[0] + x[1]*loc_dxN[1] + x[2]*loc_dxN[2]
    jac_m[0, 1] = y[0]*loc_dxN[0] + y[1]*loc_dxN[1] + y[2]*loc_dxN[2]
    jac_m[1, 0] = x[0]*loc_dyN[0] + x[1]*loc_dyN[1] + x[2]*loc_dyN[2]
    jac_m[1, 1] = y[0]*loc_dyN[0] + y[1]*loc_dyN[1] + y[2]*loc_dyN[2]

    jac_im = np.linalg.inv(jac_m)

    area = abs(jac_m[0, 0]*jac_m[1, 1] - jac_m[1, 0]*jac_m[0, 1]) / 2

    Ndx = np.zeros((3, 1))
    Ndx[0] = jac_im[0, 0]*loc_dxN[0] + jac_im[0, 1]*loc_dyN[0]
    Ndx[1] = jac_im[0, 0]*loc_dxN[1] + jac_im[0, 1]*loc_dyN[1]
    Ndx[2] = jac_im[0, 0]*loc_dxN[2] + jac_im[0, 1]*loc_dyN[2]

    Ndy = np.zeros((3, 1))
    Ndy[0] = jac_im[1, 0]*loc_dxN[0] + jac_im[1, 1]*loc_dyN[0]
    Ndy[1] = jac_im[1, 0]*loc_dxN[1] + jac_im[1, 1]*loc_dyN[1]
    Ndy[2] = jac_im[1, 0]*loc_dxN[2] + jac_im[1, 1]*loc_dyN[2]

    bmat_b1 = np.array(
        [[0, -Ndx[0, 0],         0],
         [0,         0,  -Ndy[0, 0]],
         [0, -Ndy[0, 0], -Ndx[0, 0]]])
    bmat_b2 = np.array(
        [[0, -Ndx[1, 0],         0],
         [0,         0,  -Ndy[1, 0]],
         [0, -Ndy[1, 0], -Ndx[1, 0]]])
    bmat_b3 = np.array(
        [[0, -Ndx[2, 0],         0],
         [0,         0,  -Ndy[2, 0]],
         [0, -Ndy[2, 0], -Ndx[2, 0]]])
    bmat_b = np.concatenate((bmat_b1, bmat_b2, bmat_b3), axis=1)

    bmat_s1 = np.array(
        [[Ndx[0, 0], -N[0, 0],        0],
         [Ndy[0, 0],        0, -N[0, 0]]])
    bmat_s2 = np.array(
        [[Ndx[1, 0], -N[1, 0],        0],
         [Ndy[1, 0],        0, -N[1, 0]]])
    bmat_s3 = np.array(
        [[Ndx[2, 0], -N[2, 0],        0],
         [Ndy[2, 0],        0, -N[2, 0]]])
    bmat_s = np.concatenate((bmat_s1, bmat_s2, bmat_s3), axis=1)

    return bmat_b, bmat_s, area

def NodeStress(D_mat_b, D_mat_s, xg, yg, u):
    """Computes stresses at the Gauss points and evaluates values to the nodes.

        Arg:
            D_mat_b:    X coordinates of the element  [list]
            D_mat_s:    Y coordinates of the element  [list]
            xg:         Gauss point - local X coordinate [float]
            yg:         Gauss point - local X coordinate [float]
            u:          Nodal displacements
        Returns:
            ndStress:   Nodal stress matrix
    """

    pass

def D_mat(h, E, mi):
    """Computes constitutive matrices for bending
    moments and shear forces of element.

        Arg:
            h:          Thickness of element  [m, float]
            E:          Young modulus of elasticity  [Pa, int]
            mi:         Poissons number [float]
        Returns:
            D_mat_b:    Constitutive matrix for bending moments
            D_mat_s:    Constitutive matrix for shear forces
    """

    D_mat_b = (h**3 / 12) * (E / (1 - mi**2)) * np.array(
        [[1, mi,   0],
         [mi, 1,   0],
         [0,  0,   (1-mi)/2]])

    D_mat_s = (h*5 / 6) * (E / (2*(1 + mi))) * np.array(
        [[1,  0],
         [0,  1]])

    return D_mat_b, D_mat_s


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np

def rotx(t):
    """Creates a matrix that rotates t radians around the x axis."""
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[1,  0,  0],
                     [0,  c, -s],
                     [0,  s,  c]])

def roty(t):
    """Creates a matrix that rotates t radians around the y axis."""
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[c,  0,  s],
                     [0,  1,  0],
                     [-s, 0,  c]])

def rotz(t):
    """Creates a matrix that rotates t radians around the z axis."""
    c = np.cos(t)
    s = np.sin(t)
    return np.array([[c, -s,  0],
                     [s,  c,  0],
                     [0,  0,  1]])

def rot_matrix(rx, ry, rz):
    return rotz(rz).dot(roty(ry)).dot(rotx(rx))

def yaw_pitch_roll_from_rot_matrix(R):
    """Recovers yaw, pitch roll from a rotation matrix.

    Assumes rotation to have been
    applied in the order R = R_z * R_y * R_x.
    """
    yaw = np.arctan(R[1, 0] / R[0, 0])
    pitch = np.arctan(-R[2, 0] / np.sqrt(R[2, 1]**2 + R[2, 2]**2))
    roll = np.arctan(R[2, 1] / R[2, 2])
    return yaw, pitch, roll


from scipy.integrate import quad
import numpy as np
import matplotlib.pyplot as plt

x = np.linspace(-4, 4, num = 100)

constant = 1.0 / np.sqrt(2 * np.pi)
pdf_normal_distribution = constant * np.exp((-x ** 2) / 2.0)

fig, ax = plt.subplots(figsize=(10, 5))

ax.plot(x, pdf_normal_distribution)
ax.set_ylim(0)
ax.set_title('Normal Distribution', size=20)
ax.set_ylabel('Probability Density', size=20)

plt.show()


#importing libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn import preprocessing
from sklearn.preprocessing import normalize

%matplotlib inline

#load dataset
dataset = pd.read_csv("/Users/samar/Documents/pythonwd/special-potato/AXISBANK.csv")

#Clean dataset
#adding t1 value or the next day value
dataset["t1"] = dataset["4. close"].shift(-1)

#removing last NaN - because nobody likes naan!
dataset = dataset[0:(dataset.shape[0]-1)]

#adding p1 as change between t1 and t0
dataset["p1"] = (dataset["t1"] - dataset["4. close"])

#changing p1 to 1 if change was +ve and 0 if - ve
dataset["p1"] = np.where(dataset["p1"] > 0, 1,0)

#cleaning out dates and dataframe
X = dataset[["1. open","2. high", "3. low" ,"4. close", "5. volume"]]

def normalize(df):
    G = preprocessing.StandardScaler().fit(df)
    ndf_mean = df- G.mean_
    ndf = ndf_mean/G.var_
    return ndf

X = normalize(X)

split = (int(X.shape[0]*(2/3)))
train_set_x = X[:split]
train_set_x = train_set_x.transpose()
train_set_x = train_set_x.values

test_set_x = X[split:]
test_set_x = test_set_x.transpose()
test_set_x = test_set_x.values

# similarly making y matrix with (1, m) size
Y = dataset[["p1"]]

train_set_y = Y[:split]
train_set_y = train_set_y.transpose()
train_set_y = train_set_y.values

test_set_y = Y[split:]
test_set_y = test_set_y.transpose()
test_set_y = test_set_y.values
#defining sigmoid function
def sigmoid(z):
    s = 1 / (1 + np.exp(-z))
    return s

def initialize_with_zeros(dim):
    w = np.zeros(shape=(dim, 1))
    b = 0
    return w, b

def propagate(w, b, X, Y):
    m = X.shape[1]
    A = sigmoid(np.dot(w.T, X) + b)  # compute activation
    cost = (- 1 / m) * np.sum(Y * np.log(A) + (1 - Y) * (np.log(1 - A)))  # compute cost
    dw = (1 / m) * np.dot(X, (A - Y).T)
    db = (1 / m) * np.sum(A - Y)
    cost = np.squeeze(cost)
    grads = {"dw": dw,
             "db": db}
    return grads, cost

def optimize(w, b, X, Y, num_iterations, learning_rate, print_cost = False):
    costs = []
    for i in range(num_iterations):
        grads, cost = propagate(w, b, X, Y)
        dw = grads["dw"]
        db = grads["db"]
        w = w - (learning_rate * dw)  # need to broadcast
        b = b - (learning_rate * db)
        if i % 100 == 0:
            costs.append(cost)
        if print_cost and i % 100 == 0:
            print ("Cost after iteration %i: %f" % (i, cost))
    params = {"w": w,
              "b": b}
    grads = {"dw": dw,
             "db": db}
    return params, grads, costs

def predict(w, b, X):
    m = X.shape[1]
    Y_prediction = np.zeros((1, m))
    w = w.reshape(X.shape[0], 1)
    A = sigmoid(np.dot(w.T, X) + b)
    for i in range(A.shape[1]):
        Y_prediction[0, i] = 1 if A[0, i] > 0.5 else 0
    return Y_prediction

def model(X_train, Y_train, X_test, Y_test, num_iterations=2000, learning_rate=0.5, print_cost=False):
    w, b = initialize_with_zeros(X_train.shape[0])
    parameters, grads, costs = optimize(w, b, X_train, Y_train, num_iterations, learning_rate, print_cost)
    w = parameters["w"]
    b = parameters["b"]
    Y_prediction_test = predict(w, b, X_test)
    Y_prediction_train = predict(w, b, X_train)
    print("train accuracy: {} %".format(100 - np.mean(np.abs(Y_prediction_train - Y_train)) * 100))
    print("test accuracy: {} %".format(100 - np.mean(np.abs(Y_prediction_test - Y_test)) * 100))
    d = {"costs": costs,
         "Y_prediction_test": Y_prediction_test,
         "Y_prediction_train" : Y_prediction_train,
         "w" : w,
         "b" : b,
         "learning_rate" : learning_rate,
         "num_iterations": num_iterations}
    return d

d = model(train_set_x, train_set_y, test_set_x, test_set_y, num_iterations = 2000, learning_rate = 0.005, print_cost = True)

# Plot learning curve (with costs)
costs = np.squeeze(d['costs'])
plt.plot(costs)
plt.ylabel('cost')
plt.xlabel('iterations (per hundreds)')
plt.title("Learning rate =" + str(d["learning_rate"]))
plt.show()