import pandas as pd
import numpy as np
from six import string_types

# From Penguoin Utils.py

def _export_table(table, fname):
    """Export DataFrame to .csv"""
    import os.path as op
    extension = op.splitext(fname.lower())[1]
    if extension == '':
        fname = fname + '.csv'
    table.to_csv(fname, index=None, sep=',', encoding='utf-8',
                 float_format='%.4f', decimal='.')

def reshape_data(df, id, dv='DV', rm='Time'):
    """Reshape data from human-readable to analysis shape

    See: https://deparkes.co.uk/2016/10/28/reshape-pandas-data-with-melt/

    Parameters
    ----------
    df : DataFrame
        Dataframe in original shape
    id : string
        Column to use as identifier variables. (e.g. "Subjects")
    dv : string
        Name of the dependant variables (e.g. "DV" or "Scores")
    rm : string
        Name of the measurements (e.g. "Time" or "Weekday")

    Returns
    -------
    df_reshaped : DataFrame
        Reshaped DataFrame

    Examples
    --------

    Human-readable:

    *Values represent scores at a cognitive test at different times of the day.
    Ss = subject*::

        Ss    10am  2pm   6pm
        1     3.5   3.2   2.8
        2     2.1   2.4   2.8

    Pingouin::

        Ss    Score    Time
        1     3.5      10am
        1     3.2      2pm
        1     2.8      6pm
        2     2.1      10am
        2     2.4      2pm
        2     2.8      6pm

    >>> import pandas as pd
    >>> from pingouins import reshape_data
    >>> data = {'Ss': [1, 2, 3],
    >>>        '10am': [12, 6, 5],
    >>>        '2pm': [10, 6, 11],
    >>>        '6pm': [8, 5, 7]}
    >>> df = pd.DataFrame(data, columns=['Ss', '10am', '2pm', '6pm'])
    >>> reshaped = reshape_data(df, 'Ss', dv="Score", rm="Time")
    >>> print(reshaped)
        Ss  Time  Score
        1   10am  12
        1   2pm   10
        1   6pm   8
        2   10am  6
        2   2pm   6
        2   6pm   5
        3  10am   5
        3   2pm   11
        3   6pm   7
    """
    return pd.melt(df, id_vars=id, var_name=rm, value_name=dv).sort_values(
        by=id)

def _remove_na(x, y, paired=False):
    """Remove missing values in paired and independant measurements.
    """
    x_na = np.any(np.isnan(x))
    y_na = np.any(np.isnan(y))
    if (x_na or y_na) and paired:
        ar = np.c_[x, y]
        ar = ar[~np.isnan(ar).any(axis=1)]
        x, y = ar[:, 0], ar[:, 1]
    elif (x_na or y_na) and not paired:
        x = np.array(list(filter(lambda v: v == v, x))) if x_na else x
        y = np.array(list(filter(lambda v: v == v, y))) if y_na else y
    return x, y

def _remove_rm_na(dv=None, within=None, data=None):
    """Remove subject(s) with one or more missing values in repeated
    measurements.
    """
    rm = list(data[within].dropna().unique())
    n_rm = len(rm)
    n_obs = int(data.groupby(within)[dv].count().max())
    data['ID_Subj'] = np.tile(np.arange(n_obs), n_rm)

    # Efficiently remove subjects with one or more missing values
    data = data.set_index('ID_Subj')

    # Find index with nan
    iloc_nan = pd.isnull(data).any(1).nonzero()[0]
    idx_nan = data.index[iloc_nan].values
    print('\nNote: %i subject(s) removed because of missing value(s).\n'
          % len(idx_nan))
    return data.drop(idx_nan).reset_index(drop=True)

def _check_eftype(eftype):
    """Check validity of eftype"""
    if eftype.lower() in ['none', 'hedges', 'cohen', 'glass', 'r',
                          'eta-square', 'odds-ratio', 'auc']:
        return True
    else:
        return False

def _check_dataframe(dv=None, between=None, within=None, effects=None,
                     data=None):
    """Check dataframe"""
    # Check input arguments
    if not isinstance(data, pd.DataFrame):
        raise ValueError('Data must be a pandas dataframe')
    if any(v is None for v in [dv, data]):
        raise ValueError('DV and data must be specified')
    if effects not in ['within', 'between', 'interaction', 'all']:
        raise ValueError('Effects must be: within, between, interaction, all')
    if effects == 'within' and not isinstance(within, string_types):
        raise ValueError('within must be specified when effects=within')
    elif effects == 'between' and not isinstance(between, string_types):
        raise ValueError('between must be specified when effects=between')
    elif effects == 'interaction':
        for input in [within, between]:
            if not isinstance(input, string_types):
                raise ValueError('within and between must be specified when \
                effects=interaction')

def _extract_effects(dv=None, between=None, within=None, effects=None,
                     data=None):
    """Extract main effects"""
    # Check the dataframe
    _check_dataframe(dv=dv, between=between, within=within, effects=effects,
                     data=data)

    datadic = {}
    nobs = np.array([], dtype=int)

    # Extract number of pairwise comparisons
    if effects.lower() in ['within', 'between']:
        col = within if effects == 'within' else between
        # Extract data
        labels = list(data[col].unique())
        for l in labels:
            datadic[l] = data[data[col] == l][dv]
            nobs = np.append(nobs, len(datadic[l]))

    elif effects.lower() == 'interaction':
        labels_with = list(data[within].unique())
        labels_betw = list(data[between].unique())
        for lw in labels_with:
            for l in labels_betw:
                tmp = data[data[within] == lw]
                datadic[lw, l] = tmp[tmp[between] == l][dv]
                nobs = np.append(nobs, len(datadic[lw, l]))

    dt_array = pd.DataFrame.from_dict(datadic)
    return dt_array, nobs

def gzscore(x):
    """Geometric standard (Z) score.

    Geometric Z-scores are better measures of dispersion than arithmetic
    z-scores when the sample data come from a log-normally distributed
    population.

    See https://en.wikipedia.org/wiki/Geometric_standard_deviation

    Parameters
    ----------
    x : array_like
        Array of raw values

    Returns
    -------
    gzscore : array_like
        Array of geometric z-scores (same shape as x)

    Examples
    --------
    Standardize a log-normal array

        >>> import numpy as np
        >>> from pingouin import gzscore
        >>> np.random.seed(123)
        >>> raw = np.random.lognormal(size=100)
        >>> print(raw.mean().round(3), raw.std().round(3))
            1.849 2.282
        >>> z = gzscore(raw)
        >>> print(z.mean(), z.std())
            0 0.995
    """
    from scipy.stats import gmean
    # Geometric mean
    geo_mean = gmean(x)
    # Geometric standard deviation
    gstd = np.exp(np.sqrt(np.sum((np.log(x / geo_mean))**2) / (len(x) - 1)))
    # Geometric z-score
    return np.log(x / geo_mean) / np.log(gstd)

def test_normality(*args, alpha=.05):
    """Test the normality of one or more array.

    Parameters
    ----------
    sample1, sample2,... : array_like
        Array of sample data. May be different lengths.

    Returns
    -------
    normal : boolean
        True if x comes from a normal distribution.
    p : float
        P-value.

    See Also
    --------
    test_homoscedasticity : Test equality of variance.
    test_sphericity : Mauchly's test for sphericity.

    Examples
    --------
    1. Test the normality of one array.

        >>> import numpy as np
        >>> from pingouin import test_normality
        >>> np.random.seed(123)
        >>> x = np.random.normal(size=100)
        >>> normal, p = test_normality(x, alpha=.05)
        >>> print(normal, p)
        True 0.27

    2. Test the normality of two arrays.

        >>> import numpy as np
        >>> from pingouin import test_normality
        >>> np.random.seed(123)
        >>> x = np.random.normal(size=100)
        >>> y = np.random.rand(100)
        >>> normal, p = test_normality(x, y, alpha=.05)
        >>> print(normal, p)
        [True   False] [0.27   0.0005]
    """
    from scipy.stats import shapiro
    k = len(args)
    p = np.zeros(k)
    normal = np.zeros(k, 'bool')
    for j in range(k):
        _, p[j] = shapiro(args[j])
        normal[j] = True if p[j] > alpha else False

    if k == 1:
        normal = bool(normal)
        p = float(p)

    return normal, p

def test_homoscedasticity(*args, alpha=.05):
    """Test equality of variance.

    If data are normally distributed, uses Bartlett (1937).
    If data are not-normally distributed, uses Levene (1960).

    Parameters
    ----------
    sample1, sample2,... : array_like
        Array of sample data. May be different lengths.

    Returns
    -------
    equal_var : boolean
        True if data have equal variance.
    p : float
        P-value.

    See Also
    --------
    test_normality : Test the normality of one or more array.
    test_sphericity : Mauchly's test for sphericity.

    Examples
    --------
    Test the homoscedasticity of two arrays.

        >>> import numpy as np
        >>> from pingouin import test_homoscedasticity
        >>> np.random.seed(123)
        >>> # Scale = standard deviation of the distribution.
        >>> x = np.random.normal(loc=0, scale=1., size=100)
        >>> y = np.random.normal(loc=0, scale=0.8,size=100)
        >>> print(np.var(x), np.var(y))
            1.27 0.60
        >>> equal_var, p = test_homoscedasticity(x, y, alpha=.05)
        >>> print(equal_var, p)
            False 0.0002
    """
    from scipy.stats import levene, bartlett
    k = len(args)
    if k < 2:
        raise ValueError("Must enter at least two input sample vectors.")

    # Test normality of data
    normal, _ = test_normality(*args)
    if np.count_nonzero(normal) != normal.size:
        # print('Data are not normally distributed. Using Levene test.')
        _, p = levene(*args)
    else:
        _, p = bartlett(*args)

    equal_var = True if p > alpha else False
    return equal_var, p

def test_dist(*args, dist='norm'):
    """Anderson-Darling test of distribution.

    Parameters
    ----------
    sample1, sample2,... : array_like
        Array of sample data. May be different lengths.
    dist : string
        Distribution ('norm', 'expon', 'logistic', 'gumbel')

    Returns
    -------
    from_dist : boolean
        True if data comes from this distribution.
    """
    from scipy.stats import anderson
    k = len(args)
    from_dist = np.zeros(k, 'bool')
    sig_level = np.zeros(k)
    for j in range(k):
        st, cr, sig = anderson(args[j], dist=dist)
        from_dist[j] = True if (st > cr).any() else False
        sig_level[j] = sig[np.argmin(np.abs(st - cr))]

    if k == 1:
        from_dist = bool(from_dist)
        sig_level = float(sig_level)
    return from_dist, sig_level

def test_sphericity(X, alpha=.05):
    """Mauchly's test for sphericity

    https://www.mathworks.com/help/stats/mauchlys-test-of-sphericity.html

    Warning: results can slightly differ than R or Matlab. If you can,
    always double-check your results.

    Parameters
    ----------
    X : array_like
        Data array of shape (n_observations, n_repetitions)
    alpha : float, optional
        Significance level

    Returns
    -------
    sphericity : boolean
        True if data have the sphericity property.
    W : float
        Mauchly's W statistic
    chi_sq : float
        Chi-square statistic
    ddof : int
        Degrees of freedom
    p : float
        P-value.
    eps : float
        Epsilon adjustment factor.

    See Also
    --------
    test_homoscedasticity : Test equality of variance.
    test_normality : Test the normality of one or more array.

    Examples
    --------
    Test the sphericity of an array with 30 observations *
    3 repeated measures

        >>> import numpy as np
        >>> from pingouin import test_sphericity
        >>> np.random.seed(123)
        >>> x = np.random.normal(loc=0, scale=1., size=30)
        >>> y = np.random.normal(loc=0, scale=0.8,size=30)
        >>> z = np.random.normal(loc=0, scale=0.9,size=30)
        >>> X = np.c_[x, y, z]
        >>> sphericity, W, chi_sq, ddof, p, eps = test_sphericity(X)
        >>> print(sphericity, p)
        True 0.56
        """
    from scipy.stats import chi2
    n = X.shape[0]

    # Compute the covariance matrix
    S = np.cov(X, rowvar=0)
    p = S.shape[1]
    d = p - 1

    # Orthonormal contrast matrix
    C = np.array(np.triu(np.ones((p, d))), order='F')
    C.reshape(-1, order='F')[1::p + 1] = -np.arange(d)
    C, _ = np.linalg.qr(C)
    d = C.shape[1]
    T = C.T.dot(S).dot(C)

    # Compute epsilon
    eig = np.linalg.eigvals(T)
    eps = np.sum(eig) ** 2 / ((S.shape[0] - 1) * np.sum(eig**2))
    # Alternative:
    # eps = np.trace(T) ** 2 / ((S.shape[0] - 1) * np.sum(
    # np.sum(T * T, axis=1)))

    # Mauchly's statistic
    W = np.linalg.det(T) / (np.trace(T) / (p - 1))**d

    # Chi-square statistic
    nr = n - np.linalg.matrix_rank(X)
    dd = 1 - (2 * d**2 + d + 2) / (6 * d * nr)
    chi_sq = -np.log(W) * dd * nr
    ddof = d * (d + 1) / 2 - 1
    pval = chi2.sf(chi_sq, ddof)

    # Second order approximation
    pval2 = chi2.sf(chi_sq, ddof + 4)
    w2 = (d + 2) * (d - 1) * (d - 2) * (2 * d**3 + 6 * d * d + 3 * d + 2) / \
         (288 * d * d * nr * nr * dd * dd)
    pval += w2 * (pval2 - pval)

    sphericity = True if pval > alpha else False
    return sphericity, W, chi_sq, ddof, pval, eps

def ttest(x, y, paired=False, tail='two-sided', correction='auto', r=.707):
    """T-test.

    Parameters
    ----------
    x : array_like
        First set of observations.
    y : array_like or float
        Second set of observations. If y is a single value, a one-sample T-test
        is computed.
    paired : boolean
        Specify whether the two observations are related (i.e. repeated
        measures) or independant.
    tail : string
        Specify whether to return two-sided or one-sided p-value.
    correction : string or boolean
        For unpaired two sample T-tests, specify whether or not to correct for
        unequal variances using Welch separate variances T-test. If 'auto', it
        will automatically uses Welch T-test when the sample sizes are unequal,
        as recommended by Zimmerman 2004.
    r : float
        Cauchy scale factor for computing the Bayes Factor.
        Smaller values of r (e.g. 0.5), may be appropriate when small effect
        sizes are expected a priori; larger values of r are appropriate when
        large effect sizes are expected (Rouder et al 2009).
        The default is 0.707 (= np.sqrt(2) / 2).

    Returns
    -------
    stats : pandas DataFrame
        T-test summary ::

        'T-val' : T-value
        'p-val' : p-value
        'dof' : degrees of freedom
        'cohen-d' : Cohen d effect size
        'power' : achieved power of the test ( = 1 - type II error)
        'BF10' : Bayes Factor of the alternative hypothesis

    Notes
    -----
    Missing values are automatically removed from the data. If x and y are
    paired, the entire row is removed.

    Examples
    --------
    1. One-sample T-test.

        >>> from pingouin import ttest
        >>> x = [5.5, 2.4, 6.8, 9.6, 4.2]
        >>> ttest(x, 4)
            T-val   p-val  dof  cohen-d  power   BF10
            1.397  0.2348    4    0.699  0.226  0.766

    2. Paired two-sample T-test (one-tailed).

        >>> from pingouin import ttest
        >>> pre = [5.5, 2.4, 6.8, 9.6, 4.2]
        >>> post = [6.4, 3.4, 6.4, 11., 4.8]
        >>> ttest(pre, post, paired=True, tail='one-sided')
            T-val   p-val  dof  cohen-d  power   BF10
            -2.308   0.04    4     0.28  0.132  1.864

    3. Paired two-sample T-test with missing values.

        >>> from pingouin import ttest
        >>> from numpy import nan
        >>> pre = [5.5, 2.4, nan, 9.6, 4.2]
        >>> post = [6.4, 3.4, 6.4, 11., 4.8]
        >>> ttest(pre, post, paired=True)
            T-val    p-val  dof  cohen-d  power    BF10
            -5.902  0.0097    3   -0.354  0.074  24.926

    4. Independant two-sample T-test (equal sample size).

        >>> from pingouin import ttest
        >>> import numpy as np
        >>> np.random.seed(123)
        >>> x = np.random.normal(loc=7, size=20)
        >>> y = np.random.normal(loc=4, size=20)
        >>> ttest(x, y, correction='auto')
            T-val     p-val  dof  cohen-d  power   BF10
            9.106  4.30e-11   38     2.88    1.0  1.4e8

    5. Independant two-sample T-test (unequal sample size).

        >>> from pingouin import ttest
        >>> import numpy as np
        >>> np.random.seed(123)
        >>> x = np.random.normal(loc=7, size=20)
        >>> y = np.random.normal(loc=6.5, size=15)
        >>> ttest(x, y, correction='auto')
            T-val     p-val  dof   dof-corr  cohen-d  power   BF10
            2.327     0.027   33      30.75    0.792  0.614  2.454
    """
    from scipy.stats import ttest_rel, ttest_ind, ttest_1samp
    from pingouin import ttest_power, compute_effsize
    x = np.asarray(x)
    y = np.asarray(y)

    if x.size != y.size and paired:
        print('x and y have unequal sizes. Switching to paired == False.')
        paired = False

    # Remove NA
    x, y = _remove_na(x, y, paired=paired)
    nx = x.size
    ny = y.size
    stats = pd.DataFrame({}, index=['T-test'])

    if ny == 1:
        # Case one sample T-test
        tval, pval = ttest_1samp(x, y)
        dof = nx - 1
        pval = pval / 2 if tail == 'one-sided' else pval

    if ny > 1 and paired is True:
        # Case paired two samples T-test
        tval, pval = ttest_rel(x, y)
        dof = nx - 1
        bf = bayesfactor_ttest(tval, nx, ny, paired=True, r=r)

    elif ny > 1 and paired is False:
        dof = nx + ny - 2
        # Case unpaired two samples T-test
        if correction is True or (correction == 'auto' and nx != ny):
            # Use the Welch separate variance T-test
            tval, pval = ttest_ind(x, y, equal_var=False)
            # dof are approximated using Welch–Satterthwaite equation
            vx = x.var(ddof=1)
            vy = y.var(ddof=1)
            dof_corr = (vx / nx + vy / ny)**2 / ((vx / nx)**2 / (nx - 1) +
                                                 (vy / ny)**2 / (ny - 1))
            stats['dof-corr'] = dof_corr
        else:
            tval, pval = ttest_ind(x, y, equal_var=True)

    pval = pval / 2 if tail == 'one-sided' else pval

    # Effect size and achieved power
    d = compute_effsize(x, y, paired=paired, eftype='cohen')
    power = ttest_power(d, nx, ny, paired=paired, tail=tail)

    # Bayes factor
    bf = bayesfactor_ttest(tval, nx, ny, paired=paired, tail=tail, r=r)

    # Fill output DataFrame
    stats['dof'] = dof
    stats['T-val'] = tval.round(3)
    stats['p-val'] = pval
    stats['tail'] = tail
    stats['cohen-d'] = np.abs(d).round(3)
    stats['power'] = power
    stats['BF10'] = bf

    col_order = ['T-val', 'p-val', 'dof', 'dof-corr', 'tail', 'cohen-d',
                 'power', 'BF10']
    stats = stats.reindex(columns=col_order)
    stats.dropna(how='all', axis=1, inplace=True)
    return stats

def rm_anova(dv=None, within=None, data=None, correction='auto',
             remove_na=True, detailed=False, export_filename=None):
    """One-way repeated measures ANOVA.

    Results have been tested against R and JASP.

    Parameters
    ----------
    dv : string
        Name of column containing the dependant variable.
    within : string
        Name of column containing the within factor.
    data : pandas DataFrame
        DataFrame
    correction : string or boolean
        If True, return Greenhouse-Geisser corrected p-value.
        If 'auto' (default), compute Mauchly's test of sphericity to determine
        whether the p-values needs to be corrected.
    remove_na : boolean
        If True, automatically remove from the analysis subjects with one or
        more missing values::

            Ss    x1       x2       x3
            1     5.0      4.2      nan
            2     4.6      3.6      3.9

        In this example, if remove_na == True, Ss 1 will be removed from the
        ANOVA because of the x3 missing value. If False, the two non-missing
        values will be included in the analysis.
    detailed : boolean
        If True, return a full ANOVA table
    export_filename : string
        Filename (without extension) for the output file.
        If None, do not export the table.
        By default, the file will be created in the current python console
        directory. To change that, specify the filename with full path.

    Returns
    -------
    aov : DataFrame
        ANOVA summary ::

        'Source' : Name of the within-group factor
        'ddof1' : Degrees of freedom (numerator)
        'ddof2' : Degrees of freedom (denominator)
        'F' : F-value
        'p-unc' : Uncorrected p-value
        'np2' : Partial eta-square effect size
        'p-GG-corr' : Greenhouse-Geisser corrected p-value
        'W-Mauchly' : Mauchly statistic
        'X2-Mauchly' : Chi-square statistic of the Mauchly test
        'DF-Mauchly' : Degrees of freedom for the Mauchly test
        'p-Mauchly' : p-value of the Mauchly test
        'sphericity' : sphericity of the data (boolean)

    See Also
    --------
    anova : One-way ANOVA
    mixed_anova : Two way mixed ANOVA
    friedman : Non-parametric one-way repeated measures ANOVA

    Notes
    -----
    The effect size reported in Pingouin is the partial eta-square.
    However, one should keep in mind that for one-way repeated-measures ANOVA,
    partial eta-square is the same as eta-square.

    For more details, see Bakeman 2005; Richardson 2011.

    Examples
    --------
    Compute a one-way repeated-measures ANOVA.

        >>> import pandas as pd
        >>> from pingouin import rm_anova, print_table
        >>> df = pd.read_csv('dataset.csv')
        >>> aov = rm_anova(dv='DV', within='Time', data=df, correction='auto',
                           remove_na=True, detailed=True,
                           export_filename='rm_anova.csv')
        >>> print_table(aov)
    """
    from scipy.stats import f
    # Check data
    _check_dataframe(dv=dv, within=within, data=data, effects='within')

    # Remove NaN
    if remove_na and data[dv].isnull().values.any():
        data = _remove_rm_na(dv=dv, within=within, data=data)

    # Reset index (avoid duplicate axis error)
    data = data.reset_index(drop=True)

    # Sort values (to avoid bug when creating 'Subj' column)
    data = data.sort_values(by=within, kind='mergesort')

    # Groupby
    grp_with = data.groupby(within)[dv]
    rm = list(data[within].unique())
    n_rm = len(rm)
    n_obs = int(data.groupby(within)[dv].count().max())
    grandmean = data[dv].mean()

    # Calculate sums of squares
    sstime = ((grp_with.mean() - grandmean)**2 * grp_with.count()).sum()
    sswithin = grp_with.apply(lambda x: (x - x.mean())**2).sum()
    data['Subj'] = np.tile(np.arange(n_obs), n_rm)
    grp_subj = data.groupby('Subj')[dv]
    sssubj = n_rm * np.sum((grp_subj.mean() - grandmean)**2)
    sserror = sswithin - sssubj

    # Calculate degrees of freedom
    ddof1 = n_rm - 1
    ddof2 = ddof1 * (n_obs - 1)

    # Calculate F and p-values
    mserror = sserror / (ddof2 / ddof1)
    fval = sstime / mserror
    p_unc = f(ddof1, ddof2).sf(fval)

    # Calculating partial eta-square
    # Similar to (fval * ddof1) / (fval * ddof1 + ddof2)
    np2 = sstime / (sstime + sserror)

    # Reshape and remove NAN for sphericity estimation and correction
    data_pivot = data.pivot(index='Subj', columns=within, values=dv).dropna()

    # Compute sphericity using Mauchly's test
    # Sphericity assumption only applies if there are more than 2 levels
    if correction == 'auto' or (correction is True and n_rm >= 3):
        sphericity, W_mauchly, chi_sq_mauchly, ddof_mauchly, \
            p_mauchly, eps = test_sphericity(data_pivot.values, alpha=.05)

        if correction == 'auto':
            correction = True if not sphericity else False
    else:
        correction = False

    # If required, apply Greenhouse-Geisser correction for sphericity
    if correction:
        # print('eps = ', eps)
        corr_ddof1, corr_ddof2 = [np.maximum(d * eps, 1.) for d in
                                  (ddof1, ddof2)]
        p_corr = f(corr_ddof1, corr_ddof2).sf(fval)

    # Create output dataframe
    if not detailed:
        aov = pd.DataFrame({'Source': within,
                            'ddof1': ddof1,
                            'ddof2': ddof2,
                            'F': fval,
                            'p-unc': p_unc,
                            'np2': np2
                            }, index=[0])
        if correction:
            aov['p-GG-corr'] = p_corr
            aov['W-Mauchly'] = W_mauchly
            aov['X2-Mauchly'] = chi_sq_mauchly
            aov['DF-Mauchly'] = ddof_mauchly
            aov['p-Mauchly'] = p_mauchly
            aov['sphericity'] = sphericity

        col_order = ['Source', 'ddof1', 'ddof2', 'F', 'p-unc',
                     'p-GG-corr', 'np2', 'sphericity', 'W-Mauchly',
                     'X2-Mauchly', 'DF-Mauchly', 'p-Mauchly']
    else:
        aov = pd.DataFrame({'Source': [within, 'Error'],
                            'SS': [sstime, sserror],
                            'DF': [ddof1, ddof2],
                            'MS': [sstime / ddof1, sserror / ddof2],
                            'F': [fval, np.nan],
                            'p-unc': [p_unc, np.nan],
                            'np2': [np2, np.nan]
                            })
        if correction:
            aov['p-GG-corr'] = [p_corr, np.nan]
            aov['W-Mauchly'] = [W_mauchly, np.nan]
            aov['X2-Mauchly'] = [chi_sq_mauchly, np.nan]
            aov['DF-Mauchly'] = np.array([ddof_mauchly, 0], 'int')
            aov['p-Mauchly'] = [p_mauchly, np.nan]
            aov['sphericity'] = [sphericity, np.nan]

        col_order = ['Source', 'SS', 'DF', 'MS', 'F', 'p-unc', 'p-GG-corr',
                     'np2', 'sphericity', 'W-Mauchly', 'X2-Mauchly',
                     'DF-Mauchly', 'p-Mauchly']

    aov = aov.reindex(columns=col_order)
    aov.dropna(how='all', axis=1, inplace=True)
    # Export to .csv
    if export_filename is not None:
        _export_table(aov, export_filename)
    return aov

def anova(dv=None, between=None, data=None, detailed=False,
          export_filename=None):
    """One-way ANOVA.

    Results have been tested against R and JASP.

    Parameters
    ----------
    dv : string
        Name of column containing the dependant variable.
    between : string
        Name of column containing the between factor.
    data : pandas DataFrame
        DataFrame
    detailed : boolean
        If True, return a detailed ANOVA table
    export_filename : string
        Filename (without extension) for the output file.
        If None, do not export the table.
        By default, the file will be created in the current python console
        directory. To change that, specify the filename with full path.

    Returns
    -------
    aov : DataFrame
        ANOVA summary ::

        'Source' : name of the between-group factor
        'ddof1' : degrees of freedom (numerator)
        'ddof2' : degrees of freedom (denominator)
        'F' : F-value
        'p-unc' : uncorrected p-value
        'np2' : Partial eta-square effect size

    See Also
    --------
    rm_anova : One-way repeated measures ANOVA
    mixed_anova : Two way mixed ANOVA
    kruskal : Non-parametric one-way ANOVA

    Notes
    -----
    The effect size reported in Pingouin is the partial eta-square.
    However, one should keep in mind that for one-way ANOVA
    partial eta-square is the same as eta-square and generalized eta-square.

    For more details, see Bakeman 2005; Richardson 2011.

    Examples
    --------
    Compute a one-way ANOVA.

        >>> import pandas as pd
        >>> from pingouin import anova, print_table
        >>> df = pd.read_csv('dataset.csv')
        >>> aov = anova(dv='DV', between='Group', data=df,
                        detailed=True, export_filename='anova.csv')
        >>> print_table(aov)
    """
    from scipy.stats import f

    # Check data
    _check_dataframe(dv=dv, between=between, data=data,
                     effects='between')

    # Reset index (avoid duplicate axis error)
    data = data.reset_index(drop=True)

    groups = list(data[between].unique())
    n_groups = len(groups)
    N = data[dv].size

    # Calculate sums of squares
    grp = data.groupby(between)[dv]
    # Between effect
    ssbetween = ((grp.mean() - data[dv].mean())**2 * grp.count()).sum()
    # Within effect (= error between)
    sserror = grp.apply(lambda x: (x - x.mean())**2).sum()

    # Calculate DOF, MS, F and p-values
    ddof1 = n_groups - 1
    ddof2 = N - n_groups
    msbetween = ssbetween / ddof1
    mserror = sserror / ddof2
    fval = msbetween / mserror
    p_unc = f(ddof1, ddof2).sf(fval)

    # Calculating partial eta-square
    # Similar to (fval * ddof1) / (fval * ddof1 + ddof2)
    np2 = ssbetween / (ssbetween + sserror)

    # Create output dataframe
    if not detailed:
        aov = pd.DataFrame({'Source': between,
                            'ddof1': ddof1,
                            'ddof2': ddof2,
                            'F': fval,
                            'p-unc': p_unc,
                            'np2': np2
                            }, index=[0])

        col_order = ['Source', 'ddof1', 'ddof2', 'F', 'p-unc', 'np2']
    else:
        aov = pd.DataFrame({'Source': [between, 'Within'],
                            'SS': [ssbetween, sserror],
                            'DF': [ddof1, ddof2],
                            'MS': [msbetween, mserror],
                            'F': [fval, np.nan],
                            'p-unc': [p_unc, np.nan],
                            'np2': [np2, np.nan]
                            })
        col_order = ['Source', 'SS', 'DF', 'MS', 'F', 'p-unc', 'np2']

    aov = aov.reindex(columns=col_order)
    aov.dropna(how='all', axis=1, inplace=True)
    # Export to .csv
    if export_filename is not None:
        _export_table(aov, export_filename)
    return aov

def mixed_anova(dv=None, within=None, between=None, data=None,
                correction='auto', remove_na=True, export_filename=None):
    """Mixed-design (split-plot) type II ANOVA.

    Results have been tested against R and JASP.

    Parameters
    ----------
    dv : string
        Name of column containing the dependant variable.
    within : string
        Name of column containing the within factor.
    between : string
        Name of column containing the between factor.
    data : pandas DataFrame
        DataFrame
    correction : string or boolean
        If True, return Greenhouse-Geisser corrected p-value.
        If 'auto' (default), compute Mauchly's test of sphericity to determine
        whether the p-values needs to be corrected.
    remove_na : boolean
        If True, automatically remove from the analysis subjects with one or
        more missing values::

            Ss    x1       x2       x3
            1     5.0      4.2      nan
            2     4.6      3.6      3.9

        In this example, if remove_na == True, Ss 1 will be removed from the
        ANOVA because of the x3 missing value. If False, the two non-missing
        values will be included in the analysis.
    export_filename : string
        Filename (without extension) for the output file.
        If None, do not export the table.
        By default, the file will be created in the current python console
        directory. To change that, specify the filename with full path.

    Returns
    -------
    aov : DataFrame
        ANOVA summary ::

        'Source' : Names of the factor considered
        'ddof1' : Degrees of freedom (numerator)
        'ddof2' : Degrees of freedom (denominator)
        'F' : F-values
        'p-unc' : Uncorrected p-values
        'np2' : Partial eta-square effect sizes
        'p-GG-corr' : Greenhouse-Geisser corrected p-values
        'W-Mauchly' : Mauchly statistic
        'X2-Mauchly' : Chi-square statistic of the Mauchly test
        'DF-Mauchly' : Degrees of freedom for the Mauchly test
        'p-Mauchly' : p-value of the Mauchly test
        'sphericity' : sphericity of the data (boolean)

    See Also
    --------
    anova : One-way ANOVA
    rm_anova : One-way repeated measures ANOVA

    Examples
    --------
    Compute a two-way mixed model ANOVA.

        >>> import pandas as pd
        >>> from pingouin import mixed_anova, print_table
        >>> df = pd.read_csv('dataset.csv')
        >>> aov = mixed_anova(dv='DV', within='Time', between='Group', data=df,
                             correction='auto', remove_na=False)
        >>> print_table(aov)
    """
    from scipy.stats import f
    # Check data
    _check_dataframe(dv=dv, within=within, between=between, data=data,
                     effects='interaction')
    # Remove NaN
    if remove_na and data[dv].isnull().values.any():
        data = _remove_rm_na(dv=dv, within=within, data=data)
    # Reset index (avoid duplicate axis error)
    data = data.reset_index(drop=True)

    # SUMS OF SQUARES
    grandmean = data[dv].mean()
    # Extract main effects of time and between
    mtime = rm_anova(dv=dv, within=within, data=data, correction=correction,
                     remove_na=False, detailed=True)
    mbetw = anova(dv=dv, between=between, data=data, detailed=True)
    # Extract SS total, residuals and interactions
    grp = data.groupby([between, within])[dv]
    sstotal = grp.apply(lambda x: (x - grandmean)**2).sum()
    # sst = residuals within + residuals between
    sst = grp.apply(lambda x: (x - x.mean())**2).sum()
    # Interaction
    ssinter = sstotal - (sst + mtime.loc[0, 'SS'] + mbetw.loc[0, 'SS'])
    sswg = mtime.loc[1, 'SS'] - ssinter
    sseb = sstotal - (mtime.loc[0, 'SS'] + mbetw.loc[0, 'SS'] + sswg + ssinter)

    # DEGREES OF FREEDOM
    n_obs = data.groupby(within)[dv].count().max()
    dftime = mtime.loc[0, 'DF']
    dfbetween = mbetw.loc[0, 'DF']
    dfeb = n_obs - data.groupby(between)[dv].count().count()
    dfwg = dftime * dfeb
    dfinter = mtime.loc[0, 'DF'] * mbetw.loc[0, 'DF']

    # MEAN SQUARES
    mseb = sseb / dfeb
    mswg = sswg / dfwg
    msinter = ssinter / dfinter

    # F VALUES
    fbetween = mbetw.loc[0, 'MS'] / mseb
    ftime = mtime.loc[0, 'MS'] / mswg
    finter = msinter / mswg

    # P-values
    pbetween = f(dfbetween, dfeb).sf(fbetween)
    ptime = f(dftime, dfwg).sf(ftime)
    pinter = f(dfinter, dfwg).sf(finter)

    # Effects sizes
    npsq_between = fbetween * dfbetween / (fbetween * dfbetween + dfeb)
    npsq_time = ftime * dftime / (ftime * dftime + dfwg)
    npsq_inter = ssinter / (ssinter + sswg)

    # Stats table
    aov = pd.concat([mbetw.drop(1), mtime.drop(1)], sort=False,
                    ignore_index=True)
    # Update values
    aov.rename(columns={'DF': 'DF1'}, inplace=True)
    aov.loc[0, 'F'], aov.loc[1, 'F'] = fbetween, ftime
    aov.loc[0, 'p-unc'], aov.loc[1, 'p-unc'] = pbetween, ptime
    aov.loc[0, 'np2'], aov.loc[1, 'np2'] = npsq_between, npsq_time
    aov = aov.append({'Source': 'Interaction',
                      'SS': ssinter,
                      'DF1': dfinter,
                      'MS': msinter,
                      'F': finter,
                      'p-unc': pinter,
                      'np2': npsq_inter
                      }, ignore_index=True)

    aov['DF2'] = [dfeb, dfwg, dfwg]
    col_order = ['Source', 'SS', 'DF1', 'DF2', 'MS', 'F', 'p-unc', 'np2',
                 'p-GG-corr', 'sphericity', 'W-Mauchly', 'X2-Mauchly',
                 'DF-Mauchly', 'p-Mauchly']

    aov = aov.reindex(columns=col_order)
    aov.dropna(how='all', axis=1, inplace=True)

    # Export to .csv
    if export_filename is not None:
        _export_table(aov, export_filename)
    return aov


#!/usr/bin/env python
# coding: utf-8

# # Import all the Libraries

# In[1]:

import numpy as np
import torch
import torchvision
import torchvision.transforms as transforms
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import matplotlib.pyplot as plt
from sklearn.metrics import confusion_matrix
import itertools

import warnings
warnings.filterwarnings('ignore')

from torch.utils.tensorboard import SummaryWriter
torch.set_printoptions(linewidth=120)
torch.set_grad_enabled(True)

# In[2]:

print(torch.__version__)
print(torchvision.__version__)

# # Function to print the correct number of predictions

# In[3]:

def get_num_correct(preds,labels):
    return preds.argmax(dim=1).eq(labels).sum().item()

# # The output size formula is given by
#     if its n x n input
#     O = (n-f+2p/s)+1
#
#     if its non square
#     Oh = nh-fh+2p/s +1
#     Ow = nw-fw+2p/s+1
#
#    n -> input size
#
#    p -> padding
#
#    s -> stride length
#
#    f -> filter size
#
#
#

# # Define the  Convolutional Neural Network

# In[4]:

class Network(nn.Module):
    def __init__(self):
        super(Network,self).__init__()
        self.conv1 = nn.Conv2d(in_channels=1, out_channels=6,kernel_size=5)
        self.batch_norm_1 = nn.BatchNorm2d(6)
        self.conv2 = nn.Conv2d(in_channels=6, out_channels=12, kernel_size=5)

        self.fc1 = nn.Linear(in_features=12*4*4, out_features=120)
        self.batch_norm_2 = nn.BatchNorm1d(120)
        self.fc2 = nn.Linear(in_features=120, out_features=60)
        self.out= nn.Linear(in_features=60, out_features=10)

    def forward(self,t):
        #input layer
        t=t

        #hidden conv layer
        t=self.conv1(t)
        t=F.relu(t)
        t=F.max_pool2d(t,kernel_size=2, stride=2)
        t=self.batch_norm_1(t)

        #hidden conv layer
        t=self.conv2(t)
        t=F.relu(t)
        t=F.max_pool2d(t,kernel_size=2, stride=2)

        #hidden linear layer
        t=t.reshape(-1,12*4*4)
        t=self.fc1(t)
        t=F.relu(t)
        t=self.batch_norm_2(t)

        #hidden linear layer
        t=self.fc2(t)
        t=F.relu(t)

        #output layer
        t=self.out(t)
        #t=F.softmax(t,dim=1)
        #no need of this line since loss function will perform the softmax operation
        return t

# # Import the Dataset

# In[5]:

train_set = torchvision.datasets.FashionMNIST(
    root='./data/FashionMNIST'
    ,train=True
    ,download=True
    ,transform=transforms.Compose([transforms.ToTensor()])
)

# In[6]:

train_loader_normal = torch.utils.data.DataLoader(train_set,batch_size=len(train_set),
                                               num_workers=1)
data = next(iter(train_loader_normal))
mean = data[0].mean()
std = data[0].std()
print(mean.item())
print(std.item())

# In[7]:

train_set_normal = torchvision.datasets.FashionMNIST(
    root='./data/FashionMNIST'
    ,train=True
    ,download=True
    ,transform=transforms.Compose([
        transforms.ToTensor()
        ,transforms.Normalize((mean.item(),), (std.item(),))
    ])
)
print(train_set_normal)

# In[8]:

train_loader_normal_2 = torch.utils.data.DataLoader(train_set_normal,batch_size=len(train_set),
                                               num_workers=1)
data = next(iter(train_loader_normal_2))
mean = data[0].mean()
std = data[0].std()
print("mean",mean.item(),"\t","std",std.item())

# # Check if GPU is available

# In[9]:

print(torch.cuda.is_available())

# # Run Builder class Implementation

# In[10]:

from collections import OrderedDict
from collections import namedtuple
from itertools import product
import time

class RunBuilder():
    @staticmethod
    def get_runs(params):
        Run = namedtuple('Run', params.keys())

        runs = []
        for v in product(*params.values()):
            runs.append(Run(*v))

        return runs

trainsets = {'not_normal': train_set,'normal': train_set_normal}

params = OrderedDict(
    lr = [0.01]
    ,batch_size = [1000]
    ,device = ['cuda']
    ,num_workers = [16]
    ,trainset = ['normal']
)

runs = RunBuilder.get_runs(params)
print(runs)

# # Training Loop

# In[11]:

for run in RunBuilder.get_runs(params):
    print(f'{run}')

    device = torch.device(run.device)
    network = Network().to(device)

    train_loader = torch.utils.data.DataLoader(trainsets[run.trainset],batch_size=run.batch_size,
                                               num_workers=run.num_workers)
    optimizer = optim.Adam(network.parameters(), lr=run.lr)

    images, labels = next(iter(train_loader))
    grid = torchvision.utils.make_grid(images)

    comment = f'-{run}'
    tb = SummaryWriter(comment=comment)
    tb.add_image('images', grid)
    tb.add_graph(network, images.to(getattr(run,'device','cpu')))

    for epoch in range(20):

        epoch_start = time.time()
        total_loss = 0
        total_correct = 0

        for batch in train_loader:
            images = batch[0].to(device)
            labels = batch[1].to(device)

            preds = network(images)
            loss = F.cross_entropy(preds, labels)

            optimizer.zero_grad()
            loss.backward() #calculate gradients
            optimizer.step() #Update weights

            total_loss+=loss.item() * run.batch_size
            total_correct+=get_num_correct(preds, labels)

        tb.add_scalar('Loss',total_loss, epoch)
        tb.add_scalar('Number Correct', total_correct, epoch)
        tb.add_scalar('Accuracy', total_correct/len(train_set), epoch)

        for name, weight in network.named_parameters():
            tb.add_histogram(name,weight,epoch)
            tb.add_histogram(f'{name}.grad',weight.grad,epoch)
        epoch_end = time.time()
        epoch_duration = epoch_end - epoch_start

        print("epoch",epoch,"\n","total_correct:",total_correct,"\n","loss:",
              total_loss,"\n","accuracy:",total_correct/len(train_set),
              "\n",'Duration',epoch_duration,"\n",'Device =',run.device,"\n")

    tb.close()

# # Building Confusion Matrix

# In[12]:

def get_all_preds(model,loader):
    all_preds = torch.tensor([]).to(torch.device('cuda'))
    for batch in loader:
        images = batch[0].to(torch.device('cuda'))
        labels = batch[1].to(torch.device('cuda'))

        preds = model(images)
        all_preds = torch.cat(
            (all_preds,preds),dim=0)

    return all_preds

# In[13]:

train_loader = torch.utils.data.DataLoader(train_set_normal,batch_size=1000,
                                               num_workers=16)
with torch.no_grad():
    train_preds = get_all_preds(network, train_loader)

# In[14]:

preds_correct = get_num_correct(train_preds, train_set.targets.to(torch.device('cuda')))

print('Total Correct:', preds_correct)
print('Accuracy', preds_correct / len(train_set))

# In[15]:

stacked = torch.stack(
    (
        train_set.targets.to(torch.device('cuda')),
        train_preds.argmax(dim=1)
    )
    ,dim=1
)

# In[16]:

cmt = torch.zeros(10,10, dtype=torch.int32)

for p in stacked:
    j, k =p.tolist()
    cmt[j,k] = cmt[j,k] +1

print(cmt)

# # Plotting the confusion matrix

# In[17]:

# Run this cell only when tensors are on CPU

# cm = confusion_matrix(train_set.targets, train_preds.argmax(dim=1))
# print(type(cm))
# cm

# In[18]:

def plot_confusion_matrix(cm, classes, normalize=False, title='Confusion matrix', cmap=plt.cm.Blues):
    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix, without normalization')

    print(cm)
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    fmt = '.2f' if normalize else 'd'
    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, format(cm[i, j], fmt), horizontalalignment="center", color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')

# In[19]:

names = tuple(train_set.classes)
plt.figure(figsize=(10,10))
plot_confusion_matrix(cmt, names)

# In[ ]:


import sys
import io
import unittest
import pandas as pd
import numpy as np

from datetime import datetime
from pandas.util.testing import assert_frame_equal, assert_series_equal

from MarketAnalysis import analyser

class TestAnalyser(unittest.TestCase):
  def setUp(self):
    """
    Setup testing environment, we will load the data from input/bili.csv.
    empty_analyse will be used for empty sets;
    bili will load data from input/bili.csv
    """
    self.empty = analyser.StockAnalyser('empty')
    self.bili = analyser.StockAnalyser('BILI')
    self.bili.load_csv('input/bili.csv')
    self.bili_df = self.bili.full_df()

  def test_load_csv(self):
    """
    Tests the load_csv method. If this fails, the entire test
    should fail.
    """
    self.empty.load_csv('input/bili.csv')
    res = pd.read_csv('input/bili.csv')
    res['Date'] = np.vectorize(pd.to_datetime)(res['Date'])
    # assert true if frame equal
    assert_frame_equal(res, self.empty.full_df(),)

  # @unittest.skip('skip this when debugging test unit')
  def test_load_data_from_yahoo(self):
    """
    Tests the load_data_from_yahoo method. This will compare the
    test with the prepared bili.csv, i.e. stock of BILI, with
    starting date 2018-07-20 and ending date 2019-07-19.

    Since DataReader also reads data 1 day before start date,
    we should set our start date as 2018-07-21.
    """
    data = analyser.StockAnalyser('BILI')
    start = datetime(2018,7,21)
    end = datetime(2019,7,19)
    # Setup a text trap to surpress stdout for test
    # Ref: https://codingdose.info/2018/03/22/supress-print-output-in-python/
    text_trap = io.StringIO()
    sys.stdout = text_trap

    data.load_data_from_yahoo(start_date=start, end_date=end)

    # Assert true if dataframe equals.Needs to use check_less_precise
    # as loading float from saved csv could be dodgy.
    assert_frame_equal(self.bili_df, data.full_df(), check_less_precise=3)

  def test_get_daily_return(self):
    """
    Tests get_daily_return method. We can use pandas.Series.pct_change to
    the test the method.
    """
    res = self.bili_df['Close'].pct_change()
    # Obtain df from bili using the method
    test = self.bili.get_daily_return()
    # we will be only testing on 'Daily Return' series against res
    assert_series_equal(res, test['Daily Return'], check_names=False)

  def test_rolling_mean(self):
    """
    Tests get_rolling_mean method. We can use pandas.DataFrame.rolling to
    the roll, then mean() to get the rolling mean.
    In this test, we will use 'Close' as base.
    """
    # setup list of days for MA
    day_list = [10, 20, 50, 100]
    # copy bili_df for local result test use
    res = self.bili_df.copy()
    # setup columns required
    cols = []
    cols.append('Date')
    cols.append('Close')
    for day in day_list:
      col_name = 'Moving Average for ' + repr(day) + 'days'
      cols.append(col_name)
      # Using library
      res[col_name] = res['Close'].rolling(day).mean()
    # only taking columns required
    res = res[cols]

    test = self.bili.get_rolling_mean(day_list=day_list, base='Close')

    assert_frame_equal(res,test)

if __name__ == '__main__':
  unittest.main()


#!/usr/bin/env python3

import unittest
from unittest.mock import patch
import numpy as np
from tmc import points

from tmc.utils import load, get_stdout, patch_helper

module_name="src.multiple_graphs"
main = load(module_name, "main")
ph = patch_helper(module_name)

@points('p03-09.1')
class MultipleGraphs(unittest.TestCase):

    def test_first(self):
        with patch(ph("plt.show")) as pshow,\
             patch(ph("plt.plot")) as pplot,\
             patch(ph("plt.xlabel")) as pxlabel,\
             patch(ph("plt.ylabel")) as pylabel:
            main()
            pshow.assert_called_once()
            pxlabel.assert_called_once()
            pylabel.assert_called_once()
            self.assertGreater(pplot.call_count, 0, msg="You should have called plt.plot!")
            self.assertLess(pplot.call_count, 3, msg="You should have called plt.plot at most two times!")
            if pplot.call_count == 2:
                np.testing.assert_array_equal(pplot.call_args_list[0][0][0], [2,4,6,7], err_msg="Wrong parameters to plot!")
                np.testing.assert_array_equal(pplot.call_args_list[0][0][1], [4,3,5,1], err_msg="Wrong parameters to plot!")
                np.testing.assert_array_equal(pplot.call_args_list[1][0][0], [1,2,3,4], err_msg="Wrong parameters to plot!")
                np.testing.assert_array_equal(pplot.call_args_list[1][0][1], [4,2,3,1], err_msg="Wrong parameters to plot!")
            else:
                np.testing.assert_array_equal(pplot.call_args_list[0][0], ([2,4,6,7], [4,3,5,1], [1,2,3,4], [4,2,3,1]),
                                              err_msg="Parameters to the plt.plot command were wrong")

if __name__ == '__main__':
    unittest.main()


#Content
##Functions

##Exception handling
###Try...Except statement
###Try...Except...Else statement
###Try...Except...Else...Finally statement

#Types and classes in Python
##Creating your own class
##Data attributes
##Methods

#Downloading, Reading and opening data in Python
##Download
##Reading
###Read method
###Readline method
###Readlines method
##Writing
##Appending

#Pandas
##Importing Pandas
##Reading excels with pandas
##Working with excels in pandas

#NumPy
##NumPy array or ND in one dimension
###Assigning values
###Slicing ND arrays
###Other ND attributes
##Useful NumPy tools for Data science
###Vector addition
###Vector broadcasting
###Mean of the elements
##Math functions
##Linspace

##2 Dimension NumPy array or ND

#APIs
##Definition and types
##Examples of APIs
###PyCoinGecko

#Requests Module

#Web Scraping
##Beautiful Soup objects and how to browse through them
###Tags
###Children, Parents and siblings
###Navigable String
##Filters
###find all filter
###find filter
###HTML Attributes
###Navigable strings
##Downloading and Scraping the contents of a Web Page

#How to work with tidfferent file formats
##csv, xml, json, xlsx

######If you're using PyCharm, you can select the chunk of code and run it by pressing alt+shift+e.
######Remember to import the libraries when necessary. At least in PyCharm, you will

                                        #####Functions#####
#Building your own function
def plusone(a):
    """add 1 to a"""
    b=a+1
    return b
#Using your function
print(plusone(6))

help(plusone)
#build functions with multiple parameters
def Mult(a,b):
    c=a*b
    return c
#Using your function

print(Mult(30,10))

#You can also multiply strings, but this is not recommended.
print(Mult(3," Hello world Hello"))

#Print strings in your functions
def CS():
    print('Computer Science')

CS()

#Create empty functions. Only allowed with the pass keyword, else it fails to run given the empty body.
##This will print a "None" object.

def Empt():
    pass

print(Empt())

#You can clearly see what the return function does in this example.
def Ftest(a):
    b=a+1
    print(a,"plus one equals", b)
    return b
Ftest(3)

#You can use loops in functions
##First we define our loop function
def looptions(som):
    for i,s in enumerate(som):
        print("Game", i, "is ranked:", s)

game_ratings=[8,5,10]

looptions(game_ratings)

#The following function contains a Variadic parameter, that allows us
##to input multiple elements

def GameName(*names):
    for name in names:
        print(name)

GameName('Dark Souls','Mario 64', 'Zelda: Ocarina of Time')

#You can set the scope of a function to global. Then, the element after the keyword global will be stored
## in the global enviroment

def Mario():
    global Date
    Date=1990
    return Date

Mario()

print(Date)

                        #######Exception Handling.#######
#I.E: entering numbers in a only letter input field and an error code and/or message

# The Try...Except statement will "Try" to execute its code block, but when an error occurs,
## the "except" code will run and do the exception handling once the program reaches the exception that matches
### and run the code. It should print an error message related to the specific error. You may add multiple
#### exceptions with different error codes and messages. Exceptions can be seen while you use Python I.E.
####dividing by 0 (try running: 1/0)

t=1
try:
    b=int(input("Please enter a number to divide t"))
    t=t/b
    print("Accepted. t=", t)
except ZeroDivisionError:
    print("You can't divide by 0")
except ValueError:
    print("You should only provide numbers")
except:
    print("Not a valid input")
# Try...except...else will give you the opportunity to execute another block of code if there was no exception
##or error on your code
F=2
try:
    d=int(input("Please enter a number"))
    F=F/d
    print("The new value for F is", F)

except ZeroDivisionError:
    print("You cant divide by 0")

except ValueError:
    print("Please enter a number")

else:
    if(F<1): #For more info on this one, check the else if and else elif statements.
        print("F is smaller than one: F=", F)
    elif(F>1):
        print("F is larger than 1: F=", F)
    elif(F==1):
        print("F is equal to 1: F=", F)

# If you want something to happen after the code has been executed, you can use the "Try...Except...
## Else...Finally" Statement. I.E: closing the file after it has been edited.
G=5
try:
    H=int(input("Please enter a number"))
    G=G/H
except ZeroDivisionError:
    print("You can't divide by 0")

except ValueError:
    print("Please enter a number.")
else:
    if(G<1):
        print("G is smaller than one. G is equal to:", G)
    elif(G>1):
        print("G is bigger than one. G is equal to:", G)
    elif(G==1):
        print("G is equal to 1. G=", G)
finally:
    print("Thank you for joining me in my exception handling coding.")

#Types in Python: There are many. I.E: integer, float, String, List, Dictionary, Boolean. Every time we create any of these
##you create a instance or Object of that type. You can use the type() command to find the type of your object.
type(3)

#For every type, theres a Method you can use with that class of object. Methods are functions that a class or type
##provide. It is the way you can interact with the data. One example of this, is the sort method.

ratings=[3,2.0,3.2,5.7,8.2,3,10]
print(ratings)
ratings.sort()
print(ratings)

#There's another common method: the reverse method.
print(ratings)

ratings.reverse()

print(ratings)

                                            #####Classes#####

# You can create your own types or classes
## You first create a class, then assign attributes and methods to it.
### For example, a Circle has a radius attribute.
#### To create a new class: First, you declare you want to create a new class, then the name of the class
##### and finally the class/type parent. Then, you have to define attributes, in this case radius and we'll add color too.
###### The init function tells Python you are creating a new class. The self parameter indicates the new
####### instance/object created with its class. Self will contain all the data attributes of the object.

class Circle(object ):
    def __init__(self, radius, color):
        self.radius=radius;
        self.color=color;

#We will now create a new class called rectangle, contained in its parent class, object

class Rectangle(object ):
    def __init__(self, color ,height, width):
        self.color=color;
        self.height=height;
        self.width=width;

#Now we create objects with the new classes created. We call the code by using the class name and then we have to
##input its attributes between parenthesis.

BlueCircle= Circle(10,"red" )
GreenRectangle= Rectangle("green", 5, 7 )

#You can also check the value of the attribute of each object/instance.

BlueCircle.radius

GreenRectangle.width

#You can also change their attributes

BlueCircle.radius=4

#Methods are the functions that interact and change attributes of the data.
## We can create a method to add radius to a circle:
class Ctest(object ):
    #Constructor
    def __init__(self, radius, color):
        self.radius=radius;
        self.color= color;
    #Method
    def add_radius(self, r):
        self.radius=self.radius+r
        return(self.radius)

Ctestred=Ctest(2,"red")
#To call the method, you have to input the name of the object, followed to the radius you'd like to add.
Ctestred.radius

Ctestred.add_radius(3)

Ctestred.radius

#You can get the attributes and methods of a class with the dir() function
dir(Ctest)

                                    ####Files in Python#####

#Downloading files:
import urllib.request
    url= 'https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-PY0101EN-SkillsNetwork/labs/Module%204/data/example1.txt'
    filename='Example1.txt'
    urllib.request.urlretrieve(url, filename)

#There are two different ways to open a file. The second one is better practice, because it runs the code
##and then it closes the file again.

#First, we'll use the open command by itself to create the file object. Then we will check the name of the file
##with the data name attribute as well as the mode the file is open in with the mode data attribute. Remember
###To always close the file after opening it to avoid further problems and free resources.
example1= 'Example1.txt'

File1 =open('Example1.txt', "r")

#Data attribute name
File1.name

#Data atribute mode
File1.mode

#Reading the object content.
objectcontent = File1.readlines()
print (objectcontent)
objectcontent
#Object method close
File1.close()

#Now we will use the with statement. This code will run everything within the indent. In this case, we will print
##the file content, after using the read method.
with open('Example1.txt') as File1:
    file_stuff=File1.read()
    print(file_stuff)
    File1.mode
    File1.name

print(file_stuff)

#If you input a number in the read method, you will get the first X characters of the file. As input more read methods,
##the script will read the next characters in the next line.
with open('Example1.txt') as File1:
    print(File1.read(4))
    print(File1.read(10))
    print("'Custom message.'", File1.read(20))

#You can create loops to iterate the code in every line.

with open('Example1.txt', 'r') as File1:
    i=0
    for line in File1:
        print('iteration number ', (i+1), line)
        i=i+1

#And we can save the values of the file as a list:
with open('Example1.txt') as File1:
    MyList= File1.readlines()

MyList[0]

#Writing files with the open function and the write method:

with open('C:/Users/CrsTn/PycharmProjects/Repo_CTeufel/Example2.txt', "w") as File1:
    File1.write("This is the first new line \n")
    File1.write("This will be the second new line \n")

#You can also use the for loop to write lines:

Lines=['Line number one \n', 'Line number two\n', 'Line number three \n']

with open('C:/Users/CrsTn/PycharmProjects/Repo_CTeufel/Example2.txt', "w") as File1:
    for line in Lines:
        File1.write(line)

#Appending: uses an existing file to write in it. This will add the new line.

with open('C:/Users/CrsTn/PycharmProjects/Repo_CTeufel/Example2.txt', "a") as File1:
    File1.write("Line number four \n")

#You can copy one file into another one by using 'read' mode for the original file, then
##for the copied file, you use the 'write' mode

with open("Example1.txt", "r") as readfile:
    with open("Example3.txt", "w") as writefile:
        for line in readfile:
            writefile.write(line)

#To test if the files have been written correctly, use the print function
##within the with indent.
with open("Example3.txt", "r") as writefile:
    print(writefile.readlines())

with open ("example1.txt", "r") as readfile
    with open("Example3.txt", "w") as writefile:
    for line in readlines:
        writefile.write(line)

#We can also write in files without deleting older content with the append
##mode and the write method.

lines2=['This is line four \n', 'This is line five\n']
with open("Example3.txt", "a") as writefile:
    for line in lines2:
        writefile.write(line)

with open('Example3.txt', 'r') as writefile:
    print(writefile.read())

                                                #####Libraries######
                                                ######PANDAS#######

#Pandas is a library, popular for its pre-built classes and functions for data analysis. Libraries or dependencies
##contain codes to solve problems. We will ned the xlrd dependency to install pandas
!pip install xlrd

#Importing Pandas with the import command. We will use the as statement to shorten to make the library less tedious
##to call with. The new and common abreviation will be "pd". You can also leave the library without

import pandas as pd

#Panda allows you to work with data frames and its data. To load a csv file with Pandas as a data frame, we will use
##the read_csv method. The file used as an example is one provided to us for an R 'Business Intelligence' elective
###course during my last semesters in university. I've seen it over the internet, but I'm not sure who was the first
####to publish it. Credits to that person for it.

NoShows='No shows.csv'
df=pd.read_csv(NoShows)

type(df)

#Read the headers of the data frame with the columns function
print(df.columns)

#We can make different Data frames with the values of a specific column like this:

dfgender=df[['Gender']]
##Note: when using double brackets, one will get the data in a dataframe. With one bracket, you will get a series.

#To check unique values, we can apply the method unique()
print(df['Gender'].unique())

#We could also get information out of this data frame. For example, we can count people per gender.
dfgender.value_counts(['Gender'])
df.value_counts(['Gender'])

#If we want to get all the rows with people of gender F in a new data frame, we can use the following code

df_f=df[df['Gender']=='F']

#This is a new data frame
df_f['Gender'].unique()

#To save this new data frame as a .CSV, we use the to_csv file like this
df_f.to_csv('df_females.csv')

#You can access the first, second or any specific row and column one would want to retrieve with the iloc method.
##If you ask yourself where the name of the method comes from, I'm also not sure, but I believe and have read
###it comes from integer location, therefore it only works with integers that represent the coordinates you want to call.
####There is also the 'loc' function, that works with strings, like the column name.
df.iloc[0, ] ##First row

df.iloc[0,0] ##First row, first column

df.iloc[1,5] ##second row, fifth column

#The loc method also supports names>
df.loc[20, 'PatientId']

#You can also take slices of the sample with both methods. Remember, when slicing with the """iloc""" method,
##one will get the index upper limit minus 1. Unlike iloc, the """loc""" method will count the upper limit if numbers
###used, ie, iloc[2:5, 1:3] will retrieve from row 2 to 4 and from column 1 down to 2. (Remember 0 is the first row/col)

df.loc[20:23, 'Age'] #from 20 to 23

df.iloc[0:2, 0:1 ] #from 20 to 21,

                                        ######NumPy#######
#Numpy is a library used for scientific computing and has advantages like its speed and memory. It is also the basis
## for panda.

#How to create a NumPy array
## A NumPy array or ND is similar to a list. It is normally fixed in size and every element is of the same type.

import numpy as np

a=np.array([0,1,2,3,4,])

#Let's check the type of this element
type(a)

#As you can see, it is a numpy.ndarray. One could also check the size and the number of dimension the array has with the
##.size and .ndim attribute. The shape attribute will indicate the size of the array in each direction in a tuple.
a.size
a.ndim
a.shape
#Just like lists, we can check the values in the array, starting by 0.

a[0]
a[1]

#ND arrays can also contain floats. One could see what kind of numbers the array contains with the 'dtype'. 'a' list
##contains 'int32' data or integers, while b list contains 'float64', as it contains float numbers.
b=np.array([10, 20.1, 3.5, 6.9, 5.6])

b.dtype
a.dtype

#Elements can be modified as follows. In this case, the first and the third element

b[0]=100.2
b[2]=3.6

#One can also slize ND arrays, just like lists. We will assign the new array to the letter c and we will retrieve
##the second to the third row.

c=b[1:4]
c

#And one can modify more than one value this way:
c[0:2]=23, 50
c

#Vector addition: NumPy arrays can be added as vectors in a very simple way. Just add them as you would do in simple math

d=a+b
d

#One is allowed to do a multiplication with a scalar as well:
e=d*2
e

#Hadamard product is the product between two vectors. This process is simplified by NumPy, as you'd need a function
##to do it manually, just like the scalar multiplication and addition.

f=a*b
f

#Another usefull tool is the dot product, that tells you how similar your vectors are.

dot_prod=np.dot(a,b)
dot_prod

#If you want to add a constant to every value, NumPy will make it easier. This property is called broadcasting
adding=np.array([4,2,1,3,5])
addingresult= adding+1
addingresult

#There are universal functions, which are functions that operate in ND arrays

#The mean of the elements is easily obtainable with the mean function
meanresult=addingresult.mean()
meanresult

#To find the maximum value, use the max method.

addingresult.max()

#NumPy contains pi number as well. We could notate values in radians using the np.pi function. There is also a sin
##function.
x=np.array([0, np.pi/2, np.pi])
x

y=np.sin(x)
y

#If you want to plot mathematical functions in a evenly spaced plot, you can use the linspace function, which returns
## spaced numbers in an interval. The num parameter determines the number of samples in the sequence.

np.linspace(-3,3, num=7)

import matplotlib.pyplot as plt
x=np.linspace(0,2*np.pi, 100)
y=np.sin(x)
plt.plot(x,y)
plt.show()

#Two dimensional NumPy arrays
##We will now create a 2D numpy array as follows
import numpy as np
import matplotlib.pyplot as plt

a=[[10, 13, 15], [20,23,25], [30,23,35]]
a
A=np.array(a)
A

#One can use the ndim attribute to get  the dimensions, shape attribute to get the size of every dim. and the size, which
## will obtian the number of elements in the array.
A.ndim
A.shape
A.size
type(A)
#Accessing elements in the array:
##Say we want the middle element in the last array we've created
##We can do it this way. As always, to get the row and column you want, you have to substract 1 from the actual index.
##We will also retrieve the third row completely one by one and the whole row by itself. See the results to understand.
####Hint: they're all the same after the first element obtained.

A[1,1]

A[2,0]
A[2,1]
A[2,2]

A[2]

A[2,0:3]
A[2][0:3]

#Adding, multiplying two arrays or :
##NumPy simplifies the operations with arrays/matrix, because the attributes and functions is added in the
### numpy.ndarray class.
####First, we will need another array

b=[[1,2,3],[4,5,6],[7,8,9]]
B=np.array(b)
A+B

A*B

2*B

#In multiplication when using the * symbol, means that you will obtain the multiplication of M1(i1,j1) and
##M2(i2,j2) with i1=i2 and j1=j2

C=np.array([[1,2,3], [4,5,6]])
D=np.array([[9,8,7],[6,5,4]])
E=C*D
E

#Getting the multiplication of two matrix:
np.dot(A,B)
np.dot(C,D)
#This means, you're getting the dot product of every line and column.

#To get the transposed matrix, use the T attribute
E.T

                                ######## APIs #########

#What is an API? An API helps two different softwares communicate with each other through inputs and
##outputs. It stands for Application Program Interface For example, Panda.

#There are REST APIs as well. They let two different softwares communicate through the internet, which allows you to
##access many services that are not accessible off-line. RE stands for representational, S for State and T for transfer.
###They have rules regarding to them:
####1. Communication
####2. Input or Request
####3. Output or Response

#Example of APIs

##PyCoinGecko
!pip install pycoingecko
from pycoingecko import CoinGeckoApi
cg=CoinGeckoApi()

                                           ####HTTP Protocol#####
#It is a protocol for transferring information through the internet. The REST APIs send a request communicated via HTTP
##Message. Normally, this HTTP message contains a JSON file. When you enter a Web Page, first, you send a request to
###the web server. If it arrives successfully, the server will give a response, usually by default in "index.html"

#The uniform resource locator or URL is the link to the resources on the web. It's three parts are:
## Scheme: The protocol. I.E http://
##Internet Address or base URL. I.E www.google.com
##Route: It is the location on the web server. I.E /images/IDSNlogo.png

                                    #######Requests library########
import requests

#We will use the solotodo webpage to check on prices when the knowledge is enough. First, we need to create
##an object for the URL and use the get function. Then we will check the responses of the server.
url='https://www.solotodo.cl'
r= requests.get(url)
r.status_code                   ##200 stands for ok
stbody=r.request.body
stheaders=r.headers
r.encoding

stheaders['date']

#The GET request is used to modify the request of the query. We will use httbin.org, which is a simple HTTP Request
##and Response service. The GET request is used to retrieve information from a source.
url_get='http://httpbin.org/get'
payload={"name": "Joseph", "ID":"123"}

r=requests.get(url_get, params=payload)
r.url
r.request.body
r.status_code
r.text
r.headers['Content-Type'] #Checking the type of data
r.json() #Reading the data in the data type it is formated in.

#The POST request is similar to the GET request. The main difference is, you will get the response in the body, which
##was usually empty in the GET request. The post request is used to insert or update data. The HTTBIN service will send
###Back the "posted" info or sent data  in the body of the request.

url_post= "http://httpbin.org/post"
payload={"name":"Carsten","ID":"123"}
r_post= requests.post(url_post, data=payload)

#As we can see, the requested info is the get request, but not in the post request.
r.url
r_post.url

r.body
r_post.body

                                    ########Web Scraping###########

#This are the APIs we will be utilizing for Web Scraping. Web Scraping helps the coder retrieve information and data
##from a certain Web Page. This is useful when one is in need of large for data analysis pieces of data or
### information that is constantly updated from a web page

from bs4 import BeautifulSoup
import requests

#One can store HTML code in a string. The following HTML code was created by the Cognitive Lab

html="<!DOCTYPE html><html><head><title>Page Title</title></head><body><h3><b id='boldest'>Lebron James</b></h3><p> Salary: $ 92,000,000 </p><h3> Stephen Curry</h3><p> Salary: $85,000, 000 </p><h3> Kevin Durant </h3><p> Salary: $73,200, 000</p></body></html>"

#To parse and analyze the html code, the BeautifulSoup API is useful. This API will transform the data into a complex tree
##of Python objects.
bkbsalaries= BeautifulSoup(html,'html5lib')

#If you'd like to get the HTML in the nested structure displayed, use prettify method.
print(bkbsalaries.prettify())

#Say, we are retrieving title of the page and the name of the top paid player from the HTML code.

tag_object=bkbsalaries.title
print(tag_object) #This is the page title.
type(tag_object) #As we can see, the object is a Beautiful Soup 4 object.

mostpaid= bkbsalaries.h3
print('The top paid player is', mostpaid)

#We can access child objects navigating down the branches of the tag_object. In this case we will navigate to the
##next bold object.

tag_child=mostpaid.b
print(tag_child)

#Then, we can navigate to the parent object with the parent attribute
parent_tag=tag_child.parent
parent_tag

#This is the same as
mostpaid

#To access the next sibling object, use the next_sibling att

sibling_1=mostpaid.next_sibling
sibling_1

sibling_2=sibling_1.next_sibling
sibling_2

#In the case of our HTML code, there is an id "boldest" in it. To access this id, treat the object like a dicionary or
##use the attrs attribute.

tag_child['id']

tag_child.attrs

#Or you could use the get method
tag_child.get('id')

#A string is the piece of text within the tag. NavigableString class contains text. To obtain the text or string within
##the tag, use the .string att
tag_string= tag_child.string
tag_string
#Check the type. It is a NavigableString by bs4 API

type(tag_string)

#The only difference between this object and a str python object, is that the NavigableString suppors some BeautifulSoup
##features. You can convert this NavigableString to a python string using the str function
pythstring= str(tag_string)
pythstring

#Now we will use tables. First, our table element. This one was also created by Cognitive class (thanks to them again)
table="<table><tr><td id='flight'>Flight No</td><td>Launch site</td> <td>Payload mass</td></tr><tr> <td>1</td><td><a href='https://en.wikipedia.org/wiki/Florida'>Florida<a></td><td>300 kg</td></tr><tr><td>2</td><td><a href='https://en.wikipedia.org/wiki/Texas'>Texas</a></td><td>94 kg</td></tr><tr><td>3</td><td><a href='https://en.wikipedia.org/wiki/Florida'>Florida<a> </td><td>80 kg</td></tr></table>"

#Then, we will transform this HTML table into a Beautifulsoup table

table_bs=BeautifulSoup(table,'html5lib')

#To search through a code and extract specific tags

table_rows=table_bs.find_all('tr')
table_rows[0] #this are the headers of the table for the first second and third existing columns.
type(table_rows)

#To obtain the child of the first header, in this case the table data (td) in the first column of the header,
## use the td attribute.

t_headers=table_rows[0]
t_headers.td

#One could iterate through the table to display, in our example, all the first column values with the for... in loops
for i,row in enumerate(table_rows):
    print('This is row',i,'.', ' Code:', row)

#row is a cell object within the HTML table. To get all the cell data we use the following code
table_data= table_bs.find_all('td')
print(table_data)

#This might look a little complex to read, so we'll make it more human-friendly with a for...in loop to give it a check.

for i,row in enumerate(table_rows):
    print("row", i)
    cells=table_bs.find_all('td')
    for j, cell in enumerate(cells):
        print('column',j, 'cell', cell)

#To match cells within a same row, we can use the following code.
list_input=table_bs.find_all(name=['tr','td'])
list_input

#If we want to be more specific when searching for something in an HTML code, we can search for attributes in tags. For
##example, the first td elements have an id=flight. To search for these we do the following thinga

table_bs.find_all(id='flight')

#We could search for specific clickable links too (href)
table_bs.find_all(href="https://en.wikipedia.org/wiki/Florida")

#Or we can check for any tag specific content by using true or false boolean values
table_bs.find_all(href=True)

#Then, we can use the string attribute when looking for specific strings

table_bs.find_all(string="Texas")

#While the find_all method searches for a value in every line within a document, the find method will find only the first
##element with the input. We will use two tables for this example (Ty cognitivelabs uwu). We will store their HTML
###code within the two_tables object

two_tables="<h3>Rocket Launch </h3><p><table class='rocket'><tr><td>Flight No</td><td>Launch site</td> <td>Payload mass</td></tr><tr><td>1</td><td>Florida</td><td>300 kg</td></tr><tr><td>2</td><td>Texas</td><td>94 kg</td></tr><tr><td>3</td><td>Florida </td><td>80 kg</td></tr></table></p><p><h3>Pizza Party  </h3><table class='pizza'><tr><td>Pizza Place</td><td>Orders</td> <td>Slices </td></tr><tr><td>Domino's Pizza</td><td>10</td><td>100</td></tr><tr><td>Little Caesars</td><td>12</td><td >144 </td></tr><tr><td>Papa John's </td><td>15 </td><td>165</td></tr>"

#Then we create a Beautiful Soup object with these two tables

rocketsnpizza=BeautifulSoup(two_tables,'html.parser')

#Now to search for the first table, we can use the find function

rocketsnpizza.find('table')

#To add more than to variables to our 'find' search, we can do so like this:
two_tables_bs.find("table",class_='pizza') #As we used 'class' word related to the HTML code, we add the underscore
##to make Python understand we're not meaning the keyword from Python (class)

#Now we are going to check how to download an scrap content of a Web Page
import requests
from bs4 import BeautifulSoup
url='http://www.falabella.com'

#To download the content of the webpage in text format, we use get.
data_falabella= requests.get(url).text

#Then we create a Beautiful Soup object for the Falabella Webpage
data_fb_bs= BeautifulSoup(data_falabella,'html5lib' )
#To get all the links, for example
for link in data_fb_bs.find_all('a', href=True):  #'a' stands for anchor or link.

    print(link.get('href'))

#Scraping the images of the page

for link in data_fb_bs.find_all('img'):
    print(link)
    print(link.get('src'))

#This source has a table about colors, their names and codes.
url = "https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBM-DA0321EN-SkillsNetwork/labs/datasets/HTMLColorCodes.html"

#Again, we want the code displayed as text

data_table=requests.get(url).text

#Then we transform it to a BS object
table_color_bs=BeautifulSoup(data_table,'html5lib')

#Scraping data form an HTML table
table_html=table_color_bs.find('table')

#To retrieve all the rows, we do the following

for row in table_html.find_all('tr'): #tr stands for table row. First we need every row, then we extract the column
    #now we want to get all columns in every row. A cell or column is contained within the td or table data tag
    cols=row.find_all('td')
    color_name=cols[2]
    hex_code=cols[3]
    print("{}--->{}".format(color_name,hex_code)) #The format method will let us get the name of the color and the color
                                                ###in their respective hex codes.

#Now, let's scrape data from HTML tables, then convert them into a DataFrame using BS and Pandas

import pandas as pd

url = "https://en.wikipedia.org/wiki/World_population" #This url contains info about the population in the world

world_data = requests.get(url).text

world_data_bs=BeautifulSoup(world_data, 'html5lib')

wd_table=world_data_bs.find_all('table') #HTML for table is table. EZ

#To check how many tables we've got, we can use the len method
len(wd_table)

#As an example, we will get the '10 most densely populated countries' table

for index, table in enumerate(wd_table):
    if ('10 most densely populated countries' in str(table)):
        table_index=index
print(table_index)

print(wd_table[table_index].prettify())
population_data=pd.DataFrame(columns=['Rank', 'Country', 'Population', 'Area', 'Density'])

for row in wd_table[table_index].tbody.find_all('tr'):
    col=row.find_all('td')
    if (col !=[]):
        rank=col[0].text
        country=col[1].text
        population=col[2].text.strip()
        area=col[3].text.strip()
        density=col[4].text.strip()
        population_data= population_data.append({'Rank':rank, 'Country':country, 'Population': population, 'Area':area, 'Density':density}, ignore_index=True)

population_data

#How to o scrape data from HTML tables into a DataFrame with BeautifulSoup and read_html, to make things easier.

dataframe_list = pd.read_html(url, flavor='bs4') #This function returns a list of data frames and we need to pick
                                                    #The one we need

len(dataframe_list)

population_data_read_html= pd.read_html(str(wd_table[5]), flavor='bs4')[0]

population_data_read_html

#How to scrape data from a HTML table into a Data Frame with read_html

dataframe_list=pd.read_html(url, flavor='bs4')

len(dataframe_list)

dataframe_list[5]

#Finally, we can use the match parameter to retrieve a specific table. When the table contains the provided string,
##it will be read

pd.read_html(url, match='10 most densely populated countries', flavor='bs4')[0]

                                ######Data Engineering#####

#One of the critical and basic skills of any Data Scientist is Data Engineering.
#The three steps in Data Engineering are known as ETL, that stands for:
#1. Extract - This is the step where the information is gathered from multiple sources, which can be in different file
    # formats, like JSON, CSV, XLSX, or others.
#2. Transform: This means to remove data that is not useful for analysis and to convert the data from different sources
    # from their format  to the same format for every source.
#3. Load: To load data inside a warehouse after the previous process.

                         ########How to work with different file formats#########

#there are different file formats like csv, xml, json, xlsx. Pandas will help us with this. To get the pandas API, we
##use the import function

import pandas as pd

#We can read json files too with the json API. First, we import, then we read. Remember to use the with open as command
##for opening the file then closing it. Json files are a file type similar to Python dictionaries.

import json

#For xml or extensible markup Language, we need two different APIs, since Pandas can not read this type of file.
##We will need the xml.etree.ElementrTree Api to do this.
import pandas as pd
import xml.etree.ElementTree as etree

tree=eetree.parse('fileExample.xml') #To read the file
root=tree.getroot()
columns=['Name', 'Phone Number', 'Birthday'] #To create the headers of the columns in the dataframe
df=pd.DataFrame(columns=columns) #We assign the headers to the Data Frame

for node in root:           #This loop will go through the document and collect the necessary info, for a later appending.
    name=node.find('name').text
    phonenumber=node.find('phonenumber.text')
    birthday=node.find('birthday').text

#This is how we read a csv file. First we need the PD API, then the file and finally we use the read_csv method

import pandas as pd
url ='https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-PY0101EN-SkillsNetwork/labs/Module%205/data/addresses.csv'
df = pd.read_csv(url)

df

#As the csv file contains no column names, we add them like this

df.columns=['First Name', 'Last Name', 'Location', 'City', 'State', 'Area Code']

df

#To select one column, we do this (I'm kinda tired of writing and this is the final part of the Course, so I'll be short)

df["First Name"]
type(df)
#To select different columns we use two brackets like this
df_2=df[['First Name', 'Last Name', 'Location']]
df_2

#The loc and iloc will help us with getting info from specific indexes.

df.loc[0] #First row
df.loc[[0,1,2], 'First Name'] #0th, 1ts and 2nd rows for the First name column

#The iloc uses the indexes based searching mmethod. We need to pass an integer index.

df.iloc[[0,1,2],0] #0th, 1st, 2nd row of the first column 'First Name'

#The transform function returns a self-produced dataframe with values it transformed after applying the parameters specified
##in the function. This is how it works

import pandas as pd
import numpy as np

df=pd.DataFrame(np.array([[0,1,2], [3,4,5], [6,7,8]]), columns=['a', 'b', 'c'])
df

#If we wanted to add 10 to every element in the dataframe, we can use the lambda function of python
df=df.transform(func=lambda x : x+10)
df

#Now we will find the square root for each element in the matrix.

result=df.transform(func=['sqrt'])
result

#JSON files or JavaScript Object Notation is a data interchange format with a lightweight. Easy for humans to understand
##and write.

#It is built on two structures:
    #1. A collection of name/value pairs, like dictionaries in Python. In different languages, this is known as object,
        #record, struct, dictionary, hash, table, keyed list or associative array.
    #2. An ordered list of values. This is usually known as an array, vector, list, or sequence

#JSON is a language-independent data format. Derived from Javascript, but many recent programming languages code to
##create and parse JSON formatted data and can be used in a diverse range of applications.

#Similar to dictionaries in Python, JSON uses string that contain values in a key-value mappings within {} brackets.
##The builtin package json in Python supports JSON code.

import json

#To write files in JSON:
##The concept usually used for the process of converting an object in a specific format suitabgle for web
### transmission is "Serialization".

#This is the dictionary we will use to transform into a JSON file. It was created by IBM and its cognitiveclass.

person = {
    'first_name' : 'Mark',
    'last_name' : 'abc',
    'age' : 27,
    'address': {
        "streetAddress": "21 2nd Street",
        "city": "New York",
        "state": "NY",
        "postalCode": "10021-3100"
    }
}

#To serialize in json, we can use the dump() function, wich converts the dictionary into a JSON file.
## The parameters are
    #1. dictionary- Name of the dictionary to convert
    #2. file pointer- which mode you will open it. Write, read or append, etc.

with open('person.json', 'w') as f:         #Serializing into a file
    json.dump(person, f)

#To create a JSON object from a dictionary, we use the json.dumps() to convert it.
## Its parameters are
    #1. dictionary- Name of the dictionary
    #2. indent- Number of units for indentation

json_object=json.dumps(person, indent=4) #Indent is the number of separations, we use this to make the code more readable

json_object

#To do the opposite and to read JSON to a file, we call it Deserialization and it uses the load function
## The only parameter it needs is the
    #1. File Pointer- Points to a JSON file

with open('sample.json', 'r') as openfile: #open a JSON file
    json_object=json.load(openfile) #reading the file

print(json_object)
print(type(json_object))

#To read a data from XLSX file. This file is known as a Microsoft Excel Open XML file format. Just another type of
##Spreadsheet file format. It is organized under cells and columns

import pandas as pd
import urllib.request

urllib.request.urlretrieve("https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-PY0101EN-SkillsNetwork/labs/Module%205/data/file_example_XLSX_10.xlsx", "sample.xlsx")
df = pd.read_excel("sample.xlsx")

df

#To read the XML file format. This format is known as  Extensible Markup Language. It uses tags to define objects.
##It is human-readable and machine-readable

#As Pandas include methods to read it, we can use the xml.eetree.ElementrTree to read it. It is built-in with Python.
##It will let us parse and create XML documents. It represents the XML doc as a tree, in which we can move across the nodes,
### that represent elements and sub-elements of the XML file.

import xml.etree.ElementTree as ET

#To create the file structure
employee= ET.Element('employee')
details= ET.SubElement(employee, 'details')
first= ET.SubElement(details, 'firstname')
second=ET.SubElement(details,'lastname')
third=ET.SubElement(details, 'age')
first.text='Carsten'
second.text='Teufel'
third.text='25'

#To write a XML file with the results
mydata1= ET.ElementTree(employee)
with open('new_example.xml', 'wb') as files:
    mydata1.write(files)

#To reading xml with the xml.etree.ElementTree method

import pandas as pd

import xml.etree.ElementTree as ET
import requests
url= 'https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-PY0101EN-SkillsNetwork/labs/Module%205/data/Sample-employee-XML-file.xml'
response= requests.get(url)

with open('Sample-employee-XML-file.xml', 'wb') as file:
    file.write(response.content)

#First, we need to parse the XML file and create a list of columns for our data frame, to later extract useful info
##from the XML file, then add it to a pandas Data Frame

#Example

tree=etree.parse('Sample-employee-XML-file.xml')

root= tree.getroot()
columns=['firstname', 'lastname', 'title', 'division', 'building', 'room']
dataframe=pd.DataFrame(columns=columns)
for node in root:
    firstname = node.find("firstname").text

lastname = node.find("lastname").text

title = node.find("title").text

division = node.find("division").text

building = node.find("building").text

room = node.find("room").text

dataframe = dataframe.append(pd.Series([firstname, lastname, title, division, building, room], index=columns),
                               ignore_index=True)

dataframe

#To save data, we can use the dataframe.to_csv()

dataframe.to_csv('employee.csv', index=False)

#To read a Image File, PIL will help. It's the Python Imaging Library, which provides python an interpreter with
##image editing capabilities.

from PIL import Image

import urllib.request

#To download the dataset

urllib.request.urlretrieve("https://hips.hearstapps.com/hmg-prod.s3.amazonaws.com/images/dog-puppy-on-garden-royalty-free-image-1586966191.jpg", "dog.jpg")

#Reading the image
img=Image.open('dog.jpg')

display(img)

                                    ########Data Analysis########

import pandas as pd
path = 'https://cf-courses-data.s3.us.cloud-object-storage.appdomain.cloud/IBMDeveloperSkillsNetwork-PY0101EN-SkillsNetwork/labs/Module%205/data/diabetes.csv'
df=pd.read_csv(path)

print("The first 5 rows of the dataframe")
df.head(5)

#Dimensions of the dataframe
df.shape

#For statistical overview of dataset

df.info() #This will give you the column names, how many non-null rows there are and the Dtype or datatype

df.describe() #This is for basic statistical details, like percentile, mean, Std dev. When applied to series of strings
                #the result is others.

#To check for null values (missing values) Python has two methods: isnull() and notnull()

missing_data=df.isnull()
missing_data.head(5)
missing_data.shape

#To count for missing values in each column, we do the following:

for column in missing_data.columns.values.tolist():
    print(column)
    print(missing_data[column].value_counts())
    print("")

#To correct the data format, we need to check for dtypes or data types in the df

df.dtypes

#For visualization, we will use matplotlib and seaborn. Visualization is one of the best ways to get information and
##insight from the dataset.

import matplotlib.pyplot as plt
import seaborn as sns

labels='Diabetic','Not Diabetic'
plt.pie(df['Outcome']. value_counts(), labels=labels, autopct='%0.02f%%')
plt.legend()
plt.show()


r"""
Graph editor
"""
from __future__ import absolute_import
#*****************************************************************************
#      Copyright (C) 2009   Radoslav Kirov
#
#  Distributed under the terms of the GNU General Public License (GPL)
#
#    This code is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    General Public License for more details.
#
#  The full text of the GPL is available at:
#
#                  http://www.gnu.org/licenses/
#*****************************************************************************
import sys

from .graph_generators import graphs
from sage.misc.html import html

from sage.server.support import EMBEDDED_MODE

def graph_to_js(g):
    """
    Returns a string representation of a :class:`Graph` instance
    usable by the :func:`graph_editor`.  The encoded information is
    the number of vertices, their 2D positions, and a list of edges.

    INPUT:

    - ``g`` - a :class:`Graph` instance

    OUTPUT:

    - a string

    EXAMPLES::

        sage: from sage.graphs.graph_editor import graph_to_js
        sage: G = graphs.CompleteGraph(4)
        sage: graph_to_js(G)
        'num_vertices=4;edges=[[0,1],[0,2],[0,3],[1,2],[1,3],[2,3]];pos=[[0.5,0.0],[0.0,0.4999999999999999],[0.4999999999999999,1.0],[1.0,0.5000000000000001]];'
        sage: graph_to_js(graphs.StarGraph(2))
        'num_vertices=3;edges=[[0,1],[0,2]];pos=[[0.75,0.5],[1.0,0.0],[0.0,1.0]];'
    """
    string = ''
    vertex_list = g.get_vertices().keys()
    string += 'num_vertices=' + str(len(vertex_list)) + ';'
    string += 'edges=['
    for i, e in enumerate(g.edges()):
        if(i != 0):
            string += ','
        string += '[' + str(vertex_list.index(e[0])) + ',' + str(vertex_list.index(e[1])) + ']'
    string += '];'
    string += 'pos=['
    pos = g.get_pos()
    max_x = max([i[0] for i in pos.values()])
    max_y = max([i[1] for i in pos.values()])
    min_x = min([i[0] for i in pos.values()])
    min_y = min([i[1] for i in pos.values()])
    if max_x == 0:
        max_x = 1
    if max_y == 0:
        max_y = 1
    for i, v in enumerate(vertex_list):
        if(i != 0):
            string += ','
        new_pos = [float(pos[v][0] - min_x) / (max_x - min_x),
                   1.0 - float(pos[v][1] - min_y) / (max_y - min_y)]
        string += str(new_pos)
    string += '];'
    string = string.replace(' ', '')
    return string

def graph_editor(graph=None, graph_name=None,
                 replace_input=True, **layout_options):
    """
    Opens a graph editor in the Sage notebook.

    INPUT:

    - ``graph`` - a :class:`Graph` instance (default:
      graphs.CompleteGraph(2)); the graph to edit

    - ``graph_name`` - a string (default: None); the variable name to
      use for the updated instance; by default, this function attempts
      to determine the name automatically

    - ``replace_input`` - a boolean (default: True); whether to
      replace the text in the input cell with the updated graph data
      when "Save" is clicked; if this is False, the data is **still**
      evaluated as if it had been entered in the cell

    EXAMPLES::

        sage: g = graphs.CompleteGraph(3)
        sage: graph_editor(g)                       # not tested
        sage: graph_editor(graphs.HouseGraph())     # not tested
        sage: graph_editor(graph_name='my_graph')   # not tested
        sage: h = graphs.StarGraph(6)
        sage: graph_editor(h, replace_input=False)  # not tested
    """
    import sagenb.notebook.interact
    if graph is None:
        graph = graphs.CompleteGraph(2)

    if not EMBEDDED_MODE:
        return "This graph editor only runs in the Sage notebook."

    graph.layout(save_pos = True, **layout_options)

    if graph_name is None:
        graph_name = ''
        locs = sys._getframe(1).f_locals
        for var in locs:
            if id(locs[var]) == id(graph):
                graph_name = var

    cell_id = sagenb.notebook.interact.SAGE_CELL_ID

    # TODO: Put reasonable checks for large graphs, before disaster
    # occurs (i.e., breaks browser).

    close_button = r"""<button onclick="cell_delete_output(%(cell_id)s);">Close</button>""" % locals()

    if replace_input:
        eval_strategy = r"""
    f += ' graph_editor(' + g[2] + ');'
    \$('#cell_input_%(cell_id)s').val(f);
    cell_input_resize(%(cell_id)s);
    evaluate_cell(%(cell_id)s, false);
""" % locals()
    else:
        eval_strategy = r"""
    saved_input = \$('#cell_input_%(cell_id)s').val();
    \$('#cell_input_%(cell_id)s').val(f);
    evaluate_cell(%(cell_id)s, false);
    \$('#cell_input_%(cell_id)s').val(saved_input);
    send_cell_input(%(cell_id)s);
    cell_input_resize(%(cell_id)s);
""" % locals()

    update_button = r"""<button onclick="
    var f, g, saved_input;
    g = \$('#iframe_graph_editor_%(cell_id)s')[0].contentWindow.update_sage();

    if (g[2] === '') {
        alert('You need to give a Sage variable name to the graph, before saving it.');
        return;
    }
    f = g[2] + ' = Graph(' + g[0] + '); ' + g[2] + '.set_pos(' + g[1] + '); '
    %(eval_strategy)s
">Save</button>""" % locals()

    graph_js = graph_to_js(graph)
    data_fields = """<input type="hidden" id="graph_data_%(cell_id)s" value="%(graph_js)s"><input type="hidden" id="graph_name_%(cell_id)s" value="%(graph_name)s">""" % locals()

    return html(r"""

%(data_fields)s

%(update_button)s%(close_button)s

""" % locals())

# This is commented out because the mouse_out call raises an error in
# Firebug's console when the event fires but the function itself has
# not yet been loaded.

#
%(data_fields)s


import numpy as np
import pandas as pd
import json
from pandas.io.json import json_normalize


import pytest
import numpy as np
from funkyAD.helpers import count_recursive, unpack, nodify, recursive_append
from funkyAD.base import Node

def test_count_recursive_nparray():
    x = np.array([2,3,1,0])
    assert count_recursive(x)==4

def test_count_recursive_list():
    x = [1,2,3]
    assert count_recursive(x)==3

def test_count_recursive_ndarray():
    x = np.array([[1, 2, 3], [4, 5, 6]], np.int32)
    assert count_recursive(x)==6

def test_count_recursive_invalid_input():
    x = "text"
    with pytest.raises(TypeError):
        count_recursive(x)

def test_unpack_1dlist():
    x = [1,2]
    assert unpack(x) == [1,2]

def test_unpack_2darray():
    x = np.array([[1, 2, 3], [4, 5, 6]], np.int32)
    assert unpack(x)==[1,2,3,4,5,6]

def test_unpack_3darray():
    y = np.array([[[1,2],[3,4]],[[5,6],[7,8]],[[9,10],[11,12]]])
    assert unpack(y) == [1,2,3,4,5,6,7,8,9,10,11,12]

def test_unpack_ndlist():
    x = [[1,2,],[3,4]]
    assert unpack(x)==[1,2,3,4]

def test_unpack_invalid_input():
    with pytest.raises(TypeError):
        unpack("text")

def test_nodify_nparray():
    x = np.array([1,2,3])
    seed = [1,2,3]
    assert nodify(x, seed)==[Node(1,1), Node(2,2), Node(3,3)]

def test_nodify_list():
    x = [1,2,3]
    seed = [1,2,3]
    assert nodify(x, seed)==[Node(1,1), Node(2,2), Node(3,3)]

def test_nodify_invalid_input():
    with pytest.raises(TypeError):
        nodify(3.14)

def test_nodify_text_input():
    x = "test"
    seed = [1,0,0]
    with pytest.raises(TypeError):
        nodify(x, seed)

def test_nodify_node_input():
    x = Node(1,[1,1])
    seed = [1,0]
    with pytest.raises(TypeError):
        nodify(x,seed)

def test_nodify_ndarray():
    x=np.array([np.array([1])])
    seed = [1]
    assert nodify(x,seed)==[Node(1,1)]

def test_nodify_nested_list():
    x=[[1,2],[3,4]]
    seed = [1,2,3,4]
    assert nodify(x,seed)==[[Node(1,1), Node(2,2)], [Node(3,3), Node(4,4)]]

def test_recursive_append():
    x=Node(1,1)
    x.parents = [Node(2,1)]
    trace = []
    recursive_append(x,trace)
    assert trace == [Node(1,1),Node(2,1)]