from sympy.core.containers import Tuple
from sympy.core.singleton import S
from sympy.core.symbol import Symbol
from sympy.core.sympify import SympifyError

from types import FunctionType

class TableForm:
    r"""
    Create a nice table representation of data.

    Examples
    ========

    >>> from sympy import TableForm
    >>> t = TableForm([[5, 7], [4, 2], [10, 3]])
    >>> print(t)
    5  7
    4  2
    10 3

    You can use the SymPy's printing system to produce tables in any
    format (ascii, latex, html, ...).

    >>> print(t.as_latex())
    \begin{tabular}{l l}
    $5$ & $7$ \\
    $4$ & $2$ \\
    $10$ & $3$ \\
    \end{tabular}

    """

    def __init__(self, data, **kwarg):
        """
        Creates a TableForm.

        Parameters:

            data ...
                            2D data to be put into the table; data can be
                            given as a Matrix

            headings ...
                            gives the labels for rows and columns:

                            Can be a single argument that applies to both
                            dimensions:

                                - None ... no labels
                                - "automatic" ... labels are 1, 2, 3, ...

                            Can be a list of labels for rows and columns:
                            The labels for each dimension can be given
                            as None, "automatic", or [l1, l2, ...] e.g.
                            ["automatic", None] will number the rows

                            [default: None]

            alignments ...
                            alignment of the columns with:

                                - "left" or "<"
                                - "center" or "^"
                                - "right" or ">"

                            When given as a single value, the value is used for
                            all columns. The row headings (if given) will be
                            right justified unless an explicit alignment is
                            given for it and all other columns.

                            [default: "left"]

            formats ...
                            a list of format strings or functions that accept
                            3 arguments (entry, row number, col number) and
                            return a string for the table entry. (If a function
                            returns None then the _print method will be used.)

            wipe_zeros ...
                            Do not show zeros in the table.

                            [default: True]

            pad ...
                            the string to use to indicate a missing value (e.g.
                            elements that are None or those that are missing
                            from the end of a row (i.e. any row that is shorter
                            than the rest is assumed to have missing values).
                            When None, nothing will be shown for values that
                            are missing from the end of a row; values that are
                            None, however, will be shown.

                            [default: None]

        Examples
        ========

        >>> from sympy import TableForm, Symbol
        >>> TableForm([[5, 7], [4, 2], [10, 3]])
        5  7
        4  2
        10 3
        >>> TableForm([list('.'*i) for i in range(1, 4)], headings='automatic')
          | 1 2 3
        ---------
        1 | .
        2 | . .
        3 | . . .
        >>> TableForm([[Symbol('.'*(j if not i%2 else 1)) for i in range(3)]
        ...            for j in range(4)], alignments='rcl')
            .
          . . .
         .. . ..
        ... . ...
        """
        from sympy.matrices.dense import Matrix

        # We only support 2D data. Check the consistency:
        if isinstance(data, Matrix):
            data = data.tolist()
        _h = len(data)

        # fill out any short lines
        pad = kwarg.get('pad', None)
        ok_None = False
        if pad is None:
            pad = " "
            ok_None = True
        pad = Symbol(pad)
        _w = max(len(line) for line in data)
        for i, line in enumerate(data):
            if len(line) != _w:
                line.extend([pad]*(_w - len(line)))
            for j, lj in enumerate(line):
                if lj is None:
                    if not ok_None:
                        lj = pad
                else:
                    try:
                        lj = S(lj)
                    except SympifyError:
                        lj = Symbol(str(lj))
                line[j] = lj
            data[i] = line
        _lines = Tuple(*[Tuple(*d) for d in data])

        headings = kwarg.get("headings", [None, None])
        if headings == "automatic":
            _headings = [range(1, _h + 1), range(1, _w + 1)]
        else:
            h1, h2 = headings
            if h1 == "automatic":
                h1 = range(1, _h + 1)
            if h2 == "automatic":
                h2 = range(1, _w + 1)
            _headings = [h1, h2]

        allow = ('l', 'r', 'c')
        alignments = kwarg.get("alignments", "l")

        def _std_align(a):
            a = a.strip().lower()
            if len(a) > 1:
                return {'left': 'l', 'right': 'r', 'center': 'c'}.get(a, a)
            else:
                return {'<': 'l', '>': 'r', '^': 'c'}.get(a, a)
        std_align = _std_align(alignments)
        if std_align in allow:
            _alignments = [std_align]*_w
        else:
            _alignments = []
            for a in alignments:
                std_align = _std_align(a)
                _alignments.append(std_align)
                if std_align not in ('l', 'r', 'c'):
                    raise ValueError('alignment "%s" unrecognized' %
                        alignments)
        if _headings[0] and len(_alignments) == _w + 1:
            _head_align = _alignments[0]
            _alignments = _alignments[1:]
        else:
            _head_align = 'r'
        if len(_alignments) != _w:
            raise ValueError(
                'wrong number of alignments: expected %s but got %s' %
                (_w, len(_alignments)))

        _column_formats = kwarg.get("formats", [None]*_w)

        _wipe_zeros = kwarg.get("wipe_zeros", True)

        self._w = _w
        self._h = _h
        self._lines = _lines
        self._headings = _headings
        self._head_align = _head_align
        self._alignments = _alignments
        self._column_formats = _column_formats
        self._wipe_zeros = _wipe_zeros

    def __repr__(self):
        from .str import sstr
        return sstr(self, order=None)

    def __str__(self):
        from .str import sstr
        return sstr(self, order=None)

    def as_matrix(self):
        """Returns the data of the table in Matrix form.

        Examples
        ========

        >>> from sympy import TableForm
        >>> t = TableForm([[5, 7], [4, 2], [10, 3]], headings='automatic')
        >>> t
          | 1  2
        --------
        1 | 5  7
        2 | 4  2
        3 | 10 3
        >>> t.as_matrix()
        Matrix([
        [ 5, 7],
        [ 4, 2],
        [10, 3]])
        """
        from sympy.matrices.dense import Matrix
        return Matrix(self._lines)

    def as_str(self):
        # XXX obsolete ?
        return str(self)

    def as_latex(self):
        from .latex import latex
        return latex(self)

    def _sympystr(self, p):
        """
        Returns the string representation of 'self'.

        Examples
        ========

        >>> from sympy import TableForm
        >>> t = TableForm([[5, 7], [4, 2], [10, 3]])
        >>> s = t.as_str()

        """
        column_widths = [0] * self._w
        lines = []
        for line in self._lines:
            new_line = []
            for i in range(self._w):
                # Format the item somehow if needed:
                s = str(line[i])
                if self._wipe_zeros and (s == "0"):
                    s = " "
                w = len(s)
                if w > column_widths[i]:
                    column_widths[i] = w
                new_line.append(s)
            lines.append(new_line)

        # Check heading:
        if self._headings[0]:
            self._headings[0] = [str(x) for x in self._headings[0]]
            _head_width = max([len(x) for x in self._headings[0]])

        if self._headings[1]:
            new_line = []
            for i in range(self._w):
                # Format the item somehow if needed:
                s = str(self._headings[1][i])
                w = len(s)
                if w > column_widths[i]:
                    column_widths[i] = w
                new_line.append(s)
            self._headings[1] = new_line

        format_str = []

        def _align(align, w):
            return '%%%s%ss' % (
                ("-" if align == "l" else ""),
                str(w))
        format_str = [_align(align, w) for align, w in
                      zip(self._alignments, column_widths)]
        if self._headings[0]:
            format_str.insert(0, _align(self._head_align, _head_width))
            format_str.insert(1, '|')
        format_str = ' '.join(format_str) + '\n'

        s = []
        if self._headings[1]:
            d = self._headings[1]
            if self._headings[0]:
                d = [""] + d
            first_line = format_str % tuple(d)
            s.append(first_line)
            s.append("-" * (len(first_line) - 1) + "\n")
        for i, line in enumerate(lines):
            d = [l if self._alignments[j] != 'c' else
                 l.center(column_widths[j]) for j, l in enumerate(line)]
            if self._headings[0]:
                l = self._headings[0][i]
                l = (l if self._head_align != 'c' else
                     l.center(_head_width))
                d = [l] + d
            s.append(format_str % tuple(d))
        return ''.join(s)[:-1]  # don't include trailing newline

    def _latex(self, printer):
        """
        Returns the string representation of 'self'.
        """
        # Check heading:
        if self._headings[1]:
            new_line = []
            for i in range(self._w):
                # Format the item somehow if needed:
                new_line.append(str(self._headings[1][i]))
            self._headings[1] = new_line

        alignments = []
        if self._headings[0]:
            self._headings[0] = [str(x) for x in self._headings[0]]
            alignments = [self._head_align]
        alignments.extend(self._alignments)

        s = r"\begin{tabular}{" + " ".join(alignments) + "}\n"

        if self._headings[1]:
            d = self._headings[1]
            if self._headings[0]:
                d = [""] + d
            first_line = " & ".join(d) + r" \\" + "\n"
            s += first_line
            s += r"\hline" + "\n"
        for i, line in enumerate(self._lines):
            d = []
            for j, x in enumerate(line):
                if self._wipe_zeros and (x in (0, "0")):
                    d.append(" ")
                    continue
                f = self._column_formats[j]
                if f:
                    if isinstance(f, FunctionType):
                        v = f(x, i, j)
                        if v is None:
                            v = printer._print(x)
                    else:
                        v = f % x
                    d.append(v)
                else:
                    v = printer._print(x)
                    d.append("$%s$" % v)
            if self._headings[0]:
                d = [self._headings[0][i]] + d
            s += " & ".join(d) + r" \\" + "\n"
        s += r"\end{tabular}"
        return s


"""
Selecting Data III - Access Single Values
"""

import numpy as np
import pandas as pd
wine_reviews = pd.read_csv('../../winemag-data-130k.csv')

# Print the value in the 5th column of the 7th row.
print(wine_reviews.iloc[6,5]) # Sicily & Sardinia

# Create a Series called "countries" from the 'country' column and print item at index 18.
countries = wine_reviews['country']
print(countries.iloc[18]) # Spain

# Print the 10th item in countries.
print(countries.loc[9]) # France


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import seaborn as sns
import matplotlib as mpl
import matplotlib.pyplot as plt
import numpy as np
import sklearn

# In[2]:

pollution = pd.read_excel('PM2.5climate.xlsx')
pollution.head()

# In[3]:

pollution.shape

# Because the data have been preprocess, most of the variables will not be changed.
# For data cleaning, we will check duplicated and empty variables and then make a decision whether to
# keep, change or drop the variables

# In[4]:

print('Total Missing values:', round((pollution['pm2.5'].isnull().sum()/43824)*100,2),'%')

# As seen in the exploratory data, only one attribute have 2061 missing values, which is 0.047 or 4.7% of the total dataset observations.

# ---

# ## Dropping Variables
#
# ## Missing Values

# In[5]:

pollution.drop(columns = ['No','hour', 'year', 'day'], inplace=True)
pollution.head()

# In[6]:

pollution.dropna(inplace = True)
pollution.reset_index(drop = True, inplace = True)
pollution.head()

# In[7]:

pollution['pm2.5'].isnull().sum()

# In[8]:

pollution.shape

# All the missing values have been dropped because, as observed in the exploration, the missing values are in the target variable. If they had not been dropped, that would skew our regression results because it would not give any prediction for the level of pollution.
#
# In addition, the missing values will not overall impact the dataset as it accounts for only 5% total. We chose not to replace the missing values because it was such a small portion of the dataset and replacing it could cause bias and it was a safer option to remove them than replace them.
#
# No. has been dropped because it does not give any information since it is simply the row number of the observation.
#
# Hour was dropped because it is a minute detail in pollution levels and we want to explore larger trends in the level s of pollution. We want to understand the long-run trend in the level of pollution and hour is a very short-term measure.
#
# Similarly, day is also a short-term measure that we are not interested in exploring.
#
# Year was dropped because in our exploration it was found that it was not correlated with the level fo pollution and, hence, we decided to drop it.

# Let us rename the variables for better understanding:

# In[9]:

pollution.columns

# In[10]:

names = dict(DEWP = 'dewTemp', TEMP = 'temp', PRES = 'pres', cbwd = 'windDir',
             Iws = 'windSpeed', Is = 'cumSnow', Ir = 'cumRain')
pollution.rename(columns = names, inplace = True)
pollution.columns

# In[11]:

pollution['windDir'].replace({'cv':'SW'}, inplace = True)
pollution['windDir'].value_counts()

# #### About the project
#
# The next step will be exploring the outliers in the dataset.
# First part of this visualization will be shown with outliers included and the second part the visualizations will be plotted after removing outleirs.
#
# For the data analysis, there will be two regression models shown. The first model will be the model with outliers and the second model is after removing outliers. Towards the end, we will be comparing both models and see which one is most reliable model to be used for the predictions
#

# ---

# ## Outliers

# In[12]:

from scipy import stats
import numpy as np

z2 = np.abs(stats.zscore(pollution[['month', 'pm2.5', 'dewTemp', 'temp', 'pres', 'windSpeed', 'cumSnow', 'cumRain']]))
print(z2)

# In[13]:

threshold = 3
print(np.where(z2 > 3))

# In[14]:

outliers2 = pollution[(z2 >=3)]
outliers2

# In[15]:

pd.set_option('display.max_rows', 1000)
other2 = pollution[(z2 < 3).all(axis=1)]
other2

# The first table shows the values of the outliers and the second table shows the values of the dataset without the outliers. Looking at these values, we can see that the values of cumSnow and cumRain are the same for both tables (outliers and non-outliers). We do not see a reason to find z-scores of these two columns since their values are close and removing them based on their z-scores would be unreasonable causing some bias.
#
# Hence, let us find outliers based on the z-scores of all numerical features except cumSnow and cumRain.

# In[16]:

from scipy import stats
import numpy as np

z = np.abs(stats.zscore(pollution[['month', 'pm2.5', 'dewTemp', 'temp', 'pres', 'windSpeed']]))
print(z)

# In[17]:

threshold = 3
print(np.where(z > 3))

# In[18]:

outliers = pollution[(z >=3)]
outliers.shape

# In[19]:

pd.set_option('display.max_rows', 1000)
other = pollution[(z < 3).all(axis=1)]
other

# Let us create a new dataframe without the outliers:

# In[20]:

pollution.shape

# In[21]:

pollution2 = pollution[(z < 3).all(axis=1)]
pollution2.shape

# In[22]:

pollution2.reset_index(drop = True, inplace = True)
pollution2.head(10)

# Removing outliers based on numerical features, except cumSnow and cumRain, results in fewer outliers being removed; approximately 2,000 rows have not been removed compared to the rows removed when cumSnow and cumRain were used in z-score calculations. This is interesting because it shows that values that are not much different from other observations can be removed due to minor differences. Hence, it is important to look at the outliers to determine if they should be removed for further model creation.

# In[23]:

print('The data frame without outliers is', round((pollution2.shape[0]/pollution.shape[0]),2),      '% smaller than the original dataframe')

# ---

# ## One-Hot Encoding

# Let us one-hot encode the windDir variable since it is categorical.
#
# We have to one-hot encode this variable because it has string values and we will give 0, 1 values to each value of the variable.

# ##### Dataset with Outliers

# In[24]:

from sklearn.preprocessing import OneHotEncoder

windDirection = OneHotEncoder()
windDirectionDF = windDirection.fit_transform(pollution[['windDir']])
windDirectionDF = pd.DataFrame(windDirectionDF.toarray())
windDirectionDF.columns = windDirection.get_feature_names()
windDirectionDF.head()

# In[25]:

pollution.drop(columns = ['windDir'], inplace = True)
pollution.head()

# In[26]:

pollutionDF = pd.concat([pollution, windDirectionDF], axis = 1)
pollutionDF.head()

# In[27]:

pollutionDF.columns

# In[28]:

colnames = ['month', 'dewTemp', 'temp', 'pres', 'windSpeed', 'cumSnow', 'cumRain', 'x0_NE', 'x0_NW', 'x0_SE', 'x0_SW', 'pm2.5']
pollutionDF = pollutionDF[colnames]
pollutionDF.head()

# ---

# ##### Dataset without Outliers

# In[29]:

windDirection_wo = OneHotEncoder()
windDirection_woDF = windDirection_wo.fit_transform(pollution2[['windDir']])
windDirection_woDF = pd.DataFrame(windDirection_woDF.toarray())
windDirection_woDF.columns = windDirection_wo.get_feature_names()
windDirection_woDF.head()

# In[30]:

pollution2.drop(columns = ['windDir'], inplace = True)
pollution2.head()

# In[31]:

pollution2DF = pd.concat([pollution2, windDirection_woDF], axis = 1)
pollution2DF.head()

# In[32]:

colnames = ['month', 'dewTemp', 'temp', 'pres', 'windSpeed', 'cumSnow', 'cumRain', 'x0_NE', 'x0_NW', 'x0_SE', 'x0_SW', 'pm2.5']
pollution2DF = pollution2DF[colnames]
pollution2DF.head()

# In[33]:

pollution2DF.shape

# In[34]:

pollutionDF.shape

# After one-hot encoding the variables for the datasets with and without outliers, it can be seen that the number of observations in both the dataframes have remained the same. However, the number of columns have increased because of the one-hot encoded columns.

# ---

# ## Visualizations
# ##### Dataset with Outliers

# #### Scatterplot
# Let us explore if there are any outliers in this dataset with scatterplot:

# In[37]:

fig, ax = plt.subplots(4, 2, figsize = (10, 20))
#fig.tight_layout()

ax[0,0].scatter(y = pollution['pm2.5'], x = pollution['month'])
ax[0,0].set_xlabel('Month')
ax[0,0].set_ylabel('PM2.5')
ax[0,0].set_title('Month v/s PM 2.5 levels')

ax[0, 1].scatter(y = pollution['pm2.5'], x = pollution['dewTemp'])
ax[0, 1].set_xlabel('Dew Point Temperature')
ax[0, 1].set_ylabel('PM2.5')
ax[0, 1].set_title('Dew Point v/s PM 2.5 levels')

ax[1,0].scatter(y = pollution['pm2.5'], x = pollution['temp'])
ax[1,0].set_xlabel('Temperture')
ax[1,0].set_ylabel('PM2.5')
ax[1,0].set_title('Temperature v/s PM 2.5 levels')

ax[1,1].scatter(y = pollution['pm2.5'], x = pollution['pres'])
ax[1,1].set_xlabel('Air Pressure')
ax[1,1].set_ylabel('PM2.5')
ax[1,1].set_title('Air Pressure v/s PM 2.5 levels')

ax[2,0].scatter(y = pollution['pm2.5'], x = pollution['windSpeed'])
ax[2,0].set_xlabel('Wind Speed')
ax[2,0].set_ylabel('PM2.5')
ax[2,0].set_title('Wind Speed v/s PM 2.5 levels')

ax[2,1].scatter(y = pollution['pm2.5'], x = pollution['cumSnow'])
ax[2,1].set_xlabel('Hours of Cumulated Snow')
ax[2,1].set_ylabel('PM2.5')
ax[2,1].set_title('Hours of Cumulated Snow v/s PM 2.5 levels')

ax[3,0].scatter(y = pollution['pm2.5'], x = pollution['cumRain'])
ax[3,0].set_xlabel('Hours of Cumulated Rain')
ax[3,0].set_ylabel('PM2.5')
ax[3,0].set_title('Hours of Cumulated Rain v/s PM 2.5 levels')

plt.show()

# Looking at the above dashboard, it is visible that there are a few outliers in the month, dew point, temperature, and air pressure variables. This is because there are a few points that have much higher pm2.5 levels for certain x-values. This shows that there are certain observations where there are exceptionally high levels of air pollution for a certain month in a year. Similarly, there are certain temperature, dew point temperature, and air pressure values for which there are exceptionally high air pollution levels. These high values of air pollution are probably influenced by external factors that were not controlled in this dataset. Hence, it is probably better to remove any outliers. However, let us see how many outliers there are in totality.
#
# The scatterplots of the wind speed, cumulated hours fo snow and rain are interesting because they follow a 'L' pattern because of which it is hard to distinguish if there are outliers. However, outliers in these variables will be able to be seen with the z-value calculations.

# #### Box plot

# In[38]:

fig, ax = plt.subplots(4, 2, figsize = (10, 20))
ax[0,0].boxplot(x=pollution['month'])
ax[0,0].set_title('Month')

ax[0,1].boxplot(x=pollution['pm2.5'])
ax[0,1].set_title('PM2.5 Level')

ax[1,0].boxplot(x=pollution['dewTemp'])
ax[1,0].set_title('Dew Point Temperature')

ax[1,1].boxplot(x=pollution['temp'])
ax[1,1].set_title('Temperature')

ax[2,0].boxplot(x=pollution['pres'])
ax[2,0].set_title('Air Pressure')

ax[2,1].boxplot(x=pollution['windSpeed'])
ax[2,1].set_title('Wind Speed')

ax[3,0].boxplot(x=pollution['cumSnow'])
ax[3,0].set_title('Hours of Cumulative Snow')

ax[3,1].boxplot(x=pollution['cumRain'])
ax[3,1].set_title('Hours of Cumulative Rain')

plt.show()

# The boxplot above are before the outliers were removed.
#
# There are four variables that does not seem to show outliers, these variables are month, dew point temperature, temperature and air pressure. Although they seem to not have outliers, they do have some skewness which might indicate some higher-than-normal values.

# ##### Dataset without outliers
# #### Scatterplot

# In[39]:

fig, ax = plt.subplots(4, 2, figsize = (10, 20))
#fig.tight_layout()

ax[0,0].scatter(y = pollution2['pm2.5'], x = pollution2['month'])
ax[0,0].set_xlabel('Month')
ax[0,0].set_ylabel('PM2.5')
ax[0,0].set_title('Month v/s PM 2.5 levels')

ax[0, 1].scatter(y = pollution2['pm2.5'], x = pollution2['dewTemp'])
ax[0, 1].set_xlabel('Dew Point Temperature')
ax[0, 1].set_ylabel('PM2.5')
ax[0, 1].set_title('Dew Point v/s PM 2.5 levels')

ax[1,0].scatter(y = pollution2['pm2.5'], x = pollution2['temp'])
ax[1,0].set_xlabel('Temperture')
ax[1,0].set_ylabel('PM2.5')
ax[1,0].set_title('Temperature v/s PM 2.5 levels')

ax[1,1].scatter(y = pollution2['pm2.5'], x = pollution2['pres'])
ax[1,1].set_xlabel('Air Pressure')
ax[1,1].set_ylabel('PM2.5')
ax[1,1].set_title('Air Pressure v/s PM 2.5 levels')

ax[2,1].scatter(y = pollution2['pm2.5'], x = pollution2['windSpeed'])
ax[2,1].set_xlabel('Wind Speed')
ax[2,1].set_ylabel('PM2.5')
ax[2,1].set_title('Wind Speed v/s PM 2.5 levels')

ax[3,0].scatter(y = pollution2['pm2.5'], x = pollution2['cumSnow'])
ax[3,0].set_xlabel('Hours of Cumulated Snow')
ax[3,0].set_ylabel('PM2.5')
ax[3,0].set_title('Hours of Cumulated Snow v/s PM 2.5 levels')

ax[3,1].scatter(y = pollution2['pm2.5'], x = pollution2['cumRain'])
ax[3,1].set_xlabel('Hours of Cumulated Rain')
ax[3,1].set_ylabel('PM2.5')
ax[3,1].set_title('Hours of Cumulated Rain v/s PM 2.5 levels')

plt.show()

# The scatterplot above is visualizing the dataset after the outliers are removed. Due to the large dataset, the plot in the dataset seem to be cluster and does not appears to show linear correlation between variables
#
# The graph between Temperature v/s PM2.5 level and the Air Pressure v/s PM2.5 level appears to be very interesting as they seem to follow the same pattern. Both are clustered in the middle. While the Wind Speed v/s PM 2.5 Level appears to cluster more to the left and the Dew Point v/s PM2.5 Level are the opposite, it clusters more to the left
#

# #### Box plot

# In[41]:

fig, ax = plt.subplots(4, 2, figsize = (10, 20))
ax[0,0].boxplot(x=pollution2['month'])
ax[0,0].set_title('Month')

ax[0,1].boxplot(x=pollution2['pm2.5'])
ax[0,1].set_title('PM2.5 Level')

ax[1,0].boxplot(x=pollution2['dewTemp'])
ax[1,0].set_title('Dew Point Temperature')

ax[1,1].boxplot(x=pollution2['temp'])
ax[1,1].set_title('Temperature')

ax[2,0].boxplot(x=pollution2['pres'])
ax[2,0].set_title('Air Pressure')

ax[2,1].boxplot(x=pollution2['windSpeed'])
ax[2,1].set_title('Wind Speed')

ax[3,0].boxplot(x=pollution2['cumSnow'])
ax[3,0].set_title('Hours of Cummulative Snow')

ax[3,1].boxplot(x=pollution2['cumRain'])
ax[3,1].set_title('Hours of Cummulative Rain')

plt.show()

# The boxplot above are after the ouliers removed. Similar to the previous boxplot with outlier the variables month, dew point temperature, temperature and air pressure still does not show any outliers.
#
# There are some variables that shows some adjustments or changes without the outliers. Both Hours of Cummulative Snow and Hours of Cummulative rain seems to show less points. These changes are also seen in PM2.5 level and Wind Speed

# ---

# ## Scaling Numerical Features

# ##### Dataset with Outliers

# In[42]:

pollutionDF.columns

# In[43]:

from sklearn.preprocessing import StandardScaler

cols = ['month', 'dewTemp', 'temp', 'pres', 'windSpeed', 'cumSnow', 'cumRain']
scaler = StandardScaler()
pollutionDF[cols] = scaler.fit_transform(pollutionDF[cols])
pollutionDF.head()

# ---

# ##### Dataset without outliers

# In[44]:

pollution2DF.columns

# In[45]:

cols = ['month', 'dewTemp', 'temp', 'pres', 'windSpeed', 'cumSnow', 'cumRain']
scaler = StandardScaler()
pollution2DF[cols] = scaler.fit_transform(pollution2DF[cols])
pollution2DF.head()

# We standardized the dataset with and without outliers. This is because we wanted to bring all the numerical values to the same scale. We standardized it instead of normalize the datasets because it would let us know how many standard deviations the observations are from the mean.
#
# We chose not to standardize the dependent variable for ease of interpretation.

# ---

# We have completed the process of cleaning our data and preparing it for the regression analysis and visualizations (storytelling).

# In[46]:

pollutionDF.to_excel('pollution_outliers.xlsx')
pollution2DF.to_excel('pollution_NoOutliers.xlsx')

# ---

# ### Let us now move on to Data Analysis


import numpy as np
import numbers

def check_random_state(seed):
    """Turn seed into a np.random.RandomState instance
    """
    if seed is None or seed is np.random:
        return np.random.mtrand._rand
    if isinstance(seed, (numbers.Integral, np.integer)):
        return np.random.RandomState(seed)
    if isinstance(seed, np.random.RandomState):
        return seed
    raise ValueError("%r cannot to used to seed a numpy.random \
        RandomState instance" % seed)


#!usr/bin/env python3
from timeit import timeit

# When you create a list and append to it, Python needs to reallocate memory
# for the list. To avoid reallocation on every append, the list grows in fixed
# sizes. The sizes are zero, four, eight, 16, 25, etc. However, when we create
# a list with multiplication, Python knows the size of the list in advance
# and creates it in one allocation.
# ATTENTION!
# Python because everything is a reference, this might have a surprising
# effect, look the example of the empty list.
# Make sure you initialized with immutable volumes, like numbers, strings,
# and tuples, otherwise you will have to get back to our initial
# implementation.
# The best option would be to use numpy which creates ultra fast arrays
import numpy

def alloc_list(size):
    """Alloc zeros with range"""
    return [0 for _ in range(size)]

def alloc_list_fixed(size):
    """Alloc zeros with *"""
    return [0] * size

if __name__ == '__main__':
    print([1, 2, 3] * 3)
    my_list = [[]] * 5
    print(my_list)
    my_list[0].append(1)
    print(my_list)

    print('allocating a list with loop',
          timeit('alloc_list(100)', 'from __main__ import alloc_list'))
    print('allocating a list with *',
          timeit('alloc_list_fixed(100)',
                 'from __main__ import alloc_list_fixed'))
    print('allocating a list with numpy',
          timeit('numpy.zeros(100)',
                 'from __main__ import numpy'))

# 0.6352753790000003/3.813065881 = 0.1666048788103801.
# we gain about 85% speed-up. The latter is about 6 times faster.

# CONSOLE OUTPUT:
# [1, 2, 3, 1, 2, 3, 1, 2, 3]
# [[], [], [], [], []]
# [[1], [1], [1], [1], [1]]
# allocating a list with loop 3.813065881
# allocating a list with * 0.6352753790000003
# allocating a list with numpy 1.0309689520000003

# In [7]: %run -n src/list_allocation.py
#
# In [8]: %timeit alloc_list(1000)
# 40.7 µs ± 3.57 µs per loop (mean ± std. dev. of 7 runs, 10000 loops each)
#
# In [9]: %timeit alloc_list_fixed(1000)
# 2.5 µs ± 84.3 ns per loop (mean ± std. dev. of 7 runs, 100000 loops each)
#
# In [10]: import numpy
#
# In [11]: %timeit numpy.zeros(1000)
# 1.31 µs ± 11.6 ns per loop (mean ± std. dev. of 7 runs, 1000000 loops each)


import numpy as np

# iterating over 1D array
arr1 = np.array([1, 2, 3])
for x in arr1:
    print(x)

print('-'*20)

# iterating over 2D array
arr2 = np.array([[1, 2, 3], [1, 2, 3], [1, 2, 3]])
for x in arr2:
    print(x)

print('-'*20)

# iterate on each scaler element
for x in arr2:
    for y in x:
        print(y)

# iterating over 3D array
arr3 = np.array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])

for x in arr3:
    print(x)
# Iterate down to the scalars:

for x in arr3:
    for y in x:
        for z in y:
            print(z)

## Iterating Arrays Using nditer()
arr3 = np.array([[[1, 2, 3], [4, 5, 6]], [[7, 8, 9], [10, 11, 12]]])
for x in np.nditer(arr3):
    print(x)

## Iterating Array With Different Data Types
arr1 = np.array([1, 2, 3])

for x in np.nditer(arr1, flags=['buffered'], op_dtypes=['S']):
    print(x)

## Iterating With Different Step Size

arr2 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
for x in np.nditer(arr2[:, ::2]):
    print(x)

## Enumerated Iteration Using ndenumerate()

arr1 = np.array([[1, 2, 3, 4, 5, 6, 7, 8]])
for idx, x in np.ndenumerate(arr1):
    print(idx, " ", x)

## ndenumerate() a 2D array

arr2 = np.array([[1, 2, 3, 4], [5, 6, 7, 8]])
for idx, x in np.ndenumerate(arr2):
    print(idx, x)


"""
.. _ex-electrode-pos-2d:

====================================================
How to convert 3D electrode positions to a 2D image.
====================================================

Sometimes we want to convert a 3D representation of electrodes into a 2D
image. For example, if we are using electrocorticography it is common to
create scatterplots on top of a brain, with each point representing an
electrode.

In this example, we'll show two ways of doing this in MNE-Python. First,
if we have the 3D locations of each electrode then we can use Mayavi to
take a snapshot of a view of the brain. If we do not have these 3D locations,
and only have a 2D image of the electrodes on the brain, we can use the
:class:`mne.viz.ClickableImage` class to choose our own electrode positions
on the image.
"""
# Authors: Christopher Holdgraf <choldgraf@berkeley.edu>
#          Alex Rockhill        <aprockhill@mailbox.org>
#
# License: BSD-3-Clause

# %%
from mne.io.fiff.raw import read_raw_fif
import numpy as np
from matplotlib import pyplot as plt
from os import path as op

import mne
from mne.viz import ClickableImage  # noqa: F401
from mne.viz import (plot_alignment, snapshot_brain_montage, set_3d_view)

misc_path = mne.datasets.misc.data_path()
ecog_data_fname = op.join(misc_path, 'ecog', 'sample_ecog_ieeg.fif')
subjects_dir = op.join(misc_path, 'ecog')

# We've already clicked and exported
layout_path = op.join(op.dirname(mne.__file__), 'data', 'image')
layout_name = 'custom_layout.lout'

# %%
# Load data
# ---------
#
# First we will load a sample ECoG dataset which we'll use for generating
# a 2D snapshot.

raw = read_raw_fif(ecog_data_fname)
raw.pick_channels([f'G{i}' for i in range(1, 257)])  # pick just one grid

# Since we loaded in the ecog data from FIF, the coordinates
# are in 'head' space, but we actually want them in 'mri' space.
# So we will apply the head->mri transform that was used when
# generating the dataset (the estimated head->mri transform).
montage = raw.get_montage()
trans = mne.coreg.estimate_head_mri_t('sample_ecog', subjects_dir)
montage.apply_trans(trans)

# %%
# Project 3D electrodes to a 2D snapshot
# --------------------------------------
#
# Because we have the 3D location of each electrode, we can use the
# :func:`mne.viz.snapshot_brain_montage` function to return a 2D image along
# with the electrode positions on that image. We use this in conjunction with
# :func:`mne.viz.plot_alignment`, which visualizes electrode positions.

fig = plot_alignment(raw.info, trans=trans, subject='sample_ecog',
                     subjects_dir=subjects_dir, surfaces=dict(pial=0.9))
set_3d_view(figure=fig, azimuth=20, elevation=80)
xy, im = snapshot_brain_montage(fig, raw.info)

# Convert from a dictionary to array to plot
xy_pts = np.vstack([xy[ch] for ch in raw.ch_names])

# Compute beta power to visualize
raw.load_data()
beta_power = raw.filter(20, 30).apply_hilbert(envelope=True).get_data()
beta_power = beta_power.max(axis=1)  # take maximum over time

# This allows us to use matplotlib to create arbitrary 2d scatterplots
fig2, ax = plt.subplots(figsize=(10, 10))
ax.imshow(im)
cmap = ax.scatter(*xy_pts.T, c=beta_power, s=100, cmap='coolwarm')
cbar = fig2.colorbar(cmap)
cbar.ax.set_ylabel('Beta Power')
ax.set_axis_off()

# fig2.savefig('./brain.png', bbox_inches='tight')  # For ClickableImage

# %%
# Manually creating 2D electrode positions
# ----------------------------------------
#
# If we don't have the 3D electrode positions then we can still create a
# 2D representation of the electrodes. Assuming that you can see the electrodes
# on the 2D image, we can use :class:`mne.viz.ClickableImage` to open the image
# interactively. You can click points on the image and the x/y coordinate will
# be stored.
#
# We'll open an image file, then use ClickableImage to
# return 2D locations of mouse clicks (or load a file already created).
# Then, we'll return these xy positions as a layout for use with plotting topo
# maps.

# This code opens the image so you can click on it. Commented out
# because we've stored the clicks as a layout file already.

# # The click coordinates are stored as a list of tuples
# im = plt.imread('./brain.png')
# click = ClickableImage(im)
# click.plot_clicks()

# # Generate a layout from our clicks and normalize by the image
# print('Generating and saving layout...')
# lt = click.to_layout()
# lt.save(op.join(layout_path, layout_name))  # save if we want

# # We've already got the layout, load it
lt = mne.channels.read_layout(layout_name, path=layout_path, scale=False)
x = lt.pos[:, 0] * float(im.shape[1])
y = (1 - lt.pos[:, 1]) * float(im.shape[0])  # Flip the y-position
fig, ax = plt.subplots()
ax.imshow(im)
ax.scatter(x, y, s=80, color='r')
fig.tight_layout()
ax.set_axis_off()


import numpy as np
import copy

nx=5
h=np.zeros([nx]);hn=np.zeros([nx])
print(id(h),id(hn))

h[:]=10.

hn=h               # hnとhのIDが共有されてしまう ---->これだとヤバい
print(id(h),id(hn))
hn=copy.copy(h)    # hnの別のIDが割り当てられる ---->OK
print(id(h),id(hn))
for i in np.arange(0,nx):  #これでも OK
    hn[i]=h[i]

hn[:]=h[:]+10     #これでもOK

print('h=',h)
print('hn=',hn)

hn[3]=100.
print('h=',h)
print('hn=',hn)

a=np.ones(5)
b=a
b[2]=100
print(a,b)


import numpy as np

def _ensure_matrix(x):
    """
    Ensures the vector/matrix `x` is in matrix format.

    Parameters
    ----------
    x : array_like, shape (n,)
        Vector or matrix.

    Returns
    -------
    x : array_like, shape (m, p)
        Matrix.
    """

    x = np.array(x)

    if x.ndim == 1:
        x = np.reshape(x, (x.size, 1))
    elif x.ndim > 2:
        raise ValueError('`x` must be of dimension 1 or 2.')

    return x


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import matplotlib.pyplot as plt
import scipy as sc
import pandas as pd

# #  Lecture 7 -  Wells and aquifer
#
# ## Prof. Liedl/Prof. Werth/Prof. Chahar/Prabhas is to add the text contents.
# ## Anne/Sophie is to add the numerical contents
#
# Include ipynb file for:
#
# Theis method.
#
#

# ## Transmissivity ##

# When discussing storage properties in Chapter / Lecture …, we saw that aquifers or single layers may frequently be treated as two-dimensional systems. This is justified because the lateral extension of aquifers is usually much larger than the vertical extension. Thus, vertical variations of storage properties can be replaced by some average value without adversely affecting the quantification of groundwater storage.
#
# Similar things can be done with regard to conductivity properties and this brings us to the geohydraulic parameter of **Transmissivity**. The idea is to neglect vertical variations of _hydraulic conductivity_ ($K$)  and to use vertically averaged values instead. This procedure does not eliminate horizontal variability, so _transmissivity_ may still depend on horizontal coordinates $(x, y)$.
#
# The vertically averaged $K$ value is then multiplied by the water-saturated thickness to obtain transmissivity. The concept of water-saturated thickness (or water-saturated depth) requires to distinguish whether _confined_ or _unconfined_ flow conditions prevail.
#
# In general, water-saturated thickness is the distance from the aquifer bottom to a level up to which all pores are filled with water. For _confined aquifers_, this level is equal to aquifer top and water-saturated thickness is tantamount to aquifer thickness. For _unconfined aquifers_, however, water-saturated thickness corresponds to the distance between aquifer bottom and groundwater level. We will see some illustrations below when we try to quantify _transmissivity_.
#
# The symbol $T$ is mostly used to denote transmissivity which has a dimension of $L^2T^{-1}$.
#
#

# Let us have a closer look at the confined case first. The black cuboid in Fig.[LINK] illustrates that water-saturated thickness extends from aquifer bottom to aquifer top. So, it is equal to _aquifer thickness_ $m$. Transmissivity is calculated by
#
# $$
# T_x = K_x \cdot m  \;\; \text{ and }  \;\; Ty = K_y \cdot m
# $$
#
# Here we allow for horizontal aquifer anisotropy with different hydraulic conductivities in $x-$ and $y-$ direction $(K_x \neq K_y)$.
#
# For horizontally isotropic aquifers $(K_x = K_y = K)$, transmissivity is given by $T = K  \cdot m $.
#
#
#

# In[2]:

print("We already have large text content, equations and so we can add a simple numerical example here")

# Things are a bit more complicated for _unconfined aquifers_. Fig. [LINK] illustrates that water-saturated thickness extends from the aquifer bottom to the groundwater table. It is important to note that _transmissivity_ of unconfined aquifers depends on the vertical position of the groundwater table.
#
# For instance, if the groundwater table is lowered during to a draught period, _transmissivity_ is decreasing. This is fundamentally different from the confined case where the water-saturated thickness is given by aquifer geometry only and is not affected by hydraulic head changes.
#
#

# Computing transmissivity of unconfined aquifers requires to determine the difference of hydraulic head h and the elevation of aquifer bottom $z_{bot}$. Based on this, transmissivity is given by:
#
# $$T_x = K_x \cdot (h – z_{bot})$$
# and
# $$T_y = K_y \cdot (h – z_{bot})$$
#
# As above, we are allowing for horizontal aquifer anisotropy. For an isotropic unconfined aquifer we get
# $$T = K\cdot(h – z_{bot})$$

# In[3]:

print("We already have large text content, equations and so we can add a simple numerical example here")

# Two more remarks appear to be appropriate:
#
# - First, _transmissivity_ may be computed by the given equations even if the aquifer bottom is not horizontal. This case is not covered by the figure above.
# - Second, textbooks frequently present the equation $$ T = K\cdot h $$ for transmissivity of unconfined aquifers. It is to be noted that this equation only holds if two conditions are fulfilled:
#
# - The aquifer bottom must be horizontal and
# - hydraulic head values are expressed with respect to the elevation of aquifer bottom (= reference datum).
#
#
# Finally, we can try to compute transmissivity for isotropic aquifers and check how the result depends on several quantities like aquifer bottom, aquifer top, and hydraulic head.
#

# In[4]:

# A bit more complicated numerical example here. An example from Prof. Liedl - we improve further.

print("Q1. Determine if the aquifer is confined or unconfind and compute it's Transmissivity")

# input

K = 8.5e-5 # m/s, hydraulic conductivity
Ab = 120 # m asl, aquifer bottom elevation
At = 150 # m asl, aquifer top elevation
H  = 139 # m, hydraulic head

#intermediate calculation
A_T = At - Ab # m, Aquifer thickness
S_T = np.minimum(A_T, (H - Ab))

# Results

if H < At:
   print("\n It is a unconfined aquifer")
else:
   print("\n It is a Confined aquifer")

T = K*A_T # m²/s, Transmissivity

print("\nThe transmissivity is {0:0.2e} m\u00b2/s".format(T))

# In[ ]:


#!/usr/bin/env python3

import unittest
from unittest.mock import patch
import numpy as np
import pandas as pd

from tmc import points

from tmc.utils import load, get_stdout, patch_helper

module_name="src.subsetting_by_positions"
subsetting_by_positions = load(module_name, "subsetting_by_positions")
main = load(module_name, "main")
ph = patch_helper(module_name)

@points('p04-08.1')
class SubsettingByPositions(unittest.TestCase):

    def test_shape_and_columns(self):
        df = subsetting_by_positions()
        self.assertEqual(df.shape, (10,2), msg="The returned DataFrame had wrong shape!")
        #np.testing.assert_array_equal(df.index, range(10), err_msg="Incorrect index")
        np.testing.assert_array_equal(df.columns, ["Title", "Artist"],
                                      err_msg="Incorrect column names")

    def test_called(self):
        with patch(ph("subsetting_by_positions"), wraps=subsetting_by_positions) as psbp,\
             patch(ph("pd.read_csv"), wraps=pd.read_csv) as prc:
            main()
            psbp.assert_called()
            prc.assert_called()

if __name__ == '__main__':
    unittest.main()


# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.4'
#       jupytext_version: 1.2.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# + {"deletable": true, "editable": true, "cell_type": "markdown"}
#

#

#

#
Computational Seismology

#
Reproducible Papers - Syngine Paper

#

#

#

# + {"deletable": true, "editable": true, "cell_type": "markdown"}
# ---
#
# # Figure 8: Education
#
# This notebook is part of the supplementary materials for the Syngine paper and reproduces figure 8.
#
# This notebook creates the phase relative times figure. Requires matplotlib >= 1.5 and an ObsPy version (>= 1.0) with the syngine client.
#
# ##### Authors:
# * Lion Krischer ([@krischer](https://github.com/krischer))

# + {"deletable": true, "editable": true}
# %matplotlib inline

import obspy
import numpy as np

from obspy.clients.syngine import Client

import itertools

import matplotlib.pyplot as plt
from mpl_toolkits.basemap import Basemap
plt.style.use("seaborn-paper")

# + {"deletable": true, "editable": true}
c = Client()

# + {"deletable": true, "editable": true}
# Get seismograms for various strike values.
_d_plane_crossing = []

for strike in np.linspace(85, 95, 11):
    print(strike)
    _d_plane_crossing.append((strike, c.get_waveforms(model="ak135f_5s",
        receiverlatitude=0.0, receiverlongitude=0.0,
        sourcelatitude=0.0, sourcelongitude=30.0, components="Z", units="velocity",
        sourcedepthinmeters=0.0, sourcedoublecouple=[strike, 90.0, 0.0])[0]))

# + {"deletable": true, "editable": true}
max_amp = max(np.abs(_i[1].data).max() for _i in _d_plane_crossing)

for _i, tr in enumerate(_d_plane_crossing):
    plt.plot(tr[1].data / max_amp + _i)
plt.xlim(1300, 2100)
plt.show()

# + {"deletable": true, "editable": true}
_d_depth = []

# Get seismograms for various depths.
for depth in [10, 20, 30, 40, 50, 100, 200]:
    print(depth)
    _d_depth.append((depth, c.get_waveforms(model="ak135f_5s",
        receiverlatitude=0.0, receiverlongitude=0.0,
        sourcelatitude=0.0, sourcelongitude=30.0, components="Z", units="velocity",
        sourcedepthinmeters=depth * 1000, sourcemomenttensor=[1E20, 0, 0, 0, 0, 0])[0]))

# + {"deletable": true, "editable": true}
max_amp = max(np.abs(_i[1].data).max() for _i in _d_depth)

for _i, tr in enumerate(_d_depth):
    plt.plot(tr[1].data / max_amp + _i * 0.5)
plt.xlim(0, 6000)
plt.show()

# + {"deletable": true, "editable": true}
# Plot everything.
plt.figure(figsize=(8, 3))

plt.subplot(121)

max_amp = max(np.abs(_i[1].data).max() for _i in _d_plane_crossing)

ticks = []
labels = []

for _i, tr in enumerate(_d_plane_crossing):
    plt.plot(tr[1].times(), tr[1].data / max_amp * 200.4 + _i, color="0.1")
    ticks.append(_i)
    labels.append("%i" % (90.0 - tr[0]))
plt.xlim(345, 460)
plt.text(360, 10.8, "P Phase", size="small")
plt.text(403, 10.8, "PP Phase", size="small")
plt.ylim(-2, 12)
plt.xlabel("Time since origin [s]")
plt.ylabel("$\Delta_{strike}$ from nodal plane [degree]")
plt.yticks(ticks, labels)

plt.subplot(122)
max_amp = max(np.abs(_i[1].data).max() for _i in _d_depth)

colors = ["0.7", "0.1", "0.4", "0.1", "0.1", "0.1", "0.1"]

ticks = []
labels = []
for _i, tr in enumerate(_d_depth):
    plt.plot(tr[1].times(), tr[1].data / max_amp * 14.0 + _i, color=colors[_i])
    ticks.append(_i)
    labels.append("%i" % (tr[0]))
plt.xlim(0, 1500)
plt.gca().yaxis.tick_right()
plt.xlabel("Time since origin [s]")
plt.gca().yaxis.set_label_position("right")
plt.ylabel("Event depth [km]")
plt.yticks(ticks, labels)
plt.tight_layout()
plt.ylim(-1, 7)
plt.savefig("education.pdf")
plt.show()
# -


# -*- coding: utf-8 -*-
# Copyright (C) 2020-2021 by SCICO Developers
# All rights reserved. BSD 3-clause License.
# This file is part of the SCICO package. Details of the copyright and
# user license can be found in the 'LICENSE' file distributed with the
# package.

"""Random number generation.

This module provides convenient wrappers around several `jax.random
<https://jax.readthedocs.io/en/stable/jax.random.html>`_ routines to handle
the generation and splitting of PRNG keys, as well as the generation of random
:class:`.BlockArray`.

::

   # Calls to scico.random functions always return a PRNG key
   # If no key is passed to the function, a new key is generated
   x, key = scico.random.randn((2,))
   print(x)   # [ 0.19307713 -0.52678305]

   # scico.random functions automatically split the PRNGkey and return
   # an updated key
   y, key = scico.random.randn((2,), key=key)
   print(y) # [ 0.00870693 -0.04888531]

The user is responsible for passing the PRNG key to :mod:`scico.random` functions.
If no key is passed, repeated calls to :mod:`scico.random` functions will return the same
random numbers:

::

   x, key = scico.random.randn((2,))
   print(x)   # [ 0.19307713 -0.52678305]

   # No key passed, will return the same random numbers!
   y, key = scico.random.randn((2,))
   print(y)   # [ 0.19307713 -0.52678305]

If the desired shape is a tuple containing tuples, a :class:`.BlockArray` is returned:

::

   x, key = scico.random.randn( ((1, 1), (2,)), key=key)
   print(x)  # scico.blockarray.BlockArray:
             # DeviceArray([ 1.1378784 , -1.220955  , -0.59153646], dtype=float32)

"""

__author__ = """Luke Pfister <pfister@lanl.gov>"""

import functools
import inspect
import sys
from typing import Optional, Tuple, Union

import numpy as np

import jax

from scico.blockarray import BlockArray, block_sizes
from scico.typing import BlockShape, DType, JaxArray, PRNGKey, Shape
from scico.util import is_nested

def _add_seed(fun):
    """
    Modifies a jax.random function to add a `seed` argument.

    Args:
        fun: function to be modified, e.g., jax.random.normal. Expects `key`
        to be the first argument.

    Returns:
        fun_alt: a version of `fun` supporting an optional `seed` argument that
        is used to create a `jax.random.PRNGKey` that is passed along as the `key`.
        The `key` argument may still be used, but is moved to be second-to-last.
        By default, `seed=0`. The `seed` argument is added last. Other arguments are unchanged.
    """

    # find number of arguments to fun
    num_params = len(inspect.signature(fun).parameters)

    def fun_alt(*args, key=None, seed=None, **kwargs):

        # key and seed may be in *args, look for them
        if len(args) >= num_params:  # they passed all position args including key
            key = args[num_params - 1]
        if len(args) > num_params:  # they passed all position args including key and seed
            seed = args[num_params]

        if key is not None and seed is not None:
            raise ValueError("Key and seed cannot both be specified")

        if key is None:
            if seed is None:
                seed = 0
            key = jax.random.PRNGKey(seed)

        result = fun(key, *args[: num_params - 1], **kwargs)

        key, subkey = jax.random.split(key, 2)
        return result, key

    lines = fun.__doc__.split("\n\n")
    fun_alt.__doc__ = "\n\n".join(
        lines[0:1]
        + [
            f"  Wrapped version of `jax.random.{fun.__name__} <https://jax.readthedocs.io/en/stable/jax.random.html#jax.random.{fun.__name__}>`_. "
            "The SCICO version of this function moves the `key` argument to the end of the argument list, "
            "adds an additional `seed` argument after that, and allows the `shape` argument "
            "to accept a nested list, in which case a `BlockArray` is returned. "
            "Always returns a `(result, key)` tuple.",
            "  Original docstring below.",
        ]
        + lines[1:]
    )

    return fun_alt

def _allow_block_shape(fun):
    """
    Decorates a jax.random function so that the `shape` argument may be a BlockShape.
    """

    # use inspect to find which argument number is `shape`
    shape_ind = list(inspect.signature(fun).parameters.keys()).index("shape")

    @functools.wraps(fun)
    def fun_alt(*args, **kwargs):

        # get the shape argument if it was passed
        if len(args) > shape_ind:
            shape = args[shape_ind]
        elif "shape" in kwargs:
            shape = kwargs["shape"]
        else:  # shape was not passed, call fun as normal
            return fun(*args, **kwargs)

        # if shape is not nested, call fun as normal
        if not is_nested(shape):
            return fun(*args, **kwargs)
        # shape is nested, so make a BlockArray!

        # call the wrapped fun with an shape=(size,)
        subargs = list(args)
        subkwargs = kwargs.copy()
        size = np.sum(block_sizes(shape))

        if len(subargs) > shape_ind:
            subargs[shape_ind] = (size,)
        else:  # shape must be a kwarg if not a positional arg
            subkwargs["shape"] = (size,)

        result_flat = fun(*subargs, **subkwargs)

        return BlockArray.array_from_flattened(result_flat, shape)

    return fun_alt

def _wrap(fun):
    fun_wrapped = _add_seed(_allow_block_shape(fun))
    fun_wrapped.__module__ = __name__  # so it appears in docs
    return fun_wrapped

exceptions = [  # these do not take key and shape
    "PRNGKey",
    "double_sided_maxwell",
    "fold_in",
    "permutation",
    "shuffle",
    "split",
    "weibull_min",
    "threefry_2x32",
]

func_names = [
    t[0] for t in inspect.getmembers(jax.random, inspect.isfunction) if t[0] not in exceptions
]

for name in func_names:
    setattr(sys.modules[__name__], name, _wrap(getattr(jax.random, name)))

def randn(
    shape: Union[Shape, BlockShape],
    dtype: DType = np.float32,
    key: Optional[PRNGKey] = None,
    seed: Optional[int] = None,
) -> Tuple[Union[JaxArray, BlockArray], PRNGKey]:
    """Return an array drawn from the standard normal distribution. Alias for :func:`scico.random.normal`.

    Args:
        shape:  Shape of output array.  If shape is a tuple, a DeviceArray is returned.
            If shape is a tuple of tuples, a :class:`.BlockArray` is returned.
        key:  JAX PRNGKey.  Defaults to None, in which case a new key
              is created using the seed arg.
        seed: Seed for new PRNGKey. Default: 0
        dtype: dtype for returned value.  Default to float32.  If np.complex64,
               generates an array sampled from complex normal distribution.

    Returns:
        tuple: A tuple (x, key) containing:

           - **x** : (DeviceArray):  Generated random array
           - **key** : Updated random PRNGKey.
    """
    return normal(shape, dtype, key, seed)


#!/usr/bin/env python

"""
ragged_array

A "ragged" array class -- build on numpy

The idea is to be able to store data that is essentially 2-d, but each row is
an arbitrary length, like:

1   2   3
4   5   6   7   8   9
10 11
12 13  14  15  16  17  18
19 20  21
...

This can also be extended to support higher dimensional arrays, as long as
only the last dimension is the "ragged" one.

Internally, the data is stored as an array of one less dimension, with indexes
into the array to identify the "rows".

The array can be indexed by row.

Operations can be done on the entire array just like any numpy array
  - this is one of the primary advantages of using a single numpy array
    for the internal storage.
  - Another advantage is that the data are all in a block that can be
    passed off and manipulated in C or Cython
  - operations that require slicing, specifying an axis, etc, are likely to
    fail
"""

import numpy as np

class ragged_array:

    def __init__(self, data, dtype=None):
        """
        create a new ragged array

        data should be a sequence of sequences:
        [ [1, 2, 3], [4,5,6,7], [4,2] ]

        if no dtype is provided, it will be determined by the type of the first row

        """

        # generate the arrays:
        a = []
        ind = [0]
        # flatten the data sequence:
        for row in data:
            a.extend(row)
            ind.append(ind[-1]+len(row))
        self._data_array = np.array(a, dtype=dtype)
        # note: using "np.int" for the index array should give me 32 bits on
        #       32 bit python, and 64 bits on 64 bit python.
        self._index_array = np.array(ind, dtype=np.int)

    def append(self, row):

        """
        Should this be supported?

        It does require a copy of the data array

        """
        self._data_array = np.r_[self._data_array, row]
        self._index_array  = np.r_[self._index_array, (self._data_array.shape[0],) ]

    def __len__(self):
        return len(self._index_array) - 1 # there is an extra index at the end, so that IndexArray[i+1] works

    def __getitem__(self,index):
        """
        returns a numpy array of one row.
        """
        if index > (len(self._index_array) - 1):
            raise IndexError
        if  index < 0:
             if index < - (len(self._index_array) -1 ):
                 raise IndexError
             index = len(self._index_array) -1 + index
        row = (self._data_array[self._index_array[index]:self._index_array[index+1]] )
        return row

    def __getslice__(self, i, j):
        """
        ra.__getslice__(i, j) <==> a[i:j]

        Use of negative indices is not supported.

        This a view, just like numpy arrays
        """
        ## this builds a new ragged_array, as a view onto the original
        ##  fixme: this seems like it should be more elegant.
        j = min( j, len(self) )
        rslt = ragged_array(((),),)
        rslt._data_array = self._data_array[self._index_array[i]:self._index_array[j]]
        rslt._index_array = np.r_[0, (self._index_array[i+1:j+1] - self._index_array[i])]
        return rslt

    def __string_middle(self):
        '''
        helper function that generates a list of strings fr the rows in the array
        '''
        middle = []
        if len(self._data_array) > np.get_printoptions()['threshold']:
            for row in self[:3]:
                middle.append(str(row))
            middle.append("...")
            for row in self[-3:]:
                middle.append(str(row))
        else:
            for row in self:
                middle.append(str(row))

        return middle

    def __str__(self):
        """
        present a nice string representation of the array
        """
        msg = self.__string_middle()
        return "\n".join(msg)

    def __repr__(self):
        """
        present a nice string representation of the array that is "evaluateable"
        """
        msg = ['ragged_array([']
        middle = self.__string_middle()
        middle = ",\n              ".join(middle)
        msg.append(middle)
        msg.append('])')
        msg.append("")
        return "".join(msg)

    def flat(self):
        """
        returns a flattend version of the array -- 1-d

        actually returns the internal array representation, so it shares a view with the ragged array

        """
        return self._data_array


# coding: utf-8

# In[ ]:

# This Python 3 environment comes with many helpful analytics libraries installed
# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python
# For example, here's several helpful packages to load in

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)

# Input data files are available in the "../input/" directory.
# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory

from subprocess import check_output
print(check_output(["ls", "../input"]).decode("utf8"))

# Any results you write to the current directory are saved as output.

# # Introduction
# this notebook is about exploring raw data and checking data quality then make meaningful imputation of missing values
#

# In[1]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.offline as py
py.init_notebook_mode(connected=True)
import plotly.graph_objs as go
import plotly.tools as tls

# In[2]:

loan = pd.read_csv('../input/loan.csv',dtype={'desc':np.object,'verification_status_joint':np.object})
loan.shape
print ("We have %i rows and %i cols" % (loan.shape[0],loan.shape[1]))

# In[3]:

loan.info(null_counts= True)

# To get insight of business, it is necessary to categorize variables to personal information, credit/financial records and loan information and summerize them to metadata.
# 1. role:
# 2. level:
# 3. category:

# In[4]:

Personal_info = ['member_id',
                 'addr_state' ,
                 'annual_inc',
                 'dti','emp_length',
                 'emp_title',
                 'home_ownership',
                 'zip_code']
Loan_info = ['id',
            'application_type',
            'collection_recovery_fee',
            'desc',
            'funded_amnt',
            'funded_amnt_inv',
            'grade',
            'initial_list_status',
            'installment',
            'int_rate',
            'issue_d',
            'last_pymnt_amnt',
            'last_pymnt_d',
            'loan_amnt',
            'loan_status',
            'next_pymnt_d',
            'out_prncp',
            'out_prncp_inv',
            'policy_code',
            'purpose',
            'pymnt_plan',
            'recoveries',
            'sub_grade',
            'term',
            'title',
            'total_pymnt',
            'total_pymnt_inv',
             'total_rec_int',
             'total_rec_late_fee',
             'total_rec_prncp']
Joint_info = loan.columns[loan.columns.str.endswith('_joint')].tolist()

# In[6]:

data = []
for i in loan.columns.tolist():
    # define the category of variable
    if i in Personal_info:
        category = 'personal information'
    elif i in Loan_info:
        category = 'loan information'
    elif i in Joint_info:
        category = 'unique for joint loan'
    else:
        category = 'credit records'

    # difine data level
    if i in loan.columns[loan.columns.str.endswith('_d')].tolist():
        level = 'date'
    elif loan[i].dropna().nunique() ==2:
        level = 'binary'
    elif loan[i].dtype == object:
        level = 'categorical'
    elif loan[i].dtype == float:
        level = 'numeric'
    else:
        level = 'id'

    # defining datatype
    dtype = loan[i].dtype
    f_dict = {'name':i,
            'category':category,
            'level':level,
            'dtype': dtype
            }
    data.append(f_dict)

metadata = pd.DataFrame(data,columns=['name','category','level','dtype'])

# In[11]:

metadata

# In[ ]:

metadata.to_csv(metadata,index = False)

# ### Missing Values

# illustrate missing records of each column by precentage

# In[12]:

missing = loan.isnull().sum()
missing_ratio = missing[missing != 0]/loan.shape[0] * 100
missing_ratio

# # To be continued


"""Fashion-MNIST
****************************************

This is a dataset of 60,000 28x28 grayscale images of 10 fashion categories,
along with a test set of 10,000 images. This dataset can be used as
a drop-in replacement for MNIST.

The classes are:

- T-shirt/top
- Trouser
- Pullover
- Dress
- Coat
- Sandal
- Shirt
- Sneaker
- Bag
- Ankle boot

Returns:
Tuple of NumPy arrays: ``(x_train, y_train), (x_test, y_test)``.

**x_train**: uint8 NumPy array of grayscale image data with shapes
``(60000, 28, 28)``, containing the training data.

**y_train**: uint8 NumPy array of labels (integers in range 0-9)
with shape ``(60000,)`` for the training data.

**x_test**: uint8 NumPy array of grayscale image data with shapes
(10000, 28, 28), containing the test data.

**y_test**: uint8 NumPy array of labels (integers in range 0-9)
with shape ``(10000,)`` for the test data.

Example:

.. code-block::

    (x_train, y_train), (x_test, y_test) = fashion_mnist.load_data()
    assert x_train.shape == (60000, 28, 28)
    assert x_test.shape == (10000, 28, 28)
    assert y_train.shape == (60000,)
    assert y_test.shape == (10000,)

License:
  The copyright for Fashion-MNIST is held by Zalando SE.
  Fashion-MNIST is licensed under the `MIT license <https://github.com/zalandoresearch/fashion-mnist/blob/master/LICENSE>`_

"""

import os
import gzip
from typing import Tuple
import numpy as np
from mltk.utils.path import create_user_dir
from mltk.utils.archive_downloader import download_verify_extract
from mltk.utils.logger import get_logger
from keras_preprocessing.image.utils import array_to_img

INPUT_SHAPE = (28,28)
CLASSES = [
    'tshirt',
    'trouser',
    'pullover',
    'dress',
    'coat',
    'sandal',
    'shirt',
    'sneaker',
    'bag',
    'boot'
]

def load_data() -> Tuple[Tuple[np.ndarray, np.ndarray], Tuple[np.ndarray, np.ndarray]]:
    """Download the dataset, extract, load into memory,
    and return as a tuple of numpy arrays

    Returns:
        Tuple of NumPy arrays: ``(x_train, y_train), (x_test, y_test)``
    """
    y_train_path = download_verify_extract(
        url='https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz',
        file_hash='09814CFEF5A041118CEACE42F8DAE995319D331A',
        show_progress=True,
        extract=False
    )
    x_train_path = download_verify_extract(
        url='https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz',
        file_hash='95978B76B6897F6CA69A25145D01716EFB615989',
        show_progress=True,
        extract=False
    )
    y_test_path = download_verify_extract(
        url='https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz',
        file_hash='9CAAD14E1AFF9ADAC77D3744963212D36AF15BEE',
        show_progress=True,
        extract=False
    )
    x_test_path = download_verify_extract(
        url='https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz',
        file_hash='5EDDA96C6D8C36FF915115A0E8136D370A021576',
        show_progress=True,
        extract=False
    )

    with gzip.open(y_train_path, 'rb') as lbpath:
        y_train = np.frombuffer(lbpath.read(), np.uint8, offset=8)

    with gzip.open(x_train_path, 'rb') as imgpath:
        x_train = np.frombuffer(imgpath.read(), np.uint8, offset=16).reshape(len(y_train), 28, 28)

    with gzip.open(y_test_path, 'rb') as lbpath:
        y_test = np.frombuffer(lbpath.read(), np.uint8, offset=8)

    with gzip.open(x_test_path, 'rb') as imgpath:
        x_test = np.frombuffer(
            imgpath.read(), np.uint8, offset=16).reshape(len(y_test), 28, 28)

    return (x_train, y_train), (x_test, y_test)

def load_data_directory() -> str:
    """Download the dataset, extract all sample images to a directory,
    and return the path to the directory.

    Each sample type is extract to its corresponding subdirectory, e.g.:

    ~/.mltk/datasets/fashion_mnist/tshirt
    ~/.mltk/datasets/fashion_mnist/dress
    ...

    Returns:
        Path to extract directory:
    """

    dataset_dir = f'{create_user_dir()}/datasets/fashion_mnist'

    (x_train, y_train), (x_test, y_test) = load_data()
    x_samples = np.concatenate((x_train, x_test))
    y_samples = np.concatenate((y_train, y_test))

    class_ids, class_counts = np.unique(y_samples, return_counts=True)

    expected_class_counts = {}
    for class_id, class_count in zip(class_ids, class_counts):
        expected_class_counts[CLASSES[class_id]] = class_count

    for class_id, class_label in enumerate(CLASSES):
        dataset_class_dir = f'{dataset_dir}/{class_label}'
        os.makedirs(dataset_class_dir, exist_ok=True)
        class_count = len(os.listdir(dataset_class_dir))

        if class_count != expected_class_counts[class_label]:
            get_logger().warning(f'Generating {dataset_class_dir}')
            sample_count = 0
            for x, y in zip(x_samples, y_samples):
                if class_id != y:
                    continue
                sample_path = f'{dataset_class_dir}/{sample_count}.jpg'
                sample_count += 1

                x = np.expand_dims(x, axis=-1)
                img = array_to_img(x, scale=False, dtype='uint8')
                img.save(sample_path)

    return dataset_dir


# coding: utf-8

# ----------------------------------------------------------------------------

# Disciplina Tópicos Especiais II (C318)

# Curso: Fundamentos de Machine Learning

# Professor: Ricardo Augusto

# ----------------------------------------------------------------------------

# Projeto de Machine Learning

# Tema: Predição de despesas médicas por meio de Regressão Linear

# Grupo: Emanuel Massafera Magalhães, Pedro Henrique de Almeida e Thiago Santos da Costa

#%% Formulação e Definição do Problema de ML

# Frame the problem and look at the big picture.

# i) Enquadramento do problema de ML - Aprendizagem Supervisionada

# ii) Trata-se de um problema de regressão (variável de saída: despesas médicas de pacientes dos Estados Unidos)

# iii) Regressão Múltipla: uma vez que o sistema de ML precisa lidar com várias características (features) para gerar uma predição

# iv) Regressão Univariada: uma vez que estamos gerando a predição de um único valor para a variável de saída (um valor por paciente)

# v)  Se estamos fazendo a predição de múltiplos valores para a variável de saída temos uma regressão multivariada

# vi) Não há fluxo contínuo de dados entrando no sistema de ML a ser desenvolvido - com isso não há necessidade de ajuste-rápido aos dados (online learning))

# vii) A quantidade de dados pode ser acomodada na memória (batch learning)

#%% Estrutura do Projeto de ML

# - Estrutura do projeto
#   - Importação de bibiliotecas utilizadas
#   - Importação da base de dados
#   - Manipulações iniciais nos dados
#   - Criação de conjuntos de dados de treino e teste
#   - Investigando Correlações
#   - Preparação dos dados para Modelagem
#   - Limpeza dos dados
#   - Manipulando features categóricas
#   - Feature Scaling - Pipeline de transformação
#   - Criando o modelo de regressão linear
#   - Avaliação de Desempenho - Evaluation
#   - Validação Cruzada
#   - Usando o modelo nos dados de teste

#%% Bibliotecas utilizadas no projeto

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

#%% Importação da base de dados

# Importando dados (arquivo .csv) a partir da URL (fonte)
url = 'https://raw.githubusercontent.com/stedy/Machine-Learning-with-R-datasets/master/insurance.csv'
df  = pd.read_csv(url)

# Salvando no diretório do projeto
df.to_csv('insurance.csv')

# Informações sobre o dataframe (atributo info)
df.info()

# ---------------------------
# Informações sobre o dataset

# 1. age: Número inteiro que indica a idade do beneficiário principal (excluindo acima de 64 anos, uma vez que são geralmente cobertos pelo governo).

# 2. sex: Gênero do segurado, podendo ser homem ou mulher.

# 3. bmi: Numero que indica o Índice de Massa Corporal (IMC) de segurado. É uma medida internacional utilizada para determinar se uma pessoa está no peso ideal. O IMC é calculado como sendo peso (em quilogramas) dividido pela altura (em metros) ao quadrado. Um IMC ideal está dentro do faixa de 18,5 a 24,9.

# 4. children: Número inteiro que indica o número de filhos ou dependentes cobertos pelo plano de seguro do segurado.

# 5. smoker: Indica se o segurado regularmente fuma tabaco ou não.

# 6. region: Indica o local de residência do beneficiário nos EUA, dividido em quatro regiões geográficas: nordeste, sudeste, sudoeste ou noroeste.

# 7. charges: Variável dependente que representa o valor de despesas médicas.

#%% Manipulações iniciais nos dados

# Descrição estatística do dataframe Pandas
df_stats = df.describe()
df_stats

# Extração de informações de estatística descritiva do dataframe:
# Ex: 25% dos pacientes tem idade (mediana) até 27 anos
# Ex: 75% dos pacientes tem idade (mediana) até 51 anos
# Ex: A idade média dos pacientes é de 39 anos
# Ex: O valor médio do bmi é de 30.66 kg/m²
# Ex: O valor médio dos gastos médicos medianos é de 13270 dólares

# ----------------------------------------------------------------------------
# Acessando e Manipulando o dataframe

# Indexação por rótulo - selecionando uma série Pandas a partir do dataframe
df['age']

# Indexação por rótulo - selecionando mais de uma série Pandas a partir do dataframe
df[['age', 'bmi']]

# Usando a notação ponto (.) para acesso à series de um dataframe
df.age

# Verificando a variável categórica - sex
gêneros = df['sex'].value_counts()
gêneros

# Verificando a variável categórica - smoker
fumantes = df['smoker'].value_counts()
fumantes

# Verificando a variável categórica - smoker
regiões = df['region'].value_counts()
regiões

# Visualizando histogramas relacionados com todas as variáveis do dataframe
df.hist(bins=50)
plt.show()

# Visualizando histogramas das variáveis - características - features
df['age'].hist(bins = 50)
plt.xlabel('age')
plt.ylabel('count')

df['bmi'].hist(bins = 50)
plt.xlabel('bmi')
plt.ylabel('count')

df['children'].hist(bins = 50)
plt.xlabel('children')
plt.ylabel('count')

df['charges'].hist(bins=50)
plt.xlabel('charges')
plt.ylabel('count')

# Biblioteca para análise de dados (visualizações estatísticas)
import klib
klib.dist_plot(df['age'])
klib.dist_plot(df['charges'])
klib.dist_plot(df['bmi'])
klib.dist_plot(df['children'])

# ----------------------------------------------------------------------------
# Análise conduzida a partir dos histogramas

# i) A grande maioria dos indivíduos em nossos dados tem despesas médicas
# anuais entre zero e US $ 15.000, embora a cauda da distribuição se estenda
# muito além desses picos. Como a regressão linear assume uma distribuição
# normal para a variável dependente, essa distribuição não é ideal.

# ii) Outro problema em mãos é que os modelos de regressão exigem que todos
# os recursos sejam numéricos, mas temos três features (sex, region e smoker)
# que não são numéricas.

# ----------------------------------------------------------------------------

#%% Criação de conjuntos de dados de treino e teste

# Método de Amostragem Aleatória Simples (Sklearn)
from sklearn.model_selection import train_test_split

# Função do scikit-learn train_test_split
train_set, test_set = train_test_split(df, test_size = 0.2, random_state = 42)

#%% Investigando Correlações

# Criando uma cópia do dataset de treino
insurance = train_set.copy()

# Fazendo a estimativa da matriz de correlação das variáveis do dataframe
correlation_matrix = insurance.corr()
correlation_matrix

klib.corr_plot(insurance)
klib.corr_plot(insurance, target='charges')

# Gráfico usando plt.scatter
x = insurance['age']
y = insurance['charges']
plt.scatter(x,y, alpha = 0.05)
plt.xlabel('age')
plt.ylabel('charges')
plt.title('ScatterPlot - age vs charges')
plt.grid()

x = insurance['bmi']
y = insurance['charges']
plt.scatter(x,y, alpha = 0.05)
plt.xlabel('bmi')
plt.ylabel('charges')
plt.title('ScatterPlot - bmi vs charges')
plt.grid()

# Checando qual é o valor do coeficiente de correlação (Person)
# computado entre cada feature (variável) e a característica/variável charges
correlation_matrix['charges']

# Colocando as correlações em ordem ascendente
correlation_matrix['charges'].sort_values(ascending = False)

# Função scatter_matrix (Pandas): forma alternativa de checagem da matriz de correlação
from pandas.plotting import scatter_matrix

# Selecionando atributos (features) de interesse para uso da scatter_matrix
attributes = ['charges',
              'age',
              'bmi',
              'children']
# Função scatter_matrix: plot na forma de matriz contendo gráficos de dispersão
scatter_matrix(insurance[attributes], figsize=(12, 8))

# Análise: aparentemente, a feature age tem relação estatística
# significativa (alta correlação) com os gastos médicos

#%% Preparação dos Dados para Modelagem

# Criando uma cópia do dataset de treino e removendo a variável target charges
insurance = train_set.drop('charges', axis = 1)

# Criando uma série para a variável de saída (target)
insurance_labels = train_set['charges'].copy()

#%% Limpeza dos Dados (Data Cleaning)

klib.missingval_plot(insurance)

# Não há valores faltantes no dataset

#%% Manipulando features categóricas

# Checando informações do dataset
insurance.info()

insurance_num = insurance.drop(['sex', 'smoker', 'region'], axis=1)
insurance_cat = insurance[['sex', 'smoker', 'region']]
insurance_cat.head(10)

# Aplicando OneHotEncoder para fazer a codificação (encoding) das
# variáveis sex, smoker e region
from sklearn.preprocessing import OneHotEncoder

# Criando encoder
cat_encoder = OneHotEncoder()

# Fazendo o fit (estimação de parâmetros) e transform (transformando os dados)
insurance_cat_1hot = cat_encoder.fit_transform(insurance_cat)
insurance_cat_1hot

# A saída é uma matriz esparsa (sparse matrix) - sparse matrix only stores
# the location of the nonzero elements
# Podemos converter para um array NumPy
insurance_cat_1hot.toarray()

# Lista de atributos das categorias
cat_encoder.categories_

#%% Feature Scaling - Pipeline de transformação

from sklearn.pipeline import Pipeline
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import StandardScaler
num_pipeline = Pipeline([
    ('imputer', SimpleImputer(strategy='median')),
    ('std_scaler', StandardScaler()),
    ])

from sklearn.compose import ColumnTransformer
num_attribs = list(insurance_num)
cat_attribs = ['sex', 'smoker', 'region']

full_pipeline = ColumnTransformer([
    ('num', num_pipeline, num_attribs),
    ('cat', OneHotEncoder(), cat_attribs),
    ])

insurance_prepared = full_pipeline.fit_transform(insurance)

#%% Modelagem

from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
lin_reg.fit(insurance_prepared, insurance_labels)

#%% Avaliação de Desempenho - Evaluation

# ----------------------------------------------------------------------------
# Avaliação de desempenho com todos os dados (sem separação de treino e teste)

from sklearn.metrics import mean_squared_error
insurance_predictions = lin_reg.predict(insurance_prepared)
lin_mse = mean_squared_error(insurance_labels, insurance_predictions)
lin_rmse = np.sqrt(lin_mse)
lin_rmse

# ----------------------------------------------------------------------------
# Avaliação de desempenho com separação de treino e teste
from sklearn.model_selection import train_test_split

# Matriz de Features X
X = insurance_prepared

# Variáveç Target
y = insurance_labels

# Divisão de treino e teste (sklearn) - train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.3,
                                                    random_state=42)

lin_mse = mean_squared_error(insurance_labels, insurance_predictions)
lin_rmse = np.sqrt(lin_mse)
lin_rmse

#%% Validação Cruzada

from sklearn.model_selection import cross_val_score

def display_scores(scores):
    print('Scores:', scores)
    print('Mean:', scores.mean())
    print('Standard deviation:', scores.std())

lin_scores = cross_val_score(lin_reg, insurance_prepared, insurance_labels,
scoring='neg_mean_squared_error', cv=10)
lin_rmse_scores = np.sqrt(-lin_scores)
display_scores(lin_rmse_scores)

r2_train = lin_reg.score(insurance_prepared, insurance_labels)
print('R2 no set de treino: %.2f' % r2_train)

#%% Usando o modelo nos dados de teste

insurance_test = test_set.drop('charges', axis = 1)
insurance_test_labels = test_set['charges'].copy()

insurance_test_prepared = full_pipeline.fit_transform(insurance_test)

lin_scores = cross_val_score(lin_reg, insurance_test_prepared, insurance_test_labels,
scoring='neg_mean_squared_error', cv=10)
lin_rmse_scores = np.sqrt(-lin_scores)
display_scores(lin_rmse_scores)

r2_test = lin_reg.score(insurance_test_prepared, insurance_test_labels)
print('R2 no set de teste: %.2f' % r2_test)


# Databricks notebook source
# MAGIC %run "./99-Shared-Functions-and-Settings"

# COMMAND ----------

from pyspark.sql.functions import *
from datetime import datetime

# COMMAND ----------

# MAGIC %md
# MAGIC ## 1. Data Selection
# MAGIC Since we want to predict the duration of the trip based on the pickup and dropoff locations - we'll first need to make sure our data is relevent and representative of the information we'll have at the time of prediction.

# COMMAND ----------

# Read in data from the materialized view
tripData = spark.read.table('taxi_db.taxi_trips_mat_view')

# COMMAND ----------

# MAGIC %md
# MAGIC ### 1.1. `pickup_location_id` and `dropoff_location_id`
# MAGIC First, as we examined the data, it appears that pickup_location_id and dropoff_location_id are fields that were added at some point in our data... let's look at trips that have the location by month.

# COMMAND ----------

# MAGIC %md
# MAGIC #### 1.1.1 Verify date available

# COMMAND ----------

display(tripData.groupBy('pickup_year', 'pickup_month')
                .agg(sum((col('pickup_location_id')!=0).astype('int')).alias('trips_with_location'))
                .orderBy('pickup_year', 'pickup_month'))

# COMMAND ----------

# MAGIC %md
# MAGIC Let's verify with another look - specifically at setting a cutoff of July 1st, 2016 at 12:00 AM. We'll also count the number of trips AFTER that data with no location ID.
# MAGIC
# MAGIC The chart below shows us that, indeed, the locatoin field was added after that date. Therefore, we'll filter out only those days on or after July 1st.

# COMMAND ----------

cutoff_date = datetime.strptime('2016-07-01', '%Y-%m-%d')

crosstabData = (tripData.withColumn('pickup_after_cutoff', col("pickup_datetime") >= cutoff_date) # Create new boolean column if pickup_datetime >
                .withColumn('location_present', col("pickup_location_id") != 0)
                .crosstab('pickup_after_cutoff', 'location_present').toPandas())

# We'll generate a graph to compare - you can find the function to generate this graph in the notebook 00-Shared-Functions
display(generate_crosstab(crosstabData))

# COMMAND ----------

# MAGIC %md-sandbox
# MAGIC
# MAGIC One thing to note here - matplotlib plots can be displayed inline (without using the Jupyter magic command `%matplotlib inline`)
# MAGIC
# MAGIC Instead, we need to run the `display` function on the figure item returned by matplotlib. In the [Simple Plot sample on the Matplotlib site](https://matplotlib.org/gallery/lines_bars_and_markers/simple_plot.html), we generate a figure object with the `fig, ax = plt.subplots()` command
# MAGIC
# MAGIC ```python
# MAGIC import matplotlib
# MAGIC import matplotlib.pyplot as plt
# MAGIC import numpy as np
# MAGIC # Data for plotting
# MAGIC t = np.arange(0.0, 2.0, 0.01)
# MAGIC s = 1 + np.sin(2 * np.pi * t)
# MAGIC
# MAGIC fig, ax = plt.subplots()
# MAGIC ax.plot(t, s)
# MAGIC
# MAGIC ax.set(xlabel='time (s)', ylabel='voltage (mV)',
# MAGIC        title='About as simple as it gets, folks')
# MAGIC ax.grid()
# MAGIC ```
# MAGIC
# MAGIC In Databricks, we then run `display(fig)` to show that graph inline.

# COMMAND ----------

# MAGIC %md
# MAGIC #### 1.1.2 Filter data to after July 1st, 2016
# MAGIC We'll filter out all trips prior to out cutoff date.

# COMMAND ----------

# We'll write over the Trip Data
tripData = tripData.filter(col("pickup_datetime") >= cutoff_date)

# And we'll verify that no "unknown" pickup locations are still in the data
fullRowCount = tripData.count()

goodRowCount = tripData.filter(col('pickup_location_id') == 0).count()

print('{0} rows with no pickup_location_id out of {1:,} total rows in dataset'.format(goodRowCount, fullRowCount))
assert(goodRowCount == 0) # Using an assert statement to raise an error if other rows slip through

# COMMAND ----------

# MAGIC %md
# MAGIC ### 1.2. What Data is Available at the time of booking?
# MAGIC
# MAGIC Next, we want to avoid any 'data leakage' - specifically where we include data we know now for past values that we won't know at the time of predictions in the future.
# MAGIC
# MAGIC Let's look again at the columns and a sample of the data in our dataset.

# COMMAND ----------

tripData.printSchema()

# COMMAND ----------

display(tripData)

# COMMAND ----------

# MAGIC %md
# MAGIC In looking at this, there are several features that we won't know at the time of booking. For instance, we won't know the fare (since it's partially a function of duration).
# MAGIC
# MAGIC We also won't know anything about the trip dropoff time. (However, we'll keep in the dropoff_datetime 0- since we'll need that to compute duration)
# MAGIC
# MAGIC In all, of the 51 columns in our dataset, here are the columns we won't have access to at the time of booking (for our example, we'll assume we have the dropoff location, since the driver needs to know where they are driving to):
# MAGIC * `dropoff_datetime`
# MAGIC * `fare_amount`
# MAGIC * `extra`
# MAGIC * `mta_tax`
# MAGIC * `tip_amount`
# MAGIC * `tolls_amount`
# MAGIC * `ehail_fee`
# MAGIC * `improvement_surcharge`
# MAGIC * `total_amount`
# MAGIC * `payment_type`
# MAGIC * `payment_type_description`
# MAGIC * `dropoff_year`
# MAGIC * `dropoff_month`
# MAGIC * `dropoff_day`
# MAGIC * `dropoff_hour`
# MAGIC * `dropoff_minute`
# MAGIC * `dropoff_second`
# MAGIC
# MAGIC Let's drop these columns and see what's left.

# COMMAND ----------

columns_to_drop = {'fare_amount','extra','mta_tax','tip_amount','tolls_amount',
                   'ehail_fee','improvement_surcharge','total_amount','payment_type',
                   'payment_type_description','dropoff_year','dropoff_month',
                   'dropoff_hour', 'dropoff_day','dropoff_minute','dropoff_second'}

tripData = tripData.select([column for column in tripData.columns if column not in columns_to_drop])

# COMMAND ----------

# display(tripData.describe())

# COMMAND ----------

# MAGIC %md
# MAGIC ### 1.3. Which columns are blank and which columns are derived from other columns?
# MAGIC
# MAGIC We don't want a lot of redundant data entering our model - because the effects of a particular feature can be amplified. For example - in our dataset we have the `pickup_datetime`, the `pickup_year`, the `trip_year`, etc. We'll want to be careful here not to represent the same thing multiple times. In this instance, we'll keep `pickup_year`, `pickup_month`, `pickup_day` and `pickup_hour`. I'll assume that the minute and seconds are too granular to effect the trip_duration, but we may need to revisit later.
# MAGIC
# MAGIC Similarily, the latitude and longitude values of pickup and dropoff are pseudo-encoded in the location_id. We'll drop these to avoid redundancy.
# MAGIC
# MAGIC With `pickup_location_id` and `dropoff_location_id`, we've included the lookup values here. Therefore, we'll exclude the IDs, because individual values of borough, etc. may have value there.
# MAGIC
# MAGIC In all, we'll drop:
# MAGIC * `vendor_id`  - Encoded in `vendor_abbreviation`
# MAGIC * `pickup_datetime` - Will be dropped after duration calculation
# MAGIC * `dropoff_datetime` - Will be dropped after duration calculation
# MAGIC * `rate_code_id` - Encoded in `rate_code_description`, etc.
# MAGIC * `pickup_location_id` - Encoded in `pickup_borough`, etc.
# MAGIC * `dropoff_location_id` - Encoded in `dropoff_borough`, etc.
# MAGIC * `pickup_longitude` - Blank
# MAGIC * `pickup_latitude` - Blank
# MAGIC * `dropoff_longitude` - Blank
# MAGIC * `dropoff_latitude` - Blank
# MAGIC * `vendor_description` - Encoded in `vendor_abbreviation`
# MAGIC * `trip_type_description` - Mostly blank
# MAGIC * `month_name_short` - Encoded in `pickup_year`, etc.
# MAGIC * `month_name_full` - Encoded in `pickup_year`, etc.
# MAGIC * `pickup_minute` - Too granular
# MAGIC * `pickup_second` - Too granular
# MAGIC * `trip_year` - Encoded in `pickup_year`, etc.
# MAGIC * `trip_month` - Encoded in `pickup_year`, etc.

# COMMAND ----------

columns_to_drop = columns_to_drop.union({'vendor_id', 'rate_code_id', 'pickup_location_id', 'dropoff_location_id',
                                         'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude',
                                         'trip_type', 'vendor_description', 'trip_type_description', 'month_name_short',
                                         'month_name_full', 'pickup_minute', 'pickup_second', 'trip_year', 'trip_month'})

tripData = tripData.select([column for column in tripData.columns if column not in columns_to_drop])

# COMMAND ----------

display(tripData)

# COMMAND ----------

# MAGIC %md
# MAGIC ## 2. Feature (and Label) Engineering

# COMMAND ----------

# MAGIC %md
# MAGIC ### 2.1 `is_weekend_flag` and `is_rush_hour_flag` columns

# COMMAND ----------

tripData = (tripData.withColumn('is_weekend_flag', date_format(col('pickup_datetime'), 'E').isin(['Sat', 'Sun']).astype('int'))
                    .withColumn('is_rush_hour_flag', ((col('is_weekend_flag')==0) & ((col('pickup_hour').between(7, 10)) | (col('pickup_hour').between(16, 19)))).astype('int')))

# COMMAND ----------

# MAGIC %md
# MAGIC ### 2.2 `duration` column
# MAGIC
# MAGIC We've been asked to predict "duration", but we don't have that represented anywhere. Therefore, we'll need to "engineer" that feature (or label in this instance)
# MAGIC
# MAGIC We'll use the `unix_timestamp` function in PySpark to convert our timestamps to a UNIX Timestamp (# of seconds since January 1st, 1970 at 12:00AM UTC) and then subtract to get durations in seconds.

# COMMAND ----------

tripData = tripData.withColumn('duration_minutes', round((unix_timestamp(col('dropoff_datetime'))-unix_timestamp(col('pickup_datetime')))/60, 2))

columns_to_drop = columns_to_drop.union({'pickup_datetime', 'dropoff_datetime'})
tripData = tripData.select([column for column in tripData.columns if column not in columns_to_drop])

display(tripData)

# COMMAND ----------

display(tripData.describe(['duration_minutes']))

# COMMAND ----------

# We'll filter out any negative trip or trip over 2 hours...

tripData = tripData.where(col('duration_minutes').between(0, 120))
display(tripData.describe(['duration_minutes']))

# COMMAND ----------

# MAGIC %md
# MAGIC ### 2.3 Other feature engineering?
# MAGIC There may be other features that we could engineer here, but for this lab we'll skip other feature engineering / exploration

# COMMAND ----------

# MAGIC %md
# MAGIC ## 3. Moving to another notebook
# MAGIC Now, we'll take the dataframe we created here and "transfer" it to another notebook. We can do that through the SparkSQL API by registering a temporary view.

# COMMAND ----------

tripData.createOrReplaceGlobalTempView(model_dataset_name)

# You can cache the temporary view for increase performance - but it might not be optimal in a workshop setting
# spark.catalog.cacheTable('global_temp.{0}'.format(model_dataset_name))
display(spark.read.table('global_temp.{0}'.format(model_dataset_name)))

# COMMAND ----------