# Instructor Version

import re
import io
import contextlib
import numpy as np
import matplotlib.pyplot as plt
import IPython
from IPython import get_ipython
from IPython.core.magic import register_line_magic, register_cell_magic
from IPython.display import HTML, display, YouTubeVideo

# globals
_LAST_IN = _LAST_OUT = ""
_LAST_PLOT = None
_LAST_DISPLAY = ()

# wrap show() to grab a copy of a figure before it's closed
_old_show = plt.show
def _new_show(*args, **kwargs):
    global _LAST_PLOT
    _LAST_PLOT = plt.gcf()
    _old_show(*args, **kwargs)

# wrap print() to grab a copy of whatever is printed
_old_print = print
def _new_print(*args, **kwargs):
    global _LAST_OUT
    out = io.StringIO()
    with contextlib.redirect_stdout(out):
        _old_print(*args, **kwargs)
    _LAST_OUT += out.getvalue()
    _old_print(*args, **kwargs)

# wrap display() to grab the text representation of rich output objects
_old_display = IPython.display.display
def _new_display(*args, **kwargs):
    global _LAST_DISPLAY
    _LAST_DISPLAY = args
    _old_display(*args, **kwargs)

@register_line_magic
def background(color):
    """
    Change the background color of a code cell.
    """
    js = (
        "var cell = this.closest('.jp-CodeCell');"
        "if (cell) {"
        "    var input_area = cell.querySelector('.jp-Editor');"
        "} else {"
        "    var cell = this.closest('.cell');"
        "    var input_area = cell.querySelector('.input_area');"
        "}"
        f"input_area.style.background='{color}';"
        "this.parentNode.removeChild(this);"
        )
    display(HTML(f'<img src onerror="{js}">')) # Such hackery. Much javascript.

@register_line_magic
def video(youtube_id):
    """
    Display a YouTube video.
    """
    display(YouTubeVideo(youtube_id, 720, 480, rel=0))

@register_cell_magic
def graded(line, cell):
    """
    Store input and output for testing. Also changes the background color.
    """
    global _LAST_IN, _LAST_OUT, _LAST_PLOT, _LAST_DISPLAY
    background("lavender")
    _LAST_IN = cell
    _LAST_PLOT = None # should only exist if this cell makes one
    _LAST_DISPLAY = () # clear
    _LAST_OUT = "" # reset print buffer
    plt.show = _new_show # monkey patch
    IPython.display.display = _new_display
    get_ipython().user_ns["print"] = _new_print # harder to change print
    get_ipython().run_cell(cell)
    plt.show = _old_show # restore
    IPython.display.display = _old_display
    get_ipython().user_ns["print"] = _old_print

@register_cell_magic
def tests(line, cell):
    """
    This cell runs tests.
    """
    # no_code = True
    # for line in _LAST_IN.split("\n"):
    #     if not re.match(r"\s+|\s*#.*|^$", line): no_code = False
    # if no_code:
    #     background("lightyellow")
    #     print("Nothing to test yet...")
    # else:
    result = get_ipython().run_cell(cell)
    if result.success:
        background("lightgreen")
        print("All tests passed!")
    else:
        background("pink")

def equal(A, B):
    """
    Check that A = B.
    """
    np.testing.assert_equal(A, B)

def similar(A, B, rtol=0.01, atol=1e-12):
    """
    Check that A = B with tolerance atol + rtol*B.
    """
    np.testing.assert_allclose(A, B, rtol=rtol, atol=atol)

def code_contains(*args, forbidden=False):
    """
    Check that <args> appear in the previous graded cell.
    If forbidden, check that <args> do NOT appear.
    """
    for arg in args:
        if not re.search(arg, _LAST_IN):
            if not forbidden:
                raise AssertionError(f"The previous cell doesn't include '{arg}'.")
        else:
            if forbidden:
                raise AssertionError(f"The previous cell contains the forbidden code: '{arg}'")

def printed(*args):
    """
    Check that <args> were printed by previous graded cell.
    """
    for arg in args:
        if not re.search(arg, _LAST_OUT):
            raise AssertionError(f"Printed output doesn't contain '{arg}'.")

def get_plot():
    """
    Return the figure object from the last graded cell.
    """
    return _LAST_PLOT

def plot_shown():
    """
    Check that previous graded cell created a plot.
    """
    assert _LAST_PLOT, "Plot missing. Did you use plt.show()?"

def plot_has(*args):
    """
    Check that previous graded cell created a plot with all of the <args> set.
    <args> are strings like "title" or "legend".

    Note: This function only checks the most recent figure. If you need to show
    more than one plot from a single cell, use subplots. This will also break if
    you .close() the figure before this test is run.
    """
    assert _LAST_PLOT, "Plot missing. Did you use plt.show()?"

    for arg in args:
        found = False
        for ax in _LAST_PLOT.axes:
            if hasattr(ax, "get_" + arg):
                if getattr(ax, "get_" + arg)():
                    found = True
        assert found, f"'{arg}' was not set for plot"

def plot_snapshot():
    """
    Print the state of the plot from the previous graded cell.
    """
    assert _LAST_PLOT, "Plot missing. Did you use plt.show()?"

    def inspect(obj, depth=0):
        if hasattr(obj, "get_children"):
            children = getattr(obj, "get_children")()
            for child in children:
                print("\t" * depth, child, sep="")
                inspect(child, depth+1)

    inspect(_LAST_PLOT)

def animation_shown():
    """
    Check that previous graded cell created an animation.

    Note: This function requires that display.display(video_as_html) was called
    """
    found = False
    for d in _LAST_DISPLAY:
        if hasattr(d, "data"):
            if "</video>" in d.data:
                found = True
    assert found, "Animation missing. Did you show it with display.display()?"

def widget_snapshot():
    """
    Display widget's current state for manual grading

    Note: This function requires that display.display(widget) was called
    """
    # prints string representation of EVERYTHING that was displayed
    for widget in _LAST_DISPLAY:
        s = str(widget)
        matches = re.findall(r"outputs=\(.*?\)", s)
        for match in matches:
            s = s.replace(match, "")
        print(s)

        # this might break since other things can be displayed
        # for match in matches:
        #     output = re.search(r"{'text/plain':.*?}", match)
        #     if output:
        #         display(eval(output.group()), raw=True)

def widget_has(*args):
    """
    Check that widget is contains <args>, where <args> are strings like
    "IntSlider" or "HBox".

    Note: This function requires that display.display(widget) was called
    """
    for arg in args:
        match = re.search(arg, str(_LAST_DISPLAY))
        assert match, f"Widget property '{arg}' not found.  Did you use display.display()?"


#!/usr/bin/env python
# coding: utf-8

# # Visualização de dados com _seaborn_

# No capítulo [Plotagem e formatação de gráficos](09-plotagem-matplot.ipynb), fizemos uma breve introdução à _visualização de dados_ e aprendemos a utilizar o _matplotlib_ para realizar plotagens básicas de gráficos. Neste capítulo, exploraremos a visualização de dados utilizando o módulo _seaborn_.
#
# O _seaborn_ estende as potencialidades do _matplotlib_ em, praticamente, dois aspectos: i) provendo melhor estilização dos gráficos e tornando-os visualmente "belos"; ii) compactando funções de plotagem do _matplotlib_, de modo que plotagens robustas sejam obtidas com menos instruções de código.

# Vamos começar importando as bibliotecas que utilizaremos.

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

import warnings
warnings.filterwarnings('ignore')

# Vamos reconstruir e importar alguns *DataFrames*.

# In[2]:

serie_Idade = pd.Series({'Ana':20, 'João': 19, 'Maria': 21, 'Pedro': 22, 'Túlio': 20}, name="Idade")
serie_Peso = pd.Series({'Ana':55, 'João': 80, 'Maria': 62, 'Pedro': 67, 'Túlio': 73}, name="Peso")
serie_Altura = pd.Series({'Ana':162, 'João': 178, 'Maria': 162, 'Pedro': 165, 'Túlio': 171}, name="Altura")

# In[3]:

dicionario_series_exemplo = {'Idade': serie_Idade, 'Peso': serie_Peso, 'Altura': serie_Altura}

# In[4]:

df_dict_series = pd.DataFrame(dicionario_series_exemplo);df_dict_series

# In[5]:

df_exemplo = pd.read_csv('../database/exemplo_data.csv', index_col=0)
df_exemplo['coluna_3'] = pd.Series([1,2,3,4,5,6,7,8,np.nan,np.nan],index=df_exemplo.index)
df_exemplo.index = pd.to_datetime(df_exemplo.index)
df_exemplo

# In[6]:

covid_PB = pd.read_csv('https://superset.plataformatarget.com.br/superset/explore_json/?form_data=%7B%22slice_id%22%3A1550%7D&csv=true',
                             sep=',', index_col=0)
covid_PB.head()

# In[7]:

covid_BR = pd.read_excel('../database/HIST_PAINEL_COVIDBR_25jul2020.xlsx')
covid_BR.head()

# ## Gráficos de Linha e de Dispersão
#
# Através da função `relplot`, podemos alterar o valor do parâmetro `kind` para
# obter gráficos do linha, com `kind = 'line'`, ou gráficos de dispersão, com `kind  = 'scatter'`. De modo alternativo, poderíamos também utilizar as funções `lineplot` e `scatterplot`. Entretanto, com `relplot`, diversos elementos de construção de figura existentes no *matplotlib* já são pré-configurados no _seaborn_. Isto é chamado no *seaborn* de *figure-level plot*.

# Inicialmente, vejamos como a função `lineplot` se comporta como qualquer outra do `matplotlib.pyplot`.

# In[8]:

fig, ax = plt.subplots()
ax = sns.lineplot(x="index", y="coluna_1", data=df_exemplo.reset_index(), label = 'Coluna 1')
ax = sns.lineplot(x="index", y="coluna_2", data=df_exemplo.reset_index(), label = 'Coluna 2')
ax = sns.lineplot(x="index", y="coluna_3", data=df_exemplo.reset_index(), label = 'Coluna 3')
ax.set_xlabel('Data')
ax.set_ylabel('Valor')
fig.autofmt_xdate() # auto-formata xticks tipo "data"

# Para utilizar a função `relplot`, precisaremos reorganizar o banco de dados de modo que haja apenas uma coluna de valores no eixo _y_ (`Valor`) e uma coluna com um identificador de cor/tonalidade (`Coluna`).

# In[9]:

# reorganiza para 'Coluna 1'
df_exemplo_px = pd.DataFrame(df_exemplo['coluna_1']).rename({'coluna_1':'Valor'}, axis=1)
df_exemplo_px['Coluna'] = 'Coluna 1'

# concatena 'Coluna 2'
df_exemplo_px_temp = pd.DataFrame(df_exemplo['coluna_2']).rename({'coluna_2':'Valor'}, axis=1)
df_exemplo_px_temp['Coluna'] = 'Coluna 2'
df_exemplo_px = pd.concat([df_exemplo_px, df_exemplo_px_temp])

# concatena 'Coluna 3'
df_exemplo_px_temp = pd.DataFrame(df_exemplo['coluna_3']).rename({'coluna_3':'Valor'}, axis=1)
df_exemplo_px_temp['Coluna'] = 'Coluna 3'
df_exemplo_px = pd.concat([df_exemplo_px, df_exemplo_px_temp])

df_exemplo_px.head()

# In[10]:

# 'hue' atribui as tonalidades de cor com base no identificador
p = sns.relplot(x = 'index', y='Valor', hue = 'Coluna', data=df_exemplo_px.reset_index().dropna(), kind='line')
p.fig.autofmt_xdate()
p.ax.set_xlabel('Data')
p.fig.set_size_inches(8,4) # largura, altura

# Vamos agora plotar o gráfico de óbitos por COVID-19 na Paraíba juntamente com a média aritmética móvel de 7 dias e com a média geométrica móvel de 7 dias.
#
# Utilizaremos o método `rolling` de uma *Series* ou *DataFrame* do *pandas*. Este método cria janelas móveis onde podemos aplicar uma função agregadora (tal como média ou média geométrica).

# Comentários:
#
# - A média aritmética tem a desvantagem de linearizar o efeito do crescimento ou decrescimento do número de óbitos, onde sabemos que o efeito é exponencial.
# - A média geométrica móvel tem a desvantagem de se anular se o número de óbitos em algum dos dias da janela for zero.
# - Em geral, as duas médias ficam muito próximas.

# In[11]:

# função que calcula a média geométrica
from scipy.stats import gmean

# série
covid_PB_obitos = covid_PB.obitosNovos
covid_PB_obitos = covid_PB_obitos.sort_index()
covid_PB_obitos.name = 'Óbitos'

# dataframe
covid_PB_obitos_df = pd.DataFrame(covid_PB_obitos)
covid_PB_obitos_df['Tipo'] = 'Valor nominal'

# média aritmética móvel
covid_PB_obitos_df_temp = pd.DataFrame(covid_PB_obitos.rolling(7).mean().dropna())
covid_PB_obitos_df_temp['Tipo'] = 'MM:arit - 7d'
covid_PB_obitos_df = pd.concat([covid_PB_obitos_df, covid_PB_obitos_df_temp])

# média geométrica móvel
covid_PB_obitos_df_temp = pd.DataFrame(covid_PB_obitos.rolling(7).aggregate(gmean).dropna())
covid_PB_obitos_df_temp['Tipo'] = 'MM:geom - 7d'
covid_PB_obitos_df = pd.concat([covid_PB_obitos_df, covid_PB_obitos_df_temp])

covid_PB_obitos_df.index = pd.to_datetime(covid_PB_obitos_df.index)

covid_PB_obitos_df.tail()

# In[12]:

p = sns.relplot(x = 'data', y='Óbitos',
                hue = 'Tipo',
                data=covid_PB_obitos_df.reset_index(),
                kind='line')
p.fig.autofmt_xdate()
p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# Vamos agora construir um gráfico de dispersão com o _DataFrame_ `df_exemplo_px`.

# In[13]:

# 'scatter' é padrão
p = sns.relplot(x = 'index', y='Valor',
                hue = 'Coluna',
                data=df_exemplo_px.reset_index().dropna())
p.fig.autofmt_xdate()
p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# Vamos forçar os limites de datas a ficarem dentro do mínimo (menos um dia) e do máximo (mais um dia):

# In[14]:

p = sns.relplot(x = 'index', y='Valor',
                      hue = 'Coluna',
                      data=df_exemplo_px.reset_index().dropna())

# offset de data
p.ax.set_xlim((df_exemplo_px.reset_index()['index'].min()
               - pd.DateOffset(days=1)),
              (df_exemplo_px.reset_index()['index'].max()
               + pd.DateOffset(days=1)))

p.fig.autofmt_xdate()
p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# In[15]:

# especifica cores e tamanhos
p = sns.relplot(x = 'index', y='coluna_1',
                hue = 'coluna_2',
                size = 'coluna_3',
                data=df_exemplo.reset_index().dropna())

p.ax.set_xlim((df_exemplo.reset_index()['index'].min()
               - pd.DateOffset(days=1)),
              (df_exemplo.reset_index()['index'].max()
               + pd.DateOffset(days=1)))

p.fig.autofmt_xdate()
p.ax.set_xlabel('')
p.fig.set_size_inches(8,4)

# In[16]:

covid_PB_casos_obitos = covid_PB[['obitosNovos', 'casosNovos']].sort_index()
covid_PB_casos_obitos.index = pd.to_datetime(covid_PB_casos_obitos.index)

# cor: óbitos novos
p = sns.relplot(x = 'data', y = 'casosNovos',
                hue = 'obitosNovos',
                data=covid_PB_casos_obitos.reset_index())

p.ax.set_xlim((covid_PB_casos_obitos.reset_index()['data'].min()
               - pd.DateOffset(days=5)),
              (covid_PB_casos_obitos.reset_index()['data'].max()
               + pd.DateOffset(days=5)))

p.fig.autofmt_xdate()
p.ax.set_xlabel('')
p.ax.set_ylabel('Casos COVID-19 - PB')
p.ax.set_title('Casos e Óbitos de COVID-19 na Paraíba')
p.fig.set_size_inches(8,4)

# ## Gráficos de dispersão em dados categóricos
#
# Quando há muitos valores repetidos em uma variável, os gráficos de dispersão podem não ilustrar efetivamente o comportamento dos dados. Neste caso, é interessante que o gráfico considere a repetição dos valores dentro de uma mesma categoria.

# Comentários:
#
# - Isto acontece quando o eixo horizontal contém variáveis categóricas e, assim, tem-se repetição de valores dentro de uma mesma categoria.

# Para gráficos deste tipo, utilizaremos os dados de óbitos por COVID-19 no Brasil. Agruparemos o número de óbitos por dia da semana.

# In[17]:

covid_BR_obitos = covid_BR.query('regiao == "Brasil"')[['obitosNovos','data']]

covid_BR_obitos.data = pd.to_datetime(covid_BR_obitos.data)

covid_BR_obitos['Dia'] = covid_BR_obitos.data.dt.weekday.map(
    {0:'Segunda-Feira',
     1:'Terça-Feira',
     2:'Quarta-Feira',
     3:'Quinta-Feira',
     4:'Sexta-Feira',
     5:'Sábado',
     6:'Domingo'})

covid_BR_obitos = covid_BR_obitos.set_index('data')
covid_BR_obitos

# Se quisermos determinar a ordem do eixo _x_, `relplot` não é a função ideal. Além disso, devido à sobreposição dos dados, ela definitivamente não é a ideal para variáveis categóricas. Vejamos:

# In[18]:

p = sns.relplot(x='Dia', y='obitosNovos', data=covid_BR_obitos)
p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# ## O gráfico *stripplot*
#
# O *stripplot* é um gráfico de dispersão onde a cada observação é colocado um deslocamento aleatório para evitar a sobreposição e fornecer uma ideia mais precisa da quantidade de dados.
#
# Vamos construir o *stripplot* através da função `catplot`. O *stripplot* é o gráfico padrão do `catplot` (tem o argumento `kind = 'strip'`).
#
# Podemos determinar a ordem das variáveis categóricas com o argumento *order*.

# In[19]:

p = sns.catplot(x='Dia', y='obitosNovos',
                data=covid_BR_obitos, order =
                ['Segunda-Feira', 'Terça-Feira',
                 'Quarta-Feira', 'Quinta-Feira',
                 'Sexta-Feira', 'Sábado', 'Domingo'])

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# Se colocarmos `jitter=False`, obteremos o gráfico de dispersão usual (com o detalhe de que podemos definir a ordem dos rótulos do eixo *x*).

# In[20]:

p = sns.catplot(x='Dia', y='obitosNovos',
                jitter = False,
                data=covid_BR_obitos,
                order = ['Segunda-Feira', 'Terça-Feira',
                         'Quarta-Feira', 'Quinta-Feira',
                         'Sexta-Feira', 'Sábado', 'Domingo'])

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# ## O gráfico *swarmplot*
#
# O *swarmplot* é um gráfico de dispersão onde, diferentemente do *stripplot*, nenhum dado pode ficar sobreposto. Desta forma, também fornece uma ideia mais precisa da quantidade de dados.
#
# Construiremos o *swarmplot* através da função `catplot` com o argumento `kind = 'swarm'`. Como o *swarmplot* também é um tipo do `catplot`, podemos determinar a ordem das variáveis categóricas com o argumento `order`.

# In[21]:

p = sns.catplot(x='Dia', y='obitosNovos',
                kind = 'swarm',
                data=covid_BR_obitos,
                order = ['Segunda-Feira', 'Terça-Feira',
                         'Quarta-Feira', 'Quinta-Feira',
                         'Sexta-Feira', 'Sábado', 'Domingo'])

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# ## Gráficos de Barras e Colunas
#
# Para criar gráficos de barras e colunas com o *seaborn* utilizaremos a função `catplot` com o argumento `kind=bar`. Se a variável categórica estiver no eixo *x*, o gráfico será de coluna; se a variável categórica estiver no eixo *y*, o gráfico será de barra.

# In[22]:

covid_Regioes = covid_BR[['regiao','obitosNovos']].groupby('regiao').sum().query('regiao != "Brasil"')/2

p = sns.catplot(x='regiao', y='obitosNovos',
                kind = 'bar',data=covid_Regioes.reset_index())

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# In[23]:

covid_Regioes = covid_BR[['regiao','obitosNovos']].groupby('regiao').sum().query('regiao != "Brasil"')/2

p = sns.catplot(x='obitosNovos', y='regiao',
                kind = 'bar',data=covid_Regioes.reset_index())

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# In[24]:

df_dict_series_sns = pd.DataFrame(df_dict_series.Idade).rename({'Idade':'Valor'}, axis=1)
df_dict_series_sns['Dado'] = 'Idade'

df_dict_series_sns_temp = pd.DataFrame(df_dict_series.Altura).rename({'Altura':'Valor'}, axis=1)
df_dict_series_sns_temp['Dado'] = 'Altura'
df_dict_series_sns = pd.concat([df_dict_series_sns, df_dict_series_sns_temp])

df_dict_series_sns_temp = pd.DataFrame(df_dict_series.Peso).rename({'Peso':'Valor'}, axis=1)
df_dict_series_sns_temp['Dado'] = 'Peso'
df_dict_series_sns = pd.concat([df_dict_series_sns, df_dict_series_sns_temp])

df_dict_series_sns

# In[25]:

p = sns.catplot(x='index', y='Valor',
                hue='Dado',
                data = df_dict_series_sns.reset_index(),
                kind='bar')

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# ## _Box Plot_ e plots alternativos
#
# Tanto o *BoxPlot* quanto os plots alternativos que apresentaremos aqui (*violinplot* e *boxenplot*) fazem parte do `catplot`. Para construir um
#
# - *Box Plot* utiliza-se o argumento `kind='box'`;
# - *Violin Plot* utiliza-se o argumento `kind='violin'`;
# - *Boxen Plot* (ou *letter-value plot*) utiliza-se o argumento `kind='boxen'`.
#
# ```{note}
# O *boxenplot* foi criado por Hadley Wickham (criador do *ggplot2* e da maioria dos pacotes do *tidyverse* do *R*) e colaboradores e é uma generalização do *BoxPlot* que apresenta mais quantis. Foi introduzido como [letter-value plots](https://vita.had.co.nz/papers/letter-value-plot.html). O *violinplot* recebe este nome, pois seu gráfico assemelha-se a um violino.
# ```

# In[26]:

p = sns.catplot(x='Dia', y='obitosNovos',
                kind = 'box',
                data=covid_BR_obitos,
                order = ['Segunda-Feira', 'Terça-Feira',
                         'Quarta-Feira', 'Quinta-Feira',
                         'Sexta-Feira', 'Sábado', 'Domingo'])

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# In[27]:

covid_regioes_diarios_px = covid_BR.set_index(
    'data').query('regiao != "Brasil"')[['obitosNovos', 'regiao']].reset_index().rename(
    {'obitosNovos':'Óbitos','regiao':'Região','data':'Data'},axis=1)

covid_regioes_diarios_px = covid_regioes_diarios_px.groupby(['Região','Data']).sum()/2
covid_regioes_diarios_px = covid_regioes_diarios_px.reset_index().set_index('Data')

covid_regioes_diarios_px

# In[28]:

p = sns.catplot(x='Região', y='Óbitos',
                kind = 'box',
                data=covid_regioes_diarios_px)

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# Na presença de muitos *outliers*, como é o caso do gráfico anterior, é interessante considerar uma alternativa ao *Box Plot*.
#
# Vamos ver agora o *Boxen Plot* (ou *letter-value plots*). Este plot considera os quantis: ..., 0.8%, 1.56%, 3.13%, 6.25%, 12.5%, 25%, 50%, 75%, 87.5%, 93.75%, 96.88%, 98.44%, 99.24%, ...

# In[29]:

p = sns.catplot(x='Região', y='Óbitos',
                kind = 'boxen',
                data=covid_regioes_diarios_px)

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# Porém, em um gráfico sem muitos *outliers*, o *Boxen Plot* não difere muito do *Box Plot*.

# In[30]:

p = sns.catplot(x='Dia', y='obitosNovos',
                kind = 'boxen',
                data=covid_BR_obitos,
                order = ['Segunda-Feira', 'Terça-Feira',
                         'Quarta-Feira', 'Quinta-Feira',
                         'Sexta-Feira', 'Sábado', 'Domingo'])
p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# Na presença de muitos *outliers*, também é preferível um *Violin Plot* em vez de um *Box Plot*, para tornar visível o que está ocorrendo.

# In[31]:

p = sns.catplot(x='Região', y='Óbitos',
                kind = 'violin',
                data=covid_regioes_diarios_px)

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# Muitas vezes, é interessante sobrepor um *Violin Plot* a um *Swarm Plot* para evidenciar o comportamento da distribuição dos dados.

# In[32]:

p = sns.catplot(x='Região', y='Óbitos',
                kind = 'violin',
                data=covid_regioes_diarios_px)

sns.swarmplot(x='Região', y='Óbitos',
              data=covid_regioes_diarios_px,
              ax = p.ax,
              size=4, color='k')

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# In[33]:

p = sns.catplot(x='Dia', y='obitosNovos',
                kind = 'violin',
                data=covid_BR_obitos)

sns.swarmplot(x='Dia', y='obitosNovos',
              data=covid_BR_obitos,
              ax = p.ax, size=4, color='k')

p.ax.set_xlabel('');
p.fig.set_size_inches(8,4)

# ## Histogramas
#
# O *seaborn* constrói histogramas a partir da função `histplot`, ou da função obsoleta `distplot`.
#
# Incluímos um estimador de densidade baseado em núcleo Gaussiano (_Gaussian kernel_) utilizando o argumento `kde=True`.

# In[34]:

fig, ax = plt.subplots(figsize=(8,4))
_ = sns.histplot(covid_regioes_diarios_px.query('Região=="Nordeste"')['Óbitos'],kde=True)

# Para remover o estimador, selecionamos `kde=False`.

# In[35]:

fig, ax = plt.subplots(figsize=(8,4))
_ = sns.histplot(covid_regioes_diarios_px.query('Região=="Nordeste"')['Óbitos'],kde=False)

# Para plotarmos apenas o estimador de densidade sem o histograma, usamos `distplot` com a opção `hist=False`.

# In[36]:

fig, ax = plt.subplots(figsize=(8,4))
_ = sns.distplot(covid_regioes_diarios_px.query('Região=="Nordeste"')['Óbitos'],hist=False)

# É possível plotar o histograma com o estimador para qualquer série do _DataFrame_.

# In[37]:

fig, ax = plt.subplots(figsize=(8,4))
_ = sns.histplot(df_exemplo['coluna_1'],kde=True)

# ## Distribuição conjunta e marginal
#
# O histograma permite que verifiquemos a distribuição de uma ou mais variáveis, mas sem levar outras em consideração. Para plotarmos uma distribuição conjunta, bem como a distribuição individual (marginal) de cada variável, podemos utilizar a função `jointplot`.

# In[38]:

_ = sns.jointplot(x = 'coluna_1', y = 'coluna_2', data=df_exemplo, height=7)

# O próximo exemplo usa a função `jointplot` para plotar distribuições conjuntas relativas ao número de óbitos por Covid-19 por região do Brasil durante uma certa janela temporal.

# In[39]:

covid_regioes_diarios = pd.DataFrame()

regioes = covid_BR.query('regiao != "Brasil"')['regiao'].drop_duplicates().array

for regiao in regioes:
    temp_series = covid_BR.set_index('data').query('regiao == @regiao')['obitosNovos'].groupby('data').sum()/2
    temp_series.name = 'obitos_' + regiao
    covid_regioes_diarios = pd.concat([covid_regioes_diarios, temp_series], axis=1)

covid_regioes_diarios.index = pd.to_datetime(covid_regioes_diarios.index)
covid_regioes_diarios.head()

# In[40]:

_ = sns.jointplot(x='obitos_Nordeste', y='obitos_Sudeste',
                  data = covid_regioes_diarios, height=7)

# ## Estilos e cores
#
# O *seaborn* disponibiliza 5 estilos pré-definidos: *darkgrid*, *whitegrid*, *dark*, *white* e *ticks*. Vejamos cada um deles.

# In[41]:

import matplotlib.dates as mdates
from matplotlib.ticker import FuncFormatter

# In[42]:

sns.set_style("darkgrid")
p = sns.relplot(x = 'data', y='Óbitos', hue = 'Tipo', data=covid_PB_obitos_df.reset_index(), kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# In[43]:

sns.set_style("whitegrid")
p = sns.relplot(x = 'data', y='Óbitos', hue = 'Tipo', data=covid_PB_obitos_df.reset_index(), kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# In[44]:

sns.set_style("dark")
p = sns.relplot(x = 'data', y='Óbitos', hue = 'Tipo', data=covid_PB_obitos_df.reset_index(), kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# In[45]:

sns.set_style("white")
p = sns.relplot(x = 'data', y='Óbitos', hue = 'Tipo', data=covid_PB_obitos_df.reset_index(), kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# In[46]:

sns.set_style("ticks") # A diferença com o anterior são os "ticks" no eixo x
p = sns.relplot(x = 'data', y='Óbitos', hue = 'Tipo', data=covid_PB_obitos_df.reset_index(), kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# ### Molduras
#
# Utilizamos a função `despine` para adicionar ou remover molduras em plotagens com o *seaborn*. Podemos especificar lados para adicionar e remover molduras utilizando os booleanos `True` e `False`.

# In[47]:

sns.set_style("ticks")
p = sns.relplot(x = 'data', y='Óbitos', hue = 'Tipo', data=covid_PB_obitos_df.reset_index(), kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7))
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21))
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y'))
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)
sns.despine(right=False, top=False)

# Podemos criar um tipo de "acolchoamento" aumentando a distância entre o gráfico e a moldura utilizando `offset`. Atribuindo um valor para este argumento, as bordas serão afastadas naturalmente. Um tipo de "corte estético" pode ser adicionado à moldura com `trim=False`.

# In[48]:

sns.set_style("ticks")
p = sns.relplot(x = 'data', y='Óbitos',
                hue = 'Tipo',
                data=covid_PB_obitos_df.reset_index(),
                kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)
sns.despine(right=False, top=False, offset=30)

# In[49]:

sns.set_style("ticks")
p = sns.relplot(x = 'data', y='Óbitos',
                hue = 'Tipo',
                data=covid_PB_obitos_df.reset_index(),
                kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)
sns.despine(offset=30, trim=True)

# ### Contextos e escala
#
# O *seaborn* possui contextos pré-definidos que mudam a escala do gráfico para melhor satisfazer a aplicação de interesse. Para definir o contexto, utilizamos a função `set_context`. Há 4 contextos pré-definidos: *paper*, *notebook*, *talk* e *poster*.

# In[50]:

sns.set_context("poster")
sns.set_style("ticks")
p = sns.relplot(x = 'data', y='Óbitos',
                hue = 'Tipo',
                data=covid_PB_obitos_df.reset_index(),
                kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# In[51]:

sns.set_context("talk")
sns.set_style("ticks")
p = sns.relplot(x = 'data', y='Óbitos', hue = 'Tipo', data=covid_PB_obitos_df.reset_index(), kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# In[52]:

sns.set_context("notebook")
sns.set_style("ticks")
p = sns.relplot(x = 'data', y='Óbitos', hue = 'Tipo', data=covid_PB_obitos_df.reset_index(), kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# In[53]:

sns.set_context("paper")
sns.set_style("ticks")
p = sns.relplot(x = 'data', y='Óbitos', hue = 'Tipo', data=covid_PB_obitos_df.reset_index(), kind='line')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# ### Paletas e mapas de cores
#
# É possível personalizar a paleta de cores a ser utilizada ou escolher uma da lista (extremamente extensa) de paletas disponíveis utilizando a função `set_palette`. Duas formas usuais de selecionar a paleta de cores são:
#
# - no escopo de uma instrução `with` por meio da função `color_palette`;
# - pelo argumento `palette` nas funções de construção gráfica.
#
# A função `color_palette()` aceita nomes de uma paleta do _seaborn_ (_deep_, _muted_, _bright_, _pastel_,_dark_,_colorbind_), um mapa de cores (_colormap_) do _matplotlib_, uma sequência de cores em qualquer formato aceitável pelo _matplotlib_, entre outras opções.
#
# ```{note}
# Para uma lista ampla de paletas, veja a discussão neste [post](https://medium.com/@morganjonesartist/color-guide-to-seaborn-palettes-da849406d44f) ou a [_Colormap reference_](https://matplotlib.org/stable/gallery/color/colormap_reference.html) do _matplotlib_.
# ```

# Abaixo temos alguns exemplos de paletas nativas do _seaborn_

# In[54]:

sns.color_palette()

# In[55]:

sns.color_palette('colorblind')

# In[56]:

sns.color_palette('bright')

# e de mapas do cores do _matplotlib_.

# In[57]:

sns.color_palette('Greens')

# In[58]:

sns.color_palette('turbo')

# In[59]:

sns.color_palette('cividis')

# Nos exemplos a seguir, plotamos alguns gráficos novamente com paletas diferentes.

# In[60]:

# paleta: 'BuPu'
sns.set_context("paper")
sns.set_style("ticks")
p = sns.relplot(x = 'data', y='Óbitos',
                hue = 'Tipo',
                data=covid_PB_obitos_df.reset_index(), kind='line',
                palette = 'BuPu')
p.fig.autofmt_xdate()
p.ax.xaxis.set_minor_locator(mdates.DayLocator(interval=7)) #Intervalo entre os tracinhos
p.ax.xaxis.set_major_locator(mdates.DayLocator(interval=21)) #Intervalo entre as datas
p.ax.xaxis.set_major_formatter(mdates.DateFormatter('%d/%m/%Y')) #Formato da data
p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# In[61]:

# paleta: 'mako_r'
with sns.color_palette('mako_r'):
    p = sns.catplot(x='Região', y='Óbitos',
                    kind = 'violin',
                    data=covid_regioes_diarios_px)
    p.ax.set_xlabel(''); p.fig.set_size_inches(12,4)

# In[62]:

# paleta: icefire_r
sns.set_palette('icefire_r')
_ = sns.jointplot(x='obitos_Nordeste', y='obitos_Sudeste',
                  data = covid_regioes_diarios,
                  height=6)

# ## Nota
#
# Este capítulo baseia-se nas notas de aula da Profa. Andrea Rocha (CI/UFPB), elaboradas para o mini-curso [FMECD](https://gcpeixoto.github.io/FMECD/ipynb/01a-introducao.html).


#!/usr/bin/env python3

import unittest
from unittest.mock import patch
import numpy as np
import pandas as pd

from tmc import points

from tmc.utils import load, get_out, patch_helper

module_name="src.split_date_continues"
split_date_continues = load(module_name, "split_date_continues")
main = load(module_name, "main")
ph = patch_helper(module_name)

@points('p05-01.1')
class SplitDateContinues(unittest.TestCase):

    # @classmethod
    # def setUpClass(cls):
    #     cls.df = split_date_continues()

    def setUp(self):
        self.df = split_date_continues()

    def test_shape(self):
        self.assertEqual(self.df.shape, (37128, 25), msg="Incorrect shape!")

    def test_columns(self):
        np.testing.assert_array_equal(self.df.columns[:6],
                                      ['Weekday', 'Day', 'Month', 'Year', 'Hour', 'Auroransilta'],
                                      err_msg="First six column names were incorrect!")

    def test_dtypes(self):
        np.testing.assert_array_equal(self.df.dtypes[:6],
                                      [object, int, int, int, int, float],
                                      err_msg="Incorrect column types in first six columns!")

    def test_content(self):
        value = self.df.loc[0, "Auroransilta"]
        self.assertTrue(np.isnan(value),
                         msg="Incorrect value on row 0 column Auroransilta, expected NaN got %f!" % value)
        self.assertEqual(self.df.loc[0, "Baana"], 8.0,
                         msg="Incorrect value on row 0 column Baana!")

    def test_calls(self):
        with patch(ph("split_date_continues"), wraps=split_date_continues) as psplit,\
            patch(ph("pd.read_csv"), wraps=pd.read_csv) as prc,\
            patch(ph("pd.concat"), wraps=pd.concat) as pconcat:
            main()
            psplit.assert_called_once()
            prc.assert_called_once()
            pconcat.assert_called()

if __name__ == '__main__':
    unittest.main()


import sys
import numpy as np

a = np.array([])
b = np.array([1, 2, 3])
c = np.zeros(10**6)

for obj in [a, b, c]:
  print('sys:', sys.getsizeof(obj), 'np:', obj.nbytes)

Week 1/grok/samples/2a/28.check memory usage of numpy arr1.py


#Jupyter Notebook Keyboard Shortcuts
# Do not use ''' but use """ for block comment
"""
shift + tab				check function documentation
shift + enter 				run cell, select below.
ctrl + enter 				run cell
option/alt + enter 			run cell insert below.
A insert cell above.
B insert cell below.
C copy cell.
V paste cell.
D , D delete selected cell.
"""

# Basic Practice: http://codingbat.com/python
# More Mathematical (and Harder) Practice: https://projecteuler.net/archives
# List of Practice Problems: http://www.codeabbey.com/index/task_list
# A SubReddit Devoted to Daily Practice Problems: https://www.reddit.com/r/dailyprogrammer
# A very tricky website with very few hints and touch problems (Not for beginners but still interesting) http://www.pythonchallenge.com/

# Print Code
mystring = "Hello World"
print(mystring)
print(mystring[-2]) # l

#Slice
mystring = "0123456789"
print(mystring[0])  #index starts at 0
print(mystring[3:]) #prints 3 and onwards
print(mystring[:5]) #go up to but not including 5, returning 01234
print(mystring[3:6]) #prints and includes 3 up to but not include 6
print(mystring[::]) #beginning to end, step 1, same as print(mystring)
print(mystring[2:7:2]) #location 2 up to but excl 7, step 2
print(mystring[::-1]) #end to beginning

#Concatenate
mystring1 = 'Hello'
mystring2 = 'J' + mystring1[1:] #Jello    #strings are immutable
print(mystring2) #Jello
print(mystring2 + 'is Jello')
# for loop concatenate
for value,label in zip(result,labels):
    print(label + ' : ' +str(value))
    if result[1] <= 0.05:
        print('Reject null hypothesis')
    else:
        print('Fail to reject null hypothesis')

#Multiple strings
mystring = 'x' * 10
print(mystring) #xxxxxxxxxx

#Use String Functions
mystring = 'Hello World'
mystring = mystring.upper() #use tab to see functions
mystring.split() #['Hello', 'World']
mystring.split('l') #returns a list split by l

#Inserting a string
print('This is a string {}'.format('INSERTED'))
print('I like %s' %'apples')
print('Where did {0} {2} {1}'.format('He', 'Her', 'Meet')) # He Meet Her
print('The {b} {c} {d}'.format(d='dummy', c='called', b='boy')) #The boy called dummy

#Controlling outputs and FString
result = 100/777
print("The result was {r:1.5f}".format(r=result)) # R:whitespace.decimalsF The result was 0.129
name = 'Jose'
print(f'Hello, his name is {name}') #fstring string variable injection method

# Some other ways way number formatting
print('Mean Height: ', round(np.mean(height_surveys)/12,1), ' feet')
print('Standard Deviation of Height: ', round(np.var(height_surveys)**0.5 / 12, 1), ' feet')

#######################################################################################################################
#lists
my_list = [1,2,3]
my_list2 = ['STRING', 100, 200]
my_list3 = [0]*3 #[0, 0, 0]
my_list(1:)
len(my_list + my_list2)
new_list.append(4) #adds something to the end
new_list.pop(0) # removes position zero and returns it
new_list.sort() # a function that sorts in place and returns nothing
new_list2 = sorted(my_list)
type(new_list)

#nested list
my_list=[1,2,[a,b]]
my_list[2][1]

#generated random list 10000
# Calculate stats using single sample
sample_mean = np.mean(height_surveys[3])
sample_stdev = np.var(height_surveys[3])**0.5
# Calculate standard error = sample std dev / sqrt(N) where N = 10 in this example
std_error = sample_stdev/(height_surveys[3].shape[0])**0.5
# generate random list of 10000 normally distributed samples (sample mean normal distribution)
inferred_distribution = [sample_mean + np.random.normal()*\
                 std_error for i in range(10000)]

#dictionaries have use "string" key values, do not need exact index, can't sort (because it is a mapping and not a sequence), use for vlookup but can be fancy lookup of lists/nested lookup
my_dict = {'key1':'value1','key2':'value2'}
my_dict['key1']
prices_lookup = {'apple':2.99,'oranges':2,'milk':5}
prices_lookup['apple']
prices_lookup['apple'] = 5 #reassign and mutable
fancy_dict = {'k1':123,'k2':[0,1,2,3],'k3':{'insidekey':100}}
fancy_dict['k3']['insidekey']
len(prices_lookup)
type(prices_lookup)
prices_lookup.keys()			#returns (['key1', 'key2'])
prices_lookup.values()
prices_lookup.items()   #returns the package dict_items([('key1', 'value1'), ('key2', 'value2')]) and requires for loop to unpack it

#tricky mixed up
# Getting a little tricker
d = {'k1':[{'nest_key':['this is deep',['hello']]}]}
d['k1'][0]['nest_key'][1][0] #Grab hello

#tuples are similar to lists, but IMMUTABLE, so it's good for data integrity when passing around objects that can't change once defined (can't re-assign like lists can)
t = ('a','a','b')
x = ('a',[1,2],'c')
t.count('a')
t[0] = 'NEW' #will throw an error

#sets, sets do not allow duplicate items, they are unordered collections of UNIQUE elements
myset = set()
mylist = [1,1,1,1,1,1,1,1,1,2,2,2,2,3,3,3,3]
myset = set(mylist) #list injection into a set, returns no repeats values or uniquely 1, 2, 3
myset = set()
myset.add(1) #looks like dictionary with {} but no key value pairs, so not
myset.add(2)
myset.add(2) #won't throw error and won't repeat it

#######################################################################################################################
#boolean for comparison
True
False
1 > 2
1 == 2
1 != 2
b = None #to avoid object not defined yet

#comparisons and or not
'3' == 3 #returns false due to different types
'Bye' == 'bye' #returns false due to case sensitive
3.0 == 3 #returns true because both are numbers
3 != 4 #returns true
(1 < 2) and (2 > 3)
(1 < 2) or (2 > 3)
not (1 != 2)

#######################################################################################################################
#opening files
myfile = open('file.txt')
myfile = open('C:\\Users\\Username\\Folder\\test.txt')
pwd
contents = myfile.read()  #reading it again and the cursor is moved to the end
myfile.seek(0) #resets the cursor
myfile.readlines() #each line as a separate element in the list
myfile.close()

#more formal way of opening files, use shift tab
with open('myfile.txt', mode='r') as my_new_file:
	contents = my_new_file.read()
#open as r = readonly, w = overWrite/create, a = Append, r+ read and write, w+ overwrite existing and read
with open('myfile.txt', mode='a') as my_new_file:
	my_new_file.write('Four\n')

#*************************************************************************************************************

# control flows
hungry = True
location = 'Stadium'
if hungry:
	# do x
	print('x')
elif location=='Bank':
	# do y
	print('y')
else:
	# do z
	print('z')

#*************************************************************************************************************

# for loop mod
myList = [1,2,3]
for xItem in myList:
	print(xItem)
for xItem in myList:
	if xItem % 2 == 0:
		print(xItem)
	else:
		print(f'Odd Number: {xItem}')

# tuple unpacking will print the imitated structure
myList = [(1,2),(3,4),(5,6),(7,8)]
for (a,b) in myList
	print(a) # returns 1... 3, etc
	print(b) #returns 2... 4...
for a,b in myList   # no brackets works too
	print(a)
	print(b)

# dictionary unpacking and dictionary is unordered
myDict = {'k1':1, 'k2':2, 'k3':3}
for xItem in myDict:
	print(xItem) # only returns the k1, k2...
for xItem in myDict.items():
	print(xItem) # would return the tuple ('k1', 1)
for xKey, xValue in myDict.items():
	print(xValue) # would return the content

# while statements
x = 0
while x < 5:
	print(f'X is {x}')
	x = x+1
# while statement with an else
x = 0
while x < 5:
	print(f'X is {x}')
	x = x+1
else:
	print('Code complete')

#loop management
break 		#breaks and stops out of closest loop, useful for while loop
pass		#go back to top of closest loop, skip the rest of the instructions
continue 	#do nothing

# pass: example in a 'for loop'
myList = [1,2,3]
for xItem in myList:
	#comment alone would not compile as compiler expects a tabbed action and comment is not an action
	pass
print('end of loop')

# continue: example in a 'for loop'
myString = 'hello world'
for xChar in myString:
	if xChar == 'o':
		continue 			#this would skip all the o but allow the loop to continue
	print(xChar)			#prints 'hell wrld' with each line break between the letters

# break: example in a for loop
myString = 'hello world'
for xChar in myString:
	if xChar == 'o':
		break 				#this would break out
	print(xChar)			#prints 'hell' with each line break between the letters

#*************************************************************************************************************

# range operator
for xItem in range(3,10):
	print(xItem) # prints 3 to 9 and not include 10
for xItem in range(0,10,2):
	print(xItem) # prints 2, 4, 6, 8, 10 (not incl 11)
myList = (range(0,10,2)) # generator a list using range

# query the list with for loop
index_count = 0
for letter in 'abcde':
	print(f'The index is: {index_count} and the letter is {letter})
	index_count += 1

# Access the sub components of a word or list
index_count = 0
word = 'hello'
for letter in word:
	print(word[index_count])
	index_count += 1
# Exact same but using enumerate
word = 'hello'
for item in enumerate(word):
	print(item) #returns tuples such as (0, 'h'), (1, 'e'), etc...
# Exact same but using enumerate and tuple unpacking
word = 'hello'
for index,letter in enumerate(word):
	print(index)
	print(letter) #returns tuple unpacked

#zip operator
my_list = [1, 2, 3, 4, 5, 6]
my_list2 = ['a', 'b', 'c', 'd', 'e']
for item in zip(my_list, my_list2):
	print(item) #returns tuples, (1, 'a') (2, 'b')
	print('\n') #the zip only goes to the shortest so ignores 6
my_list3 = list(zip(my_list,my_list2) #returns [(1,'a'), (2,'b'), (3,'c')]
#check if in list
2 in [1,2,3] 		#returns true
'a' in 'helloapple' #returns true
d = {'mykey':345} 	#returns true
'mykey' in d 		#returns true
345 in d.values 	#returns true
345 in d.keys 		#returns false
min(my_list) 		#returns 1
max(my_list)		#returns 6

#scramble the list
shuffle(my_list) 				#would reshuffle the list
from random import shuffle
random_list = shuffle(mylist)	#no error but doesn't return

#random integer
from random import randint
mynum = randint(0, 100) #returns a random integer

#input
results = input('What is your name')
results 		#returns always a string
type(results)	#returns string
float(results)	#cast to a number
int(results)	#cast to a number
results = int(input('Favorite Number'))

#list append
mystring = 'hello'
mylist = []
for letter in mystring:
	mylist.append(letter) # ['h', 'e', 'l', 'l', 'o']

#list comprehension append 2, flattened for loop
mylist = [letter for letter in mystring]   # ['h', 'e', 'l', 'l', 'o']
mylist = [x for x in 'word']
mylist = [num for num in range(0,11)] #generate a series
mylist = [num**2 for num in range(0,11)] #perform the math operation
mylist = [x for x in range(0,11) if x%2 == 0] #only take the even numbers

st = 'Create a list of the first letters of every word in this string'
mylist = [x[0] for x in st.split()]    # ['C', 'a', 'l', 'o', 't', 'f', 'l', 'o', 'e', 'w', 'i', 't', 's']

#another example of list to list
c = [1, 2, 3, 4, 5]
f = [(( 9/5) * temp  + 32) for temp in c]    #load the list without for loop
#same as list to list but using append
listgrab = []
for x in c:
    listgrab.append((( 9/5) * x  + 32))		#load the list with for loop

#nested loop
mylist = []
for x in [2, 4, 6]:
	for y in [100, 200]:
		mylist.append(x*y)

#fizz buzz
mylist2 = []
for x in range(1, 101):
    if x % 15 == 0:
        mylist2.append('FizzBuzz')
    elif x % 5 == 0:
        mylist2.append('Buzz')
    elif x % 3 == 0:
        mylist2.append('Fizz')
    else:
        mylist2.append(x)
mylist2

#documentation
help(mylist.insert) # not help(mylist.insert())

#functions and functions that return
def some_function():
	'''
	DOCSTRING: Some documentation
	INPUT: expected input
	OUTPUT: expected output
	'''
	print('hello')

help(some_function) # would read the docstring

def some_function2(varX):
	print('x'+varX) # would throw error if no varX provided

def some_function_provide_nothing(name='NAME'):
	print('hello '+name) # has a default name

def simple_add(num1, num2):
	return num1 + num2

result = simple_add(1,2)
print(result) # expecting 3

def dog_check(mystring):
	if 'dog' in mystring.lower():
		return True
	else:
		return False

# better statement as it is already a boolean
def dog_check(mystring):
	return 'dog' in mystring.lower()

# if starts with a vowel then add 'ay', otherwise move 1st letter to end and add ay (string concat concatenate)
def pig_latin(word):
	first_letter = word[0]
	# check if vowel
	if first_letter in 'aeiou':
		pig_word = word + 'ay'
	else:
		pig_word = word[1:] + first_letter + 'ay'
	return pig_word

# what if you want to pass parameters? any problems?
def myfunc(a, b, c=0,d=0):
	# a and b are positional arguments, passed in as a tuple with only 2 parameters
	# return 5% of the sum of (a + b)
	return sum(   (a,b,c,d)  ) * 0.05     # entered as a tuple

# arbitrary number of arguments: can now pass in as many arguments as i want
# *args is just a convention, you can do *blahblah
def myfunc(*args):
	return sum(args) * 0.05
	print args # looks just like a tuple, the * term allows you to pass in as many as you want, can loop or aggregate it

myfunc(10,20,30,40)

# arbitrary number of key word arguments, returns back a dictionary (of key value pairs)
# can do whatever you want inside your function with a list of dictionary items
# **kwargs is an arbitrary choice, but the two asterix is what's indicated to python
def myfunc(**kwargs):
	if 'fruit' in kwargs:
		print('My fruit of choice is {}'.format(kwargs['fruit']))
	else:
		print('I did not find any fruit here')

myfunc(fruit='apple', veggie = 'lettuce') # output = 'My fruit of choice is apple'

# accepting both arguments, great for outside libraries, careful of ordering
def myfunc(*args,**kwargs):
	print(args)
	print(kwargs)
	print ('I would like {} {}'.format(args[0],kwargs['food']))

myfunc(10,20,30,fruit='orange',food='eggs',animal='dog') # output = 'I would like 10 eggs'

# test to only display even
def myfunc(*args):
    mylist = []
    for x in range(0,len(args)):
        if (args[x] % 2) == 0:
            mylist.append(args[x])
    return mylist

# string modification with *args
# if even then upper case, if odd then lower case
def myfunc(*args):
    mystring = ""
    tempstring = ""
    for x in range(0,len(args[0])):
        tempstring = args[0][x]
        if x % 2 == 0:
            mystring = mystring + tempstring.upper()
        else:
            mystring = mystring + tempstring.lower()
    return mystring

# how to use sum
def blackjack(a, b, c):
	if sum([a,b,c]) <= 21:
		return sum([a,b,c])
	elif 11 in [a,b,c] and sum([a,b,c])-10 <= 21:
		return sum([a,b,c]) - 10
	else:
		return "Bust"

# how to use while loops and breaks
def summer(array_numbers):
	num = 0
	adder = True
	for num in array_numbers:
		while adder:
			if num != 6:
				total += num
				break
			else:
				adder = False
		while not adder:
			if num != 9:
				break
			else:
				adder = True
				break
	return total
#usage:
summer([1, 3, 5, 7])

#################################################################################################################
# how to enumerate an array with identifiers
###########################
# enumerate the first 10 messages (0 to 9)
for message_no, message in enumerate(messages[:10]):
	print(message_no, message)
	print('\n')

#################################################################################################################
# how to pop off array items
###########################
def codewords(nums):
	code = [1, 2, 3, 'x']
	# [2, 3, 'x']
	# [3, 'x']
	# ['x']
	for num in nums:
		if num == code[0]:
			code.pop(0)
	return len(code) == 1

# for... else combo
# find primes
def count_primes(num):

	# check for 0 and 1
	if num < 2: return 0

	# storage
	primes = [2]
	x = 3

	while x <= num:
		for y in range(3, x, 2):
			if x%y == 0:
				x += 2				# skip ahead of even numbers
				break
		else:						# this only runs if it doesn't break out of "for"
			primes.append(x)
			x += 2
	print(primes)
	return len(primes)

#################################################################################################################
# pandas dataframe basics

import pandas as pd
df = pd.read_csv('some_csv.csv')
df.columns # shows columns
df.index
len(df.index) # number of rows
len(df.columns) # number of columns
df.info() # for all the info

# add a column using apply method (this function converts Yes to 1 and else to 0)
def converter(private_school):
    if private_school == 'Yes':
        return 1
    else:
        return 0
df['Cluster'] = df['Private'].apply(converter)

# look at one entry and edit it
example_entry = personnel['Email'].iloc[0]
df.loc['Cazenovia College', 'Grad.Rate']=100 # sets that value to 100 (do not use the .at function will give slicer dataframe warning)
# use lambda function to split email
example_entry.split('@')[1] # would give hotmail.com
host = personnel['Email'].apply(lambda email: email.split('@')[1]) # would also give hotmail.com

# filter comparison AND
count_amex_and_above95 = df[(df['CC Provider']=='AMEX') & df['Purchase Price'] > 95)].count()

# unique count of classifications (already sorted)
df['Job Title'].value_counts()

# lambda filter on a text string in CC Exp Date for last 2 digits of '10/25' so 3: = 25
sum(df['CC Exp Date'].apply(lambda exp: exp[3:]=='25')) # 1033
df[ df['CC Exp Date'].apply(lambda exp: exp[3:]=='25')].count() # 1033
df[ df['CC Exp Date'].apply(lambda exp: exp[3:]=='25')].value_counts().head(5)

# lambda expression with IF statement to add new column to dataframe
df['loan_repaid'] = df['loan_status'].apply(lambda status: status=='Fully Paid')  # returns column of True/False
df['loan_repaid'] = df['loan_status'].apply(lambda x : True if (x > 30 and x < 20) else False) # returns column of True/False
df['loan_repaid'] = df['loan_status'].apply(lambda x : 1 if (x == 'Fully Paid') else 0) # returns column of 1/0
df.drop('loan_status', axis=1, inplace=True) # remove text row

# another way to create dummy columns (without lambda)
df['verification_status'].value_counts() # see what the columns will be
statuses = pd.get_dummies(df['verification_status'], drop_first=True) # drop first for machine learning but can remove

#################################################################################################################
# pandas groupby

# group by to see data
# Can also pass column names as group keys
df.groupby('column1_type').mean()

# Make some arrays for use as keys
# Remember np.array(my_list) creates a numpy array
cities = np.array(['NY','LA','LA','NY','NY'])
month = np.array(['JAN','FEB','JAN','FEB','JAN'])
#Now using the data from dataset1, group the means by city and month
df['dataset1'].groupby([cities,month]).mean()

# more group by examples on rows and columns
import pandas as pd
import numpy as np
from pandas import Series,DataFrame
animals = DataFrame(np.arange(16).reshape(4, 4),
                   columns=['W', 'X', 'Y', 'Z'],
                   index=['Dog', 'Cat', 'Bird', 'Mouse'])
# Now modify certain entries with some NAN values
animals.loc[1:2, ['W', 'Y']] = np.nan
animals

# Map Rows Index (Dog, Cat, Bird, Mouse) to Categories
animal_map = {'Dog': 'house', 'Cat': 'house', 'Bird': 'nest','Mouse': 'hole'}
# Now let's try it with a Series
animal_series = Series(animal_map)
# Now let's groupby the Series
animals.groupby(animal_series, axis=0).sum()

# Map Columns (W, X, Y, Z) to Categories
behavior_map = {'W': 'good', 'X': 'bad', 'Y': 'good', 'Z': 'bad'}
# Now let's try it with a Series
behav_series = Series(behavior_map)
# Now let's groupby the Series
animals.groupby(behav_series, axis=1).count()

#################################################################################################################
# pandas melt
df = pd.DataFrame({'A': {0: 'a', 1: 'b', 2: 'c'},
                   'B': {0: 1, 1: 3, 2: 5},
                   'C': {0: 2, 1: 4, 2: 6}})
"""
df
   A  B  C
0  a  1  2
1  b  3  4
2  c  5  6
"""
pd.melt(df, id_vars=['A'], value_vars=['B'])

"""
   A variable  value
0  a        B      1
1  b        B      3
2  c        B      5
"""

#################################################################################################################
# Pandas DataFrames - Join/Merge/Pivot Table Recommender System

# read specific columns
column_names = ['user_id', 'item_id', 'rating', 'timestamp']
df = pd.read_csv('u.data', sep='\t', names=column_names)

# get list of movie titles
movie_titles = pd.read_csv("Movie_Id_Titles")
movie_titles.head()

# merge two dataframes together using Movie ID (to get the text titles and ratings)
df = pd.merge(df,movie_titles,on='item_id')
df.head()

# create a dataframe with average ratings and sort them and take a look at the top of the data
df.groupby('title')['rating'].mean().sort_values(ascending=False).head()
df.groupby('title')['rating'].count().sort_values(ascending=False).head()
# average ratings dataframe
ratings = pd.DataFrame(df.groupby('title')['rating'].mean())
ratings.head()
# number of ratings dataframe
ratings['num of ratings'] = pd.DataFrame(df.groupby('title')['rating'].count())
ratings.head()

# visualize outlier data in number of ratings - notice a lot of films only had one person rate (distribution skewed left x-axis)
sns.set_style('white')
plt.figure(figsize=(15,8))
ratings['num of ratings'].hist(bins=70)
# visualize outlier data in average ratings - notice a lot of films only had one person rate (concentration on ends, 1 and 5)
plt.figure(figsize=(10,4))
ratings['rating'].hist(bins=70)
# create a joint plot to see how the ratings and number of ratings are distributed
sns.jointplot(x='rating',y='num of ratings',data=ratings,alpha=0.5)

# create recommendation matrix using simple correlations by user_id
# Step 1: the matrix will have all the users (each row), all titles (each column), and ratings (in the values)
moviemat = df.pivot_table(index='user_id',columns='title',values='rating')
moviemat.head() # creates a massive dataset = 944 rows × 1664 columns (excel would explode)
# understand your data - sort the values to see the highest number of ratings (ascending = False)
ratings.sort_values('num of ratings', ascending=False).head(10)

# Step 2: select top two movies and make recommendation - if someone likes this movie what else they could like (by highest corr)
# get all users and Star Wars rating (would have some NaN if no rating)
starwars_user_ratings = moviemat['Star Wars (1977)'] # rows = all users, one column = star wars rating
# get all users and Liar Liar rating (would have some NaN if no rating)
liarliar_user_ratings = moviemat['Liar Liar (1997)']
# so the .corrwith creates a correlation against two dataframes
# Compute pairwise correlation between rows or columns of DataFrame with rows or columns of Series or DataFrame.
# DataFrames are first aligned along both axes before computing the correlations.
# 944 rows × 1664 columns vs. 944 rows x 1 columns
similar_to_starwars = moviemat.corrwith(starwars_user_ratings)  # returns 1664 rows movies and 1 correlation number in a Series
similar_to_liarliar = moviemat.corrwith(liarliar_user_ratings)
similar_to_starwars # title and correlation in a series
# Put series into a dataframe
corr_starwars = pd.DataFrame(similar_to_starwars,columns=['Correlation'])
# Clean up the dataframe by dropping the NaN
corr_starwars.dropna(inplace=True)
corr_starwars.head()

# Step 3: sort data by highest correlation and pull out results that do not make sense
corr_starwars.sort_values('Correlation',ascending=False).head(10)
# look back to chart and see that the cut off is around 100 reviews, so add the number of ratings to the correlation
# the join method requires that the Index is the same (in this case it is, it is the Title)
corr_starwars = corr_starwars.join(ratings['num of ratings'])
corr_starwars.head()
# apply the filter (note ascending=false to get highest first)
corr_starwars[corr_starwars['num of ratings']>100].sort_values('Correlation',ascending=False).head()
# do the same for liar liar lovers
corr_liarliar = pd.DataFrame(similar_to_liarliar,columns=['Correlation'])
corr_liarliar.dropna(inplace=True)
corr_liarliar = corr_liarliar.join(ratings['num of ratings'])
corr_liarliar[corr_liarliar['num of ratings']>100].sort_values('Correlation',ascending=False).head()

#################################################################################################################
# lambda functions for dummies

# Example 1
x = lambda a : a + 10
print(x(5))   # 15

# Example 2a
# Compare Function vs. Lambda
# put it all in one line, get rid of return, get rid of parenthesis, get rid of name
def cube(y): return y*y*y;
g =       lambda x: x*x*x
print(cube(5)) # 125
print(g(7)) # 343

# Example 2b
# Compare if even
even = lambda num: num%2 == 0
even(4) # True
even(3) # False

# Return first letter of string
first = lambda s: s[0]
first('asdf') # a

# Return reverse order of string
reversal = lambda s: s[::-1]
reversal('asdf') # fdsa

# Multiple entries
addermulti = lambda x,y: x+y
addermulti(3,3)
# Multi variables
x = lambda a, b : a * b
print(x(5, 6))  # 30

# Example 3
# Python code to illustrate filter() with lambda()
li = [5, 7, 22, 97, 54, 62, 77, 23, 73, 61]
final_list = list(filter(lambda x: (x%2 != 0) , li))
print(final_list) # [5, 7, 97, 77, 23, 73, 61]

# Example 4 - removal of illegal character - return position 1 and onwards if first entry has # ex) #EVENTID else line
text_string.map(lambda line: line[1:] if line[0] == '#' else line)
# string tokenization token split
text_string.map(lambda line: line.split())

# Example 5
# map() with lambda()
# to get double of a list.
print("map with lambda function: ")
li = [5, 7, 22, 97, 54, 62, 77, 23, 73, 61]
final_list = list(map(lambda x: x*2 , li))
print(final_list)   # [10, 14, 44, 194, 108, 124, 154, 46, 146, 122]

# Example 6
# another map example
x = [2, 3, 4, 5, 6]
y = map(lambda v : v * 5, x)
print(list(y))

# Example 7 - to use in Spark
clean = text_string.map(lambda line: line[1:] if line[0] == '#' else line)
# create a TUPLE grab State and Amount
step1 = clean.map(lambda lst: (lst[3], lst[-1])   )
# ReKey = Reduce by Key (like GroupBy) and convert to Float so the + works
step2 = step1.reduceByKey(lambda amt1,amt2 : float(amt1) + float(amt2))
# Get rid of title (first line)
step3 = step2.filter(lambda x: not x[0]=='State')
# Sort Results by Amount
step4 = step3.sortBy(lambda stAmount: stAmount[1],ascending=False)
# Perform the action
step4.collect()

# late binding closures issue in python
# Five functions are created; instead all of them just multiply x by 4
print("with late binding closure: ")
def create_multipliers():
    return [lambda x : i * x for i in range(5)]      # () seems to fix? return (lambda x : i * x for i in range(5))
for multiplier in create_multipliers():
    print(multiplier(2))
    # five 8 are printed: 8, 8, 8, 8, 8

# late binding closures work around
print("with late binding closure fixed: ")
def create_multipliers_fix():
    return [lambda x, i=i : i * x for i in range(5)]
for multiplier in create_multipliers_fix():
    print(multiplier(2))
    # 0, 2, 4, 6,8

# creates iterable and skip the function
print("another iterable lambda: ")
another = (lambda i: i + x for x in range(5))
for multiplier in another:
    print(multiplier(1))

"""
with late binding closure:
8
8
8
8
8
with late binding closure fixed:
0
2
4
6
8
another iterable lambda:
1
2
3
4
5
"""

#################################################################################################################
# Load an image
from IPython.display import Image
url = 'http://upload.wikimedia.org/wikipedia/commons/5/56/Kosaciec_szczecinkowaty_Iris_setosa.jpg'
Image(url,width=300, height=300)

#################################################################################################################
#################################################################################################################
#################################################################################################################
#################################################################################################################
#################################################################################################################
#################################################################################################################
#################################################################################################################

# hi


# ---
# jupyter:
#   jupytext:
#     formats: notebooks//ipynb,markdown_files//md,python_scripts//py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.2'
#       jupytext_version: 1.2.1
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %% {"deletable": true, "editable": true}
# %matplotlib inline
import numpy as np
import matplotlib.pyplot as plt

# %% [markdown] {"deletable": true, "editable": true}
# # Pipelining estimators

# %% [markdown] {"deletable": true, "editable": true}
# In this section we study how different estimators maybe be chained.

# %% [markdown] {"deletable": true, "editable": true}
# ## A simple example: feature extraction and selection before an estimator

# %% [markdown] {"deletable": true, "editable": true}
# ### Feature extraction: vectorizer

# %% [markdown] {"deletable": true, "editable": true}
# For some types of data, for instance text data, a feature extraction step must be applied to convert it to numerical features.
# To illustrate we load the SMS spam dataset we used earlier.

# %% {"deletable": true, "editable": true}
import os

with open(os.path.join("datasets", "smsspam", "SMSSpamCollection")) as f:
    lines = [line.strip().split("\t") for line in f.readlines()]
text = [x[1] for x in lines]
y = [x[0] == "ham" for x in lines]

# %% {"deletable": true, "editable": true}
from sklearn.model_selection import train_test_split

text_train, text_test, y_train, y_test = train_test_split(text, y)

# %% [markdown] {"deletable": true, "editable": true}
# Previously, we applied the feature extraction manually, like so:

# %% {"deletable": true, "editable": true}
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression

vectorizer = TfidfVectorizer()
vectorizer.fit(text_train)

X_train = vectorizer.transform(text_train)
X_test = vectorizer.transform(text_test)

clf = LogisticRegression()
clf.fit(X_train, y_train)

clf.score(X_test, y_test)

# %% [markdown] {"deletable": true, "editable": true}
# The situation where we learn a transformation and then apply it to the test data is very common in machine learning.
# Therefore scikit-learn has a shortcut for this, called pipelines:

# %% {"deletable": true, "editable": true}
from sklearn.pipeline import make_pipeline

pipeline = make_pipeline(TfidfVectorizer(), LogisticRegression())
pipeline.fit(text_train, y_train)
pipeline.score(text_test, y_test)

# %% [markdown] {"deletable": true, "editable": true}
# As you can see, this makes the code much shorter and easier to handle. Behind the scenes, exactly the same as above is happening. When calling fit on the pipeline, it will call fit on each step in turn.
#
# After the first step is fit, it will use the ``transform`` method of the first step to create a new representation.
# This will then be fed to the ``fit`` of the next step, and so on.
# Finally, on the last step, only ``fit`` is called.
#
# ![pipeline](figures/pipeline.svg)
#
# If we call ``score``, only ``transform`` will be called on each step - this could be the test set after all! Then, on the last step, ``score`` is called with the new representation. The same goes for ``predict``.

# %% [markdown] {"deletable": true, "editable": true}
# Building pipelines not only simplifies the code, it is also important for model selection.
# Say we want to grid-search C to tune our Logistic Regression above.
#
# Let's say we do it like this:

# %% {"deletable": true, "editable": true}
# This illustrates a common mistake. Don't use this code!
from sklearn.model_selection import GridSearchCV

vectorizer = TfidfVectorizer()
vectorizer.fit(text_train)

X_train = vectorizer.transform(text_train)
X_test = vectorizer.transform(text_test)

clf = LogisticRegression()
grid = GridSearchCV(clf, param_grid={'C': [.1, 1, 10, 100]}, cv=5)
grid.fit(X_train, y_train)

# %% [markdown] {"deletable": true, "editable": true}
# ### What did we do wrong?

# %% [markdown] {"deletable": true, "editable": true}
# Here, we did grid-search with cross-validation on ``X_train``. However, when applying ``TfidfVectorizer``, it saw all of the ``X_train``,
# not only the training folds! So it could use knowledge of the frequency of the words in the test-folds. This is called "contamination" of the test set, and leads to too optimistic estimates of generalization performance, or badly selected parameters.
# We can fix this with the pipeline, though:

# %% {"deletable": true, "editable": true}
from sklearn.model_selection import GridSearchCV

pipeline = make_pipeline(TfidfVectorizer(),
                         LogisticRegression())

grid = GridSearchCV(pipeline,
                    param_grid={'logisticregression__C': [.1, 1, 10, 100]}, cv=5)

grid.fit(text_train, y_train)
grid.score(text_test, y_test)

# %% [markdown] {"deletable": true, "editable": true}
# Note that we need to tell the pipeline where at which step we wanted to set the parameter ``C``.
# We can do this using the special ``__`` syntax. The name before the ``__`` is simply the name of the class, the part after ``__`` is the parameter we want to set with grid-search.

# %% [markdown] {"deletable": true, "editable": true}
#

# %% [markdown] {"deletable": true, "editable": true}
# Another benefit of using pipelines is that we can now also search over parameters of the feature extraction with ``GridSearchCV``:

# %% {"deletable": true, "editable": true}
from sklearn.model_selection import GridSearchCV

pipeline = make_pipeline(TfidfVectorizer(), LogisticRegression())

params = {'logisticregression__C': [.1, 1, 10, 100],
          "tfidfvectorizer__ngram_range": [(1, 1), (1, 2), (2, 2)]}
grid = GridSearchCV(pipeline, param_grid=params, cv=5)
grid.fit(text_train, y_train)
print(grid.best_params_)
grid.score(text_test, y_test)

# %% [markdown] {"deletable": true, "editable": true}
#

#     EXERCISE:
#

#

#       Create a pipeline out of a StandardScaler and Ridge regression and apply it to the Boston housing dataset (load using ``sklearn.datasets.load_boston``). Try adding the ``sklearn.preprocessing.PolynomialFeatures`` transformer as a second preprocessing step, and grid-search the degree of the polynomials (try 1, 2 and 3).
#

#

#

# %% {"deletable": true, "editable": true}
# # %load solutions/15A_ridge_grid.py


import numpy as np

import sys
sys.path.append(".")
from ai.action.movement.movements.basic import *
import ai.actionplanner

def attacking(mars):
    rand_speed = np.random.uniform(0.2, 0.4)

    mars.setLegAngle(1, 2, 80, rand_speed)
    mars.setLegAngle(2, 2, 80, rand_speed)
    ai.actionplanner.ActionPlanner.sleep(1)
    mars.setLegAngle(1, 3, -10, rand_speed)
    mars.setLegAngle(2, 3, -10, rand_speed)
    ai.actionplanner.ActionPlanner.sleep(1)
    mars.setLegAngle(1, 1, -20, rand_speed)
    mars.setLegAngle(2, 1, -20, rand_speed)
    move_head_tail(mars,1)

    times = int(np.random.uniform(3, 5))

    for i in range(times):
        rand_speed_2 = 0.8
        rand_angle = 8

        mars.setLegAngle(3, 1, 0, rand_speed_2)
        mars.setLegAngle(4, 1, rand_angle, rand_speed_2)
        move_tail(mars)
        ai.actionplanner.ActionPlanner.sleep(0.5)
        mars.setLegAngle(3, 1, rand_angle, rand_speed_2)
        mars.setLegAngle(4, 1, 0, rand_speed_2)
        move_tail(mars)
        ai.actionplanner.ActionPlanner.sleep(0.5)

    ai.actionplanner.ActionPlanner.sleep(0.5)
    mars.setLegAngle(3, 2, -30, rand_speed)
    mars.setLegAngle(4, 2, -30, rand_speed)
    move_head_tail(mars,1)
    mars.setLegAngle(3, 3, -50, rand_speed)
    mars.setLegAngle(4, 3, -50, rand_speed)

    move_head_tail(mars,1)