# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:percent
#     notebook_metadata_filter: all
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.2'
#       jupytext_version: 1.2.3
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
#   language_info:
#     codemirror_mode:
#       name: ipython
#       version: 3
#     file_extension: .py
#     mimetype: text/x-python
#     name: python
#     nbconvert_exporter: python
#     pygments_lexer: ipython3
#     version: 3.7.6
# ---

# %% [markdown]
# # ConsPortfolioModel Documentation
#
# [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/econ-ark/DemARK/master?filepath=notebooks%2FConsPortfolioModelDoc.ipynb)

# %% {"code_folding": []}
# Setup stuff

import HARK.ConsumptionSaving.ConsPortfolioModel as cpm
import HARK.ConsumptionSaving.ConsumerParameters as param
import copy
import matplotlib.pyplot as plt
import numpy as np
import warnings
warnings.filterwarnings('ignore')
from HARK.ConsumptionSaving.ConsPortfolioModel import PortfolioSolution

# %% [markdown]
# We implement three different ways to allow portfolio choice.
#            The agent can choose
#               * any portfolio share ('continuous choice')
#               * only a specified set of portfolio shares ('discrete choice')
#                 * With probability 1 (agent always gets to choose)
#                 * With probability 0 < p < 1 (stochastic chance to choose)
#
#            We allow two choices for the description of the
#            distribution of the stochastic variable:
#            1. A generic discrete probability distribution
#               * Nodes and their probabilities are specified
#            2. A true lognormal distribution
#               * The mean return and the standard deviation are specified
#
#            In the discrete portfolio shares case, the user also must
#            input a function that *draws* from the distribution in drawRiskyFunc
#
#            Other assumptions:
#               * distributions are time constant
#               * probability of being allowed to reoptimize is time constant
#                  * If p < 1, you must specify the PortfolioSet discretely
#

# %% {"code_folding": []}
# Set up the model
# Parameters from Mehra and Prescott (1985):
Avg = 1.08 # equity premium
Std = 0.20 # standard deviation of rate-of-return shocks

RiskyDstnFunc = cpm.RiskyDstnFactory(RiskyAvg=Avg, RiskyStd=Std)       # Generates nodes for integration
RiskyDrawFunc = cpm.LogNormalRiskyDstnDraw(RiskyAvg=Avg, RiskyStd=Std) # Function to generate draws from a lognormal distribution

init_portfolio = copy.copy(param.init_idiosyncratic_shocks) # Default parameter values for inf horiz model - including labor income with transitory and permanent shocks
init_portfolio['approxRiskyDstn'] = RiskyDstnFunc
init_portfolio['drawRiskyFunc']   = RiskyDrawFunc
init_portfolio['RiskyCount']      = 2   # Number of points in the approximation; 2 points is minimum
init_portfolio['RiskyShareCount'] = 25  # How many discrete points to allow for portfolio share
init_portfolio['Rfree']           = 1.0 # Riskfree return factor (interest rate is zero)
init_portfolio['CRRA']            = 6.0 # Relative risk aversion

# Uninteresting technical parameters:
init_portfolio['aXtraMax']        = 100
init_portfolio['aXtraCount']      = 50
init_portfolio['BoroCnstArt']     = 0.0 # important for theoretical reasons
# init_portfolio['vFuncBool'] = True # We do not need value function for purposes here

init_portfolio['DiscFac'] = 0.92 # Make them impatient even wrt a riskfree return of 1.08

# Create portfolio choice consumer type
pcct = cpm.PortfolioConsumerType(**init_portfolio)

# %% {"code_folding": []}
# Solve the model under the given parameters

pcct.solve()
aMin = 0   # Minimum ratio of assets to income to plot
aMax = 10  # Maximum ratio of assets to income to plot
aPts = 100 # Number of points to plot

# Campbell-Viceira (2002) approximation to optimal portfolio share in Merton-Samuelson (1969) model
pcct.MertSamCampVicShare = pcct.RiskyShareLimitFunc(RiskyDstnFunc(init_portfolio['RiskyCount']))

# Define grid of points on which to plot
eevalgrid = np.linspace(0,aMax,aPts) # range of values of assets for the plot

# Plot portfolio share by wealth-to-income ratio
plt.plot(eevalgrid, pcct.solution[0].RiskyShareFunc[0][0](eevalgrid))
plt.axhline(pcct.MertSamCampVicShare, c='r') # The Campbell-Viceira approximation
plt.ylim(0,1.05)
plt.text((aMax-aMin)/4,0.45,r'$\uparrow $ limit as  $m \uparrow \infty$',fontsize = 22,fontweight='bold')
plt.show()

# %% {"code_folding": [0]}
# Simulate 20 years of behavior for a set of consumers initially distributed widely
SimPer = 20

pcct.track_vars = ['aNrmNow', 't_age', 'RiskyShareNow']
pcct.T_sim = SimPer
pcct.initializeSim()
pcct.simulate()
pcct.RiskyShareNow_hist

from mpl_toolkits.mplot3d import Axes3D
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_zlim(pcct.MertSamCampVicShare,1.0)
ax.scatter(pcct.aNrmNow_hist, pcct.t_age_hist, pcct.RiskyShareNow_hist)
plt.show()

# The consumers are impatient and so even if they start rich they end up as buffer stock savers
# But with all of their buffer stock savings in the stock market

# %%
# Solve the specialized / simple version for which Campbell-Viceira (2002) is a good approximation
# (as wealth approaches infinity)
# This is the version for which Campbell and Viceira provide an approximate formula
# assuming lognormally distributed shocks

init_lognormportfolio = copy.deepcopy(init_portfolio) # Use same parameter values
init_lognormportfolio['RiskyAvg']   = Avg
init_lognormportfolio['RiskyStd']   = Std
init_lognormportfolio['RiskyCount'] = 11 # Eleven is enough points to do justice to the distribution
lnpcct = cpm.LogNormalPortfolioConsumerType(**init_lognormportfolio)

lnpcct.solve()
lnpcct.MertSamCampVicShare = lnpcct.RiskyShareLimitFunc(RiskyDstnFunc(init_portfolio['RiskyCount']))

fig = plt.figure()
plt.plot(eevalgrid, lnpcct.solution[0].RiskyShareFunc[0][0](eevalgrid))
plt.axhline(lnpcct.MertSamCampVicShare, c='r')
plt.ylim(0,1.05)
plt.text((aMax-aMin)/4,lnpcct.MertSamCampVicShare-0.1,r'$\uparrow $ limit as  $m \uparrow \infty$',fontsize = 22,fontweight='bold')
plt.show()

# %%
# Again simulate a few periods -- similar results (poor buffer stock savers put all their money in the stock market)
lnpcct.track_vars = ['aNrmNow', 't_age', 'RiskyShareNow']
lnpcct.T_sim = SimPer
lnpcct.initializeSim()
lnpcct.simulate()
lnpcct.RiskyShareNow_hist

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(lnpcct.aNrmNow_hist, lnpcct.t_age_hist, lnpcct.RiskyShareNow_hist)
ax.set_zlim(lnpcct.MertSamCampVicShare,1.0)
plt.show()

# %%
# Version where only discrete values of portfolio share of risky assets are allowed

init_portfolio_prb = copy.deepcopy(init_portfolio)

init_portfolio_prb['AdjustPrb'] = 1.0
init_portfolio_prb['PortfolioDomain'] = cpm.DiscreteDomain([0.0, 0.5, 0.6, 1.0])
pcct_prb = cpm.PortfolioConsumerType(**init_portfolio_prb)
pcct_prb.solve()

eevalgrid = np.linspace(0,100,100)
plt.plot(eevalgrid, pcct_prb.solution[0].RiskyShareFunc[0][0](eevalgrid))
plt.axhline(pcct_prb.RiskyShareLimitFunc(RiskyDstnFunc(init_portfolio['RiskyCount'])), c='r')
plt.ylim(0,1.05)
plt.show()

# %%
# Version where you can choose your portfolio share only with some 0 < p < 1

init_portfolio_prb = copy.deepcopy(init_portfolio)

init_portfolio_prb['AdjustPrb'] = 0.5
init_portfolio_prb['PortfolioDomain'] = cpm.DiscreteDomain([0.0, 0.6, 1.0])
pcct_prb = cpm.PortfolioConsumerType(**init_portfolio_prb)

pcct_prb.solve()

plt.plot(eevalgrid, pcct_prb.solution[0].RiskyShareFunc[0][0](eevalgrid))
plt.show()

pcct_prb.track_vars = ['aNrmNow', 't_age', 'RiskyShareNow', 'CantAdjust']
pcct_prb.T_sim = 10
pcct_prb.AgentCount = 30
pcct_prb.initializeSim()
pcct_prb.simulate()
pcct_prb.RiskyShareNow_hist

fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.scatter(pcct_prb.aNrmNow_hist, pcct_prb.t_age_hist, pcct_prb.RiskyShareNow_hist)
plt.show()


"""
pierwsze kroki w zmiennych

"""

if __name__ == '__main__':
    '''
    a co jeżeli chcielibyśmy wpisać wartość tylko taką jaką chcemy, a jeszcze
    jej nie znamy przed uruchomieniem programu?

    w takich przypadkach, przydatna okazuje się funkcja "input"
    '''

    # a = input('podaj a: ')
    # b = input('podaj b: ')

    # print(a + b)

    '''
    dlaczego dodało 3 + 7 i wyszło 37?

    nasz skrypt dodał te dwie zmienne tak jakby były one tekstem

    trzeba skonwertować dane z klawiatury na typ liczbowy, np. "int"
    '''

    # a = int(input('podaj a: '))
    # b = int(input('podaj b: '))

    # print(a + b)

    '''
    możemy definiować różne typy zmiennych, np.
      łańcuch znaków              (string)
      liczbę całkowitą            (integer)
      wartość prawda/fałsz        (boolean)
      liczbę zmiennoprzecinkową   (float)
    '''

    i = 4
    s = "5"
    b = True
    f = .0

    '''
    lub też korzystając z wbudowanych funkcji:
    '''

    i2 = int('234')
    s2 = str(450)
    # w obu przypadkach powyżej, zajdzie konwersja typu
    b2 = bool(2)
    f2 = float(1)

    print(f2)
    print(b2)
    print(s2)
    print(i2)

    '''
    możemy sprawdzać typ zmiennej stosując funkcję "type" albo "isinstance"

    w ten sposób możemy sprawdzić czy zmienne odpowiadają temu typowi który chcemy:
    '''

    print(isinstance(i, int))
    print(type(b))
    print(isinstance(b, bool))
    print(type(f))

    if isinstance(b, bool):
        print("działam poprawnie")
    else:
        print("podałeś złe dane, spróbuj jeszcze raz")

    print(type(f) == float)
    print(type(f))
    print(type(float))
    print(float)

    if type(f) == float:
        c = 15. + f
        print(c)
    else:
        print("incorrect data type")

    '''
    co się stanie gdy to do siebie dodamy? czy kolejność ma znaczenie?
    '''

    # print(a + b)
    # print(b + a)

    '''
    nie możemy dodać do siebie tych zmiennych od razu, musimy je przekonwertować na typ który umożliwia nam ich dodawanie
    czyli na typ wspólny dla obu zmiennych (np. obie typu string, obie typu integer itp)

    dobra a co jeżeli byśmy chcieli pomnóżyć przez coś tekst?
    '''

    print("AAAAAAAAAA" * 10)

    '''
    zapisywanie danych do pliku

    jest możliwych kilka metod zapisu, najprostszą z nich jest otwarcie już istniejącego pliku i zapisanie do niego
    informacji

    plik otwieramy za pomocą wbudowanej funkcji open, w nawiasie podając nazwę pliku, oraz przekazując go do zmiennej

    mówimy tu o tak zwanym "strumieniu danych" który przypisujemy do zmiennej "file" zaznaczając, że chodzi tu o
    strumień danych zapisywanych do pliku

    następnie odwołujemy się do pliku, po kropce wywołując wewnętrzną funkcję "write" w nawiasie której podajemy dane

    dane muszą mieć format tekstowy; po zapisie zamykamy strumień wywołując wewnętrzną funkcję "close()"
    '''

    a = 5

    file = open('new_file.txt', 'w')
    file.write(str(a))
    file.close()

    '''
    aby nie trzeba było zamykać pliku samemu oraz przypisywać strumienia do zmiennej, możemy użyć również klauzuli

    "
        with (tutaj coś) as (tutaj nazwa, na jaką chcemy przechrzcić nasze coś):
            instrukcje...
    "

    korzystamy tutaj z tzw. "context manager'a"

    CM pozwala nam, w ramach operowania na danej klasie obiektu wykonać pewne funkcje, które wymagane byłyby zawsze

    w przypadku zwykłego otwarcia strumienia danych do pliku są to dwie rzeczy:
        przypisanie strumienia do zmiennej
            oraz
        po zakończonych opearacjach zamknięcie strumienia

    taką konstrukcję można wykorzystać wszędzie, np. przy połączeniach z bazą danych, gdzie w wielu
    miejscach w programie mógłbym:
        sprawdzać czy dany użytkownik istnieje w bazie
        sprawdzać czy loguje się z innego IP niż zazwyczaj
        sprawdzać czy nie został zbanowany
        sprawdzać czy pewna osoba nie chce zarejestrować się na podobny nick do zbanowanego

    w każdych z tych sytuacji musiałbym pisać kilka poleceń:
        połącz z bazą danych
        załaduj tabelę użytkownicy
            (teraz instrukcje specyficzne dla przypadku)
        zapisz zmiany
        zakończ połączenie
    te 4 rzeczy mogłyby wykonywać się automatycznie, za kulisami, oraz ZAWSZE, przy użyciu CM
    (jako człowiek mógłbym np. zapomnieć dołączyć instrukcję zamknięcia połączenia, w wyniku czego po pewnym czasie serwer
    byłby przeciążony przez niezamknięte połączenia z bazą, CM automatyzuje to i nie muszę się martwić)
    '''
    with open('new_file2.txt', 'x') as file:
        file.write('tu jest tekst')
        file.write(str(a))

    with open('new_file2.txt', 'r') as file:
        caly_tekst_z_pliku = file.read()
        print(caly_tekst_z_pliku)

    with open('new_file2.txt', 'r') as file:
        caly_tekst_z_pliku = file.read()
        print(caly_tekst_z_pliku)

    with open('new_file2.txt', 'w') as file:
        file.write('file overwritten!')

    with open('new_file2.txt', 'r') as file:
        caly_tekst_z_pliku = file.read()
        print(caly_tekst_z_pliku)

    '''
    open ma kilka możliwych trybów działania:
        po przecinku, jako kolejny argument:
            "x" - otwórz do zapisu, tworzy automatycznie plik (ZAKŁADA że go nie ma, jak znajdzie wywala błąd)
            "w" - otwórz do zapisu, nadpisuje wszystko co jest w już istniejącym pliku nową treścią
            "a" (od 'append') - otwórz do zapisu, zapisuje na koniec pliku w nowym wierszu zostawiając to co już jest
            "r" (od read) - otwórz do odczytu
            otatnie 2 wywalają błąd gdy nie znajdą pliku

    stosując naszą wiedzę z poprzedniego przykładu, napiszmy to w klauzuli try/except
    '''

    try:
        with open('new_file2.txt', 'x') as file:
            file.write('tu jest tekst')
            file.write(str(a))
    except FileExistsError:
        with open('new_file2.txt', 'a') as file:
            file.write('tu jest tekst')
            file.write(str(a))

    '''
    no dobrze mówiliśmy tu o podstawowych typach zmiennych, tak żeby móc się jakkolwiek wypowiedzieć

    a co gdy mamy wiele takich zmiennych, np. 100 czy 200 czy 1000000? Będziemy je tak wypisywać w nieskończoność?
    Z pomocą przyjść mogą nam listy i słowniki
    '''

    a = []  # deklaracja utworzenia listy
    b = {}  # deklracja utworzenia słownika

    '''
    jak z nich korzystać? jest to niezwykle proste, lecz odmienne dla obu tych typów

    najpierw listy
    '''

    c = [0, 1, 2, 3, 4, 5, 6, 8, 9, True, 12415, 124.26236, 'awphphfaw', 12 + 45]

    '''
    każdy obiekt który chcemy wpisać do tablicy, musi zostać umieszczony w niej po kolei, każdy następny po przecinku
    w ten sposób wypełniamy listę w sposób zwyczajny

    na liście mogą się znaleźć różne typy danych, a nawet skomplikowane obiekty o których będzie więcej później

    a jak w sposób zautomatyzowany wpisać liczby w tablicę, np. generowane przez jakąś funkcję kolegi?

    można skorzystać tutaj z pętli for
    '''
    # a = []  # będziemy wypełniać pustą tablicę, by np. przekazać ją do innego miejsca w programie
    pygod = 'jestem bogiem pythona'
    print(a)
    for i in range(20):
        a.append(pygod)  # każde wykonanie pętli doda do tablicy "a" tekst "jestem bogiem pythona"

    print(a)
    print(c)

    '''
    można też utworzyć listę korzystając z wbudowanej funkcji
    '''
    empty_list = list()
    list_1 = list((1, 2, 3, 4, 5))

    '''
    mamy listę z elementami, ale czy możemy się odwołać do konkretnego jej elementu?

    No pewnie że tak, odwołujemy się do niego podając numer w kolejności w jakiej element ten występuje
    w tablicy/liście, zaczynając od zera:
    '''

    print(c[0])
    print(c[5])
    print(c[10])
    # print(c[22])  # wywali błąd

    '''
    podmiana elementów również jest możliwa, również wystarczy odnieść się do elementu o pożadanym numerze, i przyrównać
    do niego nową wartość
    '''

    print(c[0])
    c[0] = pygod
    print(c[0])

    a = '''
    tekst tej pomocy zostanie przyrównany do zmiennej, jednocześnie wykonując "split"

    jest to specjalny rodzaj tworzenia tablicy z tekstu, gdzie elementami tablicy będą wyrażenia które będą rozdzielone
    po kluczu

    np. żeby otrymać listę wyrazów z tekstu który czytasz, użyjemy jako klucza znaku " " (spacji)
    '''.split(" ")
    print(a)

    '''
    Jak widać tekst został rozdzielony po spacjach i otrzymaliśmy listę, a jak ją z powrotem złączyć?

    Wykorzystamy wbudowaną w string funkcję "join", działa na takiej samej zasadzie jak split, ale odwraca jego proces

    Widzimy, też pewną prawidłowość; stosujemy "split" a także będziemy stosować "join" na już zadeklarowanym łańcuchu
    znaków; nie musimy się martwić o to, by przyrównać go do zmiennej, już sam tekst nią jest
    '''

    print(" ".join(a))

    '''
    do tworzenia list możemy użyć jeszcze jednej przydatnej formuły, jaką jest `list comprehension`

    list comprehension pozwala na skompresowanie tworzenia listy pętlą for, umieszczając ją w deklaracji jej utworzenia
    (pomiędzy nawiasami kwadratowymi)

    nie potrzebujemy więc używać a.append(`argument`), wystarczy użyć list comprehension
    '''

    l = [' coś ' for i in range(20)]
    print(l)

    '''
    w pętli for możemy użyć więcej niż jednej kolekcji do iterowania po danych; używamy funkcji
    zip(element1, element2...), która pozwala nam zadeklarować te zmienne jako 2 listy po których mamy iterować

    pamiętajmy że muszą to być listy o tej samej długości!
    '''

    poly3 = []

    polynomial1 = [1, 3, 7, 9]
    polynomial2 = [3, 4, 7, 0]

    for coeff1, coeff2 in zip(polynomial1, polynomial2):
        poly3.append(coeff1 + coeff2)

    '''
    a w list comprehension
    '''

    poly4 = [coeff1 + coeff2 for coeff1, coeff2 in zip(polynomial2, polynomial1)]

    print("poly3 = ", poly3)
    print("poly4 = ", poly4)

    '''
    mamy również możliwość tworenia czegoś co przypominałoby wielowymiarowe tablice; jak to zrobić?

    cóż jest to po prostu tablica zawarta w tablicy
    '''

    matrix = [
        [1, 2, 3],
        [1, 2, 4],
        [1, 1, 1]
    ]
    matrix2 = [
        [1, 0, 2],
        [1, 1, 0],
        [1, 0, 6]
    ]

    '''
    dostaliśmy 2 macierze 3x3 z odpowiednio wstawionymi parametrami

    mnożenie macierzy przy użyciu gwiazdki powoduje mnożenie "element po elemencie"

    mnożenie przy użyciu "@" oznacza właściwe mnożenie macierzowe, znane z akademickiego kursu matematyki
    '''
    import numpy as np

    matrix2 = np.array(matrix2)
    matrix = np.array(matrix)

    matrix3 = matrix * matrix2
    matrix4 = matrix @ matrix2
    matrix5 = matrix2 @ matrix

    print(matrix3, "\n\n", matrix4, "\n\n", matrix5)

    '''
    innym bardzo podobnym typem danych do list są tak zwane tuple. Czym one się różnią?

    Przede wszystkim tym, że nie można ich modyfikować, zmieniać ich rozmiarów ani podstawiać danych

    Przydaje się to szczególnie gdy wyjmujemy wrażliwe dane z bazy, nie jesteśmy w stanie ich zmodyfikować, ani żadne
    funkcje w programie
    '''

    tup = 1, 2, 3, 4, 5, 6
    print(tup)

    tup2 = 2,
    print(tup2)

    '''
    o powstaniu tupli decyduje nie nawias okrągły, ale przecinek, widać to wyżej,

    poza tym nie różnią się za wiele od list, jedyną różnicą jest to, że używając analogicznej metody tworzenia tupli
    jak w listach (list comprehension), W wyniku tego tworzy się tzw. wyrażenie generatorowe
    '''

    generator = (i*2 for i in range(20))

    '''
    oraz jak zawsze możemy skorzystać z wbudowanej funkcji, by stworzyć tuplę, np. przerabiając już utworzoną
    listę na tuplę, albo poprostu utworzyć pustą, choc tak naprawdę nie ma to sensu, gdyż tupli nie można modyfikować
    '''

    empty_tup = tuple()
    tup_from_list = tuple([1, 2, 3])
    print(empty_tup, tup_from_list)

    '''
    o generatorach powiemy sobie później, jak narazie powiedzmy sobie teraz o słownikach, słowniki również zapisują
    wiele danych w sobie, tak samo jak lista, czy tupla

    odmiennie od list zapis w słownikach realizuje się jako zapis w stylu klucz=wartość
    '''

    # b = {}  # deklaracja słownika

    b['klucz'] = 'wartość'
    b['s2'] = 2

    '''
    możemy też zadeklarować listy używając wbudowanej funkcji "dict(wartośći...)"
    '''

    dict_1 = dict(
        wrd=15,
        wrc=25,
        hamburger='cheeseburger',
        some_complicated_list=range(1,100)
    )
    cust_dict = dict(
        [
            ('f', 2),
            ('a', 3),
            ('x', 6)
        ],
        bambo=3
    )

    '''
    jak widać kluczem jest zawsze string, wartością może być wszystko, łącznie z innym słownikiem

    odwołanie się do wartości przebiega analogicznie
    '''
    print(b['klucz'])
    print('s2 : ', b['s2'])

    print(b)
    '''
    jeżeli chcielibyśmy zainicjować słownik z jakimiś wartościami na starcie, wpisujemy je również jako pary
    klucz wartość, ale w określony sposób; definiuje się je jako:
        `"key" : value`

    a więc wartości następują po dwukropku
    '''
    value = 3.14

    d = {'a' : 3,
         'another dict' : {
             'key': value
         },
         'something': 'else'}

    '''
    w pętli for możemy odwołać się zarówno do kluczy jak i wartości
    '''

    for key, value in d:
        print(key, ' ',  value)

    '''
    warto zaznaczyć że słowniki są wykorzystywane we frameworku Django; przekazuje się w nich tzw. kontekst, który
    będzie następnie wyświetlany na stronie
    '''


# coding: utf-8

# ---
#
# _You are currently looking at **version 1.0** of this notebook. To download notebooks and datafiles, as well as get help on Jupyter notebooks in the Coursera platform, visit the [Jupyter Notebook FAQ](https://www.coursera.org/learn/python-text-mining/resources/d9pwm) course resource._
#
# ---

# # Assignment 2 - Introduction to NLTK
#
# In part 1 of this assignment you will use nltk to explore the Herman Melville novel Moby Dick. Then in part 2 you will create a spelling recommender function that uses nltk to find words similar to the misspelling.

# ## Part 1 - Analyzing Moby Dick

# In[ ]:

import nltk
import pandas as pd
import numpy as np

# If you would like to work with the raw text you can use 'moby_raw'
with open('moby.txt', 'r') as f:
    moby_raw = f.read()

# If you would like to work with the novel in nltk.Text format you can use 'text1'
moby_tokens = nltk.word_tokenize(moby_raw)
text1 = nltk.Text(moby_tokens)

# ### Example 1
#
# How many tokens (words and punctuation symbols) are in text1?
#
# *This function should return an integer.*

# In[ ]:

def example_one():

    return len(nltk.word_tokenize(moby_raw)) # or alternatively len(text1)

example_one()

# ### Example 2
#
# How many unique tokens (unique words and punctuation) does text1 have?
#
# *This function should return an integer.*

# In[ ]:

def example_two():

    return len(set(nltk.word_tokenize(moby_raw))) # or alternatively len(set(text1))

example_two()

# ### Example 3
#
# After lemmatizing the verbs, how many unique tokens does text1 have?
#
# *This function should return an integer.*

# In[ ]:

from nltk.stem import WordNetLemmatizer

def example_three():

    lemmatizer = WordNetLemmatizer()
    lemmatized = [lemmatizer.lemmatize(w,'v') for w in text1]

    return len(set(lemmatized))

example_three()

# ### Question 1
#
# What is the lexical diversity of the given text input? (i.e. ratio of unique tokens to the total number of tokens)
#
# *This function should return a float.*

# In[ ]:

def answer_one():

    return # Your answer here

answer_one()

# ### Question 2
#
# What percentage of tokens is 'whale'or 'Whale'?
#
# *This function should return a float.*

# In[ ]:

def answer_two():

    return # Your answer here

answer_two()

# ### Question 3
#
# What are the 20 most frequently occurring (unique) tokens in the text? What is their frequency?
#
# *This function should return a list of 20 tuples where each tuple is of the form `(token, frequency)`. The list should be sorted in descending order of frequency.*

# In[ ]:

def answer_three():

    return # Your answer here

answer_three()

# ### Question 4
#
# What tokens have a length of greater than 5 and frequency of more than 150?
#
# *This function should return a sorted list of the tokens that match the above constraints. To sort your list, use `sorted()`*

# In[ ]:

def answer_four():

    return # Your answer here

answer_four()

# ### Question 5
#
# Find the longest word in text1 and that word's length.
#
# *This function should return a tuple `(longest_word, length)`.*

# In[ ]:

def answer_five():

    return # Your answer here

answer_five()

# ### Question 6
#
# What unique words have a frequency of more than 2000? What is their frequency?
#
# "Hint:  you may want to use `isalpha()` to check if the token is a word and not punctuation."
#
# *This function should return a list of tuples of the form `(frequency, word)` sorted in descending order of frequency.*

# In[ ]:

def answer_six():

    return # Your answer here

answer_six()

# ### Question 7
#
# What is the average number of tokens per sentence?
#
# *This function should return a float.*

# In[ ]:

def answer_seven():

    return # Your answer here

answer_seven()

# ### Question 8
#
# What are the 5 most frequent parts of speech in this text? What is their frequency?
#
# *This function should return a list of tuples of the form `(part_of_speech, frequency)` sorted in descending order of frequency.*

# In[ ]:

def answer_eight():

    return # Your answer here

answer_eight()

# ## Part 2 - Spelling Recommender
#
# For this part of the assignment you will create three different spelling recommenders, that each take a list of misspelled words and recommends a correctly spelled word for every word in the list.
#
# For every misspelled word, the recommender should find find the word in `correct_spellings` that has the shortest distance*, and starts with the same letter as the misspelled word, and return that word as a recommendation.
#
# *Each of the three different recommenders will use a different distance measure (outlined below).
#
# Each of the recommenders should provide recommendations for the three default words provided: `['cormulent', 'incendenece', 'validrate']`.

# In[ ]:

from nltk.corpus import words

correct_spellings = words.words()

# ### Question 9
#
# For this recommender, your function should provide recommendations for the three default words provided above using the following distance metric:
#
# **[Jaccard distance](https://en.wikipedia.org/wiki/Jaccard_index) on the trigrams of the two words.**
#
# *This function should return a list of length three:
# `['cormulent_reccomendation', 'incendenece_reccomendation', 'validrate_reccomendation']`.*

# In[ ]:

def answer_nine(entries=['cormulent', 'incendenece', 'validrate']):

    return # Your answer here

answer_nine()

# ### Question 10
#
# For this recommender, your function should provide recommendations for the three default words provided above using the following distance metric:
#
# **[Jaccard distance](https://en.wikipedia.org/wiki/Jaccard_index) on the 4-grams of the two words.**
#
# *This function should return a list of length three:
# `['cormulent_reccomendation', 'incendenece_reccomendation', 'validrate_reccomendation']`.*

# In[ ]:

def answer_ten(entries=['cormulent', 'incendenece', 'validrate']):

    return # Your answer here

answer_ten()

# ### Question 11
#
# For this recommender, your function should provide recommendations for the three default words provided above using the following distance metric:
#
# **[Edit distance on the two words with transpositions.](https://en.wikipedia.org/wiki/Damerau%E2%80%93Levenshtein_distance)**
#
# *This function should return a list of length three:
# `['cormulent_reccomendation', 'incendenece_reccomendation', 'validrate_reccomendation']`.*

# In[ ]:

def answer_eleven(entries=['cormulent', 'incendenece', 'validrate']):

    return # Your answer here

answer_eleven()

Michigan - Text Mining/Text Mining Assignment+2.py


# To add a new cell, type '# %%'
# To add a new markdown cell, type '# %% [markdown]'
# %%
from datetime import datetime
print ("Today is:" + str(datetime.now()))

# %%
def say_hello(recipient):
    return 'Hi, {}!'.format(recipient)
say_hello('Curious') + " Welcome to the Jupyter Notebooks tutorial!"

# %%
import numpy as np
def square(x):
    return x * x
print ("You can use the awesome numpy to do things like:")
x = np.random.randint(1, 10)
print ("The square of {} is: ".format(x) + str(square(x)))

# %%
import os
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style="darkgrid")

df = pd.read_csv(os.getcwd() + '/TutorialNotebooks/UserSample.csv')
df.head()

# %%
print ("Total records: " + str(len(df)))
df.dtypes

# %%


# tests.test_cluster.test_elbow
# Tests for the KElbowVisualizer
#
# Author:   Benjamin Bengfort <bbengfort@districtdatalabs.com>
# Created:  Thu Mar 23 22:30:19 2017 -0400
#
# Copyright (C) 2016 District Data Labs
# For license information, see LICENSE.txt
#
# ID: test_elbow.py [5a370c8] benjamin@bengfort.com $

"""
Tests for the KElbowVisualizer
"""

##########################################################################
## Imports
##########################################################################

import sys
import pytest
import numpy as np
import matplotlib.pyplot as plt

from ..base import VisualTestCase
from ..dataset import DatasetMixin

from scipy.sparse import csc_matrix, csr_matrix
from numpy.testing.utils import assert_array_almost_equal

from sklearn.datasets import make_blobs
from sklearn.cluster import KMeans, MiniBatchKMeans
from sklearn.feature_extraction.text import TfidfVectorizer

from yellowbrick.cluster.elbow import distortion_score
from yellowbrick.cluster.elbow import KElbowVisualizer
from yellowbrick.exceptions import YellowbrickValueError

try:
    import pandas as pd
except ImportError:
    pd = None

##########################################################################
## K-Elbow Helpers Test Cases
##########################################################################

X = np.array(
      [[-0.40020753, -4.67055317, -0.27191127, -1.49156318],
       [ 0.37143349, -4.89391622, -1.23893945,  0.48318165],
       [ 8.625142  , -1.2372284 ,  1.39301471,  4.3394457 ],
       [ 7.65803596, -2.21017215,  1.99175714,  3.71004654],
       [ 0.89319875, -5.37152317,  1.50313598,  1.95284886],
       [ 2.68362166, -5.78810913, -0.41233406,  1.94638989],
       [ 7.63541182, -1.99606076,  0.9241231 ,  4.53478238],
       [ 9.04699415, -0.74540679,  0.98042851,  5.99569071],
       [ 1.02552122, -5.73874278, -1.74804915, -0.07831216],
       [ 7.18135665, -3.49473178,  1.14300963,  4.46065816],
       [ 0.58812902, -4.66559815, -0.72831685,  1.40171779],
       [ 1.48620862, -5.9963108 ,  0.19145963, -1.11369256],
       [ 7.6625556 , -1.21328083,  2.06361094,  6.2643551 ],
       [ 9.45050727, -1.36536078,  1.31154384,  3.89103468],
       [ 6.88203724, -1.62040255,  3.89961049,  2.12865388],
       [ 5.60842705, -2.10693356,  1.93328514,  3.90825432],
       [ 2.35150936, -6.62836131, -1.84278374,  0.51540886],
       [ 1.17446451, -5.62506058, -2.18420699,  1.21385128]]
)

y = np.array([0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0])

class TestKElbowHelper(object):
    """
    Helper functions for K-Elbow Visualizer
    """

    def test_distortion_score(self):
        """
        Test the distortion score metric function
        """
        score = distortion_score(X, y)
        assert score == pytest.approx(69.10006514142941)

    @pytest.mark.parametrize("Xs", [
        csc_matrix(X), csr_matrix(X),
    ], ids=["csc", "csr"])
    def test_distortion_score_sparse_matrix_input(self, Xs):
        """
        Test the distortion score metric on a sparse array
        """
        score = distortion_score(Xs, y)
        assert score == pytest.approx(69.10006514142938)

    @pytest.mark.skipif(pd is None, reason="pandas is required")
    def test_distortion_score_pandas_input(self):
        """
        Test the distortion score metric on pandas DataFrame and Series
        """
        df = pd.DataFrame(X)
        s = pd.Series(y)

        score = distortion_score(df, s)
        assert score == pytest.approx(69.10006514142941)

##########################################################################
## KElbowVisualizer Test Cases
##########################################################################

class TestKElbowVisualizer(VisualTestCase, DatasetMixin):
    """
    K-Elbow Visualizer Tests
    """

    @pytest.mark.xfail(reason="images not close due to timing lines")
    def test_integrated_kmeans_elbow(self):
        """
        Test no exceptions for kmeans k-elbow visualizer on blobs dataset
        """
        # NOTE #182: cannot use occupancy dataset because of memory usage

        # Generate a blobs data set
        X,y = make_blobs(
            n_samples=1000, n_features=12, centers=6,
            shuffle=True, random_state=42
        )

        try:
            _, ax = plt.subplots()

            visualizer = KElbowVisualizer(KMeans(random_state=42), k=4, ax=ax)
            visualizer.fit(X)
            visualizer.poof()

            self.assert_images_similar(visualizer)
        except Exception as e:
            pytest.fail("error during k-elbow: {}".format(e))

    @pytest.mark.xfail(reason="images not close due to timing lines")
    def test_integrated_mini_batch_kmeans_elbow(self):
        """
        Test no exceptions for mini-batch kmeans k-elbow visualizer
        """
        # NOTE #182: cannot use occupancy dataset because of memory usage

        # Generate a blobs data set
        X,y = make_blobs(
            n_samples=1000, n_features=12, centers=6, shuffle=True, random_state=42
        )

        try:
            _, ax = plt.subplots()

            visualizer = KElbowVisualizer(
                MiniBatchKMeans(random_state=42), k=4, ax=ax
            )
            visualizer.fit(X)
            visualizer.poof()

            self.assert_images_similar(visualizer)
        except Exception as e:
            pytest.fail("error during k-elbow: {}".format(e))

    @pytest.mark.skip(reason="takes over 20 seconds to run")
    def test_topic_modeling_k_means(self):
        """
        Test topic modeling k-means on the hobbies corpus
        """
        corpus = self.load_corpus("hobbies")

        tfidf  = TfidfVectorizer()
        docs   = tfidf.fit_transform(corpus.data)
        visualizer = KElbowVisualizer(KMeans(), k=(4, 8))

        visualizer.fit(docs)
        visualizer.poof()

        self.assert_images_similar(visualizer)

    def test_invalid_k(self):
        """
        Assert that invalid values of K raise exceptions
        """

        with pytest.raises(YellowbrickValueError):
            KElbowVisualizer(KMeans(), k=(1, 2, 3, 'foo', 5))

        with pytest.raises(YellowbrickValueError):
            KElbowVisualizer(KMeans(), k="foo")

    def test_valid_k(self):
        """
        Assert that valid values of K generate correct k_values_:
        if k is an int, k_values_ = range(2, k+1)
        if k is a tuple of 2 ints, k_values = range(k[0], k[1])
        if k is an iterable, k_values_ = list(k)
        """
        visualizer = KElbowVisualizer(KMeans(), k=8)
        assert visualizer.k_values_ == list(np.arange(2, 8+1))

        visualizer = KElbowVisualizer(KMeans(), k=(4, 12))
        assert visualizer.k_values_ == list(np.arange(4, 12))

        visualizer = KElbowVisualizer(KMeans(), k=np.arange(10, 100, 10))
        assert visualizer.k_values_ == list(np.arange(10, 100, 10))

        visualizer = KElbowVisualizer(KMeans(),
                                      k=[10, 20, 30, 40, 50, 60, 70, 80, 90])
        assert visualizer.k_values_ == list(np.arange(10, 100, 10))

    @pytest.mark.xfail(
        sys.platform == 'win32', reason="images not close on windows"
    )
    def test_distortion_metric(self):
        """
        Test the distortion metric of the k-elbow visualizer
        """
        visualizer = KElbowVisualizer(
            KMeans(random_state=0), k=5, metric="distortion", timings=False
        )
        visualizer.fit(X)

        expected = np.array([ 69.100065, 54.081571, 43.146921, 34.978487])
        assert len(visualizer.k_scores_) == 4

        visualizer.poof()
        self.assert_images_similar(visualizer)
        assert_array_almost_equal(visualizer.k_scores_, expected)

    @pytest.mark.xfail(
        sys.platform == 'win32', reason="images not close on windows"
    )
    def test_silhouette_metric(self):
        """
        Test the silhouette metric of the k-elbow visualizer
        """
        visualizer = KElbowVisualizer(
            KMeans(random_state=0), k=5, metric="silhouette", timings=False
        )
        visualizer.fit(X)

        expected = np.array([ 0.691636,  0.456646,  0.255174,  0.239842])
        assert len(visualizer.k_scores_) == 4

        visualizer.poof()
        self.assert_images_similar(visualizer)
        assert_array_almost_equal(visualizer.k_scores_, expected)

    @pytest.mark.xfail(
        sys.platform == 'win32', reason="images not close on windows"
    )
    def test_calinski_harabaz_metric(self):
        """
        Test the calinski-harabaz metric of the k-elbow visualizer
        """
        visualizer = KElbowVisualizer(
            KMeans(random_state=0), k=5,
            metric="calinski_harabaz", timings=False
        )
        visualizer.fit(X)
        assert len(visualizer.k_scores_) == 4

        expected = np.array([
            81.662726256035683, 50.992378259195554,
            40.952179227847012, 35.939494
        ])

        visualizer.poof()
        self.assert_images_similar(visualizer)
        assert_array_almost_equal(visualizer.k_scores_, expected)

    def test_bad_metric(self):
        """
        Assert KElbow raises an exception when a bad metric is supplied
        """
        with pytest.raises(YellowbrickValueError):
            KElbowVisualizer(KMeans(), k=5, metric="foo")

    @pytest.mark.xfail(
        sys.platform == 'win32', reason="images not close on windows"
    )
    def test_timings(self):
        """
        Test the twinx double axes with k-elbow timings
        """
        visualizer = KElbowVisualizer(
            KMeans(random_state=0), k=5, timings=True
        )
        visualizer.fit(X)

        # Check that we kept track of time
        assert len(visualizer.k_timers_) == 4
        assert all([t > 0 for t in visualizer.k_timers_])

        # Check that we plotted time on a twinx
        assert hasattr(visualizer, "axes")
        assert len(visualizer.axes) == 2

        # delete the timings axes and
        # overwrite k_timers_, k_values_ for image similarity Tests
        visualizer.axes[1].remove()
        visualizer.k_timers_ = [
            0.01084589958190918, 0.011144161224365234,
            0.017028093338012695, 0.010634183883666992
        ]
        visualizer.k_values_ = [2, 3, 4, 5]

        # call draw again which is normally called in fit
        visualizer.draw()
        visualizer.poof()

        self.assert_images_similar(visualizer)


# -*- coding: utf-8 -*-
"""
Created on Tue Nov 26 08:20:33 2019

@author: junt_
"""

import numpy as np
import pandas as pd
import os

# 1) Csv, Jason, HTML, XML
# 2) Binary
# 3) Relational Databases

path = "Z://junt_//Documents//7mo//Python//Pandas//Data//pokemon.csv"

df_one = pd.read_csv(
        path,
        nrows = 10)

columnas = ['#','Name','Type 1','Total','Attack','Defense','Sp. Atk','Sp. Def','Speed']

df_dos = pd.read_csv(
        path,
        nrows = 10,
        usecols = columnas,
        index_col = '#')

df_tres = pd.read_csv(path)

save_path = "Z://junt_//Documents//7mo//Python//Pandas//Data//pokemon_completo.pickle"

df_tres.to_pickle(save_path)

df_cuatro = pd.read_pickle(save_path)


''' '''
'''
 ISC License

 Copyright (c) 2016, Autonomous Vehicle Systems Lab, University of Colorado at Boulder

 Permission to use, copy, modify, and/or distribute this software for any
 purpose with or without fee is hereby granted, provided that the above
 copyright notice and this permission notice appear in all copies.

 THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

'''

#
# Bore Angle Calculation Test
#
# Purpose:  Test the proper function of the ore Angle Calculation module.
#           Proper function is tested by
#
# Author:   Rachel Mamich
# Creation Date:  Jun. 30, 2017
#

# @cond DOXYGEN_IGNORE
from Basilisk.utilities import SimulationBaseClass
from Basilisk.simulation import bore_ang_calc
from Basilisk.utilities import macros
from Basilisk.utilities import RigidBodyKinematics
from Basilisk.simulation import spice_interface
from Basilisk.simulation import spacecraftPlus
from Basilisk.utilities import unitTestSupport
import pytest
import numpy
import os
# @endcond

path = os.path.dirname(os.path.abspath(__file__))

class ResultsStore:
    def __init__(self):
        self.PassFail = []
    def texSnippet(self):
        for i in range(len(self.PassFail)):
            snippetName = 'Result' + str(i)
            if self.PassFail[i] == 'PASSED':
                textColor = 'ForestGreen'
            elif self.PassFail[i] == 'FAILED':
                textColor = 'Red'
            texSnippet =  '\\textcolor{' + textColor + '}{'+ self.PassFail[i] + '}'
            unitTestSupport.writeTeXSnippet(snippetName, texSnippet, path)

@pytest.fixture(scope="module")
def testFixture():
    listRes = ResultsStore()
    yield listRes
    listRes.texSnippet()

# uncomment this line is this test is to be skipped in the global unit test run, adjust message as needed
# @pytest.mark.skipif(conditionstring)
# uncomment this line if this test has an expected failure, adjust message as needed
# @pytest.mark.xfail(True)
# The following 'parametrize' function decorator provides the parameters and expected results for each
#   of the multiple test runs for this test.
@pytest.mark.parametrize("boresightLoc, eulerLoc",
                         [([1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3)], [0.0, 0.0, 0.0]),
                          ([-1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3)], [0.0, 0.0, 0.0]),
                          ([1.0 / numpy.sqrt(3), -1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3)], [0.0, 0.0, 0.0]),
                          ([-1.0 / numpy.sqrt(3), -1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3)], [0.0, 0.0, 0.0]),
                          ([1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3), -1.0 / numpy.sqrt(3)], [0.0, 0.0, 0.0]),
                          ([-1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3), -1.0 / numpy.sqrt(3)], [0.0, 0.0, 0.0]),
                          ([1.0 / numpy.sqrt(3), -1.0 / numpy.sqrt(3), -1.0 / numpy.sqrt(3)], [0.0, 0.0, 0.0]),
                          ([-1.0 / numpy.sqrt(3), -1.0 / numpy.sqrt(3), -1.0 / numpy.sqrt(3)], [0.0, 0.0, 0.0]),
                          ([0.0, 0.0, 1.0], [numpy.pi / 4, numpy.pi / 4, 0.0]),
                          ([0.0, 0.0, 1.0], [3 * numpy.pi / 4, numpy.pi / 4, 0.0]),
                          ([0.0, 0.0, 1.0], [5 * numpy.pi / 4, numpy.pi / 4, 0.0]),
                          ([0.0, 0.0, 1.0], [-numpy.pi / 4, numpy.pi / 4, 0.0]),
                          ([0.0, 0.0, 1.0], [numpy.pi / 4, -numpy.pi / 4, 0.0]),
                          ([0.0, 0.0, 1.0], [3 * numpy.pi / 4, -numpy.pi / 4, 0.0]),
                          ([0.0, 0.0, 1.0], [5 * numpy.pi / 4, -numpy.pi / 4, 0.0]),
                          ([0.0, 0.0, 1.0], [-numpy.pi / 4, -numpy.pi / 4, 0.0]),
                          ([1.0, 0.0, 0.0], [0.0, 0.0, 0.0])])
# # provide a unique test method name, starting with test_
def test_bore_ang_calc(testFixture, show_plots, boresightLoc, eulerLoc):
    # each test method requires a single assert method to be called
    [testResults, testMessage] = bore_ang_calc_func(testFixture, show_plots, boresightLoc, eulerLoc)
    assert testResults < 1, testMessage

# Run unit test
def bore_ang_calc_func(testFixture, show_plots, boresightLoc, eulerLoc):
    testFailCount = 0  # zero unit test result counter
    testMessages = []  # create empty array to store test log messages

    # Create a sim module as an empty container
    unitTaskName = "unitTask"  # arbitrary name (don't change)
    unitProcessName = "TestProcess"  # arbitrary name (don't change)

    # Create a sim module as an empty container
    TotalSim = SimulationBaseClass.SimBaseClass()
    TotalSim.TotalSim.terminateSimulation()

    DynUnitTestProc = TotalSim.CreateNewProcess(unitProcessName)
    # create the dynamics task and specify the integration update time
    DynUnitTestProc.addTask(TotalSim.CreateNewTask(unitTaskName, macros.sec2nano(1.0)))

    spiceMessage = spice_interface.SpicePlanetStateSimMsg()
    stateMessage = spacecraftPlus.SCPlusStatesSimMsg()
    angMessage = bore_ang_calc.AngOffValuesSimMsg()
    vehPosition = [10000.0, 0.0, 0.0]
    sunPosition = [10000.0, 1000.0, 0.0]
    stateMessage.r_BN_N = vehPosition
    stateMessage.v_BN_N = [-365052.0511, 0.0, 0.0]
    if eulerLoc[0] == 0.0:
        stateMessage.sigma_BN = [0.0, 0.0, 0.0]
    else:
        stateMessage.sigma_BN = RigidBodyKinematics.euler3212MRP(eulerLoc)
    spiceMessage.PositionVector = sunPosition
    spiceMessage.PlanetName = "sun"
    # Inertial State output Message
    inputMessageSize = stateMessage.getStructSize()
    TotalSim.TotalSim.CreateNewMessage(unitProcessName,
                                       "inertial_state_output",
                                       inputMessageSize,
                                       2)  # number of buffers (leave at 2 as default, don't make zero)
    TotalSim.TotalSim.WriteMessageData("inertial_state_output",
                                       inputMessageSize,
                                       0,
                                       stateMessage)

    # Sun Planet Data Message
    inputMessageSize = spiceMessage.getStructSize()
    TotalSim.TotalSim.CreateNewMessage(unitProcessName,
                                       "sun_planet_data",
                                       inputMessageSize,
                                       2)  # number of buffers (leave at 2 as default, don't make zero)
    TotalSim.TotalSim.WriteMessageData("sun_planet_data",
                                       inputMessageSize,
                                       0,
                                       spiceMessage)

    # Initialize the spice modules that we are using.
    BACObject = bore_ang_calc.BoreAngCalc()
    BACObject.ModelTag = "solarArrayBoresight"
    BACObject.StateString = "inertial_state_output"
    BACObject.celBodyString = "sun_planet_data"
    BACObject.OutputDataString = "solar_array_sun_bore"
    BACObject.boreVec_B = boresightLoc  # boresight in body frame
    TotalSim.AddModelToTask(unitTaskName, BACObject)
    #
    # Configure simulation
    TotalSim.ConfigureStopTime(int(1.0 * 1E9))
    TotalSim.TotalSim.logThisMessage(BACObject.OutputDataString)

    # Execute simulation
    TotalSim.InitializeSimulation()
    TotalSim.AddVariableForLogging(BACObject.ModelTag + ".boreVecPoint", 1, 0, 2, "double")
    TotalSim.ExecuteSimulation()
    ###################################################################################################################
    #
    # Begin testing module results to truth values

    simMiss = TotalSim.pullMessageLogData(BACObject.OutputDataString + '.missAngle', range(1))
    simAz = TotalSim.pullMessageLogData(BACObject.OutputDataString + '.azimuth', range(1))
    simBoreVecPt = TotalSim.GetLogVariableData(BACObject.ModelTag + ".boreVecPoint")

    # Truth values
    dcm_BN = RigidBodyKinematics.MRP2C(stateMessage.sigma_BN)
    relPosVector = numpy.subtract(spiceMessage.PositionVector, stateMessage.r_BN_N)
    relVelVector = numpy.subtract(spiceMessage.VelocityVector, stateMessage.v_BN_N)
    magRelVelVec = numpy.sqrt(relVelVector[0] ** 2 + relVelVector[1] ** 2 + relVelVector[2] ** 2)
    if magRelVelVec == 0:
        secPointVector = numpy.zeros((1, 3))
        magSecPtVec = 0
    else:
        secPointVector = numpy.cross(relPosVector, relVelVector) / numpy.linalg.norm(numpy.cross(relPosVector,
                                                                                relVelVector))
        magSecPtVec = 1
    primPointVector = relPosVector / numpy.linalg.norm(relPosVector)  # r_p/b_N
    dcm_PoN = numpy.zeros((3, 3))
    dcm_PoN[0, 0:2] = primPointVector[0:2]
    magPrimPtVec = numpy.sqrt(primPointVector[0] ** 2 + primPointVector[1] ** 2 + primPointVector[2] ** 2)
    if magPrimPtVec != 0 and magSecPtVec != 0:
        dcm_PoN_2 = numpy.cross(primPointVector, secPointVector) / numpy.linalg.norm(
            numpy.cross(primPointVector, secPointVector))
        for i in range(3):
            dcm_PoN[2, i] = dcm_PoN_2[i]
    dcm_PoN_1 = numpy.cross(dcm_PoN_2, primPointVector)
    for i in range(3):
        dcm_PoN[1, i] = dcm_PoN_1[i]
    dcm_BPo = numpy.dot(dcm_BN, dcm_PoN.transpose())
    vecBore_B = numpy.zeros((3, 1))
    for i in range(3):
        vecBore_B[i, 0] = BACObject.boreVec_B[i]
    boreVecPoint = numpy.dot(numpy.transpose(dcm_BPo), vecBore_B)
    boreVecPoint_1 = []
    for i in range(3):
        boreVecPoint_1.append(boreVecPoint[i, 0])
    boreVecPoint_1 = numpy.array(boreVecPoint_1)

    ####################################################################################################################
    # attempt calculation in body frame
    r_B = numpy.dot(dcm_BN, stateMessage.r_BN_N)  # BN * N = B

    # Set tolersnce
    AllowTolerance = 1E-10
    boreVecPoint_final = [boreVecPoint_1]
    simBoreVecPt_final = [simBoreVecPt[0]]

    testFailCount, testMessages = unitTestSupport.compareArray(boreVecPoint_final, simBoreVecPt_final, AllowTolerance,
                                                               "FAILED: Calculating the vector boreVecPoint.",
                                                               testFailCount, testMessages)

    # Truth values
    #boreVecPoint_1 = [0.0, 1.0, 0.0]

    baselinePoint = [1.0, 0.0, 0.0]
    baselinePoint = numpy.array(baselinePoint)
    dotValue = numpy.dot(boreVecPoint_1, baselinePoint)
    r_N = numpy.dot(numpy.transpose(dcm_BN), BACObject.boreVec_B)
    baselineProj = numpy.dot(numpy.transpose(dcm_PoN), baselinePoint)
    dotValue_2 = numpy.dot(r_N, baselineProj)
    boresightMissAng = numpy.arccos(dotValue)
    boresightMissAng_2 = numpy.arccos(dotValue_2)  # boresight calc using body frame
    if boresightMissAng == numpy.pi / 2:
        simAz_final = numpy.array(simAz[-1])
        boresightAzimuth = simAz_final[-1]
        print "The miss angle is 0, therefore the miss angle is ill defined!"
    else:
        boresightAzimuth = numpy.arctan2(boreVecPoint_1[2], boreVecPoint_1[1])

    # Next Check
    AllowTolerance = 1E-10
    simMiss_final = numpy.array(simMiss[-1])
    if (boresightMissAng - simMiss_final[
        -1]) > AllowTolerance:  # Skip test days that are Sunday because of the end of a GPS week
        testFailCount += 1
        testMessages.append(
            "FAILED: Calculating the miss angle of the boresight failed with difference of: %(DiffVal)f \n" % \
            {"DiffVal": boresightMissAng - simMiss_final[-1]})
    simAz_final = numpy.array(simAz[-1])
    if (boresightAzimuth - simAz_final[-1]) > AllowTolerance:  # Skip test days that are Sunday because of the end of a GPS week
        testFailCount += 1
        testMessages.append(
            "FAILED: Calculating the azimuth angle of the boresight failed with difference of: %(DiffVal)f \n" % \
            {"DiffVal": boresightAzimuth - simAz_final[-1]})

    # print out success message if no error were found
    if testFailCount == 0:
        print "PASSED"
        testFixture.PassFail.append("PASSED")
    else:
        testFixture.PassFail.append("FAILED")

    # each test method requires a single assert method to be called
    #   this check below just makes sure no sub-test failures were found
    return [testFailCount, ''.join(testMessages)]

# This statement below ensures that the unit test scrip can be run as a
# stand-along python script
#
if __name__ == "__main__":
    bore_ang_calc_func(ResultsStore(), False,  # show_plots
                       [1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3), 1.0 / numpy.sqrt(3)], [0.0, 0.0, 0.0])


import numpy as np					# The Numpy library
import matplotlib.pyplot as plt		# Matplotlib's pyplot

import sys							# Gives access to a C-like sys library
import os							# Gives access to operations system

print(sys.argv)		# Prints any command line arguments, incl program name
print(os.getcwd())	# Prints the current working directory


from sage.misc.lazy_import import lazy_import
lazy_import('sage.doctest.control', 'run_doctests')


import numpy as np
import random
import torch

def set_seeds(seed_value, use_cuda):
    np.random.seed(seed_value)  # cpu vars
    torch.manual_seed(seed_value)  # cpu  vars
    random.seed(seed_value)  # Python
    if use_cuda:
        torch.cuda.manual_seed(seed_value)
        # torch.cuda.manual_seed_all(seed_value)  # gpu vars
        torch.backends.cudnn.deterministic = True  # needed
        torch.backends.cudnn.benchmark = False


import pytest
import numpy as np
import cv2
import S1_algotools as s1

"""-------------TEST average_above_zero---------------"""
def test_average_above_zero_expected_list():
    """Function that test if average_above_zero raise exeption when the param is not a list"""
    with pytest.raises(ValueError):
        s1.average_above_zero(15)

def test_average_above_zero_empty():
    """Function that test if average_above_zero raise exeption when the param is an empty list"""
    with pytest.raises(ValueError):
        s1.average_above_zero([])

def  test_average_above_zero_expected_number_list():
    """Function that test if average_above_zero raise exeption when the param is an not a numbers list"""
    with pytest.raises(ValueError):
        s1.average_above_zero(["e","r","r","o","r"])

def test_average_above_zero_working():
    """Function that test if average_above_zero return the correct average"""
    assert s1.average_above_zero([10,20])==15.0

"""--------------TEST max_value-----------------------"""
def test_max_value_expected_list():
    """Function that test if max_value raise exeption when the param is not a list"""
    with pytest.raises(ValueError):
        s1.max_value(15)

def test_max_value_empty():
    """Function that test if max_value raise exeption when the param is an empty list"""
    with pytest.raises(ValueError):
        s1.max_value([])

def  test_max_value_expected_number_list():
    """Function that test if max_value raise exeption when the param is an not a numbers list"""
    with pytest.raises(ValueError):
        s1.max_value(["e","r","r","o","r"])

def test_max_value_working():
    """Function that test if max_value return the max value and index"""
    assert s1.max_value([10,51,48,63,24])==(63,3)

"""--------------TEST reverse_table-------------------"""
def test_reverse_table_expected_list():
    """Function that test if reverse_table raise exeption when the param is not a list"""
    with pytest.raises(ValueError):
        s1.reverse_table(15)

def test_reverse_table_empty():
    """Function that test if reverse_table raise exeption when the param is an empty list"""
    with pytest.raises(ValueError):
        s1.reverse_table([])

def  test_reverse_table_expected_number_list():
    """Function that test if reverse_table raise exeption when the param is an not a numbers list"""
    with pytest.raises(ValueError):
        s1.reverse_table(["e","r","r","o","r"])

def test_revese_table_working():
    """Function that test if reverse_table return the reversed array"""
    assert np.array_equal(s1.reverse_table([10,15,20,12,15]),[15,12,20,15,10])==True

"""--------------TEST --------------------------"""
def test_roi_bbox_expected_ndarray():
    """Function that test if reverse_table raise exeption when the param is not a ndarray"""
    with pytest.raises(ValueError):
        s1.roi_bbox(15)

def test_roi_bbox_working():
    img=cv2.imread("img_sample.png",0)
    assert s1.roi_bbox(img)

"""--------------TEST random_fill_sparse---------------"""
def test_random_fill_sparse_working():
    """Function that test if random_fill_sparse return an array with 'X'"""
    charar=np.chararray((5, 5))
    charar[:] = '0'
    arr = s1.random_fill_sparse(charar,2)
    condition = arr == b'X'
    assert np.count_nonzero(condition)==2


#!/usr/bin/env python
# coding: utf-8

# # Handwritten mathematical sumbols

# ## Data preprocessing

# Hand_written
#
#     --> handwritten_mathematical_model.ipynb
#     --> hasy-data
#     --> hasy-data-labels
#     --> symbols.csv
#     --> hasyv2
#         --> hasy-data
#         --> hasy-data-labels
#         --> symbols.csv
#         --> verification-task
#     --> verification-task
#
# Note:- You can use hasyv2 but for this I have not used that

#Just to check
from IPython.display import Image
Image(url='hasy-data/v2-00010.png')

#importing libraries for preprocessing task
import csv
from PIL import Image as pil_image
import keras.preprocessing.image

#load all images and save there classes

imgs = []
classes = []
with open('hasy-data-labels.csv') as csvFile:
    csvReader = csv.reader(csvFile)
    i = 0
    for row in csvReader:
        if i > 0:
            img = keras.preprocessing.image.img_to_array(pil_image.open(row[0]))
            img /= 255.0
            imgs.append((row[0], row[2], img))
            classes.append(row[2])
        i += 1
print("Total number of images: ", len(imgs)) #print the total number of images

# Randomly split the data into training and test
#
# 80% -> train
#
# 20% -> test

import random
random.shuffle(imgs)
split_idx = int(0.8*len(imgs))
train = imgs[:split_idx]
test = imgs[split_idx:]

#Later take 20% from train for validation

import numpy as np

train_input = np.asarray(list(map(lambda row: row[2], train)))
test_input = np.asarray(list(map(lambda row: row[2], test)))

train_output = np.asarray(list(map(lambda row: row[1], train)))
test_output = np.asarray(list(map(lambda row: row[1], test)))

# learn about oneHotEncoder
# https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.OneHotEncoder.html
#our labels are in the text format therefore using oneHotEncoder to convert them

from sklearn.preprocessing import LabelEncoder, OneHotEncoder

label_encoder = LabelEncoder()
integer_encoded = label_encoder.fit_transform(classes)

onehot_encoder = OneHotEncoder(sparse = False)
integer_encoded = integer_encoded.reshape(len(integer_encoded), 1)
onehot_encoder.fit(integer_encoded)

train_output_int = label_encoder.transform(train_output)
train_output = onehot_encoder.transform(train_output_int.reshape(len(train_output_int), 1))
test_output_int = label_encoder.transform(test_output)
test_ouput = onehot_encoder.transform(test_output_int.reshape(len(test_output_int), 1))

num_classes = len(label_encoder.classes_)

print("Number of classes", num_classes)
print("Shape of the input image: ", np.shape(train_input[0]))

# ## Model

#importing libraries

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import RMSprop, Adam
from tensorflow.keras.layers import (
    Dense,
    Dropout,
    Flatten,
    Conv2D,
    MaxPooling2D
)

# This is very basic CNN architecture with some convolution layer followed by max pooling layer
# which is then followed by flatten and some dense layer.

#model architecture

model = Sequential([
    Conv2D(32, (3,3), activation='relu', input_shape = np.shape(train_input[0])),
    MaxPooling2D(pool_size = (2,2)),
    Conv2D(32, (3,3), activation='relu'),
    MaxPooling2D(pool_size = (2,2)),
    Flatten(),
    Dense(1024, activation = 'tanh'),
    Dropout(0.5),  #to reduce overfitting
    Dense(num_classes, activation='softmax')
])

#compile model
learning_rate = 1e-5 #1e-2 (0.01)

model.compile(
            optimizer = 'adam', #Adam(lr = learning_rate)
            loss = 'categorical_crossentropy',
            metrics = ['accuracy'])

model.summary()

# Training
BATCH_SIZE = 32
EPOCHS = 15 #10
VALIDATION_SPLIT = 0.2

history = model.fit(train_input, train_output,
                   batch_size = BATCH_SIZE,
                   epochs = EPOCHS,
                   verbose = 1,
                   validation_split = VALIDATION_SPLIT)

#Accuracy I achieved was nearly 82% after 10th epoch (Val_accuracy -> 77%)
#tuning hyparameters and changing model architecture will help you gain much more accuracy and reduce overfitting

# Plotting the graphs:
#     1. Training and validation accuracy
#     2. Training and validation loss

import matplotlib.pyplot as plt

acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(len(acc))

fig = plt.subplots(figsize=(30, 10))

plt.subplot(1, 2, 1)
plt.plot(acc)
plt.plot(val_acc)
plt.title('Training and Validation Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend(['Training Accuracy', 'Validation Accuracy'])

plt.subplot(1, 2, 2)
plt.plot(loss)
plt.plot(val_loss)
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend(['Training Loss', 'Validation Loss'])

plt.show()

# ## Saving model
model.save('mathSymbolsPredictor.model')
np.save('classes.npy', label_encoder.classes_)

# ## New predictions

import keras.models
#model_ = keras.models.load('mathSymbolsPredictor.model')

label_encoder2 = LabelEncoder()
label_encoder2.classes_ = np.load('classes.npy')

PATH = 'hasy-data/v2-00100.png'

image = keras.preprocessing.image.img_to_array(pil_image.open(PATH))
image /= 255.
predicted = (model.predict(image.reshape(1, 32, 32, 3)))
inverted = label_encoder2.inverse_transform([np.argmax(predicted)])
print(inverted[0], np.max(predicted))
(Image(url=PATH))

# # Convert your model into .tflite
# ## To deploy it on an application

RPS_SAVED_MODEL = "rps_saved_model"
tf.saved_model.save(model, RPS_SAVED_MODEL)

get_ipython().run_cell_magic('bash', '-s $RPS_SAVED_MODEL', 'saved_model_cli show --dir $1 --tag_set serve --signature_def serving_default')

loaded = tf.saved_model.load(RPS_SAVED_MODEL)
print(list(loaded.signatures.keys()))
infer = loaded.signatures["serving_default"]
print(infer.structured_input_signature)
print(infer.structured_outputs)
converter = tf.lite.TFLiteConverter.from_saved_model(RPS_SAVED_MODEL)
converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE]
tflite_model = converter.convert()

import tensorflow as tf
tflite_model_file = 'handwritten_mathematical_symbol_model_v1.tflite'


import numpy as np

# Create 2 lists containing any the same number of elements
# Save each as objects named a_list and b_list

____

____

# Using boolean operators, what is outputted when you test to see if they are equal?

____


#!/usr/bin/env python3

import numpy as np                  # numpy library (many number manipulation code)
import matplotlib.pyplot as plt     # matplotlib.pyplot used for making graphs and plots

import sys                          # gives access to sys library
import os                           # gives access to operating system

print(sys.argv)                     # prints command line arguements
print(os.getcwd())                  # prints the current directory that this code is in


"""
Fortran compiler
"""
from __future__ import absolute_import
from six import iteritems

import os
import imp
import shutil
import sys

from sage.misc.temporary_file import tmp_dir

class InlineFortran:
    def __init__(self, globals=None):
        # globals=None means: use user globals from REPL
        self.globs = globals
        self.library_paths=[]
        self.libraries=[]
        self.verbose = False

    def __repr__(self):
        return "Interface to Fortran compiler"

    def __call__(self, *args, **kwds):
        return self.eval(*args, **kwds)

    def eval(self, x, globals=None, locals=None):
        """
        Compile fortran code ``x`` and adds the functions in it to
        ``globals``.

        INPUT:

        - ``x`` -- Fortran code

        - ``globals`` -- a dict to which to add the functions from the
          fortran module

        - ``locals`` -- ignored

        EXAMPLES::

            sage: code = '''
            ....: C FILE: FIB1.F
            ....:       SUBROUTINE FIB(A,N)
            ....: C
            ....: C     CALCULATE FIRST N FIBONACCI NUMBERS
            ....: C
            ....:       INTEGER N
            ....:       REAL*8 A(N)
            ....:       DO I=1,N
            ....:          IF (I.EQ.1) THEN
            ....:             A(I) = 0.0D0
            ....:          ELSEIF (I.EQ.2) THEN
            ....:             A(I) = 1.0D0
            ....:          ELSE
            ....:             A(I) = A(I-1) + A(I-2)
            ....:          ENDIF
            ....:       ENDDO
            ....:       END
            ....: C END FILE FIB1.F
            ....: '''
            sage: fortran(code, globals())
            sage: import numpy
            sage: a = numpy.array(range(10), dtype=float)
            sage: fib(a, 10)
            sage: a
            array([  0.,   1.,   1.,   2.,   3.,   5.,   8.,  13.,  21.,  34.])

        TESTS::

            sage: os.chdir(SAGE_ROOT)
            sage: fortran.eval("SYNTAX ERROR !@#$")
            Traceback (most recent call last):
            ...
            RuntimeError: failed to compile Fortran code:...
            sage: os.getcwd() == SAGE_ROOT
            True
        """
        if globals is None:
            globals = self.globs
            if globals is None:
                from sage.repl.user_globals import get_globals
                globals = get_globals()

        from numpy import f2py

        # Create everything in a temporary directory
        mytmpdir = tmp_dir()

        try:
            old_cwd = os.getcwd()
            os.chdir(mytmpdir)

            name = "fortran_module"  # Python module name
            # if the first line has !f90 as a comment, gfortran will
            # treat it as Fortran 90 code
            if x.startswith('!f90'):
                fortran_file = name + '.f90'
            else:
                fortran_file = name + '.f'

            s_lib_path = ""
            s_lib = ""
            for s in self.library_paths:
                s_lib_path = s_lib_path + "-L%s "

            for s in self.libraries:
                s_lib = s_lib + "-l%s "%s

            log = name + ".log"
            extra_args = '--quiet --f77exec=sage-inline-fortran --f90exec=sage-inline-fortran %s %s >"%s" 2>&1'%(
                s_lib_path, s_lib, log)

            f2py.compile(x, name, extra_args = extra_args, source_fn=fortran_file)
            log_string = open(log).read()

            # Note that f2py() doesn't raise an exception if it fails.
            # In that case, the import below will fail.
            try:
                file, pathname, description = imp.find_module(name, [mytmpdir])
            except ImportError:
                raise RuntimeError("failed to compile Fortran code:\n" + log_string)
            try:
                m = imp.load_module(name, file, pathname, description)
            finally:
                file.close()

            if self.verbose:
                print(log_string)
        finally:
            os.chdir(old_cwd)

            if sys.platform != 'cygwin':
                # Do not delete temporary DLLs on Cygwin; this will cause
                # future forks of this process to fail.  Instead temporary DLLs
                # will be cleaned up upon process exit
                try:
                    shutil.rmtree(mytmpdir)
                except OSError:
                    # This can fail for example over NFS
                    pass

        for k, x in iteritems(m.__dict__):
            if k[0] != '_':
                globals[k] = x

    def add_library(self,s):
       self.libraries.append(s)

    def add_library_path(self,s):
       self.library_paths.append(s)

# An instance
fortran = InlineFortran()


"""
Miscellaneous experiments for Data Bootcamp course

Repository of materials (including this file):
* https://github.com/NYUDataBootcamp/Materials/
* https://github.com/NYUDataBootcamp/Materials/Code/Python

Written by Dave Backus, March 2015
Created with Python 3.4
"""
print('\nWelcome to Data Bootcamp!')

import datetime as dt
print('Today is', dt.date.today())

"""
Check Python version
"""
import sys

print('\nWhat version of Python are we running? \n', sys.version, '\n', sep='')

if float(sys.version_info[0]) < 3.0:
    raise Exception('Program halted, old version of Python. ' +
                    'Sorry, you need to install Anaconda again.')
else:
    print('Congratulations, Python is up to date!')
#    sys.exit(0)      # this halts execution

#%%
"""
Assignments and copies
http://stackoverflow.com/questions/10844493/dataframe-apply-in-python-pandas-alters-both-original-and-duplicate-dataframes
"""
# check 1
a = [1,2,3]
b = a
b[0] = 'WHOA!'
print('\nAfter assignment, a is', a)

# to make a copy
a = [1,2,3]
b = a.copy()
b[0] = 'WHOA!'
print('\nAfter copy, a is', a)

# check 2
import numpy as np
c = np.array([7, 3, 5])
d = c
e = 2*c - 5
print('\nAfter assignment, (d, e) are', d, e)

c[0] = 10
print(d, e)

#%%
"""
Check path of current working directory
https://docs.python.org/2/library/os.path.html
"""
import os

print('\nCurrent path:\n', os.getcwd(), sep='')

"""
Check for specific file
"""
import os

print('\nList of files in working directory:')
[print(file) for file in os.listdir()]

file = 'SQL_support_code.py'
if not os.path.isfile(file):
    raise Exception('***** Program halted, file missing *****')

#%%
"""
Reading fixed width files
http://pandas.pydata.org/pandas-docs/version/0.17.0/generated/pandas.read_fwf.html
Input file is
1234567890
2345678901
3456789012
"""
import pandas as pd

fixed = pd.read_fwf('fixedformatdata.txt',
                    colspecs=[(0,2), (3,6)],    # column n1 to n2-1, start at 0
                    names=['x1', 'x2'],
                    header=None)

print('\nFixed-format file \n', fixed)

#%%
"""
Copying files from internet to hard drive (temp directory)
"""
import urllib              # handles internet files

url1 = 'https://raw.githubusercontent.com/NYUDataBootcamp/Materials/master/'
url2 = 'Data/test.csv'
url = url1 + url2
fname = '../' + 'Temp/' + 'goo.csv'

# copy file from url to fname
urllib.request.urlretrieve(url, fname)

#%%
"""
Extract file from zip
https://docs.python.org/2/library/zipfile.html
https://pymotw.com/2/zipfile/
"""
import zipfile             # handles zip files

zfname = '../' + 'Temp/' + 'wp10245.zip'

print('\nIs this a zipfile?', zipfile.is_zipfile(zfname))

# create zipfile object
zf = zipfile.ZipFile(zfname, 'r')
print('Contents:', zf.namelist())

# extract file
inzip = zf.namelist()[0]
dir  = '../Temp/'
extract = zf.extract(inzip, path=dir)

df = pd.read_excel(extract, sheetname=1, na_values=['…', '….', ''], index_col=0,
                   encoding='utf-8')

#%%
"""
IMF's historical database on public debt
https://www.imf.org/External/pubs/cat/longres.aspx?sk=24332.0
rows are countries, columns are dates (1692-2012)
"""
import pandas as pd
import urllib              # handles internet files
import zipfile             # handles zip files
import os

# copy zip file to hard drive
print('\nCopy IMF historical debt data to hard drive')
url = 'https://www.imf.org/external/pubs/ft/wp/2010/Data/wp10245.zip'
zname = '../Temp/' + os.path.basename(url)   # strip out file name
urllib.request.urlretrieve(url, zname)       # copy file from url to disk

# extract spreadsheet in two steps
zf = zipfile.ZipFile(zname, 'r')
zf.printdir()
xlsname = zf.namelist()[0]
xls = zf.extract(xlsname)

df = pd.read_excel(xls, sheetname=1, na_values=['…', '….', ''], index_col=0,
                   encoding='utf-8')

print('Type: ', type(df))
print('Shape (dimensions): ', df.shape)
print('Column labels (variables): ', df.columns.tolist())
print('Variable types: \n', df.dtypes, sep='')

df.tail()

#%%
# select years 1980 to 2013 and ifscode
years = [year for year in range(1980, 2013)]
years_str = [str(year) for year in years]
vars = ['ifscode'] + years

some = df[vars]

#%%
"""
Shortcut:  save file to disk, read from there
"""
file = '../Temp/' + 'Debt Database Fall 2013 Vintage.xlsx'
df = pd.read_excel(file, sheetname=1, na_values=['…', '….', ''], index_col=0,
                   encoding='utf-8')
#%%
print('Type: ', type(df))
print('Shape (dimensions): ', df.shape)
print('Column labels (variables): ', df.columns.tolist())
print('Variable types: \n', df.dtypes, sep='')

# select years 1980 to 2013 and ifscode
years = [year for year in range(1980, 2013)]
#years_str = [str(year) for year in years]
vars = years
some = df[vars]

#%%
"""
unicode
http://eev.ee/blog/2015/09/12/dark-corners-of-unicode/
https://docs.python.org/3.4/library/unicodedata.html
https://docs.python.org/3/howto/unicode.html#unicode-properties
http://stackoverflow.com/questions/508558/what-charset-does-microsoft-excel-use-when-saving-files
"""
import unicodedata

# this came up in the IMF debt data (cut and paste from spreadsheet)
s = '…'
len(s)

print('Unicode category and name: ',
      unicodedata.category(s), ', ', unicodedata.name(s), sep='')

#%%
"""
urrlib version of data input from csv
"""
# copy file from url to hard drive
import urllib.request
file = 'foo.csv'
url1 = 'https://raw.githubusercontent.com/NYUDataBootcamp/Materials/master/'
url2 = 'Data/test.csv'
url = url1 + url2
urllib.request.urlretrieve(url, file)

# Sarah's version
f = urllib.request.urlopen(url)
file = 'foo_sbh.csv'
with open(file, 'wb') as local_file:
    local_file.write(f.read())

#%%
"""
World Bank WDI from zip file
File is too big, takes too long to read in class (but great stuff!)
"""
import pandas as pd
import urllib
import zipfile
import os

# this is a big file, best to test with something smaller
url  = 'http://databank.worldbank.org/data/download/WDI_csv.zip'
file = '../Temp/' + os.path.basename(url)   # strip out file name
urllib.request.urlretrieve(url, file)        # copy to disk

# see what's there
print(['Is zipfile?', zipfile.is_zipfile(file)])
zf = zipfile.ZipFile(file, 'r')
#print('List of zipfile contents (two versions)')
zf.printdir()

# extract a component
csv = zf.extract('WDI_Data.csv')        # copy to disk
df1 = pd.read_csv(csv)       # read
print(df1.columns)                      # check contents

# alternative:  open and read
csv = zf.open('WDI_Data.csv')
df2 = pd.read_csv(csv)
print(df2.columns)

#%%
"""
Penn World Table
http://www.rug.nl/research/ggdc/data/pwt/pwt-8.1
Takes about 10 seconds on home wireless network
"""
import pandas as pd

url = 'http://www.rug.nl/research/ggdc/data/pwt/v81/pwt81.xlsx'
df = pd.read_excel(url, sheetname=2)

print(df.ftypes)

#%%
"""
Maddison data
http://www.ggdc.net/maddison/maddison-project/home.htm
Takes about 10 seconds on home wireless network
"""
import pandas as pd

url = 'http://www.ggdc.net/maddison/maddison-project/data/mpd_2013-01.xlsx'
df = pd.read_excel(url) #, skiprows=1, index_col=0)

print(df.ftypes)

#%%
"""
Equality of Opportunity
http://www.equality-of-opportunity.org/index.php/data
"""
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib as mpl

url = 'http://www.equality-of-opportunity.org/images/online_data_tables.xls'
df = pd.read_excel(url, sheetname=1, skiprows=8, header=0, index_col=0)
print(df.tail())
df.tail()
#%%

# fix column labels
df.columns = [pct for pct in range(1, 101)]
print('\n', df.ftypes[0:7], sep='')

# trim to eliminate extremes
trimmed = df.iloc[6:92, 6:92]

#http://stackoverflow.com/questions/14391959/heatmap-in-matplotlib-with-pcolor
fig, ax = plt.subplots()
heatmap = ax.pcolor(trimmed, cmap=plt.cm.Blues)

#%%
"""
Changing directory, using complete paths, etc
"""
import pandas as pd
import os

file = 'test.csv'
#path1 = 'C:\\Users\\dbackus\\Dropbox\\Documents\\Classes\\Data_Bootcamp\\Code\\Python'
path2 = 'C:/Users/dbackus/Dropbox/Documents/Classes/Data_Bootcamp/Code/Python'

# check to see if this works
os.chdir('C:/Users')
print('Current working directory', os.getcwd())

os.chdir(path2)
print('Current working directory', os.getcwd())

#%%
# check to see if file exists
exists = os.path.isfile(path1+'/'+file)
print('File exists?', exists)

if exists==False:
    raise Exception('File does not exist, execution halted')

#%%
df1 = pd.read_csv(path1+'\\'+file)
print('\nBack slash (df1)\n', df1)

df1 = pd.read_csv(path1+'/'+file)
print('\nBack slash alternative (df1)\n', df1)

df2 = pd.read_csv(path2+'/'+file)
print('\nForward slash (df2)\n', df2)

#%%
# extracting path and file from combination
fullpath = path2 + '/' + file

file = os.path.basename(fullpath)
path = os.path.dirname(fullpath)
print('File and path\n', file, '\n', path)

#%%
"""
Heart attack data from data.gov

https://data.medicare.gov/api/views/c7us-v4mf/rows.csv?accessType=DOWNLOAD
https://data.medicare.gov/developers
"""
import pandas as pd
url = 'https://data.medicare.gov/api/views/c7us-v4mf/rows.csv'

df = pd.read_csv(url)

#%%
print(list(df))

"""
Appendices
"""

# in case the internet is down
#df = pd.DataFrame([['Dave', 1, 2, 3.5],
#                   ['Chase', 4, 3, 4.3],
#                   ['Spencer', 5, 6, 7.8]],
#                   columns=['name', 'x1', 'x2', 'x3'])
#print('\nurl read (df)\n', df)


import os

import numpy as np
import matplotlib.pyplot as plt

from saf.euler1d.linear.asciireader import ASCIIReader

from helpers import FIGSIZE_SIX_SUBPLOTS as figsize, savefig

# ---
# ## Helper functions

# In[2]:

def znd_read_data():
    output_dir = '_output'
    dirs = os.listdir(output_dir)
    dirs.sort()

    x = []
    rho = []
    u = []
    p = []
    lamda = []

    for d in dirs:
        dirname = os.path.join(output_dir, d)

        r = ASCIIReader(dirname)

        znd_data = r.get_znd_data()
        x.append(znd_data['x'])
        rho.append((d, znd_data['rho']))
        u.append((d, znd_data['u_lab']))
        p.append((d, znd_data['p']))
        lamda.append((d, znd_data['lamda']))

    return x, rho, u, p, lamda

def _znd_plot_quantity(x, quantity, axis, k):
    line_styles = ['-', '--', ':', '-o', '-*']
    assert k >= 0
    assert k < len(line_styles)
    if k != 3 and k != 4:
        axis.plot(x, quantity, line_styles[k])
    else:
        ls = line_styles[k]
        s1 = ls[0]
        s2 = ls[1]
        R = 10
        line, = axis.plot(x, quantity, s1)
        color = line.get_color()
        axis.plot(x[::R], quantity[::R], s2, color=color, markeredgecolor=color)

def znd_plot_data(x, rho, u, p, lamda):
    # Number of rows and columns in figures
    m, n = 3, 2
    E_ACT = 30.0
    X_LIM = -8

    fig, axes = plt.subplots(nrows=m, ncols=n, figsize=figsize)

    assert len(x) == len(rho)
    assert len(x) == len(u)
    assert len(x) == len(p)
    assert len(x) == len(lamda)

    # Density
    ax = axes[0, 0]
    for k, __ in enumerate(rho):
        cur_x = x[k]
        cur_rho = rho[k][1]
        cur_rho = cur_rho/cur_rho[-1]

        _znd_plot_quantity(cur_x, cur_rho, ax, k)
    ax.set_ylabel(r'$\bar{\rho}/\bar{\rho}_{\mathrm{s}}$')
    ax.set_xlabel(r'$x$')
    ax.set_xlim((X_LIM, 0))
    ax.set_yticks([0.0, 0.2, 0.4, 0.6, 0.8, 1.0])
    ax.set_ylim((0, 1.0))

    # Progress variable
    ax = axes[0, 1]
    for k, __ in enumerate(lamda):
        cur_x = x[k]
        cur_lamda = lamda[k][1]

        _znd_plot_quantity(cur_x, cur_lamda, ax, k)
    ax.set_ylabel(r'$\bar{\lambda}$')
    ax.set_xlabel(r'$x$')
    ax.set_xlim((X_LIM, 0))
    ax.set_ylim((0, 1.0))

    # Pressure
    ax = axes[1, 0]
    for k, __ in enumerate(p):
        cur_x = x[k]
        cur_p = p[k][1]
        cur_p = cur_p/cur_p[-1]

        _znd_plot_quantity(cur_x, cur_p, ax, k)
    ax.set_ylabel(r'$\bar{p}/\bar{p}_{\mathrm{s}}$')
    ax.set_xlabel(r'$x$')
    ax.set_xlim((X_LIM, 0))
    ax.set_ylim((0.5, 1.0))

    # Velocity
    ax = axes[1, 1]
    for k, __ in enumerate(u):
        cur_x = x[k]
        cur_label = u[k][0]
        cur_u = u[k][1]
        cur_u = cur_u/cur_u[-1]

        _znd_plot_quantity(cur_x, cur_u, ax, k)
    ax.set_ylabel(r'$\bar{u}/\bar{u}_{\mathrm{s}}$')
    ax.set_xlabel(r'$x$')
    ax.set_xlim((X_LIM, 0))
    ax.set_ylim((0.5, 1.0))

    # Temperature
    ax = axes[2, 0]
    for k, __ in enumerate(p):
        cur_x = x[k]
        cur_p = p[k][1]
        cur_p = cur_p / cur_p[-1]
        cur_rho = rho[k][1]
        cur_rho = cur_rho / cur_rho[-1]
        cur_T = cur_p / cur_rho

        _znd_plot_quantity(cur_x, cur_T, ax, k)
    ax.set_ylabel(r'$\bar{T}/\bar{T}_{\mathrm{s}}$')
    ax.set_xlabel(r'$x$')
    ax.set_xlim((X_LIM, 0))
    ax.set_ylim((0.95, 2.8))

    # Reaction rate
    ax = axes[2, 1]
    for k, __ in enumerate(p):
        cur_x = x[k]
        cur_p = p[k][1]
        cur_rho = rho[k][1]
        cur_lamda = lamda[k][1]
        r = (1 - cur_lamda) * np.exp(-E_ACT*cur_rho/(cur_p))
        r = r / np.max(r)

        _znd_plot_quantity(cur_x, r, ax, k)
    ax.set_ylabel(r'$\bar{\omega}/\bar{\omega}_{\mathrm{max}}$')
    ax.set_xlabel(r'$x$')
    ax.set_xlim((X_LIM, 0))
    ax.set_ylim((0, 1.0))

    fig.tight_layout(pad=0.1, h_pad=0.55)
    savefig('znd-solutions.pdf')

# ---
# ## Reading data

# In[3]:

x, rho, u, p, lamda = znd_read_data()

# ---
# ## Plotting ZND solutions

# The figure below contains profiles of different quantities of ZND solution.
# Solid lines is for $Q=0.5$, dashed line is for $Q=1.0$, dotted line is for $Q=10$, solid line with circle marker is for $Q=50$, solid line with star marker is for $Q=100$.

# In[4]:

znd_plot_data(x, rho, u, p, lamda)


"""
Note: for naming purposes, most tests are title with as e.g. "test_nlargest_foo"
but are implicitly also testing nsmallest_foo.
"""
from string import ascii_lowercase

import numpy as np
import pytest

import pandas as pd
import pandas._testing as tm

@pytest.fixture
def df_duplicates():
    return pd.DataFrame(
        {"a": [1, 2, 3, 4, 4], "b": [1, 1, 1, 1, 1], "c": [0, 1, 2, 5, 4]},
        index=[0, 0, 1, 1, 1],
    )

@pytest.fixture
def df_strings():
    return pd.DataFrame(
        {
            "a": np.random.permutation(10),
            "b": list(ascii_lowercase[:10]),
            "c": np.random.permutation(10).astype("float64"),
        }
    )

@pytest.fixture
def df_main_dtypes():
    return pd.DataFrame(
        {
            "group": [1, 1, 2],
            "int": [1, 2, 3],
            "float": [4.0, 5.0, 6.0],
            "string": list("abc"),
            "category_string": pd.Series(list("abc")).astype("category"),
            "category_int": [7, 8, 9],
            "datetime": pd.date_range("20130101", periods=3),
            "datetimetz": pd.date_range("20130101", periods=3, tz="US/Eastern"),
            "timedelta": pd.timedelta_range("1 s", periods=3, freq="s"),
        },
        columns=[
            "group",
            "int",
            "float",
            "string",
            "category_string",
            "category_int",
            "datetime",
            "datetimetz",
            "timedelta",
        ],
    )

class TestNLargestNSmallest:

    # ----------------------------------------------------------------------
    # Top / bottom
    @pytest.mark.parametrize(
        "order",
        [
            ["a"],
            ["c"],
            ["a", "b"],
            ["a", "c"],
            ["b", "a"],
            ["b", "c"],
            ["a", "b", "c"],
            ["c", "a", "b"],
            ["c", "b", "a"],
            ["b", "c", "a"],
            ["b", "a", "c"],
            # dups!
            ["b", "c", "c"],
        ],
    )
    @pytest.mark.parametrize("n", range(1, 11))
    def test_nlargest_n(self, df_strings, nselect_method, n, order):
        # GH#10393
        df = df_strings
        if "b" in order:

            error_msg = (
                f"Column 'b' has dtype object, "
                f"cannot use method '{nselect_method}' with this dtype"
            )
            with pytest.raises(TypeError, match=error_msg):
                getattr(df, nselect_method)(n, order)
        else:
            ascending = nselect_method == "nsmallest"
            result = getattr(df, nselect_method)(n, order)
            expected = df.sort_values(order, ascending=ascending).head(n)
            tm.assert_frame_equal(result, expected)

    @pytest.mark.parametrize(
        "columns", [["group", "category_string"], ["group", "string"]]
    )
    def test_nlargest_error(self, df_main_dtypes, nselect_method, columns):
        df = df_main_dtypes
        col = columns[1]
        error_msg = (
            f"Column '{col}' has dtype {df[col].dtype}, "
            f"cannot use method '{nselect_method}' with this dtype"
        )
        # escape some characters that may be in the repr
        error_msg = (
            error_msg.replace("(", "\\(")
            .replace(")", "\\)")
            .replace("[", "\\[")
            .replace("]", "\\]")
        )
        with pytest.raises(TypeError, match=error_msg):
            getattr(df, nselect_method)(2, columns)

    def test_nlargest_all_dtypes(self, df_main_dtypes):
        df = df_main_dtypes
        df.nsmallest(2, list(set(df) - {"category_string", "string"}))
        df.nlargest(2, list(set(df) - {"category_string", "string"}))

    def test_nlargest_duplicates_on_starter_columns(self):
        # regression test for GH#22752

        df = pd.DataFrame({"a": [2, 2, 2, 1, 1, 1], "b": [1, 2, 3, 3, 2, 1]})

        result = df.nlargest(4, columns=["a", "b"])
        expected = pd.DataFrame(
            {"a": [2, 2, 2, 1], "b": [3, 2, 1, 3]}, index=[2, 1, 0, 3]
        )
        tm.assert_frame_equal(result, expected)

        result = df.nsmallest(4, columns=["a", "b"])
        expected = pd.DataFrame(
            {"a": [1, 1, 1, 2], "b": [1, 2, 3, 1]}, index=[5, 4, 3, 0]
        )
        tm.assert_frame_equal(result, expected)

    def test_nlargest_n_identical_values(self):
        # GH#15297
        df = pd.DataFrame({"a": [1] * 5, "b": [1, 2, 3, 4, 5]})

        result = df.nlargest(3, "a")
        expected = pd.DataFrame({"a": [1] * 3, "b": [1, 2, 3]}, index=[0, 1, 2])
        tm.assert_frame_equal(result, expected)

        result = df.nsmallest(3, "a")
        expected = pd.DataFrame({"a": [1] * 3, "b": [1, 2, 3]})
        tm.assert_frame_equal(result, expected)

    @pytest.mark.parametrize(
        "order",
        [["a", "b", "c"], ["c", "b", "a"], ["a"], ["b"], ["a", "b"], ["c", "b"]],
    )
    @pytest.mark.parametrize("n", range(1, 6))
    def test_nlargest_n_duplicate_index(self, df_duplicates, n, order):
        # GH#13412

        df = df_duplicates
        result = df.nsmallest(n, order)
        expected = df.sort_values(order).head(n)
        tm.assert_frame_equal(result, expected)

        result = df.nlargest(n, order)
        expected = df.sort_values(order, ascending=False).head(n)
        tm.assert_frame_equal(result, expected)

    def test_nlargest_duplicate_keep_all_ties(self):
        # GH#16818
        df = pd.DataFrame(
            {"a": [5, 4, 4, 2, 3, 3, 3, 3], "b": [10, 9, 8, 7, 5, 50, 10, 20]}
        )
        result = df.nlargest(4, "a", keep="all")
        expected = pd.DataFrame(
            {
                "a": {0: 5, 1: 4, 2: 4, 4: 3, 5: 3, 6: 3, 7: 3},
                "b": {0: 10, 1: 9, 2: 8, 4: 5, 5: 50, 6: 10, 7: 20},
            }
        )
        tm.assert_frame_equal(result, expected)

        result = df.nsmallest(2, "a", keep="all")
        expected = pd.DataFrame(
            {
                "a": {3: 2, 4: 3, 5: 3, 6: 3, 7: 3},
                "b": {3: 7, 4: 5, 5: 50, 6: 10, 7: 20},
            }
        )
        tm.assert_frame_equal(result, expected)

    def test_nlargest_multiindex_column_lookup(self):
        # Check whether tuples are correctly treated as multi-level lookups.
        # GH#23033
        df = pd.DataFrame(
            columns=pd.MultiIndex.from_product([["x"], ["a", "b"]]),
            data=[[0.33, 0.13], [0.86, 0.25], [0.25, 0.70], [0.85, 0.91]],
        )

        # nsmallest
        result = df.nsmallest(3, ("x", "a"))
        expected = df.iloc[[2, 0, 3]]
        tm.assert_frame_equal(result, expected)

        # nlargest
        result = df.nlargest(3, ("x", "b"))
        expected = df.iloc[[3, 2, 1]]
        tm.assert_frame_equal(result, expected)


# coding: utf-8

# # Solution to Question 2

# ## Python imports
#
# Before executing code cells we load some required packages and initiate database connection.

# In[1]:

import os # directory pathfinding
if 'notebooks' in os.getcwd(): # make sure jupyter server is in project root
    get_ipython().run_line_magic('cd', '..')

# In[2]:

import altair as alt # visualization library
import numpy as np  # linear algebra
import pandas as pd  # data processing
from IPython.display import display # show interactive sliders
from ipywidgets import interact # common interactivity features
import ipywidgets as widgets # package for interactivity

# import matplotlib.pyplot as plt
# import seaborn as sns
# color = sns.color_palette()

# alt.data_transformers.enable('json') # if number of records > 5000
alt.renderers.enable('notebook') # Renderer for notebook (comment out if using jupyterlab)

import lib # project code package

# In[3]:

# create a database connection
database = os.path.realpath('../rannala_project/db/rannala_project.db')
conn = lib.create_connection(database)

# ## Data Journey
#
# Now we are ready to start exploring the dataset.

# ### First 5 rows of cleaned dataset

# In[4]:

# read in dataset
df = pd.read_sql_query(
    '''SELECT f.*,
    cm.country
    FROM q2_clean f
    INNER JOIN country_mapping cm on
    f.alpha_2 = cm.alpha_2;''', conn)
df.head()

# ### Data types in dataframe

# In[5]:

df.info()

# ### Field descriptions
#
# - **country**: Country ISO2 code in lowercase.
# - **hours_bucket**: Time difference in minutes between click and install, bucketed into hours. 0 means game was installs within - for 60 minutes, 1 means between 60-119 minutes, 2 means 120-179 minutes, and so on.
# - **installs**: Number of installs in given country and time-to-install bucket.
# - **country_total_installs**: Total number of installs in the given country, regardless of time to install.
# - **share_within_country**: Share of installs within the given time-to-install bucket among all install in the given country

# ### Some basic statistical figures

# In[6]:

df.describe()

# ### First impression from data
#
# By far most of the installs happen in the first (0) hour. One country is a much bigger player than any of the others (spoiler: the red color signifies United States).
#
# It will be useful to sometimes view the dataset without US, to see better the variance within other countries' data.

# In[7]:

# Altair chart
alt.Chart(df).mark_bar().encode(
    x='hours_bucket:O',
    y='installs:Q',
    color=alt.Color('Country:N', legend=None)
).properties(
    title='Installs by Hour Bucket (color = Country)',
)

# ### Initial filters to slice the dataset
# (after setting these filters **rerun all code cells** in notebook to reload the reporting dataframe)
#
# - Minimum hours_bucket (default = 0)
# - Minimum country_total_installs (default = 1000)
# - Countries to exclude (default = None)
#     - Hold SHIFT and click to select multiple countries for exclusion

# In[8]:

# Slider widget instance
x = 0
slider = widgets.IntSlider(description='bucket (min)',min=0,max=47,step=1)
slider.value = x
def on_change(v):
    x = v['new']
slider.observe(on_change, names='value')
display(slider)

# In[9]:

# Slider widget instance
x = 1000
slider2 = widgets.IntSlider(description='country_total_installs (min)',min=0,max=50000,step=1000)
slider2.value = x
def on_change(v):
    x = v['new']
slider2.observe(on_change, names='value')
display(slider2)

# In[10]:

# Slider widget instance
slider3 = widgets.SelectMultiple(
    options=sorted(df['Country'].unique()),
    value=[],
    #rows=10,
    description='Country',
    disabled=False
)
def on_change(v):
    value = v['new']
slider3.observe(on_change, names='value')
display(slider3)

# In[11]:

# Apply initial dataset filters to new dataframe df1
cond1 = df['hours_bucket']>= slider.value
cond2 = df['country_total_installs']>= slider2.value

df_temp = df[cond1 & cond2]

df1 = df_temp[~df_temp['Country'].isin(list(slider3.value))]

# ### Installs by Hour Bucket & Country
#
# Here the dataset has already been filtered by the three preset filters defined above.

# In[12]:

# Altair Chart
alt.Chart(df1).mark_bar().encode(
    x='hours_bucket:O',
    y='installs:Q',
    color=alt.Color('Country:N')
).properties(
    title='Installs by Hour Bucket (color = Country)',
)

# ### Further filtering in interactive mode
#
# In the following chart, a live filter with Hour Bucket slider can be interacted with - the visualization responds immediately. No need to rerun code cells. The three original dataset filters still apply so we can't see data which has been filtered away at that stage.

# In[13]:

# function that returns the interactive chart dataset
def filter_dataframe_bucket(bucket):
    df_bucket = df1[df1['hours_bucket'] >= bucket]
    #return df_bucket.describe()
    return alt.Chart(df_bucket).mark_bar().encode(
    x='hours_bucket:O',
    y='installs:Q',
    color=alt.Color('Country:N')
).properties(
    title='Installs by Hour Bucket (color = Country)',
)

# In[14]:

# Altair Chart
interact(filter_dataframe_bucket, bucket = widgets.IntSlider(description='bucket (min)',min=0,max=47,step=1,value=1), );

# Next chart repeats the same approach, this time with an interactive country selector dropdown.

# In[15]:

# function that returns the interactive chart dataset
def make_altair_chart(country):
    chart = alt.Chart(df1[df1['Country'] == country]).mark_bar().encode(
        alt.Y('Country:N', sort=alt.EncodingSortField(field="installs:Q", op="sum", order='descending')),
        alt.X('hours_bucket:O'),
        alt.Color('share_within_country:Q', scale=alt.Scale(scheme='greenblue'))
        )
    return chart

# In[16]:

# Altair Chart
interact(make_altair_chart, country = sorted(df1['Country'].unique()), );

# ### Overview of share_within_country percentage per Hour Bucket
#
# For each country it is clear that the majority of installs conclude in the first ("zero") hour after an ad is clicked.

# In[17]:

# Altair Chart
alt.Chart(df1).mark_bar().encode(
        alt.Y('Country:N', sort=alt.EncodingSortField(field="installs", op="sum", order='descending')),
        alt.X('hours_bucket:O'),
    alt.Color('share_within_country:Q')
        ).properties(
    title='Share_within_country - Installs by Hour Bucket (color = share_within_country)',
)

# ### Conclusion
#
# - Binned Hours Bucket vs Installs
#
# Lets conclude by returning to the initial dataset without any filtering (only backend cleaning applied). With a simple binned chart we can conclude that the first couple hours (especially the very first hour) after click are critical. After that it becomes very unlikely that an ad-clicker would ever start the game in their mobile.

# In[18]:

# Altair Chart
alt.Chart(df).mark_rect().encode(
    alt.X('hours_bucket:Q', bin=alt.Bin(maxbins=10)),
    alt.Y('installs:Q')
)