"""
A pytest conftest module that provides pytest fixtures for galois/fields/ tests.
"""
import json
import os
import pickle

import pytest
import numpy as np

import galois

PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data")

FIELDS = [
    pytest.param("GF(2)"),

    pytest.param("GF(2^2)"),
    pytest.param("GF(2^3)"),
    pytest.param("GF(2^8)"),
    pytest.param("GF(2^32)"),
    pytest.param("GF(2^100)"),

    pytest.param("GF(5)"),
    pytest.param("GF(7)"),
    pytest.param("GF(31)"),
    pytest.param("GF(3191)"),
    pytest.param("GF(2147483647)"),
    pytest.param("GF(36893488147419103183)"),

    pytest.param("GF(7^3)"),
    pytest.param("GF(109987^4)"),
]

FIELDS_DIFF_MODES = [
    pytest.param("GF(2)-jit-calculate"),

    pytest.param("GF(2^2)-jit-lookup"),
    pytest.param("GF(2^2)-jit-calculate"),
    pytest.param("GF(2^3)-jit-lookup"),
    pytest.param("GF(2^3)-jit-calculate"),
    pytest.param("GF(2^8)-jit-lookup"),
    pytest.param("GF(2^8)-jit-calculate"),
    pytest.param("GF(2^8, 283, 19)-jit-lookup"),
    pytest.param("GF(2^8, 283, 19)-jit-calculate"),
    pytest.param("GF(2^32)-jit-calculate"),
    pytest.param("GF(2^100)-python-calculate"),

    pytest.param("GF(5)-jit-lookup"),
    pytest.param("GF(5)-jit-calculate"),
    pytest.param("GF(7)-jit-lookup"),
    pytest.param("GF(7)-jit-calculate"),
    pytest.param("GF(31)-jit-lookup"),
    pytest.param("GF(31)-jit-calculate"),
    pytest.param("GF(3191)-jit-lookup"),
    pytest.param("GF(3191)-jit-calculate"),
    pytest.param("GF(2147483647)-jit-calculate"),
    pytest.param("GF(36893488147419103183)-python-calculate"),

    pytest.param("GF(7^3)-jit-lookup"),
    pytest.param("GF(7^3)-jit-calculate"),
    pytest.param("GF(7^3, 643, 244)-jit-lookup"),
    pytest.param("GF(7^3, 643, 244)-jit-calculate"),
    pytest.param("GF(109987^4)-python-calculate"),
]

def construct_field(folder):
    if len(folder.split("-")) >= 2:
        folder, ufunc_mode = folder.split("-", maxsplit=1)
    else:
        ufunc_mode = "auto"

    if folder == "GF(2)":
        GF = galois.GF2

    elif folder == "GF(5)":
        GF = galois.GF(5, compile=ufunc_mode)
    elif folder == "GF(7)":
        GF = galois.GF(7, compile=ufunc_mode)
    elif folder == "GF(31)":
        GF = galois.GF(31, compile=ufunc_mode)
    elif folder == "GF(3191)":
        GF = galois.GF(3191, compile=ufunc_mode)
    elif folder == "GF(2147483647)":
        GF = galois.GF(2147483647, compile=ufunc_mode)
    elif folder == "GF(36893488147419103183)":
        GF = galois.GF(36893488147419103183, compile=ufunc_mode)

    elif folder == "GF(2^2)":
        GF = galois.GF(2**2, compile=ufunc_mode)
    elif folder == "GF(2^3)":
        GF = galois.GF(2**3, compile=ufunc_mode)
    elif folder == "GF(2^8)":
        GF = galois.GF(2**8, compile=ufunc_mode)
    elif folder == "GF(2^8, 283, 19)":
        GF = galois.GF(2**8, irreducible_poly=283, primitive_element=19, compile=ufunc_mode)
    elif folder == "GF(2^32)":
        GF = galois.GF(2**32, compile=ufunc_mode)
    elif folder == "GF(2^100)":
        GF = galois.GF(2**100, compile=ufunc_mode)

    elif folder == "GF(7^3)":
        GF = galois.GF(7**3, compile=ufunc_mode)
    elif folder == "GF(7^3, 643, 244)":
        GF = galois.GF(7**3, irreducible_poly=643, primitive_element=244, compile=ufunc_mode)
    elif folder == "GF(109987^4)":
        GF = galois.GF(109987**4, compile=ufunc_mode)

    else:
        raise AssertionError(f"Test data folder {folder} not found")

    return GF, ufunc_mode, os.path.join(PATH, folder)

def read_json(field_folder, filename):
    GF, folder = field_folder
    with open(os.path.join(folder, filename), "rb") as f:
        d = json.load(f)
    return GF, d

def read_pickle(field_folder, filename):
    GF, folder = field_folder
    with open(os.path.join(folder, filename), "rb") as f:
        print(f"Loading {f}...")
        d = pickle.load(f)
    return GF, d

###############################################################################
# Fixtures for iterating over each finite field
###############################################################################

@pytest.fixture(scope="session", params=FIELDS)
def field(request):
    folder = request.param
    return construct_field(folder)[0]

@pytest.fixture(scope="session", params=FIELDS_DIFF_MODES)
def field_folder(request):
    folder = request.param
    field, ufunc_mode, folder = construct_field(folder)
    return field, folder

###############################################################################
# Fixtures for arithmetic over finite fields
###############################################################################

@pytest.fixture(scope="session")
def field_properties(field_folder):
    GF, d = read_json(field_folder, "properties.json")
    d["GF"] = GF
    d["characteristic"] = d["characteristic"]
    d["degree"] = d["degree"]
    d["order"] = d["order"]
    d["primitive_element"] = d["primitive_element"]
    d["irreducible_poly"] = galois.Poly(d["irreducible_poly"], field=galois.GF(d["characteristic"]))
    return d

@pytest.fixture(scope="session")
def field_add(field_folder):
    GF, d = read_pickle(field_folder, "add.pkl")
    d["GF"] = GF
    X, Y = np.meshgrid(d["X"], d["Y"], indexing="ij")
    d["X"] = GF(X)
    d["Y"] = GF(Y)
    d["Z"] = GF(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_subtract(field_folder):
    GF, d = read_pickle(field_folder, "subtract.pkl")
    d["GF"] = GF
    X, Y = np.meshgrid(d["X"], d["Y"], indexing="ij")
    d["X"] = GF(X)
    d["Y"] = GF(Y)
    d["Z"] = GF(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_multiply(field_folder):
    GF, d = read_pickle(field_folder, "multiply.pkl")
    d["GF"] = GF
    X, Y = np.meshgrid(d["X"], d["Y"], indexing="ij")
    d["X"] = GF(X)
    d["Y"] = GF(Y)
    d["Z"] = GF(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_divide(field_folder):
    GF, d = read_pickle(field_folder, "divide.pkl")
    d["GF"] = GF
    X, Y = np.meshgrid(d["X"], d["Y"], indexing="ij")
    d["X"] = GF(X)
    d["Y"] = GF(Y)
    d["Z"] = GF(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_additive_inverse(field_folder):
    GF, d = read_pickle(field_folder, "additive_inverse.pkl")
    d["GF"] = GF
    d["X"] = GF(d["X"])
    d["Z"] = GF(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_multiplicative_inverse(field_folder):
    GF, d = read_pickle(field_folder, "multiplicative_inverse.pkl")
    d["GF"] = GF
    d["X"] = GF(d["X"])
    d["Z"] = GF(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_scalar_multiply(field_folder):
    GF, d = read_pickle(field_folder, "scalar_multiply.pkl")
    d["GF"] = GF
    X, Y = np.meshgrid(d["X"], d["Y"], indexing="ij")
    d["X"] = GF(X)
    d["Y"] = Y
    d["Z"] = GF(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_power(field_folder):
    GF, d = read_pickle(field_folder, "power.pkl")
    d["GF"] = GF
    X, Y = np.meshgrid(d["X"], d["Y"], indexing="ij")
    d["X"] = GF(X)
    d["Y"] = Y
    d["Z"] = GF(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_log(field_folder):
    GF, d = read_pickle(field_folder, "log.pkl")
    d["GF"] = GF
    d["X"] = GF(d["X"])
    d["Z"] = d["Z"]
    return d

###############################################################################
# Fixtures for linear algebra over finite fields
###############################################################################

@pytest.fixture(scope="session")
def field_matrix_multiply(field_folder):
    GF, d = read_pickle(field_folder, "matrix_multiply.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Y"] = [GF(y) for y in d["Y"]]
    d["Z"] = [GF(z) for z in d["Z"]]
    return d

@pytest.fixture(scope="session")
def field_row_reduce(field_folder):
    GF, d = read_pickle(field_folder, "row_reduce.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Z"] = [GF(z) for z in d["Z"]]
    return d

@pytest.fixture(scope="session")
def field_lu_decompose(field_folder):
    GF, d = read_pickle(field_folder, "lu_decompose.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["L"] = [GF(l) for l in d["L"]]
    d["U"] = [GF(u) for u in d["U"]]
    return d

@pytest.fixture(scope="session")
def field_plu_decompose(field_folder):
    GF, d = read_pickle(field_folder, "plu_decompose.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["P"] = [GF(p) for p in d["P"]]
    d["L"] = [GF(l) for l in d["L"]]
    d["U"] = [GF(u) for u in d["U"]]
    return d

@pytest.fixture(scope="session")
def field_matrix_inverse(field_folder):
    GF, d = read_pickle(field_folder, "matrix_inverse.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Z"] = [GF(z) for z in d["Z"]]
    return d

@pytest.fixture(scope="session")
def field_matrix_determinant(field_folder):
    GF, d = read_pickle(field_folder, "matrix_determinant.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Z"] = GF(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_matrix_solve(field_folder):
    GF, d = read_pickle(field_folder, "matrix_solve.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Y"] = [GF(y) for y in d["Y"]]
    d["Z"] = [GF(z) for z in d["Z"]]
    return d

@pytest.fixture(scope="session")
def field_row_space(field_folder):
    GF, d = read_pickle(field_folder, "row_space.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Z"] = [GF(z) for z in d["Z"]]
    return d

@pytest.fixture(scope="session")
def field_column_space(field_folder):
    GF, d = read_pickle(field_folder, "column_space.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Z"] = [GF(z) for z in d["Z"]]
    return d

@pytest.fixture(scope="session")
def field_left_null_space(field_folder):
    GF, d = read_pickle(field_folder, "left_null_space.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Z"] = [GF(z) for z in d["Z"]]
    return d

@pytest.fixture(scope="session")
def field_null_space(field_folder):
    GF, d = read_pickle(field_folder, "null_space.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Z"] = [GF(z) for z in d["Z"]]
    return d

###############################################################################
# Fixtures for arithmetic methods over finite fields
###############################################################################

@pytest.fixture(scope="session")
def field_additive_order(field_folder):
    GF, d = read_pickle(field_folder, "additive_order.pkl")
    d["GF"] = GF
    d["X"] = GF(d["X"])
    d["Z"] = d["Z"]
    return d

@pytest.fixture(scope="session")
def field_multiplicative_order(field_folder):
    GF, d = read_pickle(field_folder, "multiplicative_order.pkl")
    d["GF"] = GF
    d["X"] = GF(d["X"])
    d["Z"] = d["Z"]
    return d

@pytest.fixture(scope="session")
def field_characteristic_poly_element(field_folder):
    GF, d = read_pickle(field_folder, "characteristic_poly_element.pkl")
    d["GF"] = GF
    d["X"] = GF(d["X"])
    d["Z"] = [galois.Poly(p, field=GF.prime_subfield) for p in d["Z"]]
    return d

@pytest.fixture(scope="session")
def field_characteristic_poly_matrix(field_folder):
    GF, d = read_pickle(field_folder, "characteristic_poly_matrix.pkl")
    d["GF"] = GF
    d["X"] = [GF(x) for x in d["X"]]
    d["Z"] = [galois.Poly(p, field=GF) for p in d["Z"]]
    return d

@pytest.fixture(scope="session")
def field_minimal_poly_element(field_folder):
    GF, d = read_pickle(field_folder, "minimal_poly_element.pkl")
    d["GF"] = GF
    d["X"] = GF(d["X"])
    d["Z"] = [galois.Poly(p, field=GF.prime_subfield) for p in d["Z"]]
    return d

# @pytest.fixture(scope="session")
# def field_minimal_poly_matrix(field_folder):
#     GF, d = read_pickle(field_folder, "minimal_poly_matrix.pkl")
#     d["GF"] = GF
#     d["X"] = [GF(x) for x in d["X"]]
#     d["Z"] = [galois.Poly(p, field=GF) for p in d["Z"]]
#     return d

@pytest.fixture(scope="session")
def field_trace(field_folder):
    GF, d = read_pickle(field_folder, "field_trace.pkl")
    d["GF"] = GF
    d["X"] = GF(d["X"])
    d["Z"] = GF.prime_subfield(d["Z"])
    return d

@pytest.fixture(scope="session")
def field_norm(field_folder):
    GF, d = read_pickle(field_folder, "field_norm.pkl")
    d["GF"] = GF
    d["X"] = GF(d["X"])
    d["Z"] = GF.prime_subfield(d["Z"])
    return d


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import datetime as dt
import missingno as msno
import plotly.express as px

# In[2]:

def get_data(week_nums):
    url = "http://web.mta.info/developers/data/nyct/turnstile/turnstile_{}.txt"
    dfs = []
    for week_num in week_nums:
        file_url = url.format(week_num)
        dfs.append(pd.read_csv(file_url))
    return pd.concat(dfs)

week_nums = [210828, 210821, 210814, 210807, 210731, 210724, 210717, 210710, 210703, 210626,210619, 210612, 210605]
turnstiles_df = get_data(week_nums)

# In[3]:

turnstiles_df.head()

# In[4]:

turnstiles_df.tail()

# In[5]:

turnstiles_df.describe()

# In[6]:

turnstiles_df.info()

# In[7]:

# From importing the data from the website, we noticed some extra spaced columns
turnstiles_df.columns

# In[8]:

# The purpose of this line of code is to fix and adjust the extra spacing in the columns
turnstiles_df.columns = [column.strip() for column in turnstiles_df.columns]
turnstiles_df.columns

# In[9]:

# Three months of Data
turnstiles_df.DATE.value_counts().sort_index()

# In[10]:

# Since our imported data's date and time columns are object dtype, we need to put the date and time in a single column and change from object to datetime dtype to help us to access the data easily
turnstiles_df["DATE_TIME"] = pd.to_datetime(turnstiles_df.DATE + " " + turnstiles_df.TIME,format="%m/%d/%Y %H:%M:%S")

# In[11]:

turnstiles_df.head()

# In[12]:

# By applying the mask concept we can filter the data for such a specefic station
mask = ((turnstiles_df["C/A"] == "R504") &
(turnstiles_df["UNIT"] == "R276") &
(turnstiles_df["SCP"] == "00-00-01") &
(turnstiles_df["STATION"] == "VERNON-JACKSON"))

turnstiles_df[mask].head()

# In[13]:

# Sanity Check to verify that "C/A", "UNIT", "SCP", "STATION", "DATE_TIME" is unique
(turnstiles_df
 .groupby(["C/A", "UNIT", "SCP", "STATION", "DATE_TIME"])
 .ENTRIES.count()
 .reset_index()
 .sort_values("ENTRIES", ascending=False))

# In[14]:

# The detailed information about duplicated rows
turnstiles_df[turnstiles_df.duplicated(subset=["C/A", "UNIT", "SCP", "STATION", "DATE_TIME"])]

# In[15]:

# Number of diplicarted rows
turnstiles_df.duplicated(subset=["C/A", "UNIT", "SCP", "STATION", "DATE_TIME"]).sum()

# In[16]:

# By applying the mask concept we can filter the data for such a specefic station and time
mask1 = ((turnstiles_df["C/A"] == "R504") &
(turnstiles_df["UNIT"] == "R276") &
(turnstiles_df["SCP"] == "00-00-01") &
(turnstiles_df["STATION"] == "VERNON-JACKSON") &
(turnstiles_df["DATE_TIME"].dt.date == dt.datetime(2021, 8, 21).date()))

# In[17]:

turnstiles_df[mask1].head()

# In[18]:

# To check if there is any NULL value
turnstiles_df.info()

# In[19]:

# To check if there is any NULL value
turnstiles_df.isna().sum()

# In[20]:

# Visualize missing values as a matrix

msno.matrix(turnstiles_df)

# In[21]:

# Handle missing values if any
turnstiles_df.dropna()

# In[22]:

# Alternative methode to handle missing values if any
turnstiles_df.fillna(0)

# In[23]:

# By applying the mask concept we can filter the data for such a specefic station and time
mask2 = ((turnstiles_df["C/A"] == "R504")&(turnstiles_df["UNIT"] == "R276")&(turnstiles_df["SCP"] == "00-00-01")&(turnstiles_df["STATION"] == "VERNON-JACKSON")&(turnstiles_df["DESC"] == "RECOVR AUD")&(turnstiles_df["DATE_TIME"].dt.date == dt.datetime(2021, 8, 16).date()))
turnstiles_df[mask2].head()

# In[24]:

turnstiles_df.DESC.value_counts()

# In[25]:

# Get rid of the duplicate entry
turnstiles_df.sort_values(["C/A", "UNIT", "SCP", "STATION", "DATE_TIME"],
                          inplace=True, ascending=False)
turnstiles_df.drop_duplicates(subset=["C/A", "UNIT", "SCP", "STATION", "DATE_TIME"], inplace=True)

# In[26]:

# Sanity Check to verify that "C/A", "UNIT", "SCP", "STATION", "DATE_TIME" is unique
(turnstiles_df.groupby(["C/A", "UNIT", "SCP", "STATION", "DATE_TIME"]).ENTRIES.count().reset_index().sort_values("ENTRIES", ascending=False)).head(5)

# In[27]:

# No more duplicated rows
turnstiles_df.duplicated(subset=["C/A", "UNIT", "SCP", "STATION", "DATE_TIME"]).sum()

# In[28]:

# The detailed information about non-duplicated rows
turnstiles_df[turnstiles_df.duplicated(subset=["C/A", "UNIT", "SCP", "STATION", "DATE_TIME"])]

# In[29]:

# The sum of number of entries for each unique turnstile at a specefict time and date
turnstiles_daily = (turnstiles_df.groupby(["C/A", "UNIT", "SCP", "STATION", "DATE","TIME"],as_index=False).ENTRIES.first())

turnstiles_daily.head(20)

# In[30]:

turnstiles_daily[["PREV_DATE", "PREV_ENTRIES"]] = (turnstiles_daily.groupby(["C/A", "UNIT", "SCP", "STATION"])["DATE", "ENTRIES"].apply(lambda grp: grp.shift(1)))

# Drop the rows for the earliest date in the df
turnstiles_daily.dropna(subset=["PREV_DATE"], axis=0, inplace=True)

# Handle missing values if any
turnstiles_daily.dropna()

turnstiles_daily[turnstiles_daily["ENTRIES"] < turnstiles_daily["PREV_ENTRIES"]].head()

# What's the deal with counter being in reverse
mask = ((turnstiles_df["C/A"] == "A011")&(turnstiles_df["UNIT"] == "R080")&(turnstiles_df["SCP"] == "01-00-00")&(turnstiles_df["STATION"] == "57 ST-7 AV")&(turnstiles_df["DATE_TIME"].dt.date == dt.datetime(2021, 8, 27).date()))
turnstiles_df[mask].head()

(turnstiles_daily[turnstiles_daily["ENTRIES"] < turnstiles_daily["PREV_ENTRIES"]].groupby(["C/A", "UNIT", "SCP", "STATION"]).size())

# In[31]:

def get_daily_counts(row, max_counter):
    counter = row["ENTRIES"] - row["PREV_ENTRIES"]
    if counter < 0:
        # Maybe counter is reversed?
        counter = -counter
    if counter > max_counter:
        # Maybe counter was reset to 0?
        print(row["ENTRIES"], row["PREV_ENTRIES"])
        counter = min(row["ENTRIES"], row["PREV_ENTRIES"])
    if counter > max_counter:
        # Check it again to make sure we're not still giving a counter that's too big
        return 0
    return counter

# If counter is > 1Million, then the counter might have been reset.
# Just set it to zero as different counters have different cycle limits
# It'd probably be a good idea to use a number even significantly smaller than 1 million as the limit!
turnstiles_daily["DAILY_ENTRIES"] = turnstiles_daily.apply(get_daily_counts, axis=1, max_counter=1000000)

turnstiles_daily.head(15)

# In[32]:

turnstiles_dailye = (turnstiles_df.groupby(["C/A", "UNIT", "SCP", "STATION", "DATE","TIME"],as_index=False).EXITS.first())
turnstiles_dailye[["PREV_DATE", "PREV_EXITS"]] = (turnstiles_dailye.groupby(["C/A", "UNIT", "SCP", "STATION"])["DATE", "EXITS"].apply(lambda grp: grp.shift(1)))
turnstiles_dailye.dropna(subset=["PREV_DATE"], axis=0, inplace=True)
turnstiles_dailye.dropna()
turnstiles_dailye[turnstiles_dailye["EXITS"] < turnstiles_dailye["PREV_EXITS"]].head()

# In[33]:

def get_daily_countsy(row, max_counter):
    counter = row["EXITS"] - row["PREV_EXITS"]
    if counter < 0:
        # Maybe counter is reversed?
        counter = -counter
    if counter > max_counter:
        # Maybe counter was reset to 0?
        print(row["EXITS"], row["PREV_EXITS"])
        counter = min(row["EXITS"], row["PREV_EXITS"])
    if counter > max_counter:
        # Check it again to make sure we're not still giving a counter that's too big
        return 0
    return counter

# If counter is > 1Million, then the counter might have been reset.
# Just set it to zero as different counters have different cycle limits
# It'd probably be a good idea to use a number even significantly smaller than 1 million as the limit!
turnstiles_dailye["DAILY_EXITS"] = turnstiles_dailye.apply(get_daily_countsy, axis=1, max_counter=1000000)

turnstiles_daily["DAILY_EXITS"]= turnstiles_dailye["DAILY_EXITS"]

turnstiles_daily

# In[34]:

turnstiles_daily['TRAFIC'] = turnstiles_daily['DAILY_EXITS'] + turnstiles_daily['DAILY_ENTRIES']
turnstiles_daily["DATE_TIME"] = pd.to_datetime(turnstiles_daily.DATE + " " + turnstiles_daily.TIME,format="%m/%d/%Y %H:%M:%S")

turnstiles_daily['DAY']=turnstiles_daily.DATE_TIME.dt.day_name()

turnstiles_daily=turnstiles_daily[['C/A','UNIT','SCP','STATION','DATE','TIME','DATE_TIME','DAY','DAILY_ENTRIES','DAILY_EXITS','TRAFIC']]

# In[35]:

turnstiles_stations = turnstiles_daily.groupby('STATION').mean().reset_index().sort_values('TRAFIC',ascending=False).head(15)

fig = px.bar(turnstiles_stations, x='STATION', y='TRAFIC',title="TOP CROWDED STATIONS", labels={'TRAFIC':'TRAFFIC (mean)'})
fig.show()

# In[36]:

turnstiles_stations = turnstiles_daily.groupby('STATION').mean().reset_index().sort_values('TRAFIC',ascending=True).head(15)

fig = px.bar(turnstiles_stations, x='STATION', y='TRAFIC',title="LEAST CROWDED STATIONS", labels={'TRAFIC':'TRAFFIC (mean)'})
fig.show()

# In[37]:

turnstile_OB =turnstiles_daily[turnstiles_daily['STATION']=='ORCHARD BEACH']
turnstile_OBb = turnstile_OB.groupby('DAY').mean().sort_values('TRAFIC',ascending=False).reset_index()
fig = px.bar(turnstile_OBb, x="DAY", y="TRAFIC",title="DAILY TRAFFIC LEAST CROWDED STATIONS", labels={'TRAFIC':'TRAFFIC (mean)'})
fig.show()

# In[38]:

turnstile_tts =turnstiles_daily[turnstiles_daily['STATION']=='TWENTY THIRD ST']
turnstile_ttss = turnstile_tts.groupby('DAY').mean().sort_values('TRAFIC',ascending=False).reset_index()
fig = px.bar(turnstile_ttss, x="DAY", y="TRAFIC",title="DAILY TRAFFIC MOST CROWDED STATIONS", labels={'TRAFIC':'TRAFFIC (mean)'})
fig.show()

# In[39]:

turnstile_OBt=turnstile_OB.groupby("TIME").mean().sort_values('TIME',ascending=True).reset_index()
fig = px.line(turnstile_OBt,x='TIME',y='TRAFIC',title="LEAST CROWDED STATIONS WITH RESPECT TO TIME", labels={'TRAFIC':'TRAFFIC (mean)'})
fig.show()

# In[ ]:

# In[ ]:


'''
Utility functions

@author Jonathan Karr, karr@mssm.edu
@date 3/22/2016
'''

import numpy as np

N_AVOGADRO = 6.022e23
# Avogadro constant

def nanminimum(x, y):
    return np.where(np.logical_or(np.isnan(y), np.logical_and(x <= y, np.logical_not(np.isnan(x)))), x, y)

def nanmaximum(x, y):
    return np.where(np.logical_or(np.isnan(y), np.logical_and(x >= y, np.logical_not(np.isnan(x)))), x, y)


import numpy as np

# build-in func enumerate
lst = np.arange(1, 10)
for i, v in enumerate(lst):
    print(f" i:{i} v:{v}")

some_list = ['foo', 'bar', 'baz']
mapping = {}

for i, v in enumerate(some_list):
    mapping[i] = v

print(f"mapping:{mapping}")


# -*- coding: utf-8 -*-
"""
Created on Wed Jun 20 09:34:37 2018

@author: SilverDoe
"""

'''
========================== DataFrame ==========================================

pandas.DataFrame( data, index, columns, dtype, copy)

Parameters :
============
1. data : data takes various forms like ndarray, series, map, lists, dict, constants
   and also another DataFrame.

2. index : For the row labels, the Index to be used for the resulting frame is
   Optional Default np.arrange(n) if no index is passed.

3. columns : For column labels, the optional default syntax is - np.arrange(n).
   This is only true if no index is passed.

4. dtype : Data type of each column.

5. copy : This command (or whatever it is) is used for copying of data, if the
   default is False.

'''

#=============== Empty DataFrame =================================================

import pandas as pd
df = pd.DataFrame()
print(df)

#============= DataFrame from Lists ============================================

# no index passed, no column names given
import pandas as pd
data = [1,2,3,4,5]
df = pd.DataFrame(data)
print(df)

# no index passed, column names given
import pandas as pd
data = [['Natsu',13],['Lisanna',9],['Happy',1]]
df = pd.DataFrame(data,columns=['Name','Age'])
print(df)

# no index passed, column names given, datatype passed
import pandas as pd
data = [['Natsu',13],['Lisanna',8],['Happy',1]]
df = pd.DataFrame(data,columns=['Name','Age'],dtype=float)
print(df)

#========== Dataframe from Dictionary of ndarrays/lists ============================================

'''
>>All the ndarrays must be of same length. If index is passed, then the length
 of the index should equal to the length of the arrays.

>> To preserve the order of the columns:
    1. use ordered doctionary, since dictionaries will not preserve the order when created.
    2. use columns index while creating the dataframe.
    3. use  reorder the columns the way you want by using df = df[list of column names in the order you want]

'''

# using arrays, No index given.
import pandas as pd
data = {'Name':['Lisanna', 'Natsu', 'Erza', 'Gray'],'Age':[15,20,23,20]}
df = pd.DataFrame(data)
print(df)

# using arrays, Index given.
import pandas as pd
data = {'Name':['Lisanna', 'Natsu', 'Erza', 'Gray'],'Age':[15,20,23,20]}
df = pd.DataFrame(data, index=['rank1','rank2','rank3','rank4'])
print(df)

'''
>>List of Dictionaries can be passed as input data to create a DataFrame. The
dictionary keys are by default taken as column names.

>>NaN (Not a Number) is appended in missing areas when using lists instead of arrays.

'''

# using lists, no index given
import pandas as pd
data = [{'a': 1, 'b': 2},{'a': 5, 'b': 10, 'c': 20}]
df = pd.DataFrame(data)
print(df)

# using lists,index given
import pandas as pd
data = [{'a': 1, 'b': 2},{'a': 5, 'b': 10, 'c': 20}]
df = pd.DataFrame(data, index=['first', 'second'])
print(df)

# using lists,index given, columns given
import pandas as pd
data = [{'a': 1, 'b': 2},{'a': 5, 'b': 10, 'c': 20}]

#With two column indices, values same as dictionary keys
df1 = pd.DataFrame(data, index=['first', 'second'], columns=['a', 'b'])
print(df1)

#With two column indices with one index with other name
df2 = pd.DataFrame(data, index=['first', 'second'], columns=['a', 'b1'])
print(df2)

#using dictionary of Series
import pandas as pd

d = {'one' : pd.Series([1, 2, 3], index=['a', 'b', 'c']),
      'two' : pd.Series([1, 2, 3, 4], index=['a', 'b', 'c', 'd'])}

df = pd.DataFrame(d)
print(df)

# using ordered dictionary to preserve the order of the columns
import numpy as np
import pandas as pd
from collections import OrderedDict

a = np.array( [ 1, 2, 3 ] )
b = np.array( [ 4, 5, 6 ] )
c = np.array( [ 7, 8, 9 ] )

nd = { 'p': pd.Series(a), 'z': pd.Series(b), 'n': pd.Series(c) } # normal dictionary
od = OrderedDict( { 'p': pd.Series(a), 'z': pd.Series(b), 'n': pd.Series(c) } ) # ordered doctionary

df = pd.DataFrame(od)
print(df)


"""
=========
Run Setup
=========

By: Jan N. van Rijn

One of the key features of the openml-python library is that is allows to
reinstantiate flows with hyperparameter settings that were uploaded before.
This tutorial uses the concept of setups. Although setups are not extensively
described in the OpenML documentation (because most users will not directly
use them), they form a important concept within OpenML distinguishing between
hyperparameter configurations.
A setup is the combination of a flow with all its hyperparameters set.

A key requirement for reinstantiating a flow is to have the same scikit-learn
version as the flow that was uploaded. However, this tutorial will upload the
flow (that will later be reinstantiated) itself, so it can be ran with any
scikit-learn version that is supported by this library. In this case, the
requirement of the corresponding scikit-learn versions is automatically met.

In this tutorial we will
    1) Create a flow and use it to solve a task;
    2) Download the flow, reinstantiate the model with same hyperparameters,
       and solve the same task again;
    3) We will verify that the obtained results are exactly the same.

.. warning:: This example uploads data. For that reason, this example
   connects to the test server at test.openml.org. This prevents the main
   server from crowding with example datasets, tasks, runs, and so on.
"""

# License: BSD 3-Clause

import numpy as np
import openml
import sklearn.ensemble
import sklearn.impute
import sklearn.preprocessing

openml.config.start_using_configuration_for_example()

###############################################################################
# 1) Create a flow and use it to solve a task
###############################################################################

# first, let's download the task that we are interested in
task = openml.tasks.get_task(6)

# we will create a fairly complex model, with many preprocessing components and
# many potential hyperparameters. Of course, the model can be as complex and as
# easy as you want it to be
model_original = sklearn.pipeline.make_pipeline(
    sklearn.impute.SimpleImputer(),
    sklearn.ensemble.RandomForestClassifier()
)

# Let's change some hyperparameters. Of course, in any good application we
# would tune them using, e.g., Random Search or Bayesian Optimization, but for
# the purpose of this tutorial we set them to some specific values that might
# or might not be optimal
hyperparameters_original = {
    'simpleimputer__strategy': 'median',
    'randomforestclassifier__criterion': 'entropy',
    'randomforestclassifier__max_features': 0.2,
    'randomforestclassifier__min_samples_leaf': 1,
    'randomforestclassifier__n_estimators': 16,
    'randomforestclassifier__random_state': 42,
}
model_original.set_params(**hyperparameters_original)

# solve the task and upload the result (this implicitly creates the flow)
run = openml.runs.run_model_on_task(
    model_original,
    task,
    avoid_duplicate_runs=False)
run_original = run.publish()  # this implicitly uploads the flow

###############################################################################
# 2) Download the flow and solve the same task again.
###############################################################################

# obtain setup id (note that the setup id is assigned by the OpenML server -
# therefore it was not yet available in our local copy of the run)
run_downloaded = openml.runs.get_run(run_original.run_id)
setup_id = run_downloaded.setup_id

# after this, we can easily reinstantiate the model
model_duplicate = openml.setups.initialize_model(setup_id)
# it will automatically have all the hyperparameters set

# and run the task again
run_duplicate = openml.runs.run_model_on_task(
    model_duplicate, task, avoid_duplicate_runs=False)

###############################################################################
# 3) We will verify that the obtained results are exactly the same.
###############################################################################

# the run has stored all predictions in the field data content
np.testing.assert_array_equal(run_original.data_content,
                              run_duplicate.data_content)

###############################################################################

openml.config.stop_using_configuration_for_example()


# pylint: disable=missing-function-docstring, missing-module-docstring/
# coding: utf-8

import pytest
import numpy as np

from pyccel.epyccel import epyccel
from pyccel.decorators import types
from pyccel.errors.errors import PyccelError

def test_func_no_args_1(language):
    '''test function with return value but no args'''
    def free_gift():
        gift = 10
        return gift

    c_gift = epyccel(free_gift, language=language)
    assert c_gift() == free_gift()
    assert isinstance(c_gift(), type(free_gift()))
    unexpected_arg = 0
    with pytest.raises(TypeError):
        c_gift(unexpected_arg)

def test_func_no_args_2(language):
    '''test function with negative return value but no args'''
    def p_lose():
        lose = -10
        return lose

    c_lose = epyccel(p_lose, language=language)
    assert c_lose() == p_lose()
    assert isinstance(c_lose(), type(p_lose()))
    unexpected_arg = 0
    with pytest.raises(TypeError):
        c_lose(unexpected_arg)

@pytest.mark.parametrize( 'language', [
        pytest.param("fortran", marks = pytest.mark.fortran),
        pytest.param("c", marks = pytest.mark.c),
        pytest.param("python", marks = pytest.mark.python),
    ]
)
def test_func_no_return_1(language):
    '''Test function with args and no return '''
    @types(int)
    def p_func(x):
        x *= 2

    c_func = epyccel(p_func, language=language)
    x = np.random.randint(100)
    assert c_func(x) == p_func(x)
    # Test type return sould be NoneType
    x = np.random.randint(100)
    assert isinstance(c_func(x), type(p_func(x)))

def test_func_no_return_2(language):
    '''Test function with no args and no return '''
    def p_func():
        x = 2
        x *= 2

    c_func = epyccel(p_func, language=language)
    assert c_func() == p_func()
    assert isinstance(c_func(), type(p_func()))
    unexpected_arg = 0
    with pytest.raises(TypeError):
        c_func(unexpected_arg)

def test_func_no_args_f1():
    def f1():
        from numpy import pi
        value = (2*pi)**(3/2)
        return value

    f = epyccel(f1)
    assert abs(f()-f1()) < 1e-13
#------------------------------------------------------------------------------
def test_decorator_f1(language):
    @types('int')
    def f1(x):
        y = x - 1
        return y

    f = epyccel(f1, language=language)

    # ...
    assert f(3) == f1(3)
    # ...

#------------------------------------------------------------------------------
def test_decorator_f2(language):
    @types('int [:]')
    def f2(x):
        y = x[0] - 1
        return y

    f = epyccel(f2, language=language)

    # ...
    x = np.array([3, 4, 5, 6], dtype=int)
    assert f(x) == f2(x)
    # ...

    # ...
    x = np.array([3, 4, 5, 6], dtype=int)
    assert f(x) == f2(x)
    # ...

#------------------------------------------------------------------------------
# Semantic error doesn't need testing in multiple languages
def test_decorator_f3():
    @types('int [:]')
    def f3(x):
        from numpy import empty_like
        y = empty_like(x)
        y[:] = x - 1
        return y

    with pytest.raises(PyccelError):
        epyccel(f3)

#------------------------------------------------------------------------------
# Semantic error doesn't need testing in multiple languages
def test_decorator_f4():
    @types('real [:,:]')
    def f4(x):
        from numpy import empty_like
        y = empty_like(x)
        y[:] = x - 1.0
        return y

    with pytest.raises(PyccelError):
        epyccel(f4)

#------------------------------------------------------------------------------
def test_decorator_f5(language):
    @types('int', 'real [:]')
    def f5(m1, x):
        x[:] = 0.
        for i in range(0, m1):
            x[i] = i * 1.

    f = epyccel(f5, language=language)

    # ...
    m1 = 3

    x = np.zeros(m1)
    f(m1, x)

    x_expected = np.zeros(m1)
    f5(m1, x_expected)

    assert np.allclose( x, x_expected, rtol=1e-15, atol=1e-15 )
    # ...

#------------------------------------------------------------------------------
def test_decorator_f6(language):
    @types('int', 'int', 'real [:,:]')
    def f6_1(m1, m2, x):
        x[:,:] = 0.
        for i in range(0, m1):
            for j in range(0, m2):
                x[i,j] = (2*i+j) * 1.

    f = epyccel(f6_1, language=language)

    # ...
    m1 = 2 ; m2 = 3

    x = np.zeros((m1,m2))
    f(m1, m2, x)

    x_expected = np.zeros((m1,m2))
    f6_1(m1, m2, x_expected)

    assert np.allclose( x, x_expected, rtol=1e-15, atol=1e-15 )
    # ...

#------------------------------------------------------------------------------
# in order to call the pyccelized function here, we have to create x with
# Fortran ordering
def test_decorator_f7(language):

    @types('int', 'int', 'real [:,:](order=F)')
    def f7(m1, m2, x):
        x[:,:] = 0.
        for i in range(0, m1):
            for j in range(0, m2):
                x[i,j] = (2*i+j) * 1.

    f = epyccel(f7, language=language)

    # ...
    m1 = 2 ; m2 = 3
    x_expected = np.zeros((m1,m2))
    f7(m1, m2, x_expected)

    x = np.zeros((m1,m2), order='F')
    f(m1, m2, x)

    assert np.allclose( x, x_expected, rtol=1e-15, atol=1e-15 )
    # ...

#------------------------------------------------------------------------------
def test_decorator_f8(language):
    @types('int','bool')
    def f8(x,b):
        a = x if b else 2
        return a

    f = epyccel(f8, language=language)

    # ...
    assert f(3,True)  == f8(3,True)
    assert f(3,False) == f8(3,False)
    # ...

@pytest.mark.parametrize( 'language', [
        pytest.param("fortran", marks = pytest.mark.fortran),
        pytest.param("c", marks = pytest.mark.c),
        pytest.param("python", marks = pytest.mark.python),
    ]
)
def test_arguments_f9(language):
    @types('int64[:]')
    def f9(x):
        x += 1

    f = epyccel(f9, language = language)

    x = np.zeros(10, dtype='int64')
    x_expected = x.copy()

    f9(x)
    f(x_expected)
    assert np.array_equal(x, x_expected)

@pytest.mark.parametrize( 'language', [
        pytest.param("fortran", marks = pytest.mark.fortran),
        pytest.param("c", marks = pytest.mark.c),
        pytest.param("python", marks = pytest.mark.python),
    ]
)
def test_arguments_f10(language):
    @types('int64[:]')
    def f10(x):
        x[:] += 1

    f = epyccel(f10, language = language)

    x = np.zeros(10, dtype='int64')
    x_expected = x.copy()

    f10(x)
    f(x_expected)
    assert np.array_equal(x, x_expected)

def test_multiple_returns_f11(language):
    @types('int', 'int', results='int')
    def ackermann(m, n):
        if m == 0:
            return n + 1
        elif n == 0:
            return ackermann(m - 1, 1)
        else:
            return ackermann(m - 1, ackermann(m, n - 1))

    f = epyccel(ackermann, language=language)
    assert f(2,3) == ackermann(2,3)

def test_multiple_returns_f12(language):
    @types('int')
    def non_negative(i):
        if i < 0:
            return False
        else:
            return True

    f = epyccel(non_negative, language=language)
    assert f(2) == non_negative(2)
    assert f(-1) == non_negative(-1)

def test_multiple_returns_f13(language):
    @types('int', 'int')
    def get_min(a, b):
        if a<b:
            return a
        else:
            return b

    f = epyccel(get_min, language=language)
    assert f(2,3) == get_min(2,3)

def test_multiple_returns_f14(language):
    @types('int', 'int')
    def g(x, y):
        return x,y,y,y,x

    f = epyccel(g, language=language)
    assert f(2,1) == g(2,1)

def test_decorator_f15(language):
    @types('bool', 'int8', 'int16', 'int32', 'int64')
    def f15(a,b,c,d,e):
        if a:
            return b + c
        else:
            return d + e

    f = epyccel(f15, language=language)
    assert f(True, np.int8(1), np.int16(2), np.int32(3), np.int64(4)) == \
           f15(True, np.int8(1), np.int16(2), np.int32(3), np.int64(4))
    assert f(False, np.int8(1), np.int16(2), np.int32(3), np.int64(4)) == \
           f15(False, np.int8(1), np.int16(2), np.int32(3), np.int64(4))

def test_decorator_f16(language):
    @types('int16')
    def f16(a):
        b = a
        return b
    f = epyccel(f16, language=language)
    assert f(np.int16(17)) == f16(np.int16(17))

def test_decorator_f17(language):
    @types('int8')
    def f17(a):
        b = a
        return b
    f = epyccel(f17, language=language)
    assert f(np.int8(2)) == f17(np.int8(2))

def test_decorator_f18(language):
    @types('int32')
    def f18(a):
        b = a
        return b
    f = epyccel(f18, language=language)
    assert f(np.int32(5)) == f18(np.int32(5))

def test_decorator_f19(language):
    @types('int64')
    def f19(a):
        b = a
        return b
    f = epyccel(f19, language=language)
    assert f(np.int64(1)) == f19(np.int64(1))

def test_decorator_f20(language):
    @types('complex')
    def f20(a):
        b = a
        return b
    f = epyccel(f20, language=language)
    assert f(complex(1, 2.2)) == f20(complex(1, 2.2))

def test_decorator_f21(language):
    @types('complex64')
    def f21(a):
        b = a
        return b
    f = epyccel(f21, language=language)
    assert f(np.complex64(1+ 2.2j)) == f21(np.complex64(1+ 2.2j))

def test_decorator_f22(language):
    @types('complex128')
    def f22(a):
        b = a
        return b
    f = epyccel(f22, language=language)
    assert f(complex(1, 2.2)) == f22(complex(1, 2.2))

##==============================================================================
## CLEAN UP GENERATED FILES AFTER RUNNING TESTS
##==============================================================================
#
#def teardown_module():
#    clean_test()