# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: "1.5"
#       jupytext_version: 1.5.1
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # MODFLOW 6: Working with MODFLOW Scalar Data
#
# This tutorial shows how to view, access, and change the underlying data
# variables for MODFLOW 6 objects in FloPy.  Interaction with a FloPy
# MODFLOW 6 model is different from other models, such as MODFLOW-2005,
# MT3D, and SEAWAT, for example.
#
# FloPy stores model data in data objects (`MFDataArray`, `MFDataList`,
# `MFDataScalar` objects) that are accessible from packages Data can be added
# to a package by using the appropriate parameters when the package is
# constructed and through package attributes.
#
# The MODFLOW 6 simulation structure is arranged in the following
# generalized way:
#
# >       Simulation --> Package --> DATA
# >
# >       Simulation --> Model --> Package (--> Package) --> DATA
#
#
# This tutorial focuses on MODFLOW Data that is a single integer or string,
# or consists of boolean flag(s).  These data are stored by FloPy in a
# MFScalar object and are referred to as MODFLOW scalar data.

# ## Introduction to MODFLOW Scalar Data
#
# MODFLOW single integer, strings, or boolean flag(s) are stored by FloPy as
# scalar data in `MFScalar` objects.  The different types of scalar data are
# described below.

# 1. Single integer values. Examples include `nrow`, `ncol`, `nlay`, and
#    `nper`.
# 2. Single string values.  Examples include `time_units` and `length_units`.
# 3. Boolean flags.  These can be found in the options section of most
#    packages.  These include perched, `nogrb`, `print_input`, and
#    `save_flows`.
# 4. Boolean flags with an additional optional flag.  These include
#    `newton under_relaxation` and `xt3d rhs`.
#
# In the following all four types of scalar data will be added to a model.
# Before adding data to your model first create a simulation (`MFSimulation`)
# and a model (`MFModel`) object in FloPy.

# package import
import os
import numpy as np
import flopy

# set up where simulation workspace will be stored
workspace = os.path.join("data", "mf6_working_with_data")
name = "example_1"
if not os.path.exists(workspace):
    os.makedirs(workspace)

# create the flopy simulation object
sim = flopy.mf6.MFSimulation(
    sim_name=name, exe_name="mf6", version="mf6", sim_ws=workspace
)

# create the flopy groundwater flow (gwf) model object
model_nam_file = "{}.nam".format(name)
gwf = flopy.mf6.ModflowGwf(sim, modelname=name, model_nam_file=model_nam_file)
# create the flopy iterative model solver (ims) package object
# (both pname and complexity are scalar data)
ims = flopy.mf6.modflow.mfims.ModflowIms(sim, pname="ims", complexity="SIMPLE")

# ## Adding MODFLOW Single Integer and String Values
#
# Single integer and string values can be assigned on construction of the
# `MFScalar` data object, and can be assigned or changed after construction.
#
# Below, a `TDIS` package is constructed with the `time_units` and `nper`
# parameters being assigned "DAYS" and "2", respectively.

# create the FloPy temporal discretization object
tdis = flopy.mf6.modflow.mftdis.ModflowTdis(
    sim,
    pname="tdis",
    time_units="DAYS",
    nper=2,
    perioddata=[(1.0, 1, 1.0), (1.0, 1, 1.0)],
)

# Next, `time_units` is reassigned a value after construction by using `TDIS`'s
# `time_units` attribute.

tdis.time_units = "MONTHS"

# ## Setting MODFLOW Boolean Flags
# Boolean flags can be assigned a True or False value. In the example below
# `nogrb` is assigned a value of True and then changed to false.
#
# For this example, first some values are first defined for the discretization
# package

nlay = 3
h = 50.0
length = 400.0
n = 10
bot = np.linspace(-h / nlay, -h, nlay)
delrow = delcol = length / (n - 1)

# Below the discretization package is created.  The MODFLOW `nogrb` option
# assigned a value of True, switching this option on.

dis = flopy.mf6.modflow.mfgwfdis.ModflowGwfdis(
    gwf,
    pname="dis",
    nogrb=True,
    nlay=nlay,
    nrow=n,
    ncol=n,
    delr=delrow,
    delc=delcol,
    top=0.0,
    botm=bot,
)

# The `nogrb` option is then switched off by setting the `DIS` package's
# `nogrb` attribute to False.

dis.nogrb = False

# Boolean flags with an additional optional flag can either be specified by:
#
# 1. Specifying the entire line as it would be displayed in the package file
#    as a string (`xt3doptions="xt3d rhs"`)
# 2. Specifying each flag name in a list (`xt3doptions=["xt3d", "rhs"]`)
#
# To turn off both flags use an empty string (`xt3doptions=""`) or an empty
# list (`xt3doptions=[]`).

# First, an `NPF` package is created.  `xt3doptions` can either be turned on or
# off, and if it is on `rhs` can optionally be turned on.  `xt3doptions` is set
# to the string "xt3d rhs", turning both options on.

# create the node property flow package with xt3doptions as single
npf = flopy.mf6.modflow.mfgwfnpf.ModflowGwfnpf(
    gwf,
    rewet_record="REWET WETFCT 1.0 IWETIT 1 IHDWET 0",
    pname="npf",
    icelltype=1,
    k=1.0,
    save_flows=True,
    xt3doptions="xt3d rhs",
)

# Next, the `rhs` option is turned off by setting `xt3doptions` to the string
# "xt3d".

npf.xt3doptions = "xt3d"

# Finally, both `xt3d` and `rhs` are turned off by setting `xt3doptions` to an
# empty string.

npf.xt3doptions = ""

# ## Retrieving MODFLOW Scalar Data
#
# MODFLOW scalar data can be retrieved with `get_data`, `repr`/`str`,
# or `get_file_entry`.
#
# | Retrieval Method    | Description           |
# | :---                |    :----              |
# | get_data    | Returns scalar value              |
# | repr/str    | Returns string with a header describing how data is stored (internal, external) with a string representation of the data on the next line          |
# | get_file_entry   | Returns string with the scalar keyword (if any) followed by a space and a string representation of the scalar value (if any).  This is the format used by the MODFLOW-6 package file.  This is the format used by the MODFLOW-6 package file.        |

# The `IMS` package's `complexity` option and the `NPF` package's
# `xt3doptions` are printed below using the different data retrieval methods
# highlighted above.

# First the complexity data is printed using the `get_data` method.

print(ims.complexity.get_data())

# The xt3doptions data can also be printed with `get_data`.

print(npf.xt3doptions.get_data())

# The complexity data is then printed with repr

print(repr(ims.complexity))

# The xt3doptions data is printed with repr

print(str(npf.xt3doptions))

# The complexity data is printed as it would appear in a MODFLOW 6 file using
# the `get_file_entry` method.

print(ims.complexity.get_file_entry())

# The xt3doptions data is printed as it would appear in a MODFLOW 6 file using
# the `get_file_entry` method.

print(npf.xt3doptions.get_file_entry())


"""
.. _tut-fnirs-processing:

Preprocessing functional near-infrared spectroscopy (fNIRS) data
================================================================

This tutorial covers how to convert functional near-infrared spectroscopy
(fNIRS) data from raw measurements to relative oxyhaemoglobin (HbO) and
deoxyhaemoglobin (HbR) concentration, view the average waveform, and
topographic representation of the response.

Here we will work with the :ref:`fNIRS motor data <fnirs-motor-dataset>`.
"""

# %%

import os.path as op
import numpy as np
import matplotlib.pyplot as plt
from itertools import compress

import mne

fnirs_data_folder = mne.datasets.fnirs_motor.data_path()
fnirs_cw_amplitude_dir = op.join(fnirs_data_folder, 'Participant-1')
raw_intensity = mne.io.read_raw_nirx(fnirs_cw_amplitude_dir, verbose=True)
raw_intensity.load_data()

# %%
# Providing more meaningful annotation information
# ------------------------------------------------
#
# First, we attribute more meaningful names to the trigger codes which are
# stored as annotations. Second, we include information about the duration of
# each stimulus, which was 5 seconds for all conditions in this experiment.
# Third, we remove the trigger code 15, which signaled the start and end
# of the experiment and is not relevant to our analysis.

raw_intensity.annotations.set_durations(5)
raw_intensity.annotations.rename({'1.0': 'Control',
                                  '2.0': 'Tapping/Left',
                                  '3.0': 'Tapping/Right'})
unwanted = np.nonzero(raw_intensity.annotations.description == '15.0')
raw_intensity.annotations.delete(unwanted)

# %%
# Viewing location of sensors over brain surface
# ----------------------------------------------
#
# Here we validate that the location of sources-detector pairs and channels
# are in the expected locations. Source-detector pairs are shown as lines
# between the optodes, channels (the mid point of source-detector pairs) are
# optionally shown as orange dots. Source are optionally shown as red dots and
# detectors as black.

subjects_dir = op.join(mne.datasets.sample.data_path(), 'subjects')

brain = mne.viz.Brain(
    'fsaverage', subjects_dir=subjects_dir, background='w', cortex='0.5')
brain.add_sensors(
    raw_intensity.info, trans='fsaverage',
    fnirs=['channels', 'pairs', 'sources', 'detectors'])
brain.show_view(azimuth=20, elevation=60, distance=400)

# %%
# Selecting channels appropriate for detecting neural responses
# -------------------------------------------------------------
#
# First we remove channels that are too close together (short channels) to
# detect a neural response (less than 1 cm distance between optodes).
# These short channels can be seen in the figure above.
# To achieve this we pick all the channels that are not considered to be short.

picks = mne.pick_types(raw_intensity.info, meg=False, fnirs=True)
dists = mne.preprocessing.nirs.source_detector_distances(
    raw_intensity.info, picks=picks)
raw_intensity.pick(picks[dists > 0.01])
raw_intensity.plot(n_channels=len(raw_intensity.ch_names),
                   duration=500, show_scrollbars=False)

# %%
# Converting from raw intensity to optical density
# ------------------------------------------------
#
# The raw intensity values are then converted to optical density.

raw_od = mne.preprocessing.nirs.optical_density(raw_intensity)
raw_od.plot(n_channels=len(raw_od.ch_names),
            duration=500, show_scrollbars=False)

# %%
# Evaluating the quality of the data
# ----------------------------------
#
# At this stage we can quantify the quality of the coupling
# between the scalp and the optodes using the scalp coupling index. This
# method looks for the presence of a prominent synchronous signal in the
# frequency range of cardiac signals across both photodetected signals.
#
# In this example the data is clean and the coupling is good for all
# channels, so we will not mark any channels as bad based on the scalp
# coupling index.

sci = mne.preprocessing.nirs.scalp_coupling_index(raw_od)
fig, ax = plt.subplots()
ax.hist(sci)
ax.set(xlabel='Scalp Coupling Index', ylabel='Count', xlim=[0, 1])

# %%
# In this example we will mark all channels with a SCI less than 0.5 as bad
# (this dataset is quite clean, so no channels are marked as bad).

raw_od.info['bads'] = list(compress(raw_od.ch_names, sci < 0.5))

# %%
# At this stage it is appropriate to inspect your data
# (for instructions on how to use the interactive data visualisation tool
# see :ref:`tut-visualize-raw`)
# to ensure that channels with poor scalp coupling have been removed.
# If your data contains lots of artifacts you may decide to apply
# artifact reduction techniques as described in :ref:`ex-fnirs-artifacts`.

# %%
# Converting from optical density to haemoglobin
# ----------------------------------------------
#
# Next we convert the optical density data to haemoglobin concentration using
# the modified Beer-Lambert law.

raw_haemo = mne.preprocessing.nirs.beer_lambert_law(raw_od)
raw_haemo.plot(n_channels=len(raw_haemo.ch_names),
               duration=500, show_scrollbars=False)

# %%
# Removing heart rate from signal
# -------------------------------
#
# The haemodynamic response has frequency content predominantly below 0.5 Hz.
# An increase in activity around 1 Hz can be seen in the data that is due to
# the person's heart beat and is unwanted. So we use a low pass filter to
# remove this. A high pass filter is also included to remove slow drifts
# in the data.

fig = raw_haemo.plot_psd(average=True)
fig.suptitle('Before filtering', weight='bold', size='x-large')
fig.subplots_adjust(top=0.88)
raw_haemo = raw_haemo.filter(0.05, 0.7, h_trans_bandwidth=0.2,
                             l_trans_bandwidth=0.02)
fig = raw_haemo.plot_psd(average=True)
fig.suptitle('After filtering', weight='bold', size='x-large')
fig.subplots_adjust(top=0.88)

# %%
# Extract epochs
# --------------
#
# Now that the signal has been converted to relative haemoglobin concentration,
# and the unwanted heart rate component has been removed, we can extract epochs
# related to each of the experimental conditions.
#
# First we extract the events of interest and visualise them to ensure they are
# correct.

events, event_dict = mne.events_from_annotations(raw_haemo)
fig = mne.viz.plot_events(events, event_id=event_dict,
                          sfreq=raw_haemo.info['sfreq'])
fig.subplots_adjust(right=0.7)  # make room for the legend

# %%
# Next we define the range of our epochs, the rejection criteria,
# baseline correction, and extract the epochs. We visualise the log of which
# epochs were dropped.

reject_criteria = dict(hbo=80e-6)
tmin, tmax = -5, 15

epochs = mne.Epochs(raw_haemo, events, event_id=event_dict,
                    tmin=tmin, tmax=tmax,
                    reject=reject_criteria, reject_by_annotation=True,
                    proj=True, baseline=(None, 0), preload=True,
                    detrend=None, verbose=True)
epochs.plot_drop_log()

# %%
# View consistency of responses across trials
# -------------------------------------------
#
# Now we can view the haemodynamic response for our tapping condition.
# We visualise the response for both the oxy- and deoxyhaemoglobin, and
# observe the expected peak in HbO at around 6 seconds consistently across
# trials, and the consistent dip in HbR that is slightly delayed relative to
# the HbO peak.

epochs['Tapping'].plot_image(combine='mean', vmin=-30, vmax=30,
                             ts_args=dict(ylim=dict(hbo=[-15, 15],
                                                    hbr=[-15, 15])))

# %%
# We can also view the epoched data for the control condition and observe
# that it does not show the expected morphology.

epochs['Control'].plot_image(combine='mean', vmin=-30, vmax=30,
                             ts_args=dict(ylim=dict(hbo=[-15, 15],
                                                    hbr=[-15, 15])))

# %%
# View consistency of responses across channels
# ---------------------------------------------
#
# Similarly we can view how consistent the response is across the optode
# pairs that we selected. All the channels in this data are located over the
# motor cortex, and all channels show a similar pattern in the data.

fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(15, 6))
clims = dict(hbo=[-20, 20], hbr=[-20, 20])
epochs['Control'].average().plot_image(axes=axes[:, 0], clim=clims)
epochs['Tapping'].average().plot_image(axes=axes[:, 1], clim=clims)
for column, condition in enumerate(['Control', 'Tapping']):
    for ax in axes[:, column]:
        ax.set_title('{}: {}'.format(condition, ax.get_title()))

# %%
# Plot standard fNIRS response image
# ----------------------------------
#
# Next we generate the most common visualisation of fNIRS data: plotting
# both the HbO and HbR on the same figure to illustrate the relation between
# the two signals.

evoked_dict = {'Tapping/HbO': epochs['Tapping'].average(picks='hbo'),
               'Tapping/HbR': epochs['Tapping'].average(picks='hbr'),
               'Control/HbO': epochs['Control'].average(picks='hbo'),
               'Control/HbR': epochs['Control'].average(picks='hbr')}

# Rename channels until the encoding of frequency in ch_name is fixed
for condition in evoked_dict:
    evoked_dict[condition].rename_channels(lambda x: x[:-4])

color_dict = dict(HbO='#AA3377', HbR='b')
styles_dict = dict(Control=dict(linestyle='dashed'))

mne.viz.plot_compare_evokeds(evoked_dict, combine="mean", ci=0.95,
                             colors=color_dict, styles=styles_dict)

# %%
# View topographic representation of activity
# -------------------------------------------
#
# Next we view how the topographic activity changes throughout the response.

times = np.arange(-3.5, 13.2, 3.0)
topomap_args = dict(extrapolate='local')
epochs['Tapping'].average(picks='hbo').plot_joint(
    times=times, topomap_args=topomap_args)

# %%
# Compare tapping of left and right hands
# ---------------------------------------
#
# Finally we generate topo maps for the left and right conditions to view
# the location of activity. First we visualise the HbO activity.

times = np.arange(4.0, 11.0, 1.0)
epochs['Tapping/Left'].average(picks='hbo').plot_topomap(
    times=times, **topomap_args)
epochs['Tapping/Right'].average(picks='hbo').plot_topomap(
    times=times, **topomap_args)

# %%
# And we also view the HbR activity for the two conditions.

epochs['Tapping/Left'].average(picks='hbr').plot_topomap(
    times=times, **topomap_args)
epochs['Tapping/Right'].average(picks='hbr').plot_topomap(
    times=times, **topomap_args)

# %%
# And we can plot the comparison at a single time point for two conditions.

fig, axes = plt.subplots(nrows=2, ncols=4, figsize=(9, 5),
                         gridspec_kw=dict(width_ratios=[1, 1, 1, 0.1]))
vmin, vmax, ts = -8, 8, 9.0

evoked_left = epochs['Tapping/Left'].average()
evoked_right = epochs['Tapping/Right'].average()

evoked_left.plot_topomap(ch_type='hbo', times=ts, axes=axes[0, 0],
                         vmin=vmin, vmax=vmax, colorbar=False,
                         **topomap_args)
evoked_left.plot_topomap(ch_type='hbr', times=ts, axes=axes[1, 0],
                         vmin=vmin, vmax=vmax, colorbar=False,
                         **topomap_args)
evoked_right.plot_topomap(ch_type='hbo', times=ts, axes=axes[0, 1],
                          vmin=vmin, vmax=vmax, colorbar=False,
                          **topomap_args)
evoked_right.plot_topomap(ch_type='hbr', times=ts, axes=axes[1, 1],
                          vmin=vmin, vmax=vmax, colorbar=False,
                          **topomap_args)

evoked_diff = mne.combine_evoked([evoked_left, evoked_right], weights=[1, -1])

evoked_diff.plot_topomap(ch_type='hbo', times=ts, axes=axes[0, 2:],
                         vmin=vmin, vmax=vmax, colorbar=True,
                         **topomap_args)
evoked_diff.plot_topomap(ch_type='hbr', times=ts, axes=axes[1, 2:],
                         vmin=vmin, vmax=vmax, colorbar=True,
                         **topomap_args)

for column, condition in enumerate(
        ['Tapping Left', 'Tapping Right', 'Left-Right']):
    for row, chroma in enumerate(['HbO', 'HbR']):
        axes[row, column].set_title('{}: {}'.format(chroma, condition))
fig.tight_layout()

# %%
# Lastly, we can also look at the individual waveforms to see what is
# driving the topographic plot above.

fig, axes = plt.subplots(nrows=1, ncols=1, figsize=(6, 4))
mne.viz.plot_evoked_topo(epochs['Left'].average(picks='hbo'), color='b',
                         axes=axes, legend=False)
mne.viz.plot_evoked_topo(epochs['Right'].average(picks='hbo'), color='r',
                         axes=axes, legend=False)

# Tidy the legend.
leg_lines = [line for line in axes.lines if line.get_c() == 'b'][:1]
leg_lines.append([line for line in axes.lines if line.get_c() == 'r'][0])
fig.legend(leg_lines, ['Left', 'Right'], loc='lower right')


"""
# Python Handout

Turn Python scripts into handouts with Markdown comments and inline figures. An
alternative to Jupyter notebooks without hidden state and using your own text
editor.
"""

import handout
import matplotlib.pyplot as plt
import numpy as np

"""Start your handout with an output directory."""

doc = handout.Handout('output')

"""
## Markdown text

Comments with triple quotes are converted to text blocks.

Text blocks support [Markdown formatting][1], for example:

- Headlines
- Hyperlinks
- Inline `code()` snippets
- **Bold** and *italic*

[1]: https://commonmark.org
"""

"""
## Print output

Write variables to our handout, same syntax as Python's `print()`:
"""
for index in range(3):
  doc.write('Iteration', index)

"""
## Inline figures

Display matplotlib figures on the handout using `display()`:
"""
fig, ax = plt.subplots(figsize=(6, 4))
ax.plot(np.arange(100))
fig.tight_layout()
doc.display(fig)  # Display the figure below this line.

"""Multiple plots are inserted right after another."""

for iteration in range(3):
  fig, ax = plt.subplots(figsize=(3, 2))
  ax.plot(np.sin(np.arange(100) / (iteration + 1)))
  doc.display(fig, width=0.33)

"""
## Exclude lines

Hide code from the handout with the `# handout=exclude` comment:
"""

# Invisible below:
value = 13  # handout=exclude

"""
## View the handout

Save the handout at the end of your script. Then open `output/index.html` in
your browser.
"""

doc.save()


import discord
import asyncio
import os
import cv2
import numpy as np
import re
import math

from PIL import Image
import requests
from io import BytesIO

client = discord.Client()
face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_alt.xml')
eye_cascade = cv2.CascadeClassifier('haarcascade_eye.xml')
img_face_replace = Image.open('face.png')

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')

@client.event
async def on_message(message):
    if message.content.startswith('!face'):
        urls = re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+', message.content)

        if len(message.attachments) > 0:
            urls = [message.attachments[0]['url']]

        if 'load' in message.content.strip(urls[0]):
            await face_load_handler(message, urls[0])
        elif 'print' in message.content.strip(urls[0]):
            await client.send_file(message.channel,
                    image_to_mem_buf(img_face_replace, 'png'))
        else:
            await face_replace_handler(message, urls[0])

#
# Loads image from url and replaces img_face_replace with given image
#
async def face_load_handler(message, url):
    response = requests.get(url)

    if response.status_code == 200 and \
            'image' in response.headers['content-type'].lower():
        global img_face_replace
        img_face_replace = Image.open(BytesIO(response.content))

        await client.send_message(message.channel, 'Image was successfully loaded.')
    else:
        await client.send_message(message.channel, 'There was an error loading the image.')

#
# Loads image from url and replaces all faces with img_face_replace
#
async def face_replace_handler(message, url):
    response = requests.get(url)
    img_file = BytesIO(response.content)
    img_final = Image.open(img_file)

    faces = retrieve_faces(BytesIO(response.content))

    if len(faces) < 1:
        x = int(img_final.width / 2) - int(img_face_replace.width / 2)
        y = img_final.height - int(img_face_replace.height / 2)

        await client.send_file(message.channel,
                image_to_mem_buf(image_place(
                    img_final,
                    img_face_replace,
                    (x, y, img_face_replace.width, img_face_replace.height),
                    0), 'png'))

        return

    for (x, y, w, h) in faces:
        eyes = retrieve_eyes_on_face(
                BytesIO(response.content), x, y, w, h)

        if len(eyes) == 2:
            # computes tilt of eyes to guestimate face tilt
            deltaX = eyes[0][0] - eyes[1][0]
            deltaY = eyes[1][1] - eyes[0][1] # swap index for image y coords
            deg = math.degrees(math.atan(deltaY / deltaX))
        elif len(eyes) == 3:
            await client.send_message(message.channel, 'Illuminati confirmed.')
        else:
            deg = 180 # for giggles

        height = h * (img_face_replace.height/img_face_replace.width)

        img_final = image_place(img_final,
                img_face_replace,
                (x, y, w, int(height)),
                deg)

    await client.send_file(message.channel, image_to_mem_buf(img_final, 'png'))

#
# Superimpose image on image given position and rotation
#
def image_place(main_image, place_image, xywh_tup, deg):
    place_image = place_image.resize((xywh_tup[2], xywh_tup[3]))
    place_image = place_image.rotate(deg, resample=Image.BICUBIC)
    main_image.paste(place_image, (xywh_tup[0], xywh_tup[1]), mask=place_image)

    return main_image

#
# Retrieves all coordinates of faces on given image
#
def retrieve_faces(image_fp):
    cv_mat_gray = mem_buf_to_cv2_mat(image_fp)

    # object detection using face cascade on cv matrix
    faces = face_cascade.detectMultiScale(cv_mat_gray, 1.1, 5)

    print("faces", faces)

    return faces

#
# Parameter: a face (x, y, w, h)
# Output: vector of eyes
#
def retrieve_eyes_on_face(image_fp, x, y, w, h):
    cv_mat_gray = mem_buf_to_cv2_mat(image_fp)
    cv_mat_face = cv_mat_gray[y:y+h, x:x+w]

    eyes = eye_cascade.detectMultiScale(cv_mat_face)

    print("eyes", eyes)

    return eyes

#
# Transforms Image in buffer to cv2 matrix
#
def mem_buf_to_cv2_mat(image_fp):
    # image file as byte array
    image_buf = np.asarray(bytearray(image_fp.read()), dtype=np.uint8)

    # image buffer to cv matrix
    cv_mat_gray = cv2.imdecode(image_buf, cv2.IMREAD_GRAYSCALE)

    return cv_mat_gray

#
# Transforms pillow Image to in-memory buffer image
#
def image_to_mem_buf(image, ftype):
    buf = BytesIO()
    buf.seek(0)
    image.save(buf, ftype)
    buf.name = '.' + ftype
    buf.seek(0)
    return buf

client.run(os.environ['DISCORD_TOKEN'])


#! /usr/bin/env python
"""
Unit tests for landlab.io.esri_ascii module.
"""
import os
import numpy as np
from numpy.testing import assert_array_equal, assert_array_almost_equal
from nose.tools import assert_equal, assert_true, assert_raises
try:
    from nose.tools import assert_is_instance, assert_list_equal, assert_is
except ImportError:
    from landlab.testing.tools import (assert_is_instance, assert_list_equal,
                                       assert_is)
from six import StringIO

from landlab.io import read_esri_ascii, read_asc_header
from landlab.io import (MissingRequiredKeyError, KeyTypeError, DataSizeError,
                        BadHeaderLineError, KeyValueError,
                        MismatchGridDataSizeError)
from landlab import RasterModelGrid

_TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), 'data')

def test_hugo_read_file_name():
    (grid, field) = read_esri_ascii(os.path.join(_TEST_DATA_DIR,
                                                 'hugo_site.asc'))

    assert_is_instance(grid, RasterModelGrid)

    assert_equal(field.size, 55 * 76)
    assert_equal(field.shape, (55 * 76, ))

def test_hugo_read_file_like():
    with open(os.path.join(_TEST_DATA_DIR, 'hugo_site.asc')) as asc_file:
        (grid, field) = read_esri_ascii(asc_file)

    assert_is_instance(grid, RasterModelGrid)

    assert_equal(field.size, 55 * 76)
    assert_equal(field.shape, (55 * 76, ))

def test_hugo_reshape():
    with open(os.path.join(_TEST_DATA_DIR, 'hugo_site.asc')) as asc_file:
        (grid, field) = read_esri_ascii(asc_file, reshape=True)

    assert_is_instance(grid, RasterModelGrid)

    assert_true(field.shape, (55, 76))

def test_4x3_read_file_name():
    (grid, field) = read_esri_ascii(os.path.join(_TEST_DATA_DIR,
                                                 '4_x_3.asc'))

    assert_is_instance(grid, RasterModelGrid)

    assert_is_instance(field, np.ndarray)
    assert_array_equal(field,
                       np.array([9., 10., 11.,
                                 6.,  7.,  8.,
                                 3.,  4.,  5.,
                                 0.,  1.,  2.]))

def test_4x3_read_file_like():
    with open(os.path.join(_TEST_DATA_DIR, '4_x_3.asc')) as asc_file:
        (grid, field) = read_esri_ascii(asc_file)

    assert_is_instance(grid, RasterModelGrid)

    assert_array_equal(field,
                       np.array([9., 10., 11.,
                                 6.,  7.,  8.,
                                 3.,  4.,  5.,
                                 0.,  1.,  2.]))

def test_4x3_shape_mismatch():
    asc_file = StringIO(
        """
nrows         4
ncols         3
xllcorner     1.
yllcorner     2.
cellsize      10.
NODATA_value  -9999
1. 2. 3. 4.
5. 6. 7. 8.
9. 10. 11. 12.
        """)
    (grid, field) = read_esri_ascii(asc_file)
    assert_equal(field.size, 12)

    asc_file = StringIO(
        """
nrows         4
ncols         3
xllcorner     1.
yllcorner     2.
cellsize      10.
NODATA_value  -9999
1. 2. 3. 4. 5. 6. 7. 8. 9. 10. 11. 12.
        """)
    (grid, field) = read_esri_ascii(asc_file)
    assert_equal(field.size, 12)

def test_4x3_size_mismatch():
    asc_file = StringIO(
        """
nrows         4
ncols         3
xllcorner     1.
yllcorner     2.
cellsize      10.
NODATA_value  -9999
1. 2. 3. 4. 5. 6. 7. 8. 9. 10.
        """)
    assert_raises(DataSizeError, read_esri_ascii, asc_file)

def test_grid_data_size_mismatch():
    asc_file = StringIO(
        """
nrows         4
ncols         3
xllcorner     1.
yllcorner     2.
cellsize      10.
NODATA_value  -9999
1. 2. 3. 4. 5. 6. 7. 8. 9. 10. 11. 12.
        """)
    rmg = RasterModelGrid((10,10),10.)
    assert_raises(MismatchGridDataSizeError, read_esri_ascii, asc_file,
                  grid=rmg)

def test_header_missing_required_key():
    asc_file = StringIO(
        """
nrows         4
xllcorner     1.
yllcorner     2.
cellsize      10.
NODATA_value  -9999
        """)
    assert_raises(MissingRequiredKeyError, read_asc_header, asc_file)

def test_header_unknown_key():
    asc_file = StringIO(
        """
nrows         4
ncols         3
xllcorner     1.
yllcorner     2.
cellsize      10.
NODATA_value  -9999
invalid_key   1
        """)
    assert_raises(BadHeaderLineError, read_asc_header, asc_file)

def test_header_missing_value():
    asc_file = StringIO(
        """
nrows         4
ncols         3
xllcorner     1.
yllcorner     2.
cellsize
NODATA_value  -9999
invalid_key   1
        """)
    assert_raises(BadHeaderLineError, read_asc_header, asc_file)

def test_header_bad_values():
    asc_file = StringIO(
        """
nrows         -4
ncols         3
xllcorner     1.
yllcorner     2.
cellsize      10.
NODATA_value  -9999
        """)
    assert_raises(KeyValueError, read_asc_header, asc_file)

def test_header_missing_mutex_key():
    asc_file = StringIO(
        """
ncols         3
nrows         4
yllcorner     2.
cellsize      10.
NODATA_value  -9999
        """)
    assert_raises(MissingRequiredKeyError, read_asc_header, asc_file)

def test_header_mutex_key():
    asc_file = StringIO(
        """
ncols         3
nrows         4
xllcenter     1.
yllcorner     2.
cellsize      10.
NODATA_value  -9999
        """)
    header = read_asc_header(asc_file)
    assert_equal(header['xllcenter'], 1.)
    # with assert_raises(KeyError):
    #    header['xllcorner']
    assert_raises(KeyError, lambda k: header[k], 'xllcorner')

    asc_file = StringIO(
        """
ncols         3
nrows         4
xllcorner     1.
yllcorner     2.
cellsize      10.
NODATA_value  -9999
        """)
    header = read_asc_header(asc_file)
    assert_equal(header['xllcorner'], 1.)
    assert_raises(KeyError, lambda k: header[k], 'xllcenter')

def test_header_missing_optional():
    asc_file = StringIO(
        """
ncols         3
nrows         4
xllcenter     1.
yllcorner     2.
cellsize      10.
        """)
    header = read_asc_header(asc_file)
    assert_raises(KeyError, lambda k: header[k], 'nodata_value')

def test_header_case_insensitive():
    asc_file = StringIO(
        """
nCoLs         3
nrows         4
Xllcenter     1.
YLLCORNER     2.
CELLSIZE      10.
NODATA_value  -999
        """)
    header = read_asc_header(asc_file)
    for key in ['ncols', 'nrows', 'xllcenter', 'yllcorner', 'cellsize',
                'nodata_value']:
        assert_true(key in header)

def test_header_wrong_type():
    asc_file = StringIO(
        """
nCoLs         3.5
nrows         4
Xllcenter     1.
YLLCORNER     2.
CELLSIZE      10.
NODATA_value  -999
        """)
    assert_raises(KeyTypeError, read_asc_header, asc_file)

def test_name_keyword():
    (grid, field) = read_esri_ascii(os.path.join(_TEST_DATA_DIR,
                                                 '4_x_3.asc'),
                                    name='air__temperature')

    assert_is_instance(grid, RasterModelGrid)

    assert_is_instance(field, np.ndarray)
    assert_array_equal(field,
                       np.array([9., 10., 11.,
                                 6.,  7.,  8.,
                                 3.,  4.,  5.,
                                 0.,  1.,  2.]))
    assert_array_almost_equal(grid.at_node['air__temperature'], field)
    assert_is(grid.at_node['air__temperature'], field)

def test_halo_keyword():
    (grid, field) = read_esri_ascii(os.path.join(_TEST_DATA_DIR, \
                                                 '4_x_3.asc'), \
                                                 halo=1)

    assert_is_instance(grid, RasterModelGrid)

    assert_is_instance(field, np.ndarray)
    assert_array_equal(field,
                       np.array([-9999., -9999., -9999., -9999., -9999.,
                                 -9999.,     9.,    10.,    11., -9999.,
                                 -9999.,     6.,     7.,     8., -9999.,
                                 -9999.,     3.,     4.,     5., -9999.,
                                 -9999.,     0.,     1.,     2., -9999.,
                                 -9999., -9999., -9999., -9999., -9999.]))

def test_halo_keyword_no_nodata_value():
    (grid, field) = read_esri_ascii(os.path.join(_TEST_DATA_DIR, \
                                                '4_x_3_no_nodata_value.asc'), \
                                                halo=1)

    assert_is_instance(grid, RasterModelGrid)

    assert_is_instance(field, np.ndarray)
    assert_array_equal(field,
                       np.array([-9999., -9999., -9999., -9999., -9999.,
                                 -9999.,     9.,    10.,    11., -9999.,
                                 -9999.,     6.,     7.,     8., -9999.,
                                 -9999.,     3.,     4.,     5., -9999.,
                                 -9999.,     0.,     1.,     2., -9999.,
                                 -9999., -9999., -9999., -9999., -9999.]))

if __name__ == '__main__':
    unittest.main()


"""
type_conversions.py
-------------------
This module provides a functions for converting data types.
By: Sebastian D. Goodfellow, Ph.D., 2018
"""

# Compatibility imports
from __future__ import absolute_import, division, print_function

# 3rd party imports
import numpy as np

def sex_type_conversion(sex):
    """Male=0 and Female=1"""
    if sex == 'male':
        return 0
    elif sex == 'female':
        return 1
    else:
        return np.nan

def embarked_type_conversion(embarked):
    """S=0, C=1, and Q=2"""
    if embarked == 'S':
        return 0
    elif embarked == 'C':
        return 1
    elif embarked == 'Q':
        return 2
    else:
        return np.nan


import numpy as np #the Numpy library
import matplotlib.pyplot as plt #Matplotlib's pyplot

import sys #gives access to a C-like sys library
import os #give access to operating system

print(sys.argv) #prinnts any command line arguments, incl program name
print(os.getcwd()) #prints current working directory


from abc import ABC, abstractmethod
import numpy as np

class Activation(ABC):

    def __call__(self, x):
        return self.forward(x)

    @abstractmethod
    def forward(self,
                x: np.ndarray):
        pass

    @abstractmethod
    def backward(self,
                 prev_error: np.ndarray):
        pass

    @abstractmethod
    def __dir__(self):
        pass


# Can't round() a Numpy array - have to use np.round().
import numpy as np  # 1.11.1
round(1)
round(np.array(1))  # <-- TypeError: type numpy.ndarray doesn't define __round__ method

# To be fair, Python can't round a list, to begin with.
round([1])  # <-- TypeError: type list doesn't define __round__ method


import numpy as np
import traceback
import json

# Test object to be used for other homeworks
class Test(object):
    def __init__(self):
        self.scores = {}

    def assertions(self, user_vals, expected_vals, test_type, test_name):
        if test_type == "type":
            try:
                assert type(user_vals) == type(expected_vals)
            except Exception as e:
                print("Type error, your type doesnt match the expected type.")
                print("Wrong type for %s" % test_name)
                print("Your type:   ", type(user_vals))
                print("Expected type:", type(expected_vals))
                return False
        elif test_type == "shape":
            try:
                assert user_vals.shape == expected_vals.shape
            except Exception as e:
                print("Shape error, your shapes doesnt match the expected shape.")
                print("Wrong shape for %s" % test_name)
                print("Your shape:    ", user_vals.shape)
                print("Expected shape:", expected_vals.shape)
                return False
        elif test_type == "closeness":
            try:
                assert np.allclose(user_vals, expected_vals,atol=1e-5)
            except Exception as e:
                print("Closeness error, your values dont match the expected values.")
                print("Wrong values for %s" % test_name)
                print("Your values:    ", user_vals)
                print("Expected values:", expected_vals)
                return False
        return True

    def print_failure(self, cur_test):
        print("*" * 50)
        print("The local autograder failed %s." % cur_test)
        print("*" * 50)
        print(" ")

    def print_name(self, cur_question):
        print("-" * 20)
        print(cur_question)

    def print_outcome(self, short, outcome):
        print(short + ": ", "PASS" if outcome else "*** FAIL ***")
        print("-" * 20)
        print()

    def get_test_scores(self):
        return sum(self.scores.values())

    def run_tests(self, section_title, test, test_score):
        test_name = section_title.split(' - ')[1]
        try:
            self.print_name(section_title)
            test_outcome = test()
            self.print_outcome(test_name, test_outcome)
        except Exception:
            traceback.print_exc()
            test_outcome = False

        if test_outcome == False:
            self.print_failure(test_name)
            self.scores[section_title] = 0
            return False
        self.scores[section_title] = test_score
        return True


from learntools.core import *
import textwrap
import numpy as np

class MenuAnalysisPlan(ThoughtExperiment):
    _solution = """You could group reviews by what menu items they mention, and then calculate the average rating
    for reviews that mentioned each item. You can tell which foods are mentioned in reviews with low scores,
    so the restaurant can fix the recipe or remove those foods from the menu."""

class SingleReviewMatch(CodingProblem):
    _var = "matches"
    _hint = ("You should set the attr keyword argument to 'LOWER' so matching is case insensitive. "
             "An easy way to make a list of phrase documents is to loop through each item in the "
             "menu and apply the model nlp(item). This is best done in a list comprehension. "
             "From there, you can add the patterns to the matcher with matcher.add and pass in "
             "the review document to perform the matching.")
    _solution = CS(textwrap.dedent("""
    import spacy
    from spacy.matcher import PhraseMatcher

    index_of_review_to_test_on = 14
    text_to_test_on = data.text.iloc[index_of_review_to_test_on]

    nlp = spacy.blank('en')
    review_doc = nlp(text_to_test_on)

    matcher = PhraseMatcher(nlp.vocab, attr='LOWER')
    menu_tokens_list = [nlp(item) for item in menu]
    matcher.add("MENU", menu_tokens_list)
    matches = matcher(review_doc)"""))

    def check(self, matches):
        correct = [(2, 3), (16, 17), (58, 59)]
        assert [(match[1], match[2]) for match in matches] == correct

class MatchAllDataset(CodingProblem):
    _var = "item_ratings"
    _hint = ("For each review, use the `nlp` model to convert the text to a document. Then "
             "use the matcher from exercise 1 to extract the item matches from the review text. "
             "The matches you get from the matcher are tuples (match_id, start, end), so you can "
             "do doc[start:end] to get the text phrase for that match. To get all of the unique "
             "items in the review, create a list of all the matched phrases, and convert that "
             "into a set. Finally for each of those items, append the review's rating to " "item_ratings. Make sure to add the item string in lowercase. ")
    _solution = CS(textwrap.dedent("""
    from collections import defaultdict

    item_ratings = defaultdict(list)

    for idx, review in data.iterrows():
        doc = nlp(review.text)
        matches = matcher(doc)

        found_items = set([doc[match[1]:match[2]].lower_ for match in matches])

        for item in found_items:
            item_ratings[item].append(review.stars)
    """))

    def check(self, item_ratings):
        correct = np.array([
            4.47058824, 4.15942029, 4.88888889, 4.33962264, 4.1796875 ,
            4.38888889, 4.40776699, 4.68      , 4.66666667, 4.44736842,
            4.48453608, 4.44444444, 4.5       , 4.23809524, 4.04761905,
            3.92      , 4.25      , 4.22      , 4.38095238, 3.8       ,
            4.        , 4.55555556, 3.4       , 4.45762712, 5.        ,
            5.        , 4.375     , 5.        , 4.54285714, 4.5       ,
            4.45454545, 4.12820513, 3.88888889, 4.30188679, 4.5       ,
            4.44444444, 4.75      , 4.48648649, 4.26315789, 4.        ,
            4.6       , 4.14285714, 5.
        ])

        means = np.array([sum(item)/len(item) for _, item in item_ratings.items()])
        means.sort()
        correct.sort()
        assert len(means) == len(correct), f"Please add items to item_ratings. You should have {len(correct)} items."
        assert np.allclose(means, correct)

class WorstReviewedItem(EqualityCheckProblem):
    _var = "worst_item"
    _hint = ("Loop through each item in item_ratings and calculate the mean, "
             "the sum of the ratings divided by the number of ratings. This is easiest "
             "using a dictionary comprehension. Then use the `sorted` function to sort "
             "the dictionary keys based on the dictionary values.")
    _solution = CS(textwrap.dedent("""
    # There are many ways to do this. Here is one.
    mean_ratings = {item: sum(ratings)/len(ratings) for item, ratings in item_ratings.items()}
    worst_item = sorted(mean_ratings, key=mean_ratings.get)[0]
    """))
    _expected = 'chicken cutlet'

class CountImportanceQuestion(ThoughtExperiment):
    _solution = """
    The less data you have for any specific item, the less you can trust that the average rating is the "real" sentiment of the customers. This is fairly common sense. If more people tell you the same thing, you're more likely to believe it. It's also mathematically sound. As the number of data points increases, the error on the mean decreases as 1 / sqrt(n).
    """

qvars = bind_exercises(globals(), [
    MenuAnalysisPlan,
    SingleReviewMatch,
    MatchAllDataset,
    WorstReviewedItem,
    CountImportanceQuestion
    ],
    var_format='q_{n}',
    )
__all__ = list(qvars)


# To add a new cell, type '# %%'
# To add a new markdown cell, type '# %% [markdown]'
# %%
# Setting up the environment.
import numpy as np
import pandas as pd
from scipy import stats
import plotly as pl

# %%
# Load the data from the John Hopkins github repo
df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/07-10-2020.csv')
dfpop = pd.read_csv('USpopCounties.csv')

# %%
# Dropping some columns and sorting

df1 = df[["Admin2", "Province_State", "Country_Region", "Confirmed", "Deaths", "Combined_Key"]] #getting the columns I want
df1 = df1[(df1["Country_Region"] == "US")] #dropping countries other than the US
df1 = df1.rename(columns={'Province_State': 'State'})
df1 = df1.rename(columns={'Admin2': 'County'})
df1 = df1.rename(columns={'Country_Region': 'Country'})
df1 = df1.rename(columns={'Combined_Key': 'County/State'})

# %%
# checking a match in column names to merge by County/State
set(df1.columns).intersection(set(dfpop.columns))

# %%
# changes to make the columns equal to be able to merge
dfpop['County/State'] = dfpop['County/State'].str.replace(r' County', '')
dfpop['County/State'] = dfpop['County/State'] + ', US'

# %%
# Converting lower case to title case
#dfpop['County/State'] = dfpop['County/State'].str.upper().str.title()

# %%
# checking  how many cells are the same and how many are different
df1['County/State'].isin(dfpop['County/State']).value_counts()

# %%
# Replacing names of counties that are different
dfpop['County/State'] = dfpop['County/State'].str.replace(r' Parish', '')
dfpop['County/State'] = dfpop['County/State'].str.replace(r' Census Area', '')
dfpop['County/State'] = dfpop['County/State'].str.replace(r'Baltimore city', 'Baltimore City')
dfpop['County/State'] = dfpop['County/State'].str.replace(r' City and Borough', '')
dfpop['County/State'] = dfpop['County/State'].str.replace(r' city', '')
dfpop['County/State'] = dfpop['County/State'].str.replace(r' Municipality', '')
dfpop['County/State'] = dfpop['County/State'].str.replace(r' Borough', '')
dfpop.at[2924,'County/State']='Fairfax City, Virginia, US'
dfpop.at[2926,'County/State']='Franklin City, Virginia, US'
dfpop.at[2895,'County/State']='Richmond City, Virginia, US'
dfpop.at[2896,'County/State']='Roanoke City, Virginia, US'
dfpop.at[1597,'County/State']='St. Louis City, Missouri, US'
dfpop['County/State'] = dfpop['County/State'].str.replace(r'New York, New York, US', 'New York City, New York, US')

# %%
# Merging dfpop into df1 to get the population count
dfpopmerged = pd.merge(left=df1, right=dfpop, how='left', left_on='County/State', right_on='County/State')

# %%
dfpopmerged.at[901,'Population']= 42628 #michigan department of corrections

Weber = dfpop.loc[dfpop['County/State']=='Weber, Utah, US', 'Population'].item()
Morgan = dfpop.loc[dfpop['County/State']=='Morgan, Utah, US', 'Population'].item()
dfpopmerged.at[2999, 'Population'] = Weber + Morgan

dfpopmerged.at[1438,'Population']= 491918 #kansas city
dfpopmerged.at[154,'Population']= 877 #bear river utah
dfpopmerged.at[780,'Population']= 1211 #dona ana new mexico

Dukes = dfpop.loc[dfpop['County/State']=='Dukes, Massachusetts, US', 'Population'].item()
Nantucket = dfpop.loc[dfpop['County/State']=='Nantucket, Massachusetts, US', 'Population'].item()
dfpopmerged.at[804, 'Population'] = Dukes + Nantucket

# %%
# there are some issues with the merge. Names of counties are different
dfpopmerged.isnull().sum()

# %%
#dfpopmerged = dfpopmerged.replace({'Population': {1: 62568}})
dfpopmerged.at[1,'Population']= 62568
dfpopmerged.at[31,'Population']= 5750
dfpopmerged.at[34,'Population']= 156505
dfpopmerged.at[47,'Population']= 25661
dfpopmerged.at[52,'Population']= 296112
dfpopmerged.at[81,'Population']= 121176
dfpopmerged.at[88,'Population']= 22714
dfpopmerged.at[104,'Population']= 40882
dfpopmerged.at[113,'Population']= 614700
dfpopmerged.at[142,'Population']= 887
dfpopmerged.at[145,'Population']= 36769
dfpopmerged.at[180,'Population']= 18040
dfpopmerged.at[184,'Population']= 13668
dfpopmerged.at[215,'Population']= 126131
dfpopmerged.at[242,'Population']= 16843
dfpopmerged.at[268,'Population']= 6399
dfpopmerged.at[297,'Population']= 248361
dfpopmerged.at[300,'Population']= 200182
dfpopmerged.at[302,'Population']= 9996
dfpopmerged.at[325,'Population']= 6868
dfpopmerged.at[382,'Population']= 9893
dfpopmerged.at[392,'Population']= 613
dfpopmerged.at[408,'Population']= 47042
dfpopmerged.at[432,'Population']= 237820
dfpopmerged.at[462,'Population']= 16153
dfpopmerged.at[545,'Population']= 17593
dfpopmerged.at[563,'Population']= 20021
dfpopmerged.at[585,'Population']= 5582
dfpopmerged.at[647,'Population']= 41512
dfpopmerged.at[665,'Population']= 27463
dfpopmerged.at[717,'Population']= 218195
dfpopmerged.at[739,'Population']= 28656
dfpopmerged.at[751,'Population']= 444094
dfpopmerged.at[752,'Population']= 7225
dfpopmerged.at[753,'Population']= 19499
dfpopmerged.at[783,'Population']= 5381
dfpopmerged.at[797,'Population']= 33636
dfpopmerged.at[799,'Population']= 99653
dfpopmerged.at[801,'Population']= 23865
dfpopmerged.at[807,'Population']= 14067
dfpopmerged.at[825,'Population']= 1419
dfpopmerged.at[861,'Population']= 20322
dfpopmerged.at[876,'Population']= 8211
dfpopmerged.at[879,'Population']= 28469
dfpopmerged.at[899,'Population']= 6638
dfpopmerged.at[965,'Population']= 22348
dfpopmerged.at[1045,'Population']= 135583
dfpopmerged.at[1083,'Population']= 49973
dfpopmerged.at[1145,'Population']= 22529 #hopewell
dfpopmerged.at[1183,'Population']= 29099 #iberia
dfpopmerged.at[1184,'Population']= 32511 #iberville
dfpopmerged.at[1218,'Population']= 15902 #jackson
dfpopmerged.at[1253,'Population']= 432943 #jefferson
dfpopmerged.at[1268,'Population']= 31368  #jefferson davis
dfpopmerged.at[1300,'Population']= 31974 #juneau
dfpopmerged.at[1309,'Population']= 491918 #kansas city
dfpopmerged.at[1317,'Population']= 58708 #kenai peninsula
dfpopmerged.at[1331,'Population']= 13901 #ketchikan
dfpopmerged.at[1359,'Population']= 12998 #kodiak island
dfpopmerged.at[1371,'Population']= 14892 #lasalle
dfpopmerged.at[1377,'Population']= 244390 #lafayette
dfpopmerged.at[1382,'Population']= 96714 #lafourche
dfpopmerged.at[1467,'Population']= 7446 #lexington
dfpopmerged.at[1480,'Population']= 46742 #lincoln
dfpopmerged.at[1505,'Population']= 140789 #livingtson
dfpopmerged.at[1540,'Population']= 82168 #lynchburg
dfpopmerged.at[1566,'Population']= 10951 #madison
dfpopmerged.at[1582,'Population']= 41085 #manasas
dfpopmerged.at[1583,'Population']= 17478 #manasas park
dfpopmerged.at[1630,'Population']= 12554 #martinsville
dfpopmerged.at[1639,'Population']= 108317 #matanuska susitna
dfpopmerged.at[1707,'Population']= 42628 #michigan department of corrections
dfpopmerged.at[1790,'Population']= 24874 #marehouse
dfpopmerged.at[1831,'Population']= 38158 #natchitoches
dfpopmerged.at[1849,'Population']= 8398748 #mew york
dfpopmerged.at[1853,'Population']= 180994 #newport news
dfpopmerged.at[1872,'Population']= 10004 #nome
dfpopmerged.at[1874,'Population']= 242742 #norfolk
dfpopmerged.at[1881,'Population']= 3981 #norton
dfpopmerged.at[1930,'Population']= 391006 #orleans
dfpopmerged.at[1953,'Population']= 153279 #ouachita
dfpopmerged.at[1954,'Population']= 4952 #ouray
dfpopmerged.at[2018,'Population']= 3266 #petersburg
dfpopmerged.at[2019,'Population']= 31346 #petersburg
dfpopmerged.at[2058,'Population']= 23197 #plaquemines
dfpopmerged.at[2067,'Population']= 21730 #pointe coupee
dfpopmerged.at[2084,'Population']= 12271 #poquoson
dfpopmerged.at[2088,'Population']= 94398 #portsmouth
dfpopmerged.at[2110,'Population']= 6203 #prince of wales-hyder
dfpopmerged.at[2134,'Population']= 18249 #radford
dfpopmerged.at[2151,'Population']= 121648 #rapides
dfpopmerged.at[2156,'Population']= 8442 #red river
dfpopmerged.at[2170,'Population']= 20192 #richland
dfpopmerged.at[2179,'Population']= 228783 #richmond city
dfpopmerged.at[2190,'Population']= 94073 #roanoke
dfpopmerged.at[2227,'Population']= 23884 #sabine
dfpopmerged.at[2234,'Population']= 24836 #salem
dfpopmerged.at[2363,'Population']= 6893 #southeast fairbanks
dfpopmerged.at[2374,'Population']= 47244 #st. bernard
dfpopmerged.at[2375,'Population']= 53100 #st. charles
dfpopmerged.at[2384,'Population']= 10132 #st. helena
dfpopmerged.at[2385,'Population']= 21096 #st. james
dfpopmerged.at[2386,'Population']= 42837 #st. john the baptist
dfpopmerged.at[2390,'Population']= 82124 #st. landry
dfpopmerged.at[2394,'Population']= 300576 #st. louis city
dfpopmerged.at[2396,'Population']= 79210 #st. martin
dfpopmerged.at[2397,'Population']= 49348 #st. mary
dfpopmerged.at[2399,'Population']= 258111 #st. tammany
dfpopmerged.at[2412,'Population']= 24932 #staunton
dfpopmerged.at[2439,'Population']= 98108 #suffolk
dfpopmerged.at[2476,'Population']= 134758 #tangipahoa
dfpopmerged.at[2492,'Population']= 4334 #tensas
dfpopmerged.at[2493,'Population']= 110461 #terrebonne
dfpopmerged.at[2613,'Population']= 22330 #union
dfpopmerged.at[2643,'Population']= 59611 #vermilion
dfpopmerged.at[2645,'Population']= 47429 #vernon
dfpopmerged.at[2651,'Population']= 449974 #virginia beach
dfpopmerged.at[2707,'Population']= 46194 #washington, louisiana
dfpopmerged.at[2747,'Population']= 22628 #waynesboro
dfpopmerged.at[2750,'Population']= 260213 #weber
dfpopmerged.at[2754,'Population']= 38340 #webster, louisiana
dfpopmerged.at[2761,'Population']= 26465 #west baton rouge
dfpopmerged.at[2762,'Population']= 10830 #west carroll
dfpopmerged.at[2763,'Population']= 15568 #west feliciana
dfpopmerged.at[2800,'Population']= 13389 #williamsburg
dfpopmerged.at[2808,'Population']= 28078 #mwinchester
dfpopmerged.at[2813,'Population']= 13904 #winn
dfpopmerged.at[2869,'Population']= 5230 #yukon koyukuk
dfpopmerged.at[2876,'Population']= 165768 #guam
dfpopmerged.at[2877,'Population']= 56882 #northern mariana
dfpopmerged.at[2878,'Population']= 3193694 #puerto rico
dfpopmerged.at[2880,'Population']= 106977 #virgin islands

# %%
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import pandas as pd

levels = ['County/State', 'State'] # levels totaled for the hierarchical chart
color_columns = 'Deaths'
value_column = 'Population'

def build_hierarchical_dataframe(df, levels, value_column, color_columns=None):
    """
    Build a hierarchy of levels for Sunburst or Treemap charts.

    Levels are given starting from the bottom to the top of the hierarchy,
    ie the last level corresponds to the root.
    """
    df_all_trees = pd.DataFrame(columns=['id', 'parent', 'value', 'color'])
    for i, level in enumerate(levels):
        df_tree = pd.DataFrame(columns=['id', 'parent', 'value', 'color'])
        dfg = df.groupby(levels[i:]).sum()
        dfg = dfg.reset_index()
        df_tree['id'] = dfg[level].copy()
        if i < len(levels) - 1:
            df_tree['parent'] = dfg[levels[i+1]].copy()
        else:
            df_tree['parent'] = 'total'
        df_tree['value'] = dfg[value_column]
        df_tree['color'] = dfg[color_columns]
        df_all_trees = df_all_trees.append(df_tree, ignore_index=True)
    total = pd.Series(dict(id='total', parent='',
                              value=df[value_column].sum(),
                              color=df[color_columns].sum()
                              ))
    df_all_trees = df_all_trees.append(total, ignore_index=True)
    return df_all_trees

df_all_trees = build_hierarchical_dataframe(dfpopmerged, levels, value_column, color_columns)
avg = df1['Confirmed'].mean()

# %%
# Creating a categorical column to use in the treemap to use colors for each state
codes = pd.DataFrame(df_all_trees['parent'].astype('category'))
codes["parent"] = codes["parent"].cat.codes
df_all_trees['codes'] = codes['parent']

# %%
fig = make_subplots(1, 1, specs=[[{"type": "domain"}]])

fig.add_trace(go.Treemap(
    labels=df_all_trees['id'],
    parents=df_all_trees['parent'],
    values=df_all_trees['value'],
    branchvalues='total',
    marker=dict(
        colors=df_all_trees['codes'],
        colorscale='matter',
        #cmid=0.5
        ),
    hovertemplate='%{label}
 Population: %{value:,.2s}
 %{percentParent:,.2%} of total<extra></extra>',
    texttemplate='%{label}
 Population %{value:,.2s}
 %{percentParent:,.2%} of total
',
    maxdepth=3,
    meta=df_all_trees['codes']
    ), 1, 1)

fig.update_layout(
    title='Population per state and county as percentage of total in the US',
    title_x=0.5,
    hoverlabel=dict(
            bgcolor="white",
            font_size=16,
            font_color="#595959",
            font_family="Arial",
            bordercolor='#595959'
            ),
    )
fig.show()

import plotly.io as pio
pio.write_html(fig, file='Index.html', auto_open=True)

# %%


"""

  December 2016 - pltrdy
  Handling datasets

  Notes: we assume that <eos>'s id is 1.

"""
import numpy as np
import os
import tensorflow as tf
from collections import Counter

# Don't change those values. I would have serious side effects on the model.
EOS = "<eos>"
IEOS=2

BOS = "<bos>"
IBOS=1

PAD = "<pad>"
IPAD=0

# Cut sentences after MAX_LEN words
MAX_LEN = 100

class SingleSentenceData:
  """
    A class meant to work on a Single Sentence.
    i.e. reading a file `input_fd` line by line

    The point is to run a (tiny) epoch on each line
    thus each batch contains only 1 sentence.
    One iterate over the lines by using `next()`
  """
  def __init__(self):
    self.batch_size = 1
    self.sentence = None

  def set_line(self, line, word_to_id):
    words = (" <bos> "+line).replace("\n", " <eos> ").split()+['<eos>']

    sentence = [word_to_id[word] for word in words if word in word_to_id]
    self.sentence = sentence
    return len(sentence)

  def read_from_file(self, input_fd, word_to_id):
    line = input_fd.readline()
    if  not line:
      return None
    return self.set_line(line, word_to_id)

  def batch_iterator(self):
    sen_len = len(self.sentence)
    x = np.zeros([1, sen_len-1]) + self.sentence[:-1]
    y = np.zeros([1, sen_len-1]) + self.sentence[1:]
    yield (x, y)

  @property
  def data(self):
    return self.sentence

  @property
  def epoch_size(self):
    return ((len(self.data) // self.batch_size) - 1)

class SentenceSet:
  """
    A class defining sentence based sub-dataset
    i.e. train, valid & text sets
    each element is a sentence
  """
  def __init__(self, raw, batch_size, shuffle=True):
    self.sentences = self._raw_to_sentences(raw)
    self.batch_size = batch_size
    self.shuffle = shuffle
    # nb sentence in data
    self.n_sentences = len(self.data)

    # n_iter aka. 'epoch_size'
    self.n_iter = self.n_sentences // self.batch_size

    self.sentences.sort(key=len)
    self.order = np.arange(self.n_iter)

    print(self)

  def __str__(self):
    return ("SentenceSet:\n"
           + "\n\t* #sentences: %d" % self.n_sentences
           + "\n\t* batch_size: %d" % self.batch_size
           + "\n\t* #iter: %d" % self.n_iter
           + "\n\t* shuffled: %s" % str(self.shuffle))

  def maybe_shuffle(self):
    if self.shuffle:
      np.random.shuffle(self.order)

  def _raw_to_sentences(self, raw_data):
    """
      Inputs:
        * raw_data: list of word indentifier.  [ int ].
      Output:
        * sentences
    """
    ieos = IEOS
    sentences = []

    sentence = []
    count = 0
    for d in raw_data:
      sentence.append(d)
      if d == ieos:
        sentences.append(sentence)
        sentence = []

    return sentences

  def batch_iterator(self):
    n_iter, batch_size = self.n_iter, self.batch_size
    if n_iter == 0:
      raise ValueError("epoch_size == 0, decrease batch_size")

    self.maybe_shuffle()

    # Batching data
    for i in range(n_iter):
      ii = self.order[i]
      batch_sentences = self.sentences[batch_size*ii:batch_size*(ii+1)]
      max_len = min(max([len(s) for s in batch_sentences]), MAX_LEN)
      x = np.zeros([batch_size, max_len])+IPAD
      y = np.zeros([batch_size, max_len])+IPAD

      for j in range(batch_size):
        s = batch_sentences[j]
        l = min(80, len(s))
        x[j][:l] = [IBOS]+s[:l-1]
        y[j][:l] = s[:l]

      yield (x, y)

  @property
  def data(self):
    return self.sentences

  @property
  def epoch_size(self):
    return ((len(self.data) // self.batch_size) - 1)

class SequenceSet:
  """
    Set of fixed size sequences (num_steps)
  """
  def __init__(self, raw_data, batch_size, num_steps):
    self.raw_data = np.array(raw_data, dtype=np.int32)
    self.num_steps = num_steps
    self.batch_size = batch_size

    self.data_len = data_len = len(raw_data)
    self.batch_len = batch_len = data_len // batch_size
    self.epoch_size = (batch_len - 1) // num_steps

    print(self)

  def __str__(self):
    return ("SequenceSet:"
            + "\n\t* #sequences: %d" % self.data_len
            + "\n\t* batch_size: %d" % self.batch_size
            + "\n\t* batch_len: %d" % self.batch_len
            + "\n\t* #iter: %d" % self.epoch_size)

  def batch_iterator(self):
    """Iterate on the raw data.
    Args:
      raw_data: one of the raw data outputs from ptb_raw_data.
      batch_size: int, the batch size.
      num_steps: int, the number of unrolls.
    Yields:
      Pairs of the batched data, each a matrix of shape [batch_size, num_steps].
      The second element of the tuple is the same data time-shifted to the
      right by one.
    Raises:
      ValueError: if batch_size or num_steps are too high.
    """
    # PTB Iterator from tensorflow.models.rnn.ptb.reader.py
    # on TensorFlow 0.11
    # https://github.com/tensorflow/tensorflow/blob/282823b877f173e6a33bbc9d4b9ad7dd8413ada6/tensorflow/models/rnn/ptb/reader.py

    raw_data = self.raw_data
    num_steps = self.num_steps
    batch_size = self.batch_size
    batch_len = self.batch_len
    epoch_size = self.epoch_size

    data = np.zeros([batch_size, batch_len], dtype=np.int32)
    for i in range(batch_size):
      data[i] = raw_data[batch_len * i:batch_len * (i + 1)]

    if epoch_size == 0:
      raise ValueError("epoch_size == 0, decrease batch_size or num_steps")

    for i in range(epoch_size):
      x = data[:, i*num_steps:(i+1)*num_steps]
      y = data[:, i*num_steps+1:(i+1)*num_steps+1]
      yield (x, y)

class Datasets:
  """
    Managing datasets
    It may actually contains 3 datasets, namely
    train, valid and test.
  """
  def __init__(self, path, batch_size=1, training=True, num_steps=1, word_to_id=None):
    if not training and word_to_id is None:
      raise ValueError("Must set 'word_to_id' when action is not 'train'")

    # Setting parameters
    self.path = path
    self.training = training
    self.batch_size = batch_size
    self.num_steps = num_steps

    # Loading from files
    train_path = os.path.join(path, "train.txt")
    valid_path = os.path.join(path, "valid.txt")
    test_path = os.path.join(path, "test.txt")

    if word_to_id is None:
      print("Building vocabulary...")
      self._build_vocab(train_path)
    else:
      self.word_to_id = word_to_id
    print("Vocabulary size: %d" % len(self.word_to_id))

    if training:
      print("Loading train set")
      self.train = self._load_set(train_path)

      print("Loading valid set")
      self.valid = self._load_set(valid_path)

    print("Loading test  set")
    self.test  = self._load_set(test_path, batch_size=1)

  def _load_set(self, path, batch_size=None):
    if not os.path.isfile(path):
      print("WARNING: File not found: '%s'" % path)
      return None

    if batch_size is None:
      batch_size = self.batch_size

    data = self._file_to_word_ids(path)

    if self.num_steps == 0:
      return SentenceSet(data, batch_size)
    else:
      return SequenceSet(data, batch_size, self.num_steps)

  def _build_vocab(self, filename):
    counts = Counter()
    with tf.gfile.GFile(filename, "r") as f:
      #for line in f:
      #  words = line.replace("\n"," ").split()
      #  counts += Counter(words)
      while True:
        chunk = f.read(int(500000000/2))
        if not chunk:
          break
        counts += Counter(chunk.replace("\n", " ").split())

    sorted_pairs = sorted(counts.items(), key=lambda x: (-x[1], x[0]))
    self.word_to_id = {e[0]: (i+3) for (i, e) in enumerate(sorted_pairs)}
    self.word_to_id[EOS] = IEOS
    self.word_to_id[BOS] = IBOS
    self.word_to_id[PAD] = IPAD

  def _file_to_word_ids(self, filename):
    d = []
    w2id = self.word_to_id
    with tf.gfile.GFile(filename, "r") as f:
      for line in f:
        ids = [w2id[w] for w in line.replace("\n"," %s " % EOS).split() if w in w2id]
        d += ids

    return d

  def train_data(self):
    return self.train.data

  @property
  def valid_data(self):
    return self.valid.data

  @property
  def test_data(self):
    return self.test.data


####################################
# author: Gonzalo Salazar
# purpose: lecture notes
# course: Computer Science 1 at Boston College
# other: N/A
####################################

# MODULES OR LIBRARIES
import random

# Interger division
7//3

# Print many values
x = 1
y = 6
v = 4
print(x, y, v)

phr1 = 'love you'
phr2 = 'I'
print(phr1, phr2)
print(phr1, phr2, end = " ")

# Assigning two variables
c, b = 2, 3
c
b
c, b

# Random library
random.randint(1, 6)

# FUNCTIONS
# Parenthesis are mandatory
def function_name():
    print('Hey!')
    return print('Python will finish the function here, anything after will not be ran')
    print('NOOOO!')

# when None appears it means that we forgot to use return in a function

def even_odd(number):
    if number%2==0:
        return 'Even'
    else:
        return 'Odd'

# SLEEP
from time import sleep, time  # any library should be loaded at the beginning of the script

sleep(1)  # it suspends the execution of the script for the specified # of seconds (e.g., 1)

h = 2
m = 1
s = 30

print("%1d% 02d:%02d" % (h,m,s))
print("%1d:%02d:%02d" % (h,m,s))
print("%3d:%02d:%02d" % (h,m,s))
print("%02d:%02d:%02d" % (h,m,s))

# % recognizes that we want to change the format of the variable
# %s - String (or any object with a string representation, like numbers)
# %d - Integers
# %f - Floating point numbers
# %.<number of digits>f - Floating point numbers with a fixed amount of digits to the right of the dot.
# %x/%X - Integers in hex representation (lowercase/uppercase)

# WHILE
# when using "continue", it jumps to the next iteration of the while, no matter what follows after the "continue"
# it skips the next lines and star from the beginning of the next iteration

# LISTS
# Shallow copy: is a pointer or a reference, so if I change the value of the reference
# it will change where it is used

my_list = [2,3,4,5,6,7]
print(my_list)

l2 = my_list  #shallow copy
print(l2)

l2[0] == "anna"
print(my_list, l2)

# Deep copy: creates a new space in memory for a list already existing, so it will
# not have a shallow copy's problem

    # one way
my_list_copy = [] #creates a new space in memory
for item in my_list:
    my_list_copy.append(item)

print(my_list, my_list_copy)
my_list_copy[2] = "gonzalo"
print(my_list, my_list_copy)

    # another way
l3 = my_list.copy()
l3[1] = 'the best'
print(l3, my_list)

# STRINGS
# split(","), separates by the comma
# split(), separates by space
# string.strip(), gets rid of the specified character at the beginning or at the end.
#   It only work on strings, it does not work if it is applied to a list.
#   In that case a for should be used to go through each word.
# string.replace(old,new), returns a copy of the original string with all the
#   letters in lower case, it does not modify the original string
# find(x), delivers the value of the index that the substring occupies equal to
#   x within the string
# find(x,start), gives the value of the index that occupies the substring equal
#   to x within the string, looking from the start position onwards

# deep copy - due to slicing/sub list
my_list = [111,222,333,444,555]
my_list_copy = my_list[:]  #slicing  #only works for a list with single elements
                           #inside. For list of lists it will be a shallow copy.
                           #In that case, we should use deepcopy from copy module

my_list_copy.remove(111)

print('after remove')
print(my_list)
print(my_list_copy)

# shallow copy - due to slicing/sub list
my_list = [111,222,333,444,555]
my_list_copy = my_list  # passing by reference

my_list_copy.remove(111)

print('after remove')
print(my_list)
print(my_list_copy)

# slicing
my_list = [111,222,333,444,555]
#           0   1   2   3   4
#          -5  -4  -3  -2  -1

# example where to start we should count from the right and to end, we should
# start from the left
my_list[-2:4]
my_list[0:1]  # last item is never included

# GENERATORS
#range(j): generates a list with values from 0 to j-1
#range(i,j): generates a list with values from i to j-1
#range(i,j,k): generates a list with values from i to j-1, and k is the step

# There is a problem with range(j): it has to generate a list with j values, for
# example if we are using it with a for, since Python has to know where are we as
# well as how many elements rest to cycle through. If the list is huge, this
# will be costly in terms of memory. Instead there is a solution provided by
# the xrange() function, which allows us to create the elements at the time they
# are used and thus saves memory.

# NUMPY (basically MATLAB in Python)
from numpy import *

## ARRAYS
# meant to manipulate vectors of data (numbers)
my_list = [323.42,643.43,656.4,47.546]
my_array = array(my_list)
print(my_list)
print(my_array)
print(3 * my_array + my_array ** 2)
my_array[3]

y = list(range(1,10))
y2 = arange(1,10)
print(y2)
a = zeros(4)
b = ones(10)
print(a, b)
linspace(1,10,10)

## SUBARRAYS (array slicing)
# We also have the issue of shallow copies here. To solve that we use copy()
a = matrix([[1,2,3],[4,5,6],[7,8,9]])
d = copy(a[2:,2:]) # deep copy
print("d: ", d)

c = copy(a[2:,2:])
c[-1] = 999
print("c: ", c)
print("a: ", a)

## MATRICES
a = zeros((4,3))
print("a: ", a)
my_matrix =  matrix([[1,2,3],[4,5,6],[7,8,9]])
print(my_matrix*2)
print(my_matrix+2)
size(my_matrix)  # number of values inside the matrix m times n
my_matrix.shape  # size of the matrix (m x n)
my_matrix.shape[0] # number of rows
my_matrix.shape[1] # number of columns
identity = eye(4)
print(identity)
my_diag = diag([1,2,3,4,5])
print(my_diag)
my_bool_mat = matrix([[True,True,False],[False, True, True]])
print(my_bool_mat)

## SYSTEM OF EQUATIONS
# e.g., solving the following system is easy (Ax=b)
# x -2y = -2
# 3x-2y =  2

print('x = ')
A = matrix([[1,-2],[3,-2]])
b = matrix([[-2],[2]])
x = linalg.solve(A,b)
print(x)

#transposing a matrix
print("x : ", x)
print("x transposed: ",x.T)

# MATPLOTLIB (2D graph library)
import matplotlib.pyplot as plt
import numpy as np

## Step 1 - Creating a figure
plt.figure()

## Step 2 - Loading data and creating the plot
x = np.array([4,6,3,10,7])
plt.plot(x)

## Step 2 - Showing the graph (make sure I run all the steps at the same time)
plt.show()

# another figure
y = np.array(np.arange(len(x)))
plt.figure()
plt.plot(x,y)
plt.title('My Graph')
plt.xlabel('x-axis')
plt.ylabel('y-axis')
plt.show()

# another figure (smooth cosine)
x = np.linspace(0,10,1000)
y = np.cos(x)
plt.figure()
plt.plot(x,y)
plt.title('My Cosine')
plt.xlabel('x-axis')
plt.ylabel('y-axis')
plt.show()

# plt.clf() clear the windo
# plt.subplots()  graphs inside a graph
# plt.axis()
# plt.ylim() sets limits to y-axis

## bar graphs
plt.figure()
x = [1,2,3,4,5]
y = [4,5,10,15,5]
plt.bar(x,y)
plt.title('My bars')
plt.xlabel('x-axis')
plt.ylabel('y-axis')
plt.show()

## histograms
plt.figure()
x = [1,2,3,4,5,5,4,5,4,3,1,4]
plt.hist(x)
plt.title('My histogram')
plt.xlabel('x-axis')
plt.ylabel('frequencies')
plt.show()

## two lines in the same graph
plt.figure()
x = np.arange(-2*np.pi,2*np.pi,0.1)
plt.plot(x,np.sin(x),label = 'graph1')
plt.plot(x,np.cos(x),label = 'graph2')

plt.legend()
plt.show()

## subfigures
nrows = 2
ncols = 1
plt.subplot(nrows,ncols,1)  # 1 is the index
plt.plot((np.array([4,6,3,10,7])))
plt.subplot(nrows,ncols,2)  # 2 is the index
plt.plot((np.random.uniform(10,15,20)))
plt.show()

## pie chart
a = np.random.uniform(1,10,7)
L = ['b1','b2','b3','b4','b5','b6','b7']
plt.pie(a, labels = L)
plt.title('A pie chart example')
plt.show()

## SORTING
# The are algorithms and they have three main reasons to be studied:
#   First, sorting algorithms illustrate many creative approaches to problem
#   solving and these approaches can be applied to solve other problems.
#   Second, sorting algorithms are good for practicing fundamental programming
#   techniques using selection statemens, loops, methods, and arraus.
#   Third, sorting algorithms are excellent examples to demonstrate algorithm
#   performance.
# The data to be sorted should be intergers, floats., words or anything,
# but we must stick to one type in order to sort
# For simplicity we may assume:
#   data to be sorted are intergers
#   data are sorted in ascending order
#   data are stored in a list or a numpy array

### Insertion Sort: it sorts a list of values by repeatedly inserting an unsorted
#                   list into a sorted sublist until the whole list is sorted.
# [2,9,5,4,8,1,6]  // Unsorted
# Pick 2 and create a new sorted list, then evaluate if 9 is bigger or smaller
# than 2, if the first case, then it goes to the right and we continue picking 5.
# Otherwise, do the opposite. As the list starts increasing, the number is evaluated
# against each value until it finds its position in the sorted list.

# Nice website to visualize algorithms
# www.visualgo.net

### Bubble Sort: similar to insertion sort, but now we compare two numbers each time
# [2,9,5,4,8,1,6]  // Unsorted
# Pick up 2,9, are these sorted? yes!, so left them as they were.
# Now compare 9 and 5, are they sorted? No, so switch and the result is  [2,5,9,4,8,1,6]
# Then continue with 9 and 4 and so on.
# We will end up with a new unsorted list, so the process is repeated with this
# new list until we get a new list. The process of traversing the list till its
# end is call a Pass (like an iteration).

### Quick Sort: it selects an element, called the pivot, in the array. Then it
# divides the array into two parts such that all the elements in the first part
# are less than or equal to the pivot and all the elements in the second part are
# greater than the pivot. Finally, it recursively apply the quick sort algorithm
# to the first part and then the second part. # The pivot should always start being
# the first item on the array.

### Merge Sort: similar to the idea used behind the quicksort (recursion). We start
# spliting a list in two different lists of the same size (in case of odd number of
# elements, the list is divided in one containing an even number of elements and
# another containing an odd number of elements). We repeat this with each new list,
# subsequently until we get lists containing just two elements.
# Then, in a second step, we switch the elements belonging to a list that is unsorted.
# After that, we start merging two lists together by comparing their intial elements
# first (to get the first value of the new list). Then we compare the first element
# of the second list with the second element of the first list, in order to now which
# one goes second, and finally we compare the remaining two elements. We repeat this
# process with each two pair of lists, and also with those new lists until we get
# a unique final list.
# [2,9,5,4,8,1,6,7]  // Unsorted
# Merge sort in practice:
# dividing part: [2,9,5,4] and [8,1,6,7] -> [2,9]  [5,4]  [8,1]  [6,7]
# switching part: [2,9]  [4,5]  [1,8]  [6,7]
# merge part: [2,9] - [4,5] -> 2>4 -> [2] -> 4<9 -> [2,4] -> 9>5 -> [2,4,5,9]
#   and [1,8] - [6,7] -> 1<6 -> [1] -> 8>6 -> [1,6] -> 8>7 -> [1,6,7,8]
#   now [2,4,5,9] - [1,6,7,8] -> 2>1 -> [1] -> 2<6 -> [1,2] -> 4<6 -> [1,2,4]
#   -> 5<6 -> [1,2,4,5] -> 9>6 -> [1,2,4,5,6] -> 9>7 -> [1,2,4,5,6,7] -> 9>8
#   -> [1,2,4,5,6,7,8,9]

### Bucket Sort: it assumes the keys are in the range from 0 to N-1. We need N buckets
# labeled 0,1,...,N-1. If an element's key is i, the element is put into the bucket i.
# Each bucket holds the elements with the same key value. An ArrayList can be used to
# implement a bucket. If a bucket already has elements inside, a new element should be
# be compared with those inside to know its corresponding order. We could decide between
# ascending or descending order. (Let's assume ascending order) At the end, all the
# elements from a bucket are extracted (in order) to construct a unique sorted list.
# Bucket sort in practice:
#[34,36,19,27,22,10,7] // Unsorted
# First item: 34
# bucket 1 - 0-9:   [  ]
# bucket 2 - 10-19: [  ]
# bucket 3 - 20-29: [  ]
# bucket 4 - 30-39: [34]
# Second item: 36, 34<36
# bucket 1 - 0-9:   [  ]
# bucket 2 - 10-19: [  ]
# bucket 3 - 20-29: [  ]
# bucket 4 - 30-39: [34,36]
# ...
# Fifth item: 22, 22<27
# bucket 1 - 0-9:   [  ]
# bucket 2 - 10-19: [19]
# bucket 3 - 20-29: [22,27]
# bucket 4 - 30-39: [34,36]
# ..
# Last item: 7
# bucket 1 - 0-9:   [7]
# bucket 2 - 10-19: [10,19]
# bucket 3 - 20-29: [22,27]
# bucket 4 - 30-39: [34,36]
# Finally: [b1,b2,b3,b4] -> [7,10,19,22,27,34,36]

### Radix Sort: it uses a similar approach to bucket sorting, but this time it sorts
# elements by its decimal, units, tens, hundreds, etc., from lower to bigger figures.
# Note that it does not require to sort elements inside each bucket.
#[331,454,230,34,343,45,59,453,345,231,9] // Unsorted
# first iteration (by units):
# bucket 0: [230]
# bucket 1: [231,331]
# bucket 2: []
# bucket 3: [343,453]
# bucket 4: [34,454]
# bucket 5: [45,345]
# bucket 6: []
# bucket 7: []
# bucket 8: []
# bucket 9: [9,59]
#
#[230,231,331,453,343,34,454,45,345,9,59] // Unsorted
# second iteration (by tens):
# bucket 0: [9]
# bucket 1: []
# bucket 2: []
# bucket 3: [230,231,331,34]
# bucket 4: [343,45,345]
# bucket 5: [453,454,59]
# bucket 6: []
# bucket 7: []
# bucket 8: []
# bucket 9: []
#
#[9,230,231,331,34,345,45,345,453,454,59] // Unsorted
# third iteration (by hundreds):
# bucket 0: [9,34,45,59]
# bucket 1: []
# bucket 2: [230,231]
# bucket 3: [331,345,345]
# bucket 4: [453,454]
# bucket 5: []
# bucket 6: []
# bucket 7: []
# bucket 8: []
# bucket 9: []
#
#[9,34,45,59,230,231,331,345,345,453,454] // Sorted

# OBJECT-ORIENTED PROGRAMMING
# Problem when accesing unknown members. Ocassionally,
# the name of an attribute or method of a class is only
# given at run time so the way to recover them is by
# using getattr(object_instance, string), where string
# is a string which contains the name of an attribute
# or method of a class.

# f = student('Bob Smith', 23)
# getattr(f,'full name') -> 'Bob Smith'
# getattr(f,'get_age') -> <method get_age of class studentClass at 010B3C2>
# getattr(f,'get_age')() -> 23
# getattr(f,'get_birthday') -> AttributeError - No method!

# Regarding AttibuteErrors, we can check whether an object
# instance has or not attributes using
# hasattr(object_instance,string)
# f = student('Bob Smith', 23)
# hasattr(f,'full name') -> True
# getattr(f,'get_age') -> True
# getattr(f,'get_birthday')() -> False


import time
import pandas as pd
import numpy as np

CITY_DATA = { 'chicago': 'chicago.csv',
              'new york city': 'new_york_city.csv',
              'washington': 'washington.csv' }
cities=('chicago','new york city','washington')
months=('january','february','march','april','may','june', 'all')
days=('sunday','monday','tuesday','wednesday','thursday','friday','saturday','all')
def get_filters():

    """
    Asks user to specify a city, month, and day to analyze.

    Returns:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    """
    print('Hello! Let\'s explore some US bikeshare data!')
    # get user input for city (chicago, new york city, washington). HINT: Use a while loop to handle invalid inputs
    while True:
        city=input('Enter a city from those cities (Chicago,New York City,Washington) \n').lower()
        if city not in cities: #check if the input is valid
            print('This input is invalid please enter a valid one')
            continue
        else:
            break

    # get user input for month (all, january, february, ... , june)
    while True:
        month=input('Enter a month from January to June or type all to view all months \n').lower()
        if month not in months: #check if the input is valid
            print('This input is invalid please enter a valid one')
            continue
        else:
            break

    #get user input for day of week (all, monday, tuesday, ... sunday)
    while True:
        day=input('Enter a day from Monday to Sunday or type all to view all weeks \n').lower()
        if day not in days: #check if the input is valid
            print('This input is invalid please enter a valid one')
            continue
        else:
            break

    print('-'*40)
    return city, month, day

def load_data(city, month, day):

    """
    Loads data for the specified city and filters by month and day if applicable.

    Args:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    Returns:
        df - Pandas DataFrame containing city data filtered by month and day
    """
    #we will load the data into a dataframe
    df=pd.read_csv(CITY_DATA[city])
    #to convert the start time column to datetime
    df['Start Time']=pd.to_datetime(df['Start Time'])
    #extract month from Start Time in order to make new column
    df['month']=df['Start Time'].dt.month
    #extract day_of_week from Start Time in order to make new column
    df['day_of_week']=df['Start Time'].dt.weekday_name

    if month != 'all':
        The_months=['january','february','march','april','may', 'june']
        month=The_months.index(month) + 1
        df=df[df['month']==month]

    if day != 'all':
        df=df[df['day_of_week']==day.title()]

    return df

def time_stats(df):
    """Displays statistics on the most frequent times of travel."""

    print('\nCalculating The Most Frequent Times of Travel...\n')
    start_time = time.time()

    # display the most common month
    popular_month =df['month'].mode()[0]
    print('The most common  month is : ', popular_month)

    # display the most common day of week
    popular_day =df['day_of_week'].mode()[0]
    print('The most common  day is : ',popular_day)

    #display the most common start hour
    df['hour']=df['Start Time'].dt.hour
    popular_hour =df['hour'].mode()[0]
    print('The most common hour of day is : ',popular_hour)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def station_stats(df):
    """Displays statistics on the most popular stations and trip."""

    print('\nCalculating The Most Popular Stations and Trip...\n')
    start_time = time.time()

    # display most commonly used start station
    popular_start_station =df['Start Station'].mode()[0]
    print('The popular start station is : ', popular_start_station)

    # display most commonly used end station
    popular_end_station =df['End Station'].mode()[0]
    print('The popular end station is : ',popular_end_station )

    # display most frequent combination of start station and end station trip
    popular_start_end_station=(df['Start Station'] + ' and ' + df['End Station']).mode()[0]
    print('The popular start and end station is : ',popular_start_end_station )

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def trip_duration_stats(df):
    """Displays statistics on the total and average trip duration."""

    print('\nCalculating Trip Duration...\n')
    start_time = time.time()

    # display total travel time
    total_travel_time=sum(df['Trip Duration'])
    print('Total travel time is :',total_travel_time)

    # display mean travel time
    mean_travel_time=df['Trip Duration'].mean()
    print('Mean travel time is :',mean_travel_time)

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def user_stats(df):
    """Displays statistics on bikeshare users."""

    print('\nCalculating User Stats...\n')
    start_time = time.time()

    # Display counts of user types
    print('The Counts of user types is :',df['User Type'].value_counts())

    #  Display counts of gender
    try:
        print('The Counts of Gender is :',df['Gender'].value_counts())
    except:
        print('there is no data for Gender in this city')

    # Display earliest, most recent, and most common year of birth
    try:
        The_earlist_year=df['Birth Year'].min()
        print('The Earlist Year is : ', The_earlist_year)
        The_most_recent_year=df['Birth Year'].max()
        print('The Most Recent Year is : ',The_most_recent_year)
        The_most_common_year=df['Birth Year'].mode()[0]
        print('The Most Common  Year is : ',The_most_common_year)
    except:
        print('there is no data for birth year in this city')

    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def raw_data_display(df): #display row data to user if he want
    ans=input('do you want to display 5 rows from the data? (yes/no) \n').lower()
    k=0
    while True :
        if ans == 'no':
            break
        if ans == 'yes':
            print(df[k:k+5])    #to print a slice containing 5 rows from the data frame
            ans=input('do you want to display another 5 rows ? (yes/no) \n').lower()
            k+=5

def main():
    while True:
        city, month, day = get_filters()
        df = load_data(city, month, day)
        time_stats(df)
        station_stats(df)
        trip_duration_stats(df)
        user_stats(df)
        raw_data_display(df)

        restart = input('\nWould you like to restart? Enter yes or no.\n')
        if restart.lower() != 'yes':
            break

if __name__ == "__main__":
	main()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#------------------------------------------------------------------------------
__author__ = 'James T. Dietrich'
__contact__ = 'james.t.dietrich@dartmouth.edu'
__copyright__ = '(c) James Dietrich 2016'
__license__ = 'MIT'
__date__ = 'Wed Nov 16 11:33:39 2016'
__version__ = '1.0'
__status__ = "initial release"
__url__ = "https://github.com/geojames/..."

"""
Name:           Week6-1_Pandas.py
Compatibility:  Python 3.5
Description:    This program does stuff

URL:            https://github.com/geojames/...

Requires:       libraries

Dev ToDo:

AUTHOR:         James T. Dietrich
ORGANIZATION:   Dartmouth College
Contact:        james.t.dietrich@dartmouth.edu
Copyright:      (c) James Dietrich 2016

"""
#------------------------------------------------------------------------------

# Pandas provides a set of labeled array data structures
# Similar to what you're used to in Excel
#   + 1-D "Series"
#   + 2-D "Data Frames"

# Import Pandas
import pandas as pd

# You also need Numpy
import numpy as np

# and Matplotlib, if you want to plot data
import matplotlib.pyplot as plt

#%% Creating Pandas data

# chances are you'll be importing data into Pandas with read_csv, which we'll
#   get to...but you'll also need to create Pandas data from scratch too

# Pandas uses two data structures:
#   Series = 1-D data (single column)
#   Data Frame = 2-D data (multiple columns)

# Terms:
#   + Column = actual data
#   + Index Column =  first column, defaults to numbers, but also you can
#           also name rows in the index column or use dates as the index column
#           * Yes, it is confusing, since data frames still have indecies like
#             arrays...
#
# The general structure of a data frame is:
#
# Index	COL1	COL2	COL3	...
# 1		data	data	data
# 2		data	data	data
# 3		data	data	data
# 4		data	data	data
# ...

# of for real data would look something like this:

# Index	temp   pres	wind    wind_dir	...
# 1		26.2   1012.3	0		nan
# 2		27.0   1011.1	5		"w"
# 3		27.8   1011.5	8		"nw"
# 4		24.3   1011.3	5		"w"
# ...

# 1-D series data are easy...
s = pd.Series([1,3,5,np.nan,6,8])

# 2-D data can be created in a couple different ways

# 1: typing in matrix vaules
df = pd.DataFrame([[1,2,3,4,5], [6,7,8,9,10]],columns=['A','B','C','D','E'])

# 2: from an exisitng Numpy Array
array = np.array([[1,2,3,4,5], [6,7,8,9,10]])
df2 = pd.DataFrame(array)

# 3: numpy array with column names
array = np.arange(1.,17.).reshape((4,4))
df3 = pd.DataFrame(array, columns=['A','B','C','D'])

# adding columns
# df['new column name'] = data to add...

colors = pd.Series(['blue','green','red','yellow'])

df3['color'] = colors

# directly...
df3['food'] = np.array(['fish','lettcue','apple','banana'])

#%% Accessing index and column name lists

df.index
df.columns
df.values
df.dtypes
df.describe()

#%% Accessng/adding/deleting column data

# access the column by name: string OR by dot notation

df3['A']

df3.A

# adding data

df3['color'] = pd.Series(['blue','green','red','yellow'])

df3['random'] = np.random.rand(4,1)

# new columns by math

df3['AxRand'] = df3.A * df3.random

# boolean tests as columns

df3['Blue_TF'] = df3.color == 'blue'

# adding one value to all rows (propogated value)

df3['ones'] = 'one'

# deleting columns

del df3['random']

# extracting values and deleting (pop out)
rand_vals = df3.pop('AxRand')

# Inserting new columns
#   insert(position, col_name, values)

df3.insert(0,'AxR',rand_vals)

#%% Date ranges

# often your data will need dates. Pandas has some nifty short cuts for adding
# dates to your data frames

# Start and end dates with frequency
#   pd.date_range(start, end, freq = 'D')

pd.date_range('1/1/2016','12/31/2018')

# Start, frequency and number or periods
#   pd.date_range(start, periods=10, freq='D')

# Frequency Codes
#   'A' = year end - 'AS' = year start
#   'M' = month end - 'MS' = month start
#   'W' = week
#   'D' = days, default
#   'H' = hours
#   'min' = minutes
#   's' = seconds
#
#   '5H' = 5 hours, '30s' = 30 seconds, '7D' = 7 days

pd.date_range('1/1/2016',periods=10, freq='D')

pd.date_range('1/1/2016',periods=10, freq='min')

# adding time (24-hour format)

pd.date_range('1/1/2016 07:30:00',periods=10, freq='30d')

#%% Data Frame Indexing

dates = pd.date_range('1/1/2016', periods=8)

df = pd.DataFrame(np.random.randn(8, 4), index=dates, columns=['A', 'B', 'C', 'D'])

# Getting Columns
df.A
df['B']

# Rows, all columns
# by classic index
df[0:4]

# by named index range
df['2016-01-02':'2016-01-04']

# using the location (loc) function
#   dates index, row 0
df.loc[dates[0]]

#   row and column
df.loc[2:4,['A','B']]

#   index and columns
df.loc[dates[2],'A']    # or is you have a generic index: df.loc[index[2],'A']

# Pure index locations (iloc) function
# rows and columns, like arrays
df.iloc[3:5,0:2]

#%% Data Frame Math

# Math on an entire Data Frame
df2 = df * 3

# In place replacement (not recommended, but good for recalculating and overwriting)
df.B = df.B * 2

# better to create a new column
df['Btimes2'] = df.B * 2

# math between columns
df['convert'] = ((df.A * 9/5) + 32) - df.C

# math for specific indecies
df['indexMath'] = df.D[2:5] ** 2

# math with boolean masks
#   only apply the equation to positive vaules in A
df['convertPos'] = ((df.A[df.A >=0] * 9/5) + 32) - df.C[df.A >=0]

#   Apply a different equation to the negative values
df.convertPos[df.A < 0] = ((df.A[df.A < 0] * 9/5) + 32) + df.D[df.A < 0]

#%% Reduction

dates = pd.date_range('1/1/2016', periods=8)
df = pd.DataFrame(np.random.randn(8, 4), index=dates, columns=['A', 'B', 'C', 'D'])

# Basic stats (default is columns, and to skip NaN values)
#       for row stats add (axis=0)
df.mean()
df.std()
df.median()

# Getting statistics for a specific column
df.A.mean()

# Sums (again by column)
df.sum()
df.B.sum()

#%% Combining / Adding Values

df1 = pd.DataFrame({'A': ['A0', 'A1', 'A2', 'A3'],
                    'B': ['B0', 'B1', 'B2', 'B3'],
                    'C': ['C0', 'C1', 'C2', 'C3'],
                    'D': ['D0', 'D1', 'D2', 'D3']},
                    index=[0, 1, 2, 3])

df2 = pd.DataFrame({'A': ['A4', 'A5', 'A6', 'A7'],
                    'B': ['B4', 'B5', 'B6', 'B7'],
                    'C': ['C4', 'C5', 'C6', 'C7'],
                    'D': ['D4', 'D5', 'D6', 'D7']},
                    index=[4, 5, 6, 7])

series = pd.Series(['X10', 'X11', 'X12', 'X13'], name='X')

result = pd.concat([df1, df2])
result2 = pd.concat([df1, df2], axis = 1)

result = df1.append(df2)

result = pd.concat([df1,series],axis=1)

# combining data with matching indicies
dates = pd.date_range('1/1/2016', periods=8)
df = pd.DataFrame(np.random.randn(8, 4), index=dates, columns=['A', 'B', 'C', 'D'])

dates = pd.date_range('1/4/2016', periods=8)
df2 = pd.DataFrame(np.random.randn(8, 2), index=dates, columns=['E', 'F'])

df3 = pd.concat([df, df2], axis=1)

#%% Converting Pandas to Numpy

# single column
np_from_pd = pd.np.array(df3.A)

# whole thing
np_from_pd = pd.np.array(df3)

# using numpy santax
array = np.array(df3.values)

# Monster statments with multiple instructions
np_with_reshape = pd.np.array(df3.A).reshape((2,2))