#!/usr/bin/env python
# coding: utf-8

#
#
# # Earth Analytics Education - EA  Python Course Spring 2021

# ## Important  - Assignment Guidelines
#
# 1. Before you submit your assignment to GitHub, make sure to run the entire notebook with a fresh kernel. To do this first, **restart the kernel** (in the menubar, select Kernel$\rightarrow$Restart & Run All)
# 2. Always replace the `raise NotImplementedError()` code with your code that addresses the activity challenge. If you don't replace that code, your notebook will not run.
#
# ```
# # YOUR CODE HERE
# raise NotImplementedError()
# ```
#
# 3. Any open ended questions will have a "YOUR ANSWER HERE" within a markdown cell. Replace that text with your answer also formatted using Markdown.
# 4. **DO NOT RENAME THIS NOTEBOOK File!** If the file name changes, the autograder will not grade your assignment properly.
# 6. When you create a figure, comment out `plt.show()` to ensure the autograder can grade your plots. For figure cells, DO NOT DELETE the code that says `DO NOT REMOVE LINE BELOW`.
#
# ```
# ### DO NOT REMOVE LINE BELOW ###
# student_plot1_ax = nb.convert_axes(plt)
# ```
#
# * Only include the package imports, code, and outputs that are required to run your homework assignment.
# * Be sure that your code can be run on any operating system. This means that:
#    1. the data should be downloaded in the notebook to ensure it's reproducible
#    2. all paths should be created dynamically using the `os.path.join`
#
# ## Follow to PEP 8 Syntax Guidelines & Documentation
#
# * Run the `autopep8` tool on all cells prior to submitting (HINT: hit shift + the tool to run it on all cells at once!
# * Use clear and expressive names for variables.
# * Organize your code to support readability.
# * Check for code line length
# * Use comments and white space sparingly where it is needed
# * Make sure all python imports are at the top of your notebook and follow PEP 8 order conventions
# * Spell check your Notebook before submitting it.
#
# For all of the plots below, be sure to do the following:
#
# * Make sure each plot has a clear TITLE and, where appropriate, label the x and y axes. Be sure to include UNITS in your labels.
#

# ### Add Your Name Below
# **Your Name:** Avra Saslow

#

# ---

# # Week 04 and 05 Homework - Automate NDVI Workflow
#
# For this assignment, you will write code to generate a plot of the mean normalized difference vegetation index (NDVI) for two different sites in the United States across one year of data:
#
# * San Joaquin Experimental Range (SJER) in Southern California, United States
# * Harvard Forest (HARV) in the Northeastern United States
#
# The data that you will use for this week is available from **earthpy** using the following download:
#
# `et.data.get_data('ndvi-automation')`
#
# ## Assignment Goals
#
# Your goal in this assignment is to create the most efficient and concise workflow that you can that allows for:
#
# 1. The code to scale if you added new sites or more time periods to the analysis.
# 2. Someone else to understand your workflow.
# 3. The LEAST and most efficient (i.e. runs fast, minimize repetition) amount of code that completes the task.
#
# ### HINTS
#
# * Remove values outside of the landsat valid range of values as specified in the metadata, as needed.
# * Keep any output files SEPARATE FROM input files. Outputs should be created in an outputs directory that is created in the code (if needed) and/or tested for.
# * Use the functions that we demonstrated during class to make your workflow more efficient.
# * BONUS - if you  chose - you can export your data as a csv file. You will get bonus points for doing this.
#
#
# ## Assignment Requirements
#
# Your submission to the GitHub repository should include:
# * This Jupyter Notebook file (.ipynb) with:
#     * The code to create a plot of mean NDVI across a year for  2 NEON Field Sites:
#         * NDVI on the x axis and formatted dates on the y for both NEON sites on one figure/axis object
#     * The **data should be cleaned to remove the influence of clouds**. See the [earthdatascience website for an example of what your plot might look like with and without removal of clouds](https://www.earthdatascience.org/courses/earth-analytics-python/create-efficient-data-workflows/).
# * BONUS: Create one output `.csv` file that has 3 columns - NDVI, Date and Site Name - with values for SJER and HARV.
#
# Your notebook should:
# * Have *at least* 2 well documented and well named functions with docstrings.
# * Include a Markdown cell at the top of the notebook that outlines the overall workflow using pseudocode (i.e. plain language, not code)
# * Include additional Markdown cells throughout the notebook to describe:
#     * the data that you used - and where it is from
#     * how data are being processing
#     * how the code is optimized to run fast and be more concise

# # Replace this cell with your pseudocode  for this workflow
#
# If you happen to be a diagram person a diagram is ok too
#
#

# # Psuedocode for just HARV site
# 1. Go within 'ndvi-automation' folder four levels down to access tif files for HARV
# 2. Extract and sort bands 4-5
# 3. Open up the bands with function open_clean_bands
#     - this will use rxr to open the raster, and crop_boundary as a crop extent
# 4. Calculate NDVI
# 5. Obtain QA data from landsat files for cloud mask
#     -extract all files from a tif folder that end in "pixel.tif"
#     -open up that qa data with rxr
# 6. Create cloud mask from ep cloud pixels
#     -refer to textbook for this
# 7. Get the mean of the new masked xarray
# 8. Create df with 3 columns: mean, the site name, and the date in datetime
#
# # Psuedocode for both sites
# Mostly the same as above, except we can't just name the tif file for a specific site. So...
# 1. One for loop for each site directory
# 2. A nested loop for each landsat file directory
# 4. Extract just files for bands 4-5
# 3. Another nested for loop for each band in band folder
# 3. Open up the bands with function open_clean_bands
# 4. Calculate NDVI
# 5. Obtain QA data from landsat files for cloud mask
#     - using cloud mask function
# 6. I'll have already created cloud mask from ep cloud pixels - don't need to repeat because it doesn't change
# 7. Get the mean of the new masked xarray
# 8. Create df with 3 columns: mean, the site name, and the date in datetime
#     - create list to do so
#
#

# In[1]:

# Autograding imports - do not modify this cell
import matplotcheck.autograde as ag
import matplotcheck.notebook as nb
import matplotcheck.timeseries as ts
from datetime import datetime

# In[2]:

# Import needed packages in PEP 8 order
# and no unused imports listed (10 points total)

# YOUR CODE HERE
import os
from glob import glob

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import geopandas as gpd
import rioxarray as rxr
import xarray as xr
from rasterio.plot import plotting_extent
import earthpy as et
import earthpy.mask as em
import earthpy.spatial as es
import earthpy.plot as ep

#set working directory

data = et.data.get_data('ndvi-automation')

os.chdir(os.path.join(et.io.HOME,
                      "earth-analytics",
                      "data"))

# In[3]:

# DO NOT MODIFY THIS CELL
# Tests that the working directory is set to earth-analytics/data

path = os.path.normpath(os.getcwd())
student_wd_parts = path.split(os.sep)

if student_wd_parts[-2:] == ['earth-analytics', 'data']:
    print("\u2705 Great - it looks like your working directory is set correctly to ~/earth-analytics/data")
else:
    print("\u274C Oops, the autograder will not run unless your working directory is set to earth-analytics/data")

# # Figure 1: Plot 1 - Mean NDVI For Each Site Across the Year (50 points)
#
# Create a plot of the mean normalized difference vegetation index (NDVI) for the two different sites in the United States across the year:
#
# * NDVI on the x axis and formatted dates on the y for both NEON sites on one figure/axis object.
# * Each site should be identified with a different color in the plot and legend.
# * The final plot **data should be cleaned to remove the influence of clouds**.
# * Be sure to include appropriate title and axes labels.
#
# Add additional cells as needed for processing data (e.g. defining functions, etc), but be sure to:
# * follow the instructions in the code cells that have been provided to ensure that you are able to use the sanity check tests that are provided.
# * include only the plot code in the cell identified for the final plot code below

# ## Task 1:
#
# In the cell below, create a single dataframe containing MEAN NDVI, the site name,
# and the date of the data for the HARV site
# scene `HARV/landsat-crop/LC080130302017031701T1-SC20181023151837`.  The column names for the  final
# DataFrame should be`mean_ndvi`, and `site`, and the data should be **indexed on the date**.
#
# Use the functions that we reviewed in class (or create your own versions of them) to implement your code
#
# ### In the Cell below Place  All Functions Needed to Run this Notebook (20 points)

# In[4]:

### DO NOT REMOVE THIS LINE OR EDIT / MOVE THIS CELL ###
start_time = datetime.now()

# _This first code block does quite a lot of the exploration of ndvi-automation folder structure. It sets path variables for the HARV site, creates the crop extent for that specific site, and opens the specific tif folder for LC080130302017031701T1-SC20181023151837. Finally, it extracts just the bands 4-5 to calculate the NDVI._
#
# _Because this is just one site, there isn't really a better way to build a function or a for loop that will optimize this workflow._

# In[5]:

#----------------------------------------------
#exploration of folder structure
#----------------------------------------------

# list both site directories
site_path = os.path.join("ndvi-automation", "sites")

# Get a list of both site directories
sites = glob(site_path + "/*/")
#sites

#specifically create df for HARV site
site_name = 'HARV'

#----------------------------------------------
#open shp boundary file from vector directory
#----------------------------------------------

# go into vector directory to get shp file
vector_dir = os.path.join(site_path, site_name,
                          "vector")

site_boundary_path = os.path.join(vector_dir,  site_name + "-crop.shp")
bound = gpd.read_file(site_boundary_path)

bound

#----------------------------------------------
#open tif files from landsat directory
#----------------------------------------------

# In the landsat directory, get files
landsat_dir = os.path.join(site_path, site_name, "landsat-crop")
landsat_folder = os.path.join(landsat_dir, "LC080130302017031701T1-SC20181023151837")

# Open bands in a sorted format
band_files = sorted(glob(os.path.join(landsat_folder, "*band*[4-5].tif")))

band_files

# _The following code block is where my two functions reside - the code was first tested outside of the function and then added after it was confirmed it worked on one site._
#
# _open_clean_bands takes one of the band files I just extracted above, opens it up, clips it to HARV's crop extent, cleans it up, and returns that same band for future use._
#
# _cloud_mask takes the qa file from a tif folder, opens it up using rxr, clips it to HARV's crop extent as well, and then, using an input of mask values, crops whatever NDVI array is given to a specific cloud mask._
#
# _It was important to me not to over complicate these functions. They should be useful and simple to help automate my workflow, and not try to do a billion things at once._

# In[6]:

# In this cell place all of the functions needed to run your notebook
# You will be graded here on function application, docstrings, efficiency so ensure
# All functions are placed here!

# YOUR CODE HERE

def open_clean_bands(band_path,
                     crop_bound,
                     valid_range=None):
    """Open and mask a single landsat band using a pixel_qa layer.

    Parameters
    -----------
    band_path : string
        A path to the array to be opened
    crop_bound : GeoPandas DataFrame
        A data from that tells us the extent of the site of interest
    valid_range : tuple (optional)
        A tuple of min and max range of values for the data. Default = None

    Returns
    -----------
    arr : xarray DataArray
        An xarray DataArray with values that should be
        masked set to 1 for True (Boolean)
    """

    band = (rxr.open_rasterio(band_path, masked=True)
        .rio.clip(crop_bound.geometry, from_disk=True)
        .squeeze())

    # Specify valid range of values
    if valid_range:
        mask = ((band <= 0) | (band > 10000))
        band = band.where(~mask, np.nan)

    return band

def cloud_mask(ndvi_array, site_folder, crop_bound, masked_values):
    """ This function masks clouds from a landsat band using a
    pixel_qa layer.

     Parameters
    -----------
    ndvi_array: xarray DataArray
        An xarray DataArray with ndvi values
    site_folder : string
        A path to the site folder where QA array is
    crop_bound : GeoPandas DataFrame
        The crop extent of the area of interest
    masked_values : list
        A list of all values to be masked

    Returns
    -----------
    arr : xarray DataArray
        An xarray DataArray with ndvi values, masked to the specific values
    """

    #path to specific QA files - they end in pixel.tif
    qa_path = glob(os.path.normpath(os.path.join(site_folder, "*pixel*.tif")))

    #open path with rxr using crop extent of specific site
    qa_file = rxr.open_rasterio(
            qa_path[0], masked=True).rio.clip(crop_bound.geometry,
                                              from_disk=True).squeeze()

    #crop the NDVI where NOT masked values are (i.e., where there isn't cloud cover)
    ndvi_clean_crop = ndvi_array.where(~qa_file.isin(masked_values))

    return ndvi_clean_crop

# _This is the bulk of the processing for the HARV site. This code block loops through the bands, opens and cleans them, calculates the NDVI, and then finds the associative qa_path from the tif folder. It uses this to create a cloud mask, and then after the raster has been masked, it calculates the mean NDVI, and builds a dataframe from the site name, the date, and the mean NDVI. Because it doesn't have to do this with more than one site, it's pretty streamlined as is._

# In[7]:

# Create dataframe of mean NDVI in this cell using the functions created above
# Important: to use the ungraded tests below as a sanity check,
# name your columns: mean_ndvi and site
# Call the dataframe at the end of the cell so the tests run on it!
# Be sure that the date column is an index of type date
# HINT: the time series lessons may help you remember how to do this!

# YOUR CODE HERE

#----------------------------------------------------
#loop through each band file to open and clean bands
#----------------------------------------------------

bands = []
for aband in band_files:

    #run open_clean_bands function
    cleaned_band = open_clean_bands(band_path=aband,
                                    crop_bound=bound,
                                    valid_range=(0, 10000))
    bands.append(cleaned_band)

#----------------------------------------------
#calculate NDVI
#----------------------------------------------

# NDVI = (NIR-RED)/(NIR+RED)
ndvi_xr = (bands[1] - bands[0]) / (bands[1] + bands[0])
#ndvi_xr.plot()

#------------------------------------------------
#obtain QA data from landsat files for cloud mask
#------------------------------------------------

qa_path = glob(os.path.normpath(os.path.join(landsat_folder, "*pixel*.tif")))

qa_file = rxr.open_rasterio(qa_path[0], masked=True).rio.clip(bound.geometry,
                                                              from_disk=True).squeeze()

#------------------------------------------------
#create cloud mask from ep cloud pixels
#------------------------------------------------

high_cloud_confidence = em.pixel_flags["pixel_qa"]["L8"]["High Cloud Confidence"]
cloud = em.pixel_flags["pixel_qa"]["L8"]["Cloud"]
cloud_shadow = em.pixel_flags["pixel_qa"]["L8"]["Cloud Shadow"]

all_masked_values = cloud_shadow + cloud + high_cloud_confidence

#mask ndvi with cloud mask
ndvi_clean_crop = ndvi_xr.where(~qa_file.isin(all_masked_values))
#ndvi_clean_crop.plot()

#----------------------------------------------
#get mean of xarray
#----------------------------------------------

ndvi_mean = ndvi_clean_crop.mean()
#type(ndvi_mean)

#convert mean to a float instead of xarray
ndvi_mean_value = ndvi_mean.item()

#----------------------------------------------
#create df with site, date, and mean NDVI
#----------------------------------------------

#slice up the path into its components to utilize different names in the path
slice_path = landsat_folder.split(os.sep)

#site is the third slice
site = slice_path[2]

#the file name (with date) is the fifth slice
file_string = slice_path[4]

#the date is the the file name - before 01T1. Year, Month, Day
date = file_string[10:18]

#convert that date from string to datetime
date_time = datetime.strptime(date, '%Y%m%d').strftime('%m/%d/%Y')

#create dataframe
ndvi_df = pd.DataFrame([[site, date_time, ndvi_mean_value]], columns=['site', 'date', 'mean_ndvi'])

#make sure date is in datetime format
ndvi_df['date'] = pd.to_datetime(ndvi_df['date'])

#set date as index
ndvi_df.set_index("date", inplace = True)

ndvi_df

# In[8]:

# This cell  is testing your data output above

student_ndvi_ts_single_site = _

single_scene_points = 0

# Ensure the data is stored in a dataframe.
if isinstance(student_ndvi_ts_single_site, pd.DataFrame):
    print('\u2705 Your data is stored in a DataFrame!')
    single_scene_points += 1
else:
    print('\u274C It appears your data is not stored in a DataFrame. ',
          'To see what type of object your data is stored in, check its type with type(object)')

# Ensure that the date column is the index
if isinstance(student_ndvi_ts_single_site.index, pd.core.indexes.datetimes.DatetimeIndex):
    print('\u2705 You have the index set to the date column!')
    single_scene_points += 2
else:
    print('\u274C You do not have the index set to the date column.')

# Ensure that the date column is datetime
if isinstance(student_ndvi_ts_single_site.index[0], pd._libs.tslibs.timestamps.Timestamp):
    print('\u2705 The data in your date column is datetime!')
    single_scene_points += 2
else:
    print('\u274C The data in your date column is not datetime.')

# Ensure the site name is correct
if student_ndvi_ts_single_site.site.values[0] == 'HARV':
    print('\u2705 You have the correct site name!')
    single_scene_points += 5
else:
    print('\u274C You do not have the correct site name.')

if np.allclose(0.281131628228094, student_ndvi_ts_single_site.mean_ndvi.values[0]):
    print('\u2705 You have the correct mean NDVI value!')
    single_scene_points += 5
else:
    print('\u274C You do not have the correct mean ndvi value.')

print("\n \u27A1 You received {} out of 15 points for creating a dataframe.".format(
    single_scene_points))
single_scene_points

# ## Task 2:
#
# In the cell below, process all of the landsat scenes. Create a DataFrame that contains the following
# information for each scene
#
#
# |   | index  | site  | mean_ndvi  |
# |---|---|---|---|
# | Date  |   |   |   |
# |  2017-01-07  | 0  | SJER  | .4  |
#
# Be sure to call your dataframe at the end of the cell to ensure autograding works.
# HINT: FOR THIS STEP, leave any rows containing missing values (`NAN`).

# _This code snipped abstracts a lot of the work done above. It's the same concepts (open up site, open up landsat folder, open up band files, open and clean bands, mask for cloud cover, calculate NDVI, create df), but it does so with nested for loops so that it can be used with any site. While the code is concise, three for loops seems like not the most efficient way to do this, but I'm not sure what a more efficient method would be - perhaps another function for opening up each level of directories._

# In[9]:

# Create dataframe of NDVI including the cleaning data to deal with clouds

# Important: to use the ungraded tests below as a sanity check,
# name your columns: mean_ndvi and site
# Don't forget to set date as the index and make the values of type datetime

# YOUR CODE HERE

#I already have my sites variable from task 1, so no need to re-import

#both_sites_list is where all of the data will be stored and eventually put into a df
both_sites_list = []

#----------------------------------------------------
#for loop to go into each of the two site directories
#----------------------------------------------------
for site in sites:
    #path_parts gives me each independent string that makes up the path name
    path_parts = site.split(os.sep)
    site_name = path_parts[2]
    vector_dir = os.path.join(site_path, site_name,
                              "vector")

    #open crop boundary for each site
    site_boundary_path = os.path.join(vector_dir,  site_name + "-crop.shp")
    crop_bound = gpd.read_file(site_boundary_path)

    #open up the landsat directories for each site
    landsat_dir = os.path.join(site_path, site_name, "landsat-crop")
    landsat_folders = sorted(glob(os.path.join(landsat_dir, "*")))

    #---------------------------------------------------------------------
    #secondary for loop to go through each folder in the landsat directory
    #---------------------------------------------------------------------

    for dirs in landsat_folders:
        print("Processing", dirs) #good way to check where I am in the for loop

        #open bands required for NDVI calculation
        band_files = sorted(glob(os.path.join(dirs, "*band*[4-5].tif")))

        #---------------------------------------------------------------------
        #third for loop to go through each band in band files
        #---------------------------------------------------------------------

        #like in task 1, create empty list to save bands to
        bands = []

        for aband in band_files:
            print("Opening", aband) #good way to check where I am in the for loop

            #open and clean bands using function
            cleaned_band = open_clean_bands(band_path=aband,
                                            crop_bound=crop_bound,
                                            valid_range=(0, 10000))
            #add cleaned bands to bands list
            bands.append(cleaned_band)

        #calculate NDVI
        both_sites_ndvi_xr = (bands[1] - bands[0]) / (bands[1] + bands[0])

        #I've already created the cloud mask in task 1, so I can just run the function
        both_sites_ndvi_clean_crop = cloud_mask(ndvi_array = both_sites_ndvi_xr,
                                          site_folder = dirs,
                                          crop_bound = crop_bound,
                                          masked_values = all_masked_values)

        #find ndvi mean, convert to a float
        both_sites_ndvi_mean = both_sites_ndvi_clean_crop.mean(skipna = True)
        both_sites_ndvi_mean_value = both_sites_ndvi_mean.item()

        #splice path_parts again now that we're in the directory and have each tif file name
        final_path_parts = dirs.split(os.sep)
        both_sites_site_name = final_path_parts[2]
        both_sites_file_string = final_path_parts[4]
        both_sites_date = both_sites_file_string[10:18]
        both_sites_date_time = datetime.strptime(both_sites_date, '%Y%m%d').strftime('%m/%d/%Y')

        both_sites_list.append([both_sites_site_name, both_sites_date_time, both_sites_ndvi_mean_value ])

#create dataframe from both_sites_list
both_sites_ndvi_df = pd.DataFrame(both_sites_list, columns=['site', 'date', 'mean_ndvi'])
both_sites_ndvi_df['date'] = pd.to_datetime(both_sites_ndvi_df['date'])
both_sites_ndvi_df.set_index("date", inplace = True)
both_sites_ndvi_df

# In[10]:

# Last sanity check before creating your plot (10 points)

# Ensure that you call your dataframe at the bottom of the cell above
# and that it has columns called: mean_ndvi and site

# Ensure the data is stored in a dataframe.
student_ndvi_df = _

df_points = 0

if isinstance(student_ndvi_df, pd.DataFrame):
    print('\u2705 Your data is stored in a DataFrame!')
    df_points +=2
else:
    print('\u274C It appears your data is not stored in a DataFrame. ',
          'To see what type of object your data is stored in, check its type with type(object)')

# Check that dataframe contains the appropriate number of NAN values
if student_ndvi_df.mean_ndvi.isna().sum() == 15:
    print('\u2705 Correct number of masked data values!')
    df_points +=2
else:
    print('\u274C The amount of null data in your dataframe is incorrect.')

# Ensure that the date column is the index
if isinstance(student_ndvi_df.index, pd.core.indexes.datetimes.DatetimeIndex):
    print('\u2705 You have the index set to the date column!')
    df_points +=3
else:
    print('\u274C You do not have the index set to the date column.')

# Ensure that the date column is datetime
if isinstance(student_ndvi_df.index[0], pd._libs.tslibs.timestamps.Timestamp):
    print('\u2705 The data in your date column is datetime!')
    df_points +=3
else:
    print('\u274C The data in your date column is not datetime.')

# Output for timer, # DO NOT MODIFY
end_time = datetime.now()
total_time = end_time - start_time
print(
    "Your total run time for processing the data was {0}.".format(total_time))

print("\n \u27A1 You received {} out of 10 points for creating a dataframe.".format(
    df_points))

df_points

# _Finally, this code block plots the data, grouped by site, and plots the mean NDVI by the date. It also drops all NaNs in the process. I think it's a fairly efficient way to plot._

# In[11]:

# Add only the plot code to this cell

# This is the final figure of mean NDVI
# for both sites across the year
# with data cleaned to deal with clouds

# YOUR CODE HERE
f, ax = plt.subplots(figsize=(15, 6))

for site, df in both_sites_ndvi_df.dropna().groupby('site'):
    ax.plot(df['mean_ndvi'], 'd-',  label = site)

ax.set(title = "Mean NDVI for HARV and SJER Sites (2017-2018)",
       xlabel = 'Date',
       ylabel = 'Mean NDVI')

plt.legend(bbox_to_anchor=(0.99,0.99))

### DO NOT REMOVE LINES BELOW ###
final_masked_solution = nb.convert_axes(plt, which_axes="current")

# In[12]:

# Ignore this cell for the autograding tests

# In[13]:

# Ignore this cell for the autograding tests

# # Question 1 (10 points)
#
# Imagine that you are planning NEON’s upcoming flight season to capture remote sensing data in these locations and want to ensure that you fly the area when the vegetation is the most green.
#
# When would you recommend the flights take place for each site?
#
# Answer the question in 2-3 sentences in the Markdown cell below.

# Well, the higher the NDVI, the more green the vegetation at each site is. So the highest NDVI for SJER takes place in April, and the highest NDVI for HARV takes places around early July. It's also important to note that these data points aren't regular in their timing. Perhaps there's even a better time for both of these sites in the longer gaps between the diamonds on the plot.
#
#

# # Question 2 (10 points)
#
# How could you modify your workflow to look at vegetation changes over time in each site?
#
# Answer the question in 2-3 sentences in the Markdown cell below.

# I might try other types of statistics to see if that changes the way the NDVI looks over the year...I might create
# two graphs for each site, with the mean, mode and max NDVI calculated. That way I could tell if maybe the mean is creating some error in the way I interpret the data.
#
#
#
#

# # Do not edit this cell! (10 points)
#
# The notebook includes:
# * additional Markdown cells throughout the notebook to describe:
#     * the data that you used - and where it is from
#     * how data are being processing
#     * how the code is optimized to run fast and be more concise

# # Do not edit this cell! (20 points)
#
# The notebook will also be checked for overall clean code requirements as specified at the **top** of this notebook. Some of these requirements include (review the top cells for more specifics):
#
# * Notebook begins at cell [1] and runs on any machine in its entirety.
# * PEP 8 format is applied throughout (including lengths of comment and code lines).
# * No additional code or imports in the notebook that is not needed for the workflow.
# * Notebook is fully reproducible. This means:
#    * reproducible paths using the os module.
#    * data downloaded using code in the notebook.
#    * all imports at top of notebook.

# ## BONUS - Export a  .CSV File to Share (10 points possible)
#
# This is optional - if you export a **.csv** file with the columns specified above: Site, Date and NDVI Value you can get an additional 10 points.
#
# * FULL CREDIT: File exists in csv format and contains the columns specified.
# We will check your github repo for this file!
#

# In[14]:

both_sites_ndvi_df.to_csv("/Users/avrasaslow/Documents/Earth_Lab/"
                          "Semester2/ea-2022-04-ndvi-automation-AvraSaslow/HARV_SJER_NDVI.csv")

# In[15]:

HARV_SJER_NDVI = pd.read_csv("HARV_SJER_NDVI.csv")
HARV_SJER_NDVI

# In[ ]:


#! /usr/bin/env python
##################################################################################
# A template to help create readable figures with Matplotlib and Scipy
#
# It may actually look worse on screen. The saved version should be better.
#
# This will not do anyting on it's own. Copy and paste into your script, then...
#
# You will have to fill in:
#   - the actual things to plot
#   - the axis labels - be sure to include units
#   - the legend labels
#   - the filename to save the figure as
#
# Created: 4/19/13 - Joshua Vaughan (joshua.vaughan@louisiana.edu)
#
# Modified:
#   *
#
##################################################################################

import numpy as np
import matplotlib.pyplot as plt

# Make the figure pretty, then plot the results
#   "pretty" parameters selected based on pdf output, not screen output
#   Many of these setting could also be made default by the .matplotlibrc file

# Example data so that code will show a plot...
# It's best to remove and fill in your own
x1 = np.linspace(0, 5, 501)
x2 = x1
x3 = x1
x4 = x1

y1 = sin(x1)
y2 = 0.5*sin(x2)
y3 = 0.75*sin(x3)
y4 = 1.25*sin(x4)

#-----  Copy from here down into your code, replacing items as needed ----------------

# Set the plot size - 3x2 aspect ratio is best
fig = plt.figure(figsize=(6,4))
ax = plt.gca()
plt.subplots_adjust(bottom=0.17, left=0.17, top=0.96, right=0.96)

# Change the axis units font
plt.setp(ax.get_ymajorticklabels(),fontsize=18)
plt.setp(ax.get_xmajorticklabels(),fontsize=18)

ax.spines['right'].set_color('none')
ax.spines['top'].set_color('none')

ax.xaxis.set_ticks_position('bottom')
ax.yaxis.set_ticks_position('left')

# Turn on the plot grid and set appropriate linestyle and color
ax.grid(True,linestyle=':', color='0.75')
ax.set_axisbelow(True)

# Define the X and Y axis labels
plt.xlabel('X label (units)', fontsize=22, weight='bold', labelpad=5)
plt.ylabel('Y label (units)', fontsize=22, weight='bold', labelpad=10)

plt.plot(x1, y1, linewidth=2, linestyle='-', label=r'Data 1')
plt.plot(x2, y2, linewidth=2, linestyle='--', label=r'Data 2')
plt.plot(x3, y3, linewidth=2, linestyle='-.', label=r'Data 3')
plt.plot(x4, y4, linewidth=2, linestyle=':', label=r'Data 4')

# uncomment below and set limits if needed
# plt.xlim(0,5)
# plt.ylim(0,10)

# Create the legend, then fix the fontsize
leg = plt.legend(loc='upper right', ncol = 1, fancybox=True)
ltext  = leg.get_texts()
plt.setp(ltext,fontsize=18)

# Adjust the page layout filling the page using the new tight_layout command
plt.tight_layout(pad=0.5)

# save the figure as a high-res pdf in the current folder
# plt.savefig('plot_filename.pdf')

# show the figure
plt.show()


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   '@webio':
#     lastCommId: a8ab2762cccf499696a7ef0a86be4d18
#     lastKernelId: 261999dd-7ee7-4ad4-9a26-99a84a77979b
#   cite2c:
#     citations:
#       6202365/8AH9AXN2:
#         URL: http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory.pdf
#         author:
#         - family: Carroll
#           given: Christopher
#         container-title: Manuscript, Department of Economics, Johns Hopkins University
#         id: 6202365/8AH9AXN2
#         issued:
#           month: 2
#           year: 2019
#         note: "Available at http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory\
#           \ \nCitation Key: carrollBufferStockTheory \nbibtex*[extra=bibtex:carrollBufferStockTheory]"
#         title: Theoretical Foundations of Buffer Stock Saving
#         type: article-journal
#       6202365/TGG4U7J4:
#         author:
#         - family: Clarida
#           given: Richard H.
#         container-title: International Economic Review
#         issued:
#           date-parts:
#           - - 1987
#         page: "339\u2013351"
#         title: Consumption, Liquidity Constraints, and Asset Accumulation in the Face
#           of Random Fluctuations in Income
#         type: article-journal
#         volume: XXVIII
#       undefined:
#         URL: http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory.pdf
#         author:
#         - family: Carroll
#           given: Christopher
#         container-title: Manuscript, Department of Economics, Johns Hopkins University
#         issued:
#           date-parts:
#           - - '2019'
#             - 2
#         note: "Available at http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory\
#           \ \nCitation Key: carrollBufferStockTheory \nbibtex*[extra=bibtex:carrollBufferStockTheory]"
#         title: Theoretical Foundations of Buffer Stock Saving
#         type: article-journal
#   jupytext:
#     formats: ipynb,py:percent
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.1'
#       jupytext_version: 0.8.3
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
#   language_info:
#     codemirror_mode:
#       name: ipython
#       version: 3
#     file_extension: .py
#     mimetype: text/x-python
#     name: python
#     nbconvert_exporter: python
#     pygments_lexer: ipython3
#     version: 3.6.7
#   varInspector:
#     cols:
#       lenName: 16
#       lenType: 16
#       lenVar: 40
#     kernels_config:
#       python:
#         delete_cmd_postfix: ''
#         delete_cmd_prefix: 'del '
#         library: var_list.py
#         varRefreshCmd: print(var_dic_list())
#       r:
#         delete_cmd_postfix: ') '
#         delete_cmd_prefix: rm(
#         library: var_list.r
#         varRefreshCmd: 'cat(var_dic_list()) '
#     types_to_exclude:
#     - module
#     - function
#     - builtin_function_or_method
#     - instance
#     - _Feature
#     window_display: false
# ---

# %% [markdown]
# # Buffer Stock Saving in HARK and dolo
#

# %% [markdown]
# This notebook compares the solutions to a standard buffer stock saving model obtained by the [Econ-ARK/HARK](https://github.com/econ-ark/HARK) toolkit and the [dolo](https://github.com/EconForge/dolo) modeling system.

# %% {"code_folding": [0]}
# This cell does some setup and imports generic tools

Generator=True # Is this notebook the master or is it generated?
# Import related generic python packages
import numpy as np
from time import clock
mystr = lambda number : "{:.4f}".format(number)

# This is a jupytext paired notebook that autogenerates BufferStockTheory.py
# which can be executed from a terminal command line via "ipython BufferStockTheory.py"
# But a terminal does not permit inline figures, so we need to test jupyter vs terminal
# Google "how can I check if code is executed in the ipython notebook"

from IPython import get_ipython # In case it was run from python instead of ipython
def in_ipynb():
    try:
        if str(type(get_ipython())) == "<class 'ipykernel.zmqshell.ZMQInteractiveShell'>":
            return True
        else:
            return False
    except NameError:
        return False

# Determine whether to make the figures inline (for spyder or jupyter)
# vs whatever is the automatic setting that will apply if run from the terminal
if in_ipynb():
    # %matplotlib inline generates a syntax error when run from the shell
    # so do this instead
    get_ipython().run_line_magic('matplotlib', 'inline')
else:
    get_ipython().run_line_magic('matplotlib', 'auto')
    print('You appear to be running from a terminal')
    print('By default, figures will appear one by one')
    print('Close the visible figure in order to see the next one')

# Import the plot-figure library matplotlib

import matplotlib.pyplot as plt

# In order to use LaTeX to manage all text layout in our figures, we import rc settings from matplotlib.
from matplotlib import rc
plt.rc('font', family='serif')

# LaTeX is huge and takes forever to install on mybinder
# so if it is not installed then do not use it
from distutils.spawn import find_executable
iflatexExists=False
if find_executable('latex'):
    iflatexExists=True

plt.rc('font', family='serif')
plt.rc('text', usetex=iflatexExists)

# The warnings package allows us to ignore some harmless but alarming warning messages
import warnings
warnings.filterwarnings("ignore")

# The tools for navigating the filesystem
import sys
import os

sys.path.insert(0, os.path.abspath('../../lib')) # REMARKs directory is two down from root

from HARK.utilities import plotFuncsDer, plotFuncs
from copy import copy, deepcopy

# Define (and create, if necessary) the figures directory "Figures"
if Generator:
    my_file_path = os.path.dirname(os.path.abspath("BufferStockTheory.ipynb")) # Find pathname to this file:
    Figures_HARK_dir = os.path.join(my_file_path,"Figures/") # LaTeX document assumes figures will be here
    Figures_HARK_dir = os.path.join(my_file_path,"/tmp/Figures/") # Uncomment to make figures outside of git path
    if not os.path.exists(Figures_HARK_dir):
        os.makedirs(Figures_HARK_dir)

# %% [markdown]
# ## [The Problem](http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory/#The-Problem)
#
# [This paper](http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory/#The-Problem) defines a buffer stock saving model and calibrates parameters:
#
# | Parameter | Description | Code | Value |
# | :---: | ---         | ---  | :---: |
# | $\newcommand{\PermGroFac}{\Gamma}\PermGroFac$ | Permanent Income Growth Factor | $\texttt{PermGroFac}$ | 1.03 |
# | $\newcommand{\Rfree}{\mathrm{\mathsf{R}}}\Rfree$ | Interest Factor | $\texttt{Rfree}$ | 1.04 |
# | $\newcommand{\DiscFac}{\beta}\DiscFac$ | Time Preference Factor | $\texttt{DiscFac}$ | 0.96 |
# | $\newcommand{\CRRA}{\rho}\CRRA$ | Coeﬃcient of Relative Risk Aversion| $\texttt{CRRA}$ | 2 |
# | $\newcommand{\UnempPrb}{\wp}\UnempPrb$ | Probability of Unemployment | $\texttt{UnempPrb}$ | 0.005 |
# | $\newcommand{\IncUnemp}{\mu}\IncUnemp$ | Income when Unemployed | $\texttt{IncUnemp}$ | 0. |
# | $\newcommand{\PermShkStd}{\sigma_\psi}\PermShkStd$ | Std Dev of Log Permanent Shock| $\texttt{PermShkStd}$ | 0.1 |
# | $\newcommand{\TranShkStd}{\sigma_\theta}\TranShkStd$ | Std Dev of Log Transitory Shock| $\texttt{TranShkStd}$ | 0.1 |
#
# For a microeconomic consumer with 'Market Resources' (net worth plus current income) $M_{t}$, end-of-period assets $A_{t}$ will be the amount remaining after consumption of $C_{t}$.
# \begin{eqnarray}
# A_{t}   &=&M_{t}-C_{t}  \label{eq:DBCparts} \\
# %B_{t+1}   & = & A_{t} R \notag \\
# \end{eqnarray}
#
# The consumer's permanent noncapital income $P$ grows by a predictable factor $\PermGroFac$ and is subject to an unpredictable lognormally distributed multiplicative shock $\mathbb{E}_{t}[\psi_{t+1}]=1$,
# \begin{eqnarray}
# P_{t+1} & = & P_{t} \PermGroFac \psi_{t+1}
# \end{eqnarray}
#
# and actual income is permanent income multiplied by a logormal multiplicative transitory shock, $\mathbb{E}_{t}[\theta_{t+1}]=1$, so that next period's market resources are
# \begin{eqnarray}
# %M_{t+1} &=& B_{t+1} +P_{t+1}\theta_{t+1},  \notag
# M_{t+1} &=& A_{t}R +P_{t+1}\theta_{t+1}.  \notag
# \end{eqnarray}
#
# When the consumer has a CRRA utility function $u(c)=\frac{c^{1-\rho}}{1-\rho}$, the paper shows that the problem can be written in terms of ratios of money variables to permanent income, e.g. $m_{t} \equiv M_{t}/P_{t}$, and the Bellman form of [the problem reduces to](http://econ.jhu.edu/people/ccarroll/papers/BufferStockTheory/#The-Related-Problem):
#
# \begin{eqnarray*}
# v_t(m_t) &=& \max_{c_t}~~ u(c_t) + \beta~\mathbb{E}_{t} [(\Gamma\psi_{t+1})^{1-\rho} v_{t+1}(m_{t+1}) ] \\
# & s.t. & \\
# a_t &=& m_t - c_t \\
# m_{t+1} &=& R/(\Gamma \psi_{t+1}) a_t + \theta_{t+1} \\
# \end{eqnarray*}
#
# and the Euler equation for this model is
#
# \begin{eqnarray*}
# c_{t}^{-\rho} & = & R \beta \mathbb{E}_{t}[(\Gamma \psi c_{t+1})^{-\rho})] \\
# 0 & = & R \beta \mathbb{E}_{t}[(\Gamma \psi c_{t+1}/c_{t})^{-\rho})]-1
# \end{eqnarray*}
#
#
# For the purposes of this notebook, the paper's baseline parameterization is changed as follows:
#
# 1. The unemployment (zero-income event) shocks are turned off
# 2. An explicit liqudity constraint is added ($c_{t} \leq m_{t}$)

# %% [markdown]
# # Dolo
#
# [Installation instructions](https://github.com/EconForge/dolo/wiki/Installation) for dolo involve a number of dependencies, including the dolo language "dolang."  This notebook assumes all these have been installed.

# %%
from dolo import *

# %% [markdown]
# Dolo defines models using "model files" whose syntax is specified in the documentation

# %%
model_dolo = yaml_import("../models/bufferstock.yaml")
print( model_dolo )

# %%
# Set a maximum value of the market resources ratio m for use in both models
max_m = 500
model_dolo.data['calibration']['max_m'] = max_m
model_dolo.data['domain']['m'] = [0,'max_m']

# Obtain the decision rule by time iteration
dr = time_iteration(model_dolo,tol=1e-08,verbose=True)

# %% [markdown]
# # HARK
# The HARK tool used here is $\texttt{ConsIndShockModel.py}$.  For an introduction to this module, see the [ConsIndShockModel.ipynb](https://econ-ark.org/notebooks) notebook at the [Econ-ARK](https://econ-ark.org) website.

# %% {"code_folding": [0]}
# Define a parameter dictionary with baseline parameter values

# Set the baseline parameter values
PermGroFac = 1.03
Rfree      = 1.04
DiscFac    = 0.96
CRRA       = 2.00
UnempPrb   = 0.00
IncUnemp   = 0.0
PermShkStd = 0.1
TranShkStd = 0.1
# Import default parameter values
import HARK.ConsumptionSaving.ConsumerParameters as Params

# Make a dictionary containing all parameters needed to solve the model
base_params = Params.init_idiosyncratic_shocks

# Set the parameters for the baseline results in the paper
# using the variable values defined in the cell above
base_params['PermGroFac'] = [PermGroFac]   # Permanent income growth factor
base_params['Rfree']      = Rfree          # Interest factor on assets
base_params['DiscFac']    = DiscFac        # Time Preference Factor
base_params['CRRA']       = CRRA           # Coefficient of relative risk aversion
base_params['UnempPrb']   = UnempPrb       # Probability of unemployment (e.g. Probability of Zero Income in the paper)
base_params['IncUnemp']   = IncUnemp       # Induces natural borrowing constraint
base_params['PermShkStd'] = [PermShkStd]   # Standard deviation of log permanent income shocks
base_params['TranShkStd'] = [TranShkStd]   # Standard deviation of log transitory income shocks

# Some technical settings that are not interesting for our purposes
base_params['LivPrb']       = [1.0]   # 100 percent probability of living to next period
base_params['CubicBool']    = True    # Use cubic spline interpolation
base_params['T_cycle']      = 1       # No 'seasonal' cycles
base_params['BoroCnstArt']  = None    # No artificial borrowing constraint

from HARK.utilities import plotFuncsDer, plotFuncs
from HARK.ConsumptionSaving.ConsIndShockModel import IndShockConsumerType
# %%
# Create a model identical to the dolo model
# Start with the HARK baseline parameters and modify
# to be like the dolo model
base_params_dolo = dict(base_params)
base_params_dolo['BoroCnstArt']  = 0.0    # Liquidity constraint at 0
base_params_dolo['UnempPrb']     = 0      # No point-mass on unemployment state
base_params_dolo['TranShkCount'] = 5      # Default number of nodes in dolo
base_params_dolo['PermShkCount'] = 5
base_params_dolo['aXtraMax']     = max_m  # Use same maximum
base_params_dolo['aXtraCount']   = 100    # How dense to make the grid
base_params_dolo['DiscFac']      = 0.96
#base_params_dolo['CubicBool']    = False
model_HARK = IndShockConsumerType(**base_params_dolo,cycles=0) # cycles=0 indicates infinite horizon

# %%
# Solve the HARK model
model_HARK.updateIncomeProcess()
model_HARK.solve()
model_HARK.UnempPrb = 0.05
model_HARK.unpackcFunc()

# %%

# Plot the results: Green is perfect foresight, red is HARK, black is dolo
tab = tabulate(model_dolo, dr, 'm')
plt.plot(tab['m'],tab['c'])     # This is pretty cool syntax
m = tab.iloc[:,2]
c_m  = model_HARK.cFunc[0](m)
# cPF uses the analytical formula for the perfect foresight solution
cPF = (np.array(m)-1+1/(1-PermGroFac/Rfree))*((Rfree-(Rfree * DiscFac)**(1/CRRA))/Rfree)
plt.plot(tab['m'],c_m,color="red")
plt.plot(m,cPF,color="green")


import pandas as pd
import numpy as np
obj = pd.Series(range(5), index=list('aabbc'))
obj
obj.index
obj.index.is_unique()
obj.index.is_unique
obj['a']
obj['b']
df = pd.DataFrame(np.random.randn(4, 3), index=list('aabb'))
df
df.loc['a']
df.loc['b']


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import os
from sympy import *
import pandas as pd
import numpy as np
import scipy.fftpack
import matplotlib as mpl
import matplotlib.pyplot as plt
plt.style.use("seaborn-paper")

def find_nearest(array, value):
    array = np.asarray(array)
    idx = (np.abs(array - value)).argmin()
    return idx

# ## Canvas palette

# In[2]:

#Canvas for single plot
x = np.linspace(0,10,100)
y = np.sin(x)
plt.figure(figsize=[14,6])
plt.grid(True)
plt.title("Change-me!",fontsize=20)
plt.plot(x,y,label="testvalue")
plt.legend(fontsize=16)
plt.xlabel("XLABEL (unit)",fontsize=18)
plt.ylabel("YLABEL (unit)",fontsize=18)
plt.show()

# In[3]:

#Canvas for side by side
fig, axes = plt.subplots(nrows=1, ncols=2, figsize=(14,6))
fig.suptitle("test",y=1.05,fontsize=20)

axes[0].grid(True)
axes[0].plot(x,y,label="testvalue")
axes[0].legend(fontsize=16)
axes[0].set_title("TESTTITLE",fontsize=18)
axes[0].set_xlabel("XLABEL (unit)",fontsize=18)
axes[0].set_ylabel("YLABEL (unit)",fontsize=18)
axes[0].legend(fontsize=16)
axes[0].tick_params(axis='both', which='major', labelsize=15)

axes[1].grid(True)
axes[1].plot(x,y,label="testvalue")
axes[1].legend(fontsize=16)
axes[1].set_title("TESTTITLE",fontsize=18)
axes[1].set_xlabel("XLABEL (unit)",fontsize=18)
axes[1].set_ylabel("YLABEL (unit)",fontsize=18)
axes[1].legend(fontsize=16)
axes[1].tick_params(axis='both', which='major', labelsize=15)

fig.tight_layout()
plt.show()

# In[4]:

#Canvas for side by side
fig, axes = plt.subplots(nrows=2, ncols=4, figsize=(14,6))
fig.suptitle("test",y=1.05,fontsize=20)

axes[0,0].grid(True)
axes[0,0].plot(x,y,label="testvalue")
axes[0,0].legend(fontsize=16)
axes[0,0].set_title("TESTTITLE",fontsize=18)
axes[0,0].set_xlabel("XLABEL (unit)",fontsize=18)
axes[0,0].set_ylabel("YLABEL (unit)",fontsize=18)
axes[0,0].legend(fontsize=16)
axes[0,0].tick_params(axis='both', which='major', labelsize=15)

axes[0,1].grid(True)
axes[0,1].plot(x,y,label="testvalue")
axes[0,1].legend(fontsize=16)
axes[0,1].set_title("TESTTITLE",fontsize=18)
axes[0,1].set_xlabel("XLABEL (unit)",fontsize=18)
axes[0,1].set_ylabel("YLABEL (unit)",fontsize=18)
axes[0,1].legend(fontsize=16)
axes[0,1].tick_params(axis='both', which='major', labelsize=15)

fig.tight_layout()
plt.show()

# ## Read data

# In[5]:

#Folder and paths definitions
main_path  = os.getcwd()
datafolder_path = main_path+"/results"
results_dir = "/output_py"
output_dir = main_path+results_dir
try:
    os.mkdir(output_dir)
except OSError:
    print ("Creation of the directory %s failed" % results_dir)
else:
    print ("Successfully created the directory %s " % results_dir)

# In[ ]:

# In[6]:

#Simulation parameters
N = 5000
T = 10000
n_runs = 20
dt = .01
freq = "gfreq"
MF = "MF"

if(freq =="gfreq"):
    freq_plot="$\\mathcal{N}(0,1)$ natural freqs"
else:
    freq_plot="Uniformly distributed freqs $\\in[-.5,.5]$"
if(MF =="MF"):
    MF_plot="MeanField"
else:
    MF_plot="non-MeanField"

# In[13]:

#OutputFileNames
#S/N --> |r(t)|/sigma(r(t))
sn_name = "S_N"
#(Mod&Phase)(t)
modphase_name = "ModPhase_t"
#Spectrum
spectrum_name = "Spectrum"
#r_inf
rinf_name = "r_inf"
#Configuration-specific name
config_name= "/N%d_nruns%d_freq=%s_"%(N,n_runs,freq)

# In[19]:

#K for simulation
K_r0 = np.arange(0,1.4,.2) #da 0 a 1.2 a step di .2, note the last step is not included!
K_r1 = np.arange(1.21,2.01,.01)
K_r2 = np.arange(2.1,5.1,.1)
Kvalues = np.concatenate((K_r0,K_r1,K_r2))
Kvalues = np.unique(Kvalues, axis=0)
print(len(Kvalues))
Kvalues

# In[9]:

#Create dataframe dictionary. For each entry, first value is the K of the dataframe (second value)
data = []
for i in range(0,len(Kvalues)):
    filename = datafolder_path + "/%s_uphase_N%d_%s_T%d_dt%.4f_nruns%d_K%.3f.tsv"%(freq,N,MF,T,dt,n_runs,Kvalues[i])
    #cols refers to timestep, avgmod, stdmod, avgphase,stdphase (of order parameter)
    df = pd.read_csv(filename,sep="\t",header=None)
    data.append([format(Kvalues[i],'.3f'),df])

# In[ ]:

# ## Plots

# In[18]:

#Plot settings
alph = 1
tmax =100

selected_index = [0,3,17,21,25,47]
fig = plt.figure(figsize=[14,6])
plt.grid(True)
plt.title("N = %d, dt = %.3f, %s, n_runs = %d" % (N,dt,freq_plot,n_runs),fontsize=20)
for i in selected_index:
    plt.plot(data[i][1][0],data[i][1][1]/data[i][1][2],ls='--',marker='.',markersize=.5,label="K=%s"%(data[i][0]),alpha=alph)
    plt.semilogy()
    plt.legend(fontsize=16)
plt.xlabel("t",fontsize=18)
plt.ylabel("$\\frac{|r(t)|}{\\sigma(r(t))}$",fontsize=20,rotation=0)
plt.xlim(0,tmax)

fig.tight_layout()
plt.savefig(output_dir+config_name+sn_name)
plt.show()

# In[11]:

#Plot settings
alph = 1
tmax =80

fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(14,6))

fig.suptitle("N = %d, dt = %.3f, %s, n_runs = %d"%(N,dt,freq_plot, n_runs),y=.95,fontsize=20)

axes[0].grid(True)

for i in selected_index:
    axes[0].errorbar(data[i][1][0],data[i][1][1],yerr=data[i][1][2],ls='--',linewidth=.5,fmt='.',markersize=.05, elinewidth=.5, capthick=.5,label="K=%s"%(data[i][0]),alpha=alph)
    axes[1].errorbar(data[i][1][0],data[i][1][3],yerr=data[i][1][4],ls='--',linewidth=.5,fmt='.',markersize=.05, elinewidth=.5, capthick=.5,label="K=%s"%(data[i][0]),alpha=alph)

axes[0].legend(fontsize=16)
axes[0].set_title("|r(t)|",fontsize=18)
axes[0].set_xlabel("t",fontsize=18)
axes[0].set_ylabel("",fontsize=18)
axes[0].legend(fontsize=16,ncol=2)
axes[0].set_xlim(0,tmax)
axes[0].tick_params(axis='both', which='major', labelsize=15)

axes[1].grid(True)
axes[1].legend(fontsize=16)
axes[1].set_title("Arg(r(t))",fontsize=18)
axes[1].set_xlabel("t",fontsize=18)
axes[1].set_ylabel("",fontsize=18)
axes[1].legend(fontsize=16,ncol=2)
axes[1].set_xlim(0,tmax)
axes[1].tick_params(axis='both', which='major', labelsize=15)
fig.tight_layout()

plt.subplots_adjust(top=.85)
plt.savefig(output_dir+config_name+modphase_name)
plt.show()

# In[12]:

max_K_in_plot = 8
max_n_of_K_in_plot = 5
idx_max_K_in_plot = find_nearest(Kvalues,max_K_in_plot)

fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(14,6))
fig.suptitle("$\\mathcal{F}(r(t))$,\nN = %d, dt = %.3f, %s, n_runs = %d"%(N,dt,freq_plot, n_runs),y=1,fontsize=20)
counter1 = 0
counter2 = 0
for i in selected_index:
    if(counter1<len(selected_index)/2):
        axes[0,counter1].grid(True)
        im = axes[0,counter1].specgram(data[i][1][1],Fs=1/dt)
        axes[0,counter1].set_title("K = %s"%(data[i][0]),fontsize=18)
        axes[0,counter1].set_xlabel("t",fontsize=18)
        axes[0,counter1].set_ylabel("Freq.",fontsize=18)
        axes[0,counter1].tick_params(axis='both', which='major', labelsize=15)
        counter1 = counter1+1
    else:
        axes[1,counter2].grid(True)
        im = axes[1,counter2].specgram(data[i][1][1],Fs=1/dt)
        axes[1,counter2].set_title("K = %s"%(data[i][0]),fontsize=18)
        axes[1,counter2].set_xlabel("t",fontsize=18)
        axes[1,counter2].set_ylabel("Freq.",fontsize=18)
        axes[1,counter2].tick_params(axis='both', which='major', labelsize=15)
        counter2 = counter2+1

fig.tight_layout()
plt.subplots_adjust(top=.85)

plt.savefig(output_dir+config_name+spectrum_name)

plt.show()

# In[20]:

#for r_inf evaluation
Kval_list = []
r_inf = []
r_inf_err = []
last_percent = .9
for i in range(0,len(Kvalues)):
    r_inf.append([np.mean(data[i][1][1][int(len(data[i][1][1])*last_percent):])])
    r_inf_err.append([np.std(data[i][1][1][int(len(data[i][1][1])*last_percent):])])
    Kval_list.append([Kvalues[i]])

# # ADD ERRORBARS

# In[ ]:

fig = plt.figure(figsize=[14,6])
plt.grid(True)
plt.title("N = %d, dt = %.3f, %s, n_runs = %d"%(N,dt,freq_plot, n_runs),y=1,fontsize=20)
#plt.errorbar(Kval_list,r_inf,y_err=r_inf_err,label="testvalue")
plt.errorbar(Kval_list,r_inf,ls='--',linewidth=.5,fmt='.',markersize=5, elinewidth=.5, capthick=.5,label="Average on last %d steps"%((1-last_percent)*T+1))
plt.legend(fontsize=16)
plt.xlabel("K",fontsize=18)
plt.ylabel("$r_{\\infty}$",fontsize=18,rotation=0)

fig.tight_layout()
plt.savefig(output_dir+config_name+rinf_name)

plt.show()


"""
Functions for georeferenced gridded data I/O. Configured to read geotiffs in as ndarrays
and print ndarrays to geotiffs. Geotiffs can be converted to PNG and JPEG images.

Author: Gina O'Neil
"""

from osgeo import gdal, gdal_array, ogr
import numpy as np
import subprocess
import sys
import scipy
from scipy import stats
import os
import time

def gtiff_to_arr(path_to_file, dtype):
    """
    :param path_to_file: filepath to input geotiff
    :param dtype: datatype of pixels
    :return: ndarray
    """

    if path_to_file[-3:] != "tif":
        print ("Wetland ID tool is only configured to process geotiffs. \n")
        return None

    else:
        print ("Reading in %s as an array..." %(os.path.basename(path_to_file) + '\n'))
        #TODO: look into benefits of switch to xarray dataframe stucture

        #get geotiff metadata
        tif_ds = gdal.Open(os.path.join(path_to_file), gdal.GA_ReadOnly)

        driver = tif_ds.GetDriver()

        prj = tif_ds.GetProjection()

        ncol = tif_ds.RasterXSize

        nrow = tif_ds.RasterYSize

        ext = tif_ds.GetGeoTransform()

        n_bands = tif_ds.RasterCount

        pixel_res = ext[1]

        #NOTE: all tiffs must be read in as float arrays in order to set missing values to np.nan \
        #this could be changed if there is a method to create geotiffs such that masked elements are NaN

        #prepare empty array with target size tif_ds.GetRasterBand(1).DataType

        if dtype == 'float':
            gdal_type = gdal.GDT_Float32
        elif dtype == 'int':
            gdal_type = gdal.GDT_Int32
        elif dtype == 'byte':
            gdal_type = gdal.GDT_Byte

        tif_as_array = np.zeros((tif_ds.RasterYSize, tif_ds.RasterXSize, tif_ds.RasterCount), \
                       gdal_array.GDALTypeCodeToNumericTypeCode(gdal_type))

        print ('Array created from %s has shape:' %(os.path.basename(path_to_file)))
        print (tif_as_array.shape, '\n')

        #populate the empty array
        if n_bands > 1:
            for b in range(tif_as_array.shape[2]):
                tif_as_array[:, :, b] = tif_ds.GetRasterBand(b + 1).ReadAsArray()
        else:
            tif_as_array[:,:,0] = tif_ds.GetRasterBand(1).ReadAsArray()
            tif_as_array = tif_as_array[:,:,0]

            #save tiff meta data
        tif_meta = { 'driver' : driver, 'prj' : prj, 'ncol' : ncol, 'nrow' : nrow, 'ext' : ext, 'nbands' : n_bands, 'pix_res' : pixel_res }
    tif_ds = None

    return tif_as_array, tif_meta

def arr_to_gtiff(data, data_meta, fpath, fname, dtype='float', nodata=-9999):
    """
    :param data: ndarray
    :param data_meta: (dict) georeferenced meta data for ndarray
    :param fpath: output path
    :param fname: output filename
    :param dtype: target gdal data type
    :param nodata: gdal no data value
    :return: file path to output geotiff
   """

    print ("Writing array to geotiff: %s..."%(fname) + '\n')
    if dtype == 'float':
        gdal_type = gdal.GDT_Float32
    elif dtype == 'int':
        gdal_type = gdal.GDT_Int32
    elif dtype == 'byte':
        gdal_type = gdal.GDT_Byte
    else:
        sys.exit("Datatype not recognized, system exiting.....")

    saveas = os.path.join(fpath, fname)
    driver = data_meta['driver']
    ncol, nrow = data_meta['ncol'], data_meta['nrow']
    prj = data_meta['prj']
    ext = data_meta['ext']
    n_bands = data_meta['nbands']
    out_raster_ds = driver.Create(saveas, ncol, nrow, n_bands, gdal_type, ['COMPRESS=LZW'])
    out_raster_ds.SetProjection(prj)
    out_raster_ds.SetGeoTransform(ext)

    if n_bands > 1:
        for b in range(n_bands):
            out_raster_ds.GetRasterBand(b + 1).WriteArray(data[:, :, b])
            band = out_raster_ds.GetRasterBand(b + 1)
            band.SetNoDataValue(nodata)
    else:
        out_raster_ds.GetRasterBand(1).WriteArray(data)
        band = out_raster_ds.GetRasterBand(1)
        band.SetNoDataValue(nodata)
    # Close dataset
    out_raster_ds = None

    cmd_info = 'gdalinfo.exe -stats \"%s\"'%(saveas)
    subprocess.call(cmd_info, shell = False)
    return saveas

def gtiff_to_img(tif_in, fpath, fname, img_type, no_data_val):
    """
    :param tif_in: filepath to geotiff to be converted
    :param fpath: (str) img out file path
    :param fname: (str) img out filename
    :param img_type: (str) "JPG" or "PNG", n_bands > 1 should use JPG
    :return: filepath to new img
    """
    imgout = os.path.join(fpath, fname)

    if img_type == "JPG":
        list_options = [
            '-ot Byte',
            '-of JPEG',
            '-scale',  # inputs are scaled to 0-255
            '-co QUALITY=100 TILED=YES'#,
            #'-a_nodata {}'.format(no_data_val)
        ]
        options_string = " ".join(list_options)

    elif img_type == "PNG":
        list_options = [
            '-ot Byte',
            '-of PNG'#,
            # '-scale 0 1 1 2',  # change here to assign different values to gt classes
            #'-a_nodata {}'.format(no_data_val)
        ]
        options_string = " ".join(list_options)

    else:
        print ("Only JPG or PNG images can be created.")
        return ""

    gdal.Translate(imgout, tif_in, options=options_string)

    print ("Converted {} to {} image!".format(os.path.basename(tif_in), img_type))

    return imgout

# TODO: def array_to_img(data, datameta, fpath, fname, dtype='float', nodata=-9999, ext):

def clean_array(arr, no_data=None):
    """
    :param arr: Ndarray
    :param no_data: no data value if known, otherwise will be guessed by taking mode of corner values
    :return: clean array where no data values are masked
    """
    if no_data==None:
        if np.ndim(arr) > 2:
            nan_val_list = [arr[0,0,0], arr[-1,-1,0], arr[0,-1,0], arr[-1,0,0] ]
            nan_val_mode = stats.mode(nan_val_list, axis=0)
            nan_val = nan_val_mode[0].item()
            print ("Detected %f to be a NaN Value." %(nan_val))
            tif_arr_mask = np.ma.masked_values(arr, nan_val)

        else:
            nan_val_list = [arr[0,0], arr[-1,-1], arr[0,-1], arr[-1,0] ]
            nan_val_mode = stats.mode(nan_val_list, axis=0)
            nan_val = nan_val_mode[0].item()
            print ("Detected %f to be a NaN Value." %(nan_val))
            tif_arr_mask = np.ma.masked_values(arr, nan_val)
    else:
        nan_val = no_data
        tif_arr_mask = np.ma.masked_values(arr, nan_val)

    tif_arr_clean = tif_arr_mask.reshape(np.shape(arr))

    return tif_arr_clean

def clip_geotif(tif_in, fpath, clip_bounds, suf = "_c.tif", no_data=-9999.):
    """
    :param tif_in: input geotif
    :param fpath: output filepath
    :param clip_bounds: shapefile to use as clipping bounds
    :param suf: suffix to add to tif_in base name
    :param no_data: optional no data value
    :return: filepath to clipped geotif
    """
    tif_ds = gdal.Open(tif_in, gdal.GA_ReadOnly)
    tif_name = os.path.basename(tif_in)
    ext = tif_ds.GetGeoTransform()
    pix_res = float(ext[1])
    tif_out = os.path.join(fpath, tif_name[:-4] + suf)

    cmd = "gdalwarp.exe -cutline \"%s\" -dstnodata %d -tr %f %f -overwrite -r bilinear \
        -crop_to_cutline \"%s\" \"%s\"" %(clip_bounds, no_data, pix_res, pix_res, tif_in, tif_out)

    cmd_info = 'gdalinfo.exe -stats \"%s\"'%(tif_out)

    subprocess.call(cmd)
    subprocess.call(cmd_info)

    print ("%s has been clipped!" %(tif_name))
    return tif_out

def rasterize_simple(shp_in, fpath, fname, out_tif_val, pix_res):
    """
    :param shp_in: shapefile to be rasterised
    :param fpath: output filepath
    :param fname: output fileanme
    :param out_tif_val: value to burn into new raster
    :param pix_res: output pixel resolution
    :return: filepath to new geotif raster
    """

    tif_out = os.path.join(fpath, fname)

    cmd = "gdal_rasterize -burn %f -a_nodata -9999. -ot Float32 -tr %f %f %s %s" \
    %(out_tif_val, pix_res, pix_res, shp_in, tif_out)

    cmd_info = 'gdalinfo.exe -stats \"%s\"'%(tif_out)

    subprocess.call(cmd, shell = True)
    subprocess.call(cmd_info, shell = True)

    print ("%s has been created! \n" %(tif_out))

    return tif_out

def rasterize_opts(shp_in, fpath, fname, out_tif_val, pix_res, ext, outter_vals):
    """
    :param shp_in: shapefile to be rasterised
    :param fpath: output filepath
    :param fname: output fileanme
    :param out_tif_val: value to burn into new raster
    :param pix_res: output pixel resolution
    :param ext: ext of output raster, list of xmin, ymin, xmax, ymax
    :param outter_vals: pixel values for pixels outside of shapefile but within extents
    :return: filepath to new geotif raster
    """

    tif_out = os.path.join(fpath, fname)

    cmd = "gdal_rasterize -init %f -burn %f -a_nodata -9999. -ot Float32 -co COMPRESS=LZW -te %f %f %f %f -tr %f %f %s %s" \
    %(outter_vals, out_tif_val, ext[0], ext[2], ext[1], ext[3], pix_res, pix_res, shp_in, tif_out)
    print(cmd)

    cmd_info = 'gdalinfo.exe -stats \"%s\"'%(tif_out)

    subprocess.call(cmd, shell = True)
    subprocess.call(cmd_info, shell = True)

    print ("%s has been created! \n" %(tif_out))

    return tif_out

def create_verif(wetlands_shp, bounds_shp, fpath, pix_res):
    """
    :param wetlands_shp: wetlands shapefile
    :param bounds_shp: limits shapefile
    :param fpath: output filepath
    :param pix_res: output pixel resolution
    :return: filepath to new verification raster
    """

    inDriver = ogr.GetDriverByName("ESRI Shapefile")
    inDataSource = inDriver.Open(bounds_shp, 0)
    inLayer = inDataSource.GetLayer()
    bounds_ext = inLayer.GetExtent()
    verif_tif = rasterize_opts(wetlands_shp, fpath, "verif.tif", 0., pix_res, bounds_ext, 1.)

    return verif_tif


# -*- coding: utf-8 -*-
r"""
Elements with labels.

This module implements a simple wrapper class for pairs consisting of
an "element" and a "label".
For representation purposes (``repr``, ``str``, ``latex``), this pair
behaves like its label, while the element is "silent".
However, these pairs compare like usual pairs (i.e., both element and
label have to be equal for two such pairs to be equal).
This is used for visual representations of graphs and posets
with vertex labels.
"""

from sage.misc.latex import latex

class ElementWithLabel(object):
    """
    Auxiliary class for showing/viewing :class:`Poset`s with
    non-injective labelings.
    For hashing and equality testing the resulting object behaves
    like a tuple ``(element, label)``.
    For any presentation purposes it appears just as ``label`` would.

    EXAMPLES::

        sage: P = Poset({1: [2,3]})
        sage: labs = {i: P.rank(i) for i in range(1, 4)}
        sage: print(labs)
        {1: 0, 2: 1, 3: 1}
        sage: print(P.plot(element_labels=labs))
        Graphics object consisting of 6 graphics primitives

        sage: from sage.misc.element_with_label import ElementWithLabel
        sage: W = WeylGroup("A1")
        sage: P = W.bruhat_poset(facade=True)
        sage: D = W.domain()
        sage: v = D.rho() - D.fundamental_weight(1)
        sage: nP = P.relabel(lambda w: ElementWithLabel(w, w.action(v)))
        sage: list(nP)
        [(0, 0), (0, 0)]
    """
    def __init__(self, element, label):
        """
        Construct an object that wraps ``element`` but presents itself
        as ``label``.

        TESTS::

            sage: from sage.misc.element_with_label import ElementWithLabel
            sage: e = ElementWithLabel(1, 'a')
            sage: e
            'a'
            sage: e.element
            1
        """
        self.element = element
        self.label = label

    def _latex_(self):
        """
        Return the latex representation of ``self``,
        which is just the latex representation of the label.

        TESTS::

            sage: var('a_1')
            a_1
            sage: from sage.misc.element_with_label import ElementWithLabel
            sage: e = ElementWithLabel(1, a_1)
            sage: latex(e)
            a_{1}
        """
        return latex(self.label)

    def __str__(self):
        """
        Return the string representation of ``self``, which is just
        the string representation of the label.

        TESTS::

            sage: var('a_1')
            a_1
            sage: from sage.misc.element_with_label import ElementWithLabel
            sage: e = ElementWithLabel(1, a_1)
            sage: str(e)
            'a_1'
        """
        return str(self.label)

    def __repr__(self):
        """
        Return the representation of ``self``, which is just
        the representation of the label.

        TESTS::

            sage: var('a_1')
            a_1
            sage: from sage.misc.element_with_label import ElementWithLabel
            sage: e = ElementWithLabel(1, a_1)
            sage: repr(e)
            'a_1'
        """
        return repr(self.label)

    def __hash__(self):
        """
        Return the hash of the labeled element ``self``,
        which is just the hash of ``self.element``.

        TESTS::

            sage: from sage.misc.element_with_label import ElementWithLabel
            sage: a = ElementWithLabel(1, 'a')
            sage: b = ElementWithLabel(1, 'b')
            sage: d = {}
            sage: d[a] = 'element 1'
            sage: d[b] = 'element 2'
            sage: print(d)
            {'a': 'element 1', 'b': 'element 2'}
            sage: a = ElementWithLabel("a", [2,3])
            sage: hash(a) == hash(a.element)
            True
        """
        return hash(self.element)

    def __eq__(self, other):
        """
        Two labeled elements are equal if and only if both of their
        constituents are equal.

        TESTS::

            sage: from sage.misc.element_with_label import ElementWithLabel
            sage: a = ElementWithLabel(1, 'a')
            sage: b = ElementWithLabel(1, 'b')
            sage: x = ElementWithLabel(1, 'a')
            sage: a == b
            False
            sage: a == x
            True
            sage: 1 == a
            False
            sage: b == 1
            False
        """
        if not (isinstance(self, ElementWithLabel) and
                isinstance(other, ElementWithLabel)):
            return False
        return self.element == other.element and self.label == other.label

    def __ne__(self, other):
        """
        Two labeled elements are not equal if and only if first or second
        constituents are not equal.

        TESTS::

            sage: from sage.misc.element_with_label import ElementWithLabel
            sage: a = ElementWithLabel(1, 'a')
            sage: b = ElementWithLabel(1, 'b')
            sage: x = ElementWithLabel(1, 'a')
            sage: a != b
            True
            sage: a != x
            False
        """
        return not(self == other)


import time
import pandas as pd
import numpy as np

CITY_DATA = { 'chicago': 'chicago.csv',
              'new york city': 'new_york_city.csv',
              'washington': 'washington.csv' }

def get_filters(city, month, day):
    """def  Asks user to specify a city, month, and day to analyze.

    Returns:
        (str) city - name of the city to analyze
        (str) month - name of the month to filter by, or "all" to apply no month filter
        (str) day - name of the day of week to filter by, or "all" to apply no day filter
    """
    print('Hello! Let\'s explore some US bikeshare data!')

    # TO DO: get user input for city (chicago, new york city, washington). HINT: Use a while loop to handle invalid inputs
    while True:
        city = input('Which of the following cities you want to ckeck? chicago, new york city, or washington? ' ).lower()
        if city not in CITY_DATA:
            print('Ops, no data for this city')
            continue
        else:
            break

    # TO DO: get user input for month (all, january, february, ... , june)
    months = ['january', 'february', 'march', 'april', 'may', 'june', 'all']
    while True:
        month = input('which month do you choose? january, february, march, april, may, june or all: ').lower()
        if month not in months:
            print('Ops, Wrong Input')
            continue
        else:
            break
    # TO DO: get user input for day of week (all, monday, tuesday, ... sunday)
    days = ['Sunday','Monday','Tuesday','Wednesday','Thursday','Friday','All']
    while True:
        day = input('which day of the week do you choose? sunday, monday, tuesday, wednesday, thursday, friday, saterday or all: ').title()
        if day not in days:
            print('Ops, Wrong Input')
            continue
        else:
            break

    print('Your Choice for city is: ' , city)
    print('Your Choice for month is: ' , month)
    print('Your choice for day is: ' , day)
    print('_'*40)
    return city, month, day

def load_data(city, month, day):

    """Loads data for the specified city and filters by month and day if applicable.

    Args:
    (str) city - name of the city to analyze
    (str) month - name of the month to filter by, or "all" to apply no month filter
    (str) day - name of the day of week to filter by, or "all" to apply no day filter
    Returns:
    df - Pandas DataFrame containing city data filtered by month and day"""

    # load data file into a dataframe
    df = pd.read_csv(CITY_DATA[city])
    # convert the Start Time column to datetime
    df['Start Time'] = pd.to_datetime(df['Start Time'])
    # extract month and day of week from Start Time to create new columns
    df['month'] = df['Start Time'].dt.month
    df['day'] = df['Start Time'].dt.weekday_name
    # filter by month if applicable
    if month != 'all':
        # use the index of the months list to get the corresponding int
        months = ['january', 'february', 'march', 'april', 'may', 'june']
        month = months.index(month) + 1
        # filter by month to create the new dataframe
        df = df[df['month'] == month]

    # filter by day of week if applicable
    if day != 'All':
        # filter by day of week to create the new dataframe
        df = df[df['day'] == day]

    return df

def time_stats(df):
    """Displays statistics on the most frequent times of travel."""

    print('\nCalculating The Most Frequent Times of Travel...\n')
    start_time = time.time()

    # TO DO: display the most common month
    popular_month = df['month'].mode()[0]
    # TO DO: display the most common day of week
    popular_day = df['day'].mode()[0]
    # TO DO: extract hour from Start Time column and display the most common start hour
    df['hour'] = df['Start Time'].dt.hour
    popular_hour = df['hour'].mode()[0]

    print('Most Popular Month:', popular_month)
    print('Most Popular Day:', popular_day)
    print('Most Popular Hour:', popular_hour)
    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def station_stats(df):
    """Displays statistics on the most popular stations and trip."""
    print('\nCalculating The Most Popular Stations and Trip...\n')
    start_time = time.time()
    # TO DO: display most commonly used start station
    popular_start_station = df['Start Station'].mode()[0]
    # TO DO: display most commonly used end station
    popular_End_station = df['End Station'].mode()[0]
    # TO DO: display most frequent combination of start station and end station trip
    df['popular_combination_trip'] = df['Start Station'] + '&' + df['End Station']
    popular_combination_trip = df['popular_combination_trip'].mode()[0]

    print('Most Popular Start Station:', popular_start_station)
    print('Most Popular End Station:', popular_End_station)
    print('Most Frequent Combination of Start Station and End Station Trip:', popular_combination_trip)
    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def trip_duration_stats(df):
    """Displays statistics on the total and average trip duration."""

    print('\nCalculating Trip Duration...\n')
    start_time = time.time()
    # TO DO: display total travel time
    total_travels = df['Trip Duration'].sum()
    # TO DO: display mean travel time
    average_trips_time = df['Trip Duration'].mean()

    print('Total Travels/trips Time is:', total_travels)
    print('Average Travels/trips Time is:', average_trips_time)
    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def user_stats(df):
    """Displays statistics on bikeshare users."""
    print('\nCalculating User Stats...\n')
    start_time = time.time()
    # TO DO: Display counts of user types
    user_types = df['User Type'].value_counts()
    # TO DO: Display counts of gender
    if 'Gender' in df:
        gender_types = df['Gender'].value_counts()
        print('counts of Users Genders is:' , gender_types)
    else:
        print('there is no gender data for this city')
    # TO DO: Display earliest, most recent, and most common year of birth
    if 'Birth Year' in df:
        earliest_birth = int(df['Birth Year'].min())
        recent_birth = int(df['Birth Year'].max())
        popular_birth = int(df['Birth Year'].mode()[0])
        print('Youngest Users are born in {}, and the eldest Users Birth is {}, while Most Users born in {}.'.format(recent_birth , earliest_birth , popular_birth))
    else:
        print('there is no Birth Year data for this city')

    print('Count of Users Types is: ', user_types)
    print("\nThis took %s seconds." % (time.time() - start_time))
    print('-'*40)

def data(df):
    raw_input = 0
    #asking for raw input data by using while loop.
    while True:
        #confirming if the user wants to see the data of not by using input.
        raw = input('Do you want to view 5 rows of the data? type: yes or no? ').lower()
        if raw == 'no':
            break
        elif raw == 'yes':
            raw_input += 5
            #printing 5 rows of the data
            print(df.iloc[raw_input : raw_input + 5])
            break
        else:
            print('Wrong input')
    while True:
        #starting a new loop to add 5 more rows everytime the user asks to.
        more = input('Do you want to view 5 more rows? type: yes or no? ').lower()
        if more == 'no':
            break
        elif more == 'yes':
            raw_input += 5
            print(df.iloc[raw_input : raw_input + 5])
            continue
        else:
            print('Wrong Input')

    return raw_input

def main():
    city = []
    month = []
    day = []
    while True:
        city, month, day = get_filters(city, month, day)
        df = load_data(city, month, day)

        time_stats(df)
        station_stats(df)
        trip_duration_stats(df)
        user_stats(df)
        data(df)

        restart = input('\nWould you like to restart? Enter yes or no.\n')
        if restart.lower() != 'yes':
            break

if __name__ == "__main__":
	main()


# Lecture code for Week 2
# inspired by https://www.programiz.com/python-programming/tuple

my_string = 'baldan'
type(my_string)

mystr1 = "Nice"
mystr2 = "day"

myFullStr = mystr1 + " " + mystr2

my_int = -15
type(my_int)

my_float = 15.78
type(my_float)

my_bool = False # True
type(my_bool)

import datetime
today = datetime.date.today()
today.year
today.month
today.day

datetime.datetime.strptime('2017-01-22', "%Y-%m-%d") # convert string to datetime (strptime)
datetime.datetime.strptime('2017/01/22', "%Y/%m/%d")

today.strftime("%Y/%m/%d") # convert datetime to string (strptime)

# list
my_list_str = ['bat','bold','suren']
my_list_str[2]
my_list_str[:1]
my_list_str[:2]
my_list_str[1:]
my_list_str[::-1]
my_list_str[::-2]
my_list_str[-1]
my_list_str[-2]

my_list_num = [15, 15.2, 4.7]
type(my_list_num)

# tuple
my_tuple = (1,2,3)

# set
my_set1 = {1,2,3}
my_set2 = {7,8,9}
my_set.add(2)
my_set1 | my_set2 # join
my_set1 & my_set2 # intersection
my_set2 - my_set1

# dictionary . json

my_dict = {"name": "Bat", "age":25,"country":"Mongolia"}
person2 = {"name": "Bold", "age":23,"country":"Mongolia","city":"Darkhan"}

big_dict = {"Younger": person1, "Older": person2}

# Numerical package - numpy (scipy)

import numpy as np

# Pandas
import pandas as pd

# Pandas read examples:
# https://www.datacamp.com/community/tutorials/importing-data-into-pandas

# be careful with backward slash \
df = pd.read_excel("C:/Users/sugarkhuu/Documents/python/repo/Introduction_Python/week2/data.xlsx")
# pd.read_excel("C:\\Users\\sugarkhuu\\Documents\\python\\repo\\Introduction_Python\\week2\\data.xlsx")

df.dtypes
df.head()
df.tail()

df.describe()

df['age']
df['firstName']

# iloc
df.iloc[2,1]
df.iloc[2:5,1:3]

# loc
df.loc[5,"lastName"]
df.loc[5:8,"lastName"]
df.loc[5:8,("lastName","firstName")]

# filter
df[df["age"]<27]
df[df["age"]<27][["firstName","lastName","salary","age"]]
df[df["age"]>=27][["firstName","lastName","salary","age"]]

df[(df["age"]<27) & (df["salary"]<2.0) & (df["gender"] != "M")]

df.groupby('gender')['salary'].mean()
df.groupby("gender")['salary'].max()
df.groupby("gender")['salary'].min()

df.groupby(['gender','politicalView'])['age'].mean()

df.groupby(['gender','politicalView'])['age'].mean()
df.groupby(['gender','politicalView'])[['age','salary']].mean()
df.groupby(['gender','politicalView'])[['age','salary']].max()

df.groupby('A').agg({'B': ['min', 'max'], 'C': 'sum'})

df.groupby(['gender','politicalView']).agg({'age': ["mean","max"], 'salary': ["max","count"]})

df.groupby(['gender','politicalView']).agg({'age': ["mean","max"],  \
    "salary": ["mean","max"] })

df = df.sort_values(by="yearsInCompany")

df.sort_values(by="gender", inplace=True)
df["name"] = np.arange(10)
del df["name"]

df[["age","salary"]]

df.columns # column names


"""
This file contains code that will kick off training and testing processes
"""
import os
import json

from experiments.UNetExperiment import UNetExperiment
from data_prep.HippocampusDatasetLoader import LoadHippocampusData
import numpy as np

class Config:
    """
    Holds configuration parameters
    """
    def __init__(self):
        self.name = "Basic_unet"
        self.root_dir = r"/home/workspace/src/data"
        self.n_epochs = 5
        self.learning_rate = 0.0002
        self.batch_size = 32 # TODO Let's bump this up a smidge (was 8)
        self.patch_size = 64
        self.test_results_dir = "/home/workspace/out/" # Unclear is this is the best place for this (update: arbitrary, but fine)

if __name__ == "__main__":
    # Get configuration

    # TASK: Fill in parameters of the Config class and specify directory where the data is stored and
    # directory where results will go
    c = Config()

    # Load data
    print("Loading data...")

    # TASK: LoadHippocampusData is not complete. Go to the implementation and complete it.
    data = LoadHippocampusData(c.root_dir, y_shape = c.patch_size, z_shape = c.patch_size)

    # Create test-train-val split
    # In a real world scenario you would probably do multiple splits for
    # multi-fold training to improve your model quality

    keys = range(len(data))

    # Here, random permutation of keys array would be useful in case if we do something like
    # a k-fold training and combining the results.

    split = dict()

    # TASK: create three keys in the dictionary: "train", "val" and "test". In each key, store
    # the array with indices of training volumes to be used for training, validation
    # and testing respectively.
    # <YOUR CODE GOES HERE>

    # FIXME typically we would use some randomization in making these splits, however I don't believe we
    # have reason to suspect there is any particular pattern to this dataset

#     split['train'] = keys[:int(len(keys)*0.6)]
#     split['val'] = keys[int(len(keys)*0.6):int(len(keys)*0.9)]
#     split['test'] = keys[int(len(keys)*0.9):]

    # DONE change of heart: update with randomized sampling method
    key_lenth = len(keys)
    split['train'] = np.random.choice(keys, int(key_lenth*0.6), replace=False)
    split['val'] = np.random.choice(keys, int(key_lenth*0.2), replace=False)
    split['test'] = np.random.choice(keys, int(key_lenth*0.2), replace=False)

    # Set up and run experiment

    # TASK: Class UNetExperiment has missing pieces. Go to the file and fill them in
    exp = UNetExperiment(c, split, data)

    # You could free up memory by deleting the dataset
    # as it has been copied into loaders
    # del dataset

    # run training
    exp.run()

    # prep and run testing

    # TASK: Test method is not complete. Go to the method and complete it
    results_json = exp.run_test()

    results_json["config"] = vars(c)

    print("Write to: ", exp.out_dir, "results.json")

    with open(os.path.join(exp.out_dir, "results.json"), 'w') as out_file:
        json.dump(results_json, out_file, indent=2, separators=(',', ': '))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@program: makeh5.py

@task: Create dummy datasets
"""

import h5py
import numpy as np

A = np.array([1, 2])
B = np.array([[[1, 2], [3, 4]],
              [[5, 6], [7, 8]]])
c = 42
d = 1.23
e = []

with h5py.File('./ex_dataset.h5', 'w') as hf:
    gf1 = hf.create_group('arrays')
    gf1.create_dataset('A', data=A)
    gf1.create_dataset('B', data=B)
    gf2 = hf.create_group('scalars')
    gf2.create_dataset('c', data=c)
    gf2.create_dataset('d', data=d)
    gf2.create_dataset('e', data=e)


# -*- coding: utf-8 -*-
"""
Created on Thu Jun 11 09:38:15 2020

@author: 66IN
"""

import pandas as pd

import numpy as np

s = pd.Series([1,2,3,4,np.nan,5,6])   #using series in pandas

print(s)

# example of dataframe :

d = pd.date_range('20200301',periods=10)

print(d)

df = pd.DataFrame(np.random.randn(10,4),index=d,columns=['A','B','C','D'])

print(df)

df1 = pd.DataFrame({'A':[1,2,3,4],

                    'B':'Aditya'})

print(df1)

#functions in python

print(df.head())
print(df.tail())

print(df.to_numpy())


# -*- coding: utf-8 -*-
"""
Created on Mon Oct  8 15:36:02 2018

@author: derobest
"""
import numpy as np
def test_basicTrue():
    """ one of the simplest test that does nothing except saying it works..."""
    assert True

#testing session 1 functions
def load_S1_script():
    """
        utility function that tris to load the script written along the first lesson
        @throws an ImportError exception if the script file does not exist
        @return the script as a loaded module
    """
    S1_script_filename='assignments/Session1/S1_algotools.py'
    import imp
    s1_algotools=imp.load_source('session_1_script', S1_script_filename)
    return  s1_algotools

def test_session1script_exists():
    try:
        load_S1_script()
        assert True
    #except  ImportError,e:
    except  ImportError:
        print('Expected script not found, carrefuly check the assignement instructions ')
        assert False

    """
    Test en indiquant le résultat dans le test
     """
def test_average():
    test_tab=[1,2,3,8]
    assert load_S1_script().average_above_zero(test_tab) == 3.5

    """
    Tests en utilisant la fonction intégrée mean à numpy.
    """
def check_S1_selective_average(testList):
    ##
    # utility function that asserts if load_S1_script().average_above_zero works fine
    # @param testList a list of values onto average_above_zero is applied
    # @test ensures the function returns the correct average value
    #another way to process the positive elements average to compare with
    positive_elements_float_array=np.array([i for i in testList if i > 0], dtype=float)
    reference_average_value=np.mean(positive_elements_float_array)
    assert load_S1_script().average_above_zero(testList) ==reference_average_value

def test_S1_selective_average_non_zeros_values():
    ##
    # @test validates average_above_zero works fine with integer values >0
    check_S1_selective_average([1,2,3,4,-7])

def test_S1_selective_average_with_zeros_values():
    ##
    # @test validates average_above_zero works fine with integer values >=0
    check_S1_selective_average([0,1,2,3,4,-7])

def test_S1_selective_average_with_negative_values():
    ##
    # @test validates average_above_zero works fine with integer values <=0
    #check_S1_selective_average([0,-7])
    check_S1_selective_average([8,7,-7])

def test_S1_selective_average_with_empty_list():
    ##
    # @test validates average_above_zero works fine with an empty list
    try:
        check_S1_selective_average([])
        assert False
    except ValueError:
        assert True

##max_value
    # @test validates max_value works  with input which is not a list
def test_S1_max_value_not_a_list():
    try:
        test_MVList=100
        load_S1_script().max_value(test_MVList)
        assert False
    except ValueError:
        assert True

def test_S1_max_value_empty_list():
    # @test validates max_value works  with input which is an empty list
    try:
        test_MVList=[]
        load_S1_script().max_value(test_MVList)
        assert False
    except ValueError:
        assert True

def test_S1_max_value_standard_list():
    # @test validates max_value works  with input which is an empty list
    test_MVList=[1,2,3,15,9]
    assert load_S1_script().max_value(test_MVList) == (15,3)

def test_S1_max_value_negative_value():
    # @test validates max_value works  with input which is an empty list
    test_MVList=[1,2,-100,15,9]
    assert load_S1_script().max_value(test_MVList) == (15,3)

##Reverse_table
def test_S1_revertTable_empty():
    testList=[]
    assert testList==load_S1_script().reverse_table(testList)

def test_S1_revertTable_singleElement():
    testList=[1]
    assert testList==load_S1_script().reverse_table(testList)

def test_S1_revertTable_evenElement():###################################################
    testList=[1,2]
    #import copy
    #testList_copy=copy.deepcopy(testList)
    #print('reverseTable:{inL}=>{out}'.format(inL=testList_copy,out=load_S1_script().reverse_table(testList)))
    assert[2,1]==load_S1_script().reverse_table(testList)

def test_S1_revertTable_oddElement():
    testList=[1,2,3]
    assert[3,2,1]==load_S1_script().reverse_table(testList)

def test_S1_revertTable_error_list():
    try:
        testList='a';
        load_S1_script().reverse_table(testList)
        assert False
    except ValueError:
        assert True

##roi_box
def test_S1_roiBBox_regular():
    inputMat=np.zeros((5,6),dtype=np.bool)
    inputMat[2:4,3:5]=np.ones((2,2),dtype=np.bool)
#    assert (load_S1_script().roi_bbox(inputMat)==[[2,3],[2,4],[3,3],[3,4]])
    np.testing.assert_array_equal(load_S1_script().roi_bbox(inputMat), [[2,3],[2,4],[3,3],[3,4]])

##fill_sparse
def test_S1_RandomFillSparse_regular():
    matrix = np.array([["","","","","",""],["","","","","",""],["","","","","",""]])
    k=4
    randomX=load_S1_script().random_fill_sparse(matrix,k)

    cpt=0
    for nlig in range(len(randomX)):
            for ncol in range (len(randomX[0])):
                if randomX[nlig][ncol] == 'X':
                    cpt+= 1
    assert cpt==k

##remove_whiteSpace
def test_S1_RemoveWhiteSpace_regular():
    word = 'st ri ng'
    assert 'string'==load_S1_script().remove_whitespace(word)

#shuffle
def test_S1_Shuffle():
    list= ["pomme","pêche","poire","abricot"]
    shuffledList = load_S1_script().remove_whitespace(list)
    assert len(list)==len(shuffledList)

def test_S1_SortSelective():
    list=[10,15,7,1,3,3,9]
    expectedSortedList=[1,3,3,7,9,10,15]
    assert expectedSortedList==load_S1_script().sort_selective(list)

def test_S1_SortBubble():
    list=[10,15,7,1,3,3,9]
    expectedSortedList=[1,3,3,7,9,10,15]
    assert expectedSortedList==load_S1_script().sort_bubble(list)


"""
@brief      test log(time=93s)
"""

import sys
import os
import unittest
import numpy as np
from pyquickhelper.loghelper.flog import fLOG

try:
    import src
except ImportError:
    path = os.path.normpath(
        os.path.abspath(
            os.path.join(
                os.path.split(__file__)[0],
                "..",
                "..")))
    if path not in sys.path:
        sys.path.append(path)
    import src

class TestShpinxGallery(unittest.TestCase):

    def test_numpy_random(self):
        fLOG(
            __file__,
            self._testMethodName,
            OutputPrint=__name__ == "__main__")

        np.random.seed(42)

if __name__ == "__main__":
    unittest.main()