# -*- coding: utf-8 -*-
# The line above is used when I need to run this file as a shell script, as a cron job

#%%[markdown]
#
# # HW Web Scraping
# ## Calling on all Weather enthusiasts

# Reference : class file WS_sample_Stock.py
#
# Okay, I want to collect weather info for my family living at different zip codes.
# Feel free to replace the zipcode list with your favorites.
#
# Ideally, I can put the codes on pythonanywhere.com,
# so that it can be set as a cron job on a reliable server
# instead of on my own laptop.
# As we did in class, we can use weather.gov to get the weather info
# for the different zip codes. But the way that website is designed,
# we cannot encode the zip code into the url. So we used
# selenium to automate. But we cannot install selenium on
# pythonanywhere.com. That is using too much resources on the public server.
#
# Good thing there are so many options out there. I found
# [this site](url = 'https://www.wunderground.com/weather/us/dc/20052') works okay.
#
# Now, get the codes to work, with a list of 10 zip codes with the state abbreviation
# ( so the list/tuple should looks like this: ('dc/20052' , 'ny/10001' , 'ca/90210' , ... ) ),
# automatically pull the forecast temperature high and low for the day
# at 6am at our Washington-DC time?
# The stock portfolio example (WS_sample_Stock.py) in class is adaptable to
# handle this task.
#
# Have the codes tested and working on your computer. If you are interested, you can
# deploy it pythonanhywhere.com. I can show you how. Deploying to pythonanywhere.com is
# an optional exercise. Simply run your
# codes on your computer for a few days, or 2-3 times a day to get
# the temp info at different times to make sure it works.
# # The csv file will eventually looks like this
# Date,dc/20052,ny/10001,ca/90210,nj/07069,va/22207,il/60007,tx/77001,az/85001,pa/19019,tx/78006
# 2022-03-03,53° | 34°,50° | 34°,--° | 53°,51° | 29°,53° | 34°,30° | 22°,74° | 56°,71° | 49°,53° | 32°,71° | 51°
# 2022-03-02,53° | 52°,73° | 39°,68° | --°,62° | 32°,48° | 31°,40° | 25°,69° | 52°,72° | 52°,65° | 37°,66° | 56°
# 2022-03-01,53° | 35°,49° | 34°,--° | 53°,51° | 29°,52° | 34°,30° | 21°,74° | 55°,--° | 47°,53° | 32°,71° | 51°
#
# Some of the temperatures might come up as --°, which is fine. Don't worry about that.
# That's all you need to submit, the working codes and a sample data csv file.
#
# Imagine you can do something similar to track your favorite sport team's statistics, or air ticket price, ...
# What get you excited?
#
# Of course, whenever there is an API available, use that instead of webscraping. It's much more reliable.

#%%

from matplotlib.pyplot import axis
import numpy as np
import requests
import datetime
import pandas as pd
import requests
from bs4 import BeautifulSoup

def getUrl(zipcode):
  url = 'https://www.wunderground.com/weather/us/'+zipcode # zipcode should be in the format:  stateAbbreviation/five-digit-zipcode like dc/20052
  return url

def getSoup(url,parser=''):
  # ######  QUESTION 1      QUESTION 1      QUESTION 1   ##########

  # write your codes here
  p = parser if (parser == 'lxml' or parser == 'html.parser') else 'html5lib'
  r = requests.get(url)
  s = BeautifulSoup(r.content, p)

  # ######  END of QUESTION 1    ###   END of QUESTION 1   ##########
  return s  # return some soup

def getTempHiLo(soup): # get the block of values of hi-lo temperature on this site
  # ######  QUESTION 2      QUESTION 2      QUESTION 2   ##########

  # write your codes here
  hi_lo = soup.find(class_='hi-lo').text

  # ######  END of QUESTION 2    ###   END of QUESTION 2   ##########
  return hi_lo # return the text for the hi-lo temperatures

def getDailyTemp(filename):
  # the source file header has the list of zip codes I want to keep track.
  # I am using this source file to keep track of the list of zipcodes below:
  zipcodes = ['dc/20052' , 'ny/10001' , 'ca/90210', 'nj/07069', 'va/22207', 'il/60007', 'tx/77001', 'az/85001', 'pa/19019', 'tx/78006']

  # I will use the date string as the key for my dataframe
  tday = datetime.datetime.today()
  tdaystr = tday.strftime('%Y-%m-%d')  # default format is 'yyyy-mm-dd 00:00:00'

  # open file, import df from file, with first row as header, first column as index.
  df_last = pd.read_csv(filename, header=0, index_col=0, date_parser=lambda x: datetime.datetime.strptime(x, '%Y-%m-%d') )

  df_new = pd.DataFrame(columns=df_last.columns ) # set a new empty dataframe for new day's data
  df_new.index.name = df_last.index.name # set the index name
  df_new = df_new.append(pd.Series(name=tdaystr, dtype='object')) # add a blank row with today's date as index

  # ######  QUESTION 3      QUESTION 3      QUESTION 3   ##########

  # write your codes here
  for i, zipcode in enumerate(df_last.columns):
      zip_url = getUrl(zipcode)
      zip_soup = getSoup(zip_url, 'html.parser')
      zip_temp = getTempHiLo(zip_soup)

      df_new.iloc[0, i] = zip_temp
  df_last = pd.concat([df_new, df_last])
  # You can run the current codes to see what is the df_new and df_last look like.
  # Need to get the new Temperatures for each location/zip, and put them in the new data frame
  # Then insert that df_new to the top of df_last.

  # ######  END of QUESTION 3    ###   END of QUESTION 3   ##########

  df_last.index = pd.to_datetime(df_last.index, format='%Y-%m-%d') # row index = 'Date', fixing formatting issue. Without this line, the dates on dataframe will be recorded as 'yyyy-mm-dd 00:00:00' to the datafame

  df_last = df_last.iloc[0:25,:]   # trim number of rows to max 25, before saving to file to prevent file growing too big
  df_last.to_csv(filename, encoding='utf_8_sig')  # saving/updating csv file    # df_last.to_csv(filename, sep='\t')

  return None

# make sure the folder is correct
source = 'weatherDaily.csv'
# run one time to test getting weather data for today
getDailyTemp(source)

#%%


#!/usr/bin/env python3

import os
import re
import pandas as pd
import numpy as np
import statsmodels
import matplotlib.pyplot as plt
import matplotlib

import unittest
from unittest.mock import patch, MagicMock

from tmc import points
from tmc.utils import load, get_out, patch_helper, spy_decorator

module_name="src.regression"
regression = load(module_name, "regression")
main = load(module_name, "main")
ph = patch_helper(module_name)

def get_exercises(nb):
    pattern = r"^#\s*exercise\s+(\d+)\s*"
    result = {}
    for cell in nb.cells:
        if cell.cell_type == "code" and re.match(pattern, cell.source):
            m = re.match(pattern, cell.source)
            n = int(m.group(1))
            assert n not in result, "Overlapping exercise number %i" % n
            result[n] = cell.source
    return result

inputnb = "src/project_notebook_regression_analysis.ipynb"
exercises = {}
# UNCOMMENT THE BELOW THREE LINES
import nbformat
nb = nbformat.read(inputnb, as_version=4)
exercises = get_exercises(nb)

#print("Read %i exercises:" % len(exercises))
#print(exercises.keys())

def find_interaction(var1, var2, params):
    interaction1 = var1 + ":" + var2
    interaction2 = var2 + ":" + var1
    if interaction1 in params:
        return interaction1
    elif interaction2 in params:
        return interaction2
    else:
        return None

class Regression(unittest.TestCase):

    def check_explanators(self, explanators, params):
        for e in explanators:
            if type(e) == tuple:
                assert len(e) == 2
                var1, var2 = e
                val = find_interaction(var1, var2, params)
                self.assertNotEqual(val, None,
                                 msg="Model parameters are missing interaction between variable %s and %s!" % e)
            else:
                self.assertTrue(e in params, msg="Model parameter %s missing!" % e)

    @points('p08-01.0')
    def test_00_imports(self):
        exec(exercises[0])
        self.assertTrue("statsmodels" in locals(), msg="Module statsmodels was not loaded!")
        self.assertTrue("pd" in locals(), msg="Module pandas was not loaded!")

    @points('p08-01.1')
    def test_01_load(self):
        #exec(exercises[0])
        describe_method = spy_decorator(pd.core.frame.DataFrame.describe, "describe")
        with patch("pandas.read_csv", wraps=pd.read_csv) as prc,\
             patch.object(pd.core.frame.DataFrame, "describe", new=describe_method):
            exec(exercises[1])
            prc.assert_called()
            #fram=int()
            self.assertTrue("fram" in locals(), msg="DataFrame 'fram' was not loaded!")
            myfram = locals()["fram"]
            self.assertIsInstance(myfram, pd.core.frame.DataFrame,
                                  msg="'fram' is not a DataFrame object!")
            self.assertEqual(myfram.shape, (1394, 14),
                             msg="The read DataFrame had incorrect shape!")
            #koe.assert_called()
            describe_method.mock.assert_called()

    @points('p08-01.2')
    def test_02_rescale(self):
        exec(exercises[2])
        self.assertTrue("rescale" in locals(), msg="Could not find function 'rescale'!")
        s = pd.Series(np.random.randn(10)*6 + 3)
        myrescale = locals()["rescale"]
        s2 = myrescale(s)
        mean = s2.mean()
        sigma = s2.std()
        self.assertAlmostEqual(mean, 0, msg="Expected rescale to return series having expectation 0!")
        self.assertAlmostEqual(sigma, 0.5,
                               msg="Expected rescale to return series having standard deviation 0.5!")

    @points('p08-01.3')
    def test_03_rescaled_variables(self):
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        myfram = locals()["fram"]
        for variable in "sAGE sFRW sSBP sDBP sCHOL sCIG".split():
            self.assertIn(variable, myfram,
                          msg="Expected rescaled variable %s in the DataFrame!" % variable)
            mean = myfram[variable].mean()
            std = myfram[variable].std()
            self.assertAlmostEqual(mean, 0,
                                   msg="Expected variable %s to have expectation 0!" % variable)
            self.assertAlmostEqual(std, 0.5,
                                   msg="Expected variable %s to have standard deviation 0.5!" % variable)
    @points('p08-01.4')
    def test_04_sbp1(self):
        exec(exercises[0])
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        exec(exercises[4])
        myfit = locals()["fit"]
        self.assertAlmostEqual(myfit.params.Intercept, 150.0199, places=2, msg="Incorrect intercept!")
        self.assertAlmostEqual(myfit.params.sFRW, 17.7205, places=2,
                               msg="Incorrect coefficient for sFRW!")
        self.assertAlmostEqual(myfit.params.sCHOL, 4.9169, places=2,
                               msg="Incorrect coefficient for sCHOL!")
        self.assertAlmostEqual(myfit.params["SEX[T.male]"], -4.0659, places=2,
                               msg="Incorrect coefficient for gender!")

    @points('p08-01.5')
    def test_05_sbp_with_age(self):
        exec(exercises[0])
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        exec(exercises[5])
        myfit = locals()["fit"]
        self.assertAlmostEqual(myfit.params.sAGE, 8.1332, places=2,
                               msg="Incorrect coefficient for sAGE!")
        self.assertEqual(len(myfit.params), 5, msg="Expected four explanatory variables and an intercept!")

    @points('p08-01.6')
    def test_06_sbp_with_interactions(self):
        exec(exercises[0])
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        exec(exercises[6])
        myfit = locals()["fit"]
        self.assertIn("sAGE", myfit.params, msg="Missing explanatory variable sAGE!")
        self.assertAlmostEqual(myfit.params.sAGE, 10.218851215627154, places=2,
                               msg="Incorrect coefficient for sAGE!")
        interaction = find_interaction("sFRW", "sAGE", myfit.params)
        found = interaction is not None
        self.assertAlmostEqual(myfit.params[interaction], -2.0865742749328025, places=2,
                               msg="Incorrect coefficient for sFRW:sAGE!")
        self.assertEqual(len(myfit.params), 11, msg="Expected ten explanatory variables and an intercept!")

    @points('p08-01.7')
    def test_07_sbp_with_interactions_visualization(self):
        plot_method = spy_decorator(pd.core.frame.DataFrame.plot.scatter, "scatter")
        with patch("matplotlib.pyplot.scatter", wraps=matplotlib.pyplot.scatter) as pltscatter,\
             patch.object(pd.core.frame.DataFrame.plot, "scatter", new=plot_method),\
             patch("statsmodels.graphics.regressionplots.abline_plot",
                   wraps=statsmodels.graphics.regressionplots.abline_plot) as abplot:
            exec(exercises[0])
            exec(exercises[1])
            exec(exercises[2])
            exec(exercises[3])
            exec(exercises[6])
            exec(exercises[7])
            myfit = locals()["fit"]
            #abplot.assert_called()
            self.assertEqual(abplot.call_count, 3, msg="Expected abline_plot to be called three times!")
            self.assertTrue(pltscatter.call_count > 0 or plot_method.mock.call_count > 0,
                            msg="Expected call to make a scatter plot!")

    @points('p08-01.8')
    def test_08_sbp_with_cigarets(self):
        plot_method = spy_decorator(pd.core.frame.DataFrame.plot.scatter, "scatter")
        with patch("matplotlib.pyplot.scatter", wraps=matplotlib.pyplot.scatter) as pltscatter,\
             patch.object(pd.core.frame.DataFrame.plot, "scatter", new=plot_method),\
             patch("statsmodels.graphics.regressionplots.abline_plot",
                   wraps=statsmodels.graphics.regressionplots.abline_plot) as abplot:
            exec(exercises[0])
            exec(exercises[1])
            exec(exercises[2])
            exec(exercises[3])
            exec(exercises[8])
            myfit = locals()["fit"]
            self.assertIn("sCIG", myfit.params, msg="Missing explanatory variable sCIG!")
            self.assertAlmostEqual(myfit.params.sCIG, 3.7733, places=2,
                                   msg="Incorrect coefficient for sCIG!")
            interaction = find_interaction("sFRW", "sCIG", myfit.params)

            found = interaction is not None
            self.assertTrue(found, msg="Missing explanatory variable sFRW:sCIG!")
            self.assertAlmostEqual(myfit.params[interaction], 3.6765, places=2,
                                   msg="Incorrect coefficient for sFRW:sCIG!")
            self.assertEqual(len(myfit.params), 16, msg="Expected 15 explanatory variables and an intercept!")
            #abplot.assert_called()
            self.assertEqual(abplot.call_count, 3, msg="Expected abline_plot to be called three times!")
            self.assertTrue(pltscatter.call_count > 0 or plot_method.mock.call_count > 0,
                            msg="Expected call to make a scatter plot!")

    @points('p08-01.9')
    def test_09_high_blood_pressure(self):
        exec(exercises[0])
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        exec(exercises[9])
        myfram = locals()["fram"]
        myfit = locals()["fit"]
        self.assertIn("HIGH_BP", myfram.columns, msg="No variable HIGH_BP in DataFrame 'fram'!")
        self.assertEqual(int, myfram["HIGH_BP"].dtype, msg="Use type 'int' for variable HIGH_BP!")
        self.assertEqual(len(myfit.params), 4, msg="Expected 3 explanatory variables and an intercept!")
        found = "error_rate_orig" in locals()
        self.assertTrue(found, msg="Result variable 'error_rate_orig' missing!")
        myerror_rate_orig = locals()["error_rate_orig"]
        self.assertAlmostEqual(myerror_rate_orig, 0.35581061692969873, places=4, msg="Incorrect error rate!")

    @points('p08-01.10')
    def test_10_high_blood_pressure2(self):
        exec(exercises[0])
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        exec(exercises[9])
        exec(exercises[10])
        myfram = locals()["fram"]
        myfit = locals()["fit"]
        self.assertIn("HIGH_BP", myfram.columns, msg="No variable HIGH_BP in DataFrame 'fram'!")
        self.assertEqual(int, myfram["HIGH_BP"].dtype, msg="Use type 'int' for variable HIGH_BP!")
        self.assertEqual(len(myfit.params), 7, msg="Expected 6 explanatory variables and an intercept!")
        found = "error_rate" in locals()
        self.assertTrue(found, msg="Result variable 'error_rate' missing!")
        myerror_rate = locals()["error_rate"]
        self.assertAlmostEqual(myerror_rate, 0.3278335724533716, places=4, msg="Incorrect error rate!")

    @points('p08-01.11')
    def test_11_high_blood_pressure3(self):
        scatter_method = spy_decorator(matplotlib.axes.Axes.scatter, "scatter")
        plot_method = spy_decorator(matplotlib.axes.Axes.plot, "plot")
#        with patch("matplotlib.pyplot.scatter", wraps=matplotlib.pyplot.scatter) as pltscatter:
        with patch("matplotlib.pyplot.subplots", wraps=matplotlib.pyplot.subplots) as psubplots,\
             patch.object(matplotlib.axes.Axes, "plot", new=plot_method),\
             patch.object(matplotlib.axes.Axes, "scatter", new=scatter_method):
            exec(exercises[0])
            exec(exercises[1])
            exec(exercises[2])
            exec(exercises[3])
            exec(exercises[9])
            exec(exercises[10])
            exec(exercises[11])
            myfram = locals()["fram"]
            psubplots.assert_called()
            self.assertEqual(scatter_method.mock.call_count, 2,
                          msg="Expected scatter method to be called twice!")
            self.assertEqual(plot_method.mock.call_count, 6,
                             msg="Expected plot method to be called six times!")

            (args1, kwargs1), (args2, kwargs2)  = scatter_method.mock.call_args_list
            self.assertIn(len(args1[0]), [663, 731], msg="Incorrect number of points in subfigure 1!")
            self.assertEqual(len(args1[0]), len(args1[1]), msg="Incorrect number of points in subfigure 1!")

            self.assertIn(len(args2[0]), [663, 731], msg="Incorrect number of points in subfigure 2!")
            self.assertEqual(len(args2[0]), len(args2[1]), msg="Incorrect number of points in subfigure 2!")

    @points('p08-01.12')
    def test_12_train_test_split(self):
        orig = pd.DataFrame(np.random.randn(10,10))
        df = orig.copy()
        exec(exercises[0])
        exec(exercises[12])
        self.assertTrue("train_test_split" in locals(),
                        msg="Could not find function 'train_test_split' %s, %s!" % (locals(), exercises[12]))
        mytrain_test_split = locals()["train_test_split"]
        np.random.seed(1)
        train1, test1 = mytrain_test_split(df, train_fraction=0.8)
        self.assertTrue(orig.equals(df), msg="The train_test_split function should not modify the original DataFrame!")
        self.assertEqual(len(train1), 8, msg="Expected training set to have size 0.8*originalsize!")
        self.assertEqual(len(test1), 2, msg="Expected training set to have size 0.2*originalsize!")
        np.random.seed(1)
        train2, test2 = mytrain_test_split(df, train_fraction=0.8)
        self.assertTrue(train1.equals(train2),
                        msg="If called twice with the same parameters and same seed, the result should be the same!")
        self.assertTrue(test1.equals(test2),
                        msg="If called twice with the same parameters and same seed, the result should be the same!")

    @points('p08-01.13')
    def test_13_cross_validation(self):
        exec(exercises[0])
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        exec(exercises[9])
        exec(exercises[12])
        #mytrain_test_split = locals()["train_test_split"]
        #with patch("mytrain_test_split", wraps=mytrain_test_split) as psplit:
        exec(exercises[13])
        self.assertIn("error_model", locals(), msg="Could not find variable 'error_model'!")
        myerror_model = locals()["error_model"]
        #myerror_null = locals()["error_null"]
        self.assertAlmostEqual(np.mean(myerror_model), 0.3311827956989248,
                               places=4, msg="Incorrect mean error rate!")
        #self.assertAlmostEqual(np.mean(myerror_null), x,
        #                       places=4, msg="Incorrect mean error rate by null model!")

    @points('p08-01.14')
    def test_14_chd(self):
        exec(exercises[0])
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        exec(exercises[14])
        myfram = locals()["fram"]
        self.assertIn("hasCHD", myfram.columns, msg="No variable hasCHD in DataFrame 'fram'!")
        self.assertEqual(int, myfram["hasCHD"].dtype, msg="Use type 'int' for variable hasCHD!")
        self.assertAlmostEqual(myfram.hasCHD.mean(), 0.22022955523672882, places=4, msg="Variable hasCHD has incorrect mean!")

    @points('p08-01.15')
    def test_15_chd2(self):
        exec(exercises[0])
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        exec(exercises[14])
        exec(exercises[15])
        myfram = locals()["fram"]
        myfit = locals()["fit"]
        self.assertIn("hasCHD", myfram.columns, msg="No variable hasCHD in DataFrame 'fram'!")
        self.assertEqual(int, myfram["hasCHD"].dtype, msg="Use type 'int' for variable hasCHD!")
        self.assertEqual(len(myfit.params), 7, msg="Expected 6 explanatory variables and an intercept!")
        self.check_explanators("sCHOL sCIG sFRW".split() +
                               [("sCHOL", "sCIG"), ("sCHOL", "sFRW"), ("sCIG", "sFRW")], myfit.params)
        found = "error_rate" in locals()
        self.assertTrue(found, msg="Result variable 'error_rate' missing!")
        myerror_rate = locals()["error_rate"]
        self.assertAlmostEqual(myerror_rate, 0.22022955523672882, places=4, msg="Incorrect error rate!")

    @points('p08-01.16')
    def test_16_chd_visualization(self):
        with patch("matplotlib.pyplot.scatter", wraps=matplotlib.pyplot.scatter) as pltscatter,\
            patch("matplotlib.pyplot.plot", wraps=matplotlib.pyplot.plot) as pltplot:
            exec(exercises[0])
            exec(exercises[1])
            exec(exercises[2])
            exec(exercises[3])
            exec(exercises[14])
            exec(exercises[15])
            exec(exercises[16])
            pltscatter.assert_called()
            pltplot.assert_called()

    @points('p08-01.17')
    def test_17_chd_prediction(self):
        exec(exercises[0])
        exec(exercises[1])
        exec(exercises[2])
        exec(exercises[3])
        exec(exercises[14])
        exec(exercises[15])
        exec(exercises[16])
        exec(exercises[17])
        self.assertTrue("point" in locals(), msg="Could not find variable 'point'!")
        mypoint = locals()["point"]
        self.assertIsInstance(mypoint, dict, msg="Expected variable 'point' to be a dictionary!")
        for v in "sCHOL sCIG sFRW".split():
            self.assertIn(v, mypoint, msg="The point dictionary does not contain value for %s!" % v)
        self.assertTrue("predicted" in locals(), msg="Could not find variable 'predicted'!")
        mypredicted = locals()["predicted"]
        self.assertIsInstance(mypredicted, float,
                              msg="Expected the variable 'predicted' to have type 'float', got %s!" % type(mypredicted))
        self.assertAlmostEqual(mypredicted, 0.2161616602504101, places=4, msg="Incorrect predicted probability for the point!")

if __name__ == '__main__':
    unittest.main()


#!/usr/bin/env python
# coding: utf-8

#
#
# # Earth Analytics Education - EA  Python Course Spring 2021

# ## Important  - Assignment Guidelines
#
# 1. Before you submit your assignment to GitHub, make sure to run the entire notebook with a fresh kernel. To do this first, **restart the kernel** (in the menubar, select Kernel$\rightarrow$Restart & Run All)
# 2. Always replace the `raise NotImplementedError()` code with your code that addresses the activity challenge. If you don't replace that code, your notebook will not run.
#
# ```
# # YOUR CODE HERE
# raise NotImplementedError()
# ```
#
# 3. Any open ended questions will have a "YOUR ANSWER HERE" within a markdown cell. Replace that text with your answer also formatted using Markdown.
# 4. **DO NOT RENAME THIS NOTEBOOK File!** If the file name changes, the autograder will not grade your assignment properly.
# 6. When you create a figure, comment out `plt.show()` to ensure the autograder can grade your plots. For figure cells, DO NOT DELETE the code that says `DO NOT REMOVE LINE BELOW`.
#
# ```
# ### DO NOT REMOVE LINE BELOW ###
# student_plot1_ax = nb.convert_axes(plt)
# ```
#
# * Only include the package imports, code, and outputs that are required to run your homework assignment.
# * Be sure that your code can be run on any operating system. This means that:
#    1. the data should be downloaded in the notebook to ensure it's reproducible
#    2. all paths should be created dynamically using the `os.path.join`
#
# ## Follow to PEP 8 Syntax Guidelines & Documentation
#
# * Run the `autopep8` tool on all cells prior to submitting (HINT: hit shift + the tool to run it on all cells at once!
# * Use clear and expressive names for variables.
# * Organize your code to support readability.
# * Check for code line length
# * Use comments and white space sparingly where it is needed
# * Make sure all python imports are at the top of your notebook and follow PEP 8 order conventions
# * Spell check your Notebook before submitting it.
#
# For all of the plots below, be sure to do the following:
#
# * Make sure each plot has a clear TITLE and, where appropriate, label the x and y axes. Be sure to include UNITS in your labels.
#

# ### Add Your Name Below
# **Your Name:**
#
# Jensen Widtfeldt

#

# ---

# # Week 04 and 05 Homework - Automate NDVI Workflow
#
# For this assignment, you will write code to generate a plot of the mean normalized difference vegetation index (NDVI) for two different sites in the United States across one year of data:
#
# * San Joaquin Experimental Range (SJER) in Southern California, United States
# * Harvard Forest (HARV) in the Northeastern United States
#
# The data that you will use for this week is available from **earthpy** using the following download:
#
# `et.data.get_data('ndvi-automation')`
#
# ## Assignment Goals
#
# Your goal in this assignment is to create the most efficient and concise workflow that you can that allows for:
#
# 1. The code to scale if you added new sites or more time periods to the analysis.
# 2. Someone else to understand your workflow.
# 3. The LEAST and most efficient (i.e. runs fast, minimize repetition) amount of code that completes the task.
#
# ### HINTS
#
# * Remove values outside of the landsat valid range of values as specified in the metadata, as needed.
# * Keep any output files SEPARATE FROM input files. Outputs should be created in an outputs directory that is created in the code (if needed) and/or tested for.
# * Use the functions that we demonstrated during class to make your workflow more efficient.
# * BONUS - if you  chose - you can export your data as a csv file. You will get bonus points for doing this.
#
#
# ## Assignment Requirements
#
# Your submission to the GitHub repository should include:
# * This Jupyter Notebook file (.ipynb) with:
#     * The code to create a plot of mean NDVI across a year for  2 NEON Field Sites:
#         * NDVI on the x axis and formatted dates on the y for both NEON sites on one figure/axis object
#     * The **data should be cleaned to remove the influence of clouds**. See the [earthdatascience website for an example of what your plot might look like with and without removal of clouds](https://www.earthdatascience.org/courses/earth-analytics-python/create-efficient-data-workflows/).
# * BONUS: Create one output `.csv` file that has 3 columns - NDVI, Date and Site Name - with values for SJER and HARV.
#
# Your notebook should:
# * Have *at least* 2 well documented and well named functions with docstrings.
# * Include a Markdown cell at the top of the notebook that outlines the overall workflow using pseudocode (i.e. plain language, not code)
# * Include additional Markdown cells throughout the notebook to describe:
#     * the data that you used - and where it is from
#     * how data are being processing
#     * how the code is optimized to run fast and be more concise

# # Replace this cell with your pseudocode  for this workflow
#
# If you happen to be a diagram person a diagram is ok too
#
#

# ## Psuedo-workflow! ##
#
# 1. Gather and open the data
# - Download data
# - List files in download
# - Sort / filter for right files
#
# 2. Calculate the NDVI and stats
# - Open the raster data and crop bands as needed
# - Calculate NDVI
# - Calculate other key metrics if needed
# - Save into sharable form (CSV)
# 3. Do for other sites
# - Use functions and loops for new site
# - Rinse and repeat!
#

# In[1]:

# Autograding imports - do not modify this cell
import matplotcheck.autograde as ag
import matplotcheck.notebook as nb
import matplotcheck.timeseries as ts
from datetime import datetime

# In[2]:

# Import needed packages in PEP 8 order
# and no unused imports listed (10 points total)

import os
import re
from glob import glob

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

import geopandas as gpd
import rioxarray as rxr
import xarray as xr
from rasterio.plot import plotting_extent
import earthpy as et
import earthpy.mask as em
import earthpy.spatial as es
import earthpy.plot as ep

# Get the data!
data = et.data.get_data('ndvi-automation')
os.chdir(os.path.join(et.io.HOME,
                     "earth-analytics",
                     "data"))

# In[3]:

# DO NOT MODIFY THIS CELL
# Tests that the working directory is set to earth-analytics/data

path = os.path.normpath(os.getcwd())
student_wd_parts = path.split(os.sep)

if student_wd_parts[-2:] == ['earth-analytics', 'data']:
    print("\u2705 Great - it looks like your working directory is set correctly to ~/earth-analytics/data")
else:
    print("\u274C Oops, the autograder will not run unless your working directory is set to earth-analytics/data")

# # Figure 1: Plot 1 - Mean NDVI For Each Site Across the Year (50 points)
#
# Create a plot of the mean normalized difference vegetation index (NDVI) for the two different sites in the United States across the year:
#
# * NDVI on the x axis and formatted dates on the y for both NEON sites on one figure/axis object.
# * Each site should be identified with a different color in the plot and legend.
# * The final plot **data should be cleaned to remove the influence of clouds**.
# * Be sure to include appropriate title and axes labels.
#
# Add additional cells as needed for processing data (e.g. defining functions, etc), but be sure to:
# * follow the instructions in the code cells that have been provided to ensure that you are able to use the sanity check tests that are provided.
# * include only the plot code in the cell identified for the final plot code below

# ## Task 1:
#
# In the cell below, create a single dataframe containing MEAN NDVI, the site name,
# and the date of the data for the HARV site
# scene `HARV/landsat-crop/LC080130302017031701T1-SC20181023151837`.  The column names for the  final
# DataFrame should be`mean_ndvi`, and `site`, and the data should be **indexed on the date**.
#
# Use the functions that we reviewed in class (or create your own versions of them) to implement your code
#
# ### In the Cell below Place  All Functions Needed to Run this Notebook (20 points)

# In[4]:

### DO NOT REMOVE THIS LINE OR EDIT / MOVE THIS CELL ###
start_time = datetime.now()

# In[5]:

# In this cell place all of the functions needed to run your notebook
# You will be graded here on function application, docstrings, efficiency so ensure
# All functions are placed here!

def open_clean_bands(band_path,
                    crop_extent,
                    valid_range=None):

    """"Open and mask a landsat band with squeeze.

    Parameters
    ----------
    band_path : string
        Path to the array you use
    valid_range : tuple
        A range for min and max values for the data.

    Returns
    -------
    band : xarray DataArray
        An xarray with invalid values that are masked
    """
    band = (rxr.open_rasterio(band_path, masked=True)
           .rio.clip(crop_extent.geometry, from_disk=True)
           .squeeze())

    # specify the valid range
    if valid_range:
            mask = (band <= 0) | (band > 10000)
            band = band.where(~mask, np.nan)

    return band

# Function 2: Mask cloud bands and crop

def mask_crop_ndvi(all_band_paths,
                  crop_bound,
                  pixel_qa_path,
                  vals):
    """Open a landsat band, mask potential clouds, and calculate NDVI.

    Parameters
    -----------
    all_band_paths : list
        a list for the xarray objects (using landsat  bands 4 and 5)
    crop_bound: gpd GeoDataFrame
        A geopandas dataframe  to crop the raster data (rasterio)
    pixel_qa_path: xarray DataArray
        An xarray DataArray with pixel qa values
    vals: list
        A list of values needed to create the cloud mask

    Returns
    -----------
    ndvi_mark : Xarray Dataset
        a cropped and masked xarray object containing NDVI values

    """
    # open all bands
    bands = []
    for band_path in all_band_paths:
        band = open_clean_bands(
            band_path=band_path,
            crop_extent=crop_bound,
            valid_range=(0, 10000))
        bands.append(band)

    # open and mask cloud layer
    cl_mask =  (rxr.open_rasterio(pixel_qa_path[0], masked=True)
                    .rio.clip(crop_bound.geometry, from_disk=True)
                    .squeeze())

    # final NVDI calcs
    ndvi_xr = (bands[1]-bands[0]) / (bands[1]+bands[0])

    # apply cloud mask to NDVI
    ndvi_mask = ndvi_xr.where(~cl_mask.isin(vals))

    return ndvi_mask

# ## Create code to navigate directories for Figure 1 ##

# In[6]:

# Background code prior to functions

# Navigate the site data
path = os.path.join("ndvi-automation",
                   "sites")
all_sites = glob(path + "/*/")

# define path to HARV sites
site_name = os.path.basename(os.path.normpath(all_sites[0]))

#Open shapefile for first site
vector_dir = os.path.join(all_sites[0], "vector")
site_boundary_path = os.path.join(vector_dir,
                                  site_name + "-crop.shp")
crop_bound = gpd.read_file(site_boundary_path)
crop_bound.plot()
plt.show()

# explore HARV landsat paths
HARV_landsat_dirs = sorted(glob(os.path.join(
                            all_sites[0], "landsat-crop", "*")))

# pick the right directory for HARV LC080130302017031701T1-SC20181023151837
HARV_dir = HARV_landsat_dirs[4]

# grab the bands needed for NDVI
HARV_band_paths = sorted(glob(os.path.join(HARV_dir,
                                           "*band*[4-5].tif")))

# get components
HARV_path = os.path.normpath(HARV_dir)
HARV_path_components = HARV_path.split(os.sep)
HARV_date = HARV_path_components[-1][10:18]

# In[7]:

# Test Function 1 with a loop!
bands = []
for band_path in HARV_band_paths:
    band = open_clean_bands(
            band_path=band_path,
            crop_extent=crop_bound,
            valid_range=(0, 10000))
    bands.append(band)

# calculate NDVI
ndvi_2 = es.normalized_diff(bands[1], bands[0])
ep.plot_bands(ndvi_2,
          cmap="Greys",
          vmin=-1)
ndvi_2_mean = ndvi_2.mean()
print(ndvi_2_mean)

# test Function 2 with another loop for HARV
# prep by creating cloud masks for functions to deal with pesky clouds

high_cloud_confidence = em.pixel_flags[
                        "pixel_qa"]["L8"]["High Cloud Confidence"]
cloud = em.pixel_flags[
    "pixel_qa"]["L8"]["Cloud"]
cloud_shadow = em.pixel_flags[
            "pixel_qa"]["L8"]["Cloud Shadow"]

all_masked_values = cloud_shadow + cloud + high_cloud_confidence

# Prep by open cloud mask layer for HARV
HARV_pixel_qa_path = glob(os.path.join(HARV_dir, "*qa*"))

#Now use with a for loop to generate NDVI for HARV site
HARV_ndvi_clean = []

for band_path in HARV_band_paths:
    ndvi_clean = mask_crop_ndvi(all_band_paths=HARV_band_paths,
                                crop_bound=crop_bound,
                                pixel_qa_path=HARV_pixel_qa_path,
                                vals=all_masked_values)
    site=HARV_path_components[2]
    date=HARV_band_paths[0][-27:-19]
    mean_ndvi=ndvi_clean.mean().values
    # create output
    output = [site,date,mean_ndvi]
    HARV_ndvi_clean.append(output)

#create dataframe from output and set date index
HARV_df = pd.DataFrame(HARV_ndvi_clean,
                       columns=["site","date","mean_ndvi"])
HARV_df['date'] = pd.to_datetime(HARV_df['date'],
                                 format='%Y-%m-%d')
HARV_df_indexed = HARV_df.set_index("date")

# test view the final cropped and cleaned NDVI data
ndvi_clean.plot.imshow(vmin=-1,
                      vmax=1)

# In[8]:

# Create dataframe of mean NDVI in this cell using the functions created above
# Important: to use the ungraded tests below as a sanity check,
# name your columns: mean_ndvi and site
# Call the dataframe at the end of the cell so the tests run on it!
# Be sure that the date column is an index of type date
# HINT: the time series lessons may help you remember how to do this!

#clean and view the HARV dataframe
HARV_df_final = HARV_df_indexed[:-1]
HARV_df_final

# In[9]:

# This cell  is testing your data output above

student_ndvi_ts_single_site = _

single_scene_points = 0

# Ensure the data is stored in a dataframe.
if isinstance(student_ndvi_ts_single_site, pd.DataFrame):
    print('\u2705 Your data is stored in a DataFrame!')
    single_scene_points += 1
else:
    print('\u274C It appears your data is not stored in a DataFrame. ',
          'To see what type of object your data is stored in, check its type with type(object)')

# Ensure that the date column is the index
if isinstance(student_ndvi_ts_single_site.index, pd.core.indexes.datetimes.DatetimeIndex):
    print('\u2705 You have the index set to the date column!')
    single_scene_points += 2
else:
    print('\u274C You do not have the index set to the date column.')

# Ensure that the date column is datetime
if isinstance(student_ndvi_ts_single_site.index[0], pd._libs.tslibs.timestamps.Timestamp):
    print('\u2705 The data in your date column is datetime!')
    single_scene_points += 2
else:
    print('\u274C The data in your date column is not datetime.')

# Ensure the site name is correct
if student_ndvi_ts_single_site.site.values[0] == 'HARV':
    print('\u2705 You have the correct site name!')
    single_scene_points += 5
else:
    print('\u274C You do not have the correct site name.')

if np.allclose(0.281131628228094, student_ndvi_ts_single_site.mean_ndvi.values[0]):
    print('\u2705 You have the correct mean NDVI value!')
    single_scene_points += 5
else:
    print('\u274C You do not have the correct mean ndvi value.')

print("\n \u27A1 You received {} out of 15 points for creating a dataframe.".format(
    single_scene_points))
single_scene_points

# ## Task 2:
#
# In the cell below, process all of the landsat scenes. Create a DataFrame that contains the following
# information for each scene
#
#
# |   | index  | site  | mean_ndvi  |
# |---|---|---|---|
# | Date  |   |   |   |
# |  2017-01-07  | 0  | SJER  | .4  |
#
# Be sure to call your dataframe at the end of the cell to ensure autograding works.
# HINT: FOR THIS STEP, leave any rows containing missing values (`NAN`).

# In[10]:

# Create dataframe of NDVI including the cleaning data to deal with clouds

# Important: to use the ungraded tests below as a sanity check,
# name your columns: mean_ndvi and site
# Don't forget to set date as the index and make the values of type datetime

# Create dataframe by loping through file paths
# Create empty list for dataframe
ndvi_list = []

# loop for each site
for site_dir in all_sites:
    print("Looping through", site_dir)
    asite = os.path.normpath(site_dir).split(os.sep)[-1]
    print("Working through", asite)

    # define crop_bound for each site
    site_boundary_path = os.path.join(path, asite,
                                      "vector", asite + "-crop.shp")
    site_crop_bound = gpd.read_file(site_boundary_path)
    site_crop_bound.plot()
    plt.show()

    #get a list of subdirectories for each site
    new_path=os.path.join(site_dir, "landsat-crop")
    all_dirs=glob(new_path + "/*/")

    # loop through the subdirectories to get the data!
    for single_dir in all_dirs:

        # pull out date from subdirectory name
        scene_date = single_dir.split(os.sep)[-2][-29:-21]

        # Create path for the pixel_qa_layer for each subdirectory scene
        scene_pixel_qa_path = glob(os.path.join(single_dir, "*qa*"))

        # define band paths used for NDVI calcs in each subdirectory
        total_band_paths = sorted(glob(os.path.join(single_dir,
                                                    "*band*[4-5].tif")))

        # calc NDVI
        ndvi = mask_crop_ndvi(all_band_paths=total_band_paths,
                           crop_bound=site_crop_bound,
                           pixel_qa_path=scene_pixel_qa_path,
                           vals=all_masked_values)
        mean_ndvi = ndvi.mean(skipna=True).item()
        # create output
        output = [asite, scene_date, mean_ndvi]
        #append
        ndvi_list.append(output)

#create dataframe
ndvi_df = pd.DataFrame(ndvi_list,
                       columns=["site","date","mean_ndvi"])
ndvi_df['date'] = pd.to_datetime(ndvi_df['date'], format='%Y-%m-%d')
ndvi_df_indexed = ndvi_df.set_index("date")

ndvi_df_indexed

# In[11]:

# Last sanity check before creating your plot (10 points)

# Ensure that you call your dataframe at the bottom of the cell above
# and that it has columns called: mean_ndvi and site

# Ensure the data is stored in a dataframe.
student_ndvi_df = _

df_points = 0

if isinstance(student_ndvi_df, pd.DataFrame):
    print('\u2705 Your data is stored in a DataFrame!')
    df_points +=2
else:
    print('\u274C It appears your data is not stored in a DataFrame. ',
          'To see what type of object your data is stored in, check its type with type(object)')

# Check that dataframe contains the appropriate number of NAN values
if student_ndvi_df.mean_ndvi.isna().sum() == 15:
    print('\u2705 Correct number of masked data values!')
    df_points +=2
else:
    print('\u274C The amount of null data in your dataframe is incorrect.')

# Ensure that the date column is the index
if isinstance(student_ndvi_df.index, pd.core.indexes.datetimes.DatetimeIndex):
    print('\u2705 You have the index set to the date column!')
    df_points +=3
else:
    print('\u274C You do not have the index set to the date column.')

# Ensure that the date column is datetime
if isinstance(student_ndvi_df.index[0], pd._libs.tslibs.timestamps.Timestamp):
    print('\u2705 The data in your date column is datetime!')
    df_points +=3
else:
    print('\u274C The data in your date column is not datetime.')

# Output for timer, # DO NOT MODIFY
end_time = datetime.now()
total_time = end_time - start_time
print(
    "Your total run time for processing the data was {0}.".format(total_time))

print("\n \u27A1 You received {} out of 10 points for creating a dataframe.".format(
    df_points))

df_points

# In[12]:

# Add only the plot code to this cell

# This is the final figure of mean NDVI
# for both sites across the year
# with data cleaned to deal with clouds

# Create plot
fig, ax = plt.subplots(figsize=(12, 12))
fig.suptitle("Annual NDVI Comparison\n SJER and HARV Sites", fontsize = 24)

# Loops for each subplot

#subplot 1
for site, df in ndvi_df_indexed.dropna().groupby('site'):
    if site == "HARV":
        loc = "HARV"
        color = "blue"
    else:
        loc = "SJER"
        color = "orange"
    ax.plot(df.index,
             df.mean_ndvi,
             label=loc,
             color=color,
                marker="o")
    ax.legend(bbox_to_anchor=(1.05, 1), loc='upper left',
              prop={'size': 11})
    ax.set(xlabel = "Date",
           ylabel = "NDVI")

### DO NOT REMOVE LINES BELOW ###
final_masked_solution = nb.convert_axes(plt, which_axes="current")

# In[13]:

# Ignore this cell for the autograding tests

# In[14]:

# Ignore this cell for the autograding tests

# # Question 1 (10 points)
#
# Imagine that you are planning NEON’s upcoming flight season to capture remote sensing data in these locations and want to ensure that you fly the area when the vegetation is the most green.
#
# When would you recommend the flights take place for each site?
#
# Answer the question in 2-3 sentences in the Markdown cell below.

# ## Answer ##
# For the HARV site, the data shows highest vegetation density from May to October. For the SJER site, March and April show the highest vegetation amounts.

# # Question 2 (10 points)
#
# How could you modify your workflow to look at vegetation changes over time in each site?
#
# Answer the question in 2-3 sentences in the Markdown cell below.

# ## Answer ##
#
# To look for vegetation changes over time, you can compare the NDVI for the same month year-over-year. A higher NDVI for the same month year-over-year would indicate that the selected year has a higher vegetation year than previous years

# # Do not edit this cell! (10 points)
#
# The notebook includes:
# * additional Markdown cells throughout the notebook to describe:
#     * the data that you used - and where it is from
#     * how data are being processing
#     * how the code is optimized to run fast and be more concise

# # Do not edit this cell! (20 points)
#
# The notebook will also be checked for overall clean code requirements as specified at the **top** of this notebook. Some of these requirements include (review the top cells for more specifics):
#
# * Notebook begins at cell [1] and runs on any machine in its entirety.
# * PEP 8 format is applied throughout (including lengths of comment and code lines).
# * No additional code or imports in the notebook that is not needed for the workflow.
# * Notebook is fully reproducible. This means:
#    * reproducible paths using the os module.
#    * data downloaded using code in the notebook.
#    * all imports at top of notebook.

# ## BONUS - Export a  .CSV File to Share (10 points possible)
#
# This is optional - if you export a **.csv** file with the columns specified above: Site, Date and NDVI Value you can get an additional 10 points.
#
# * FULL CREDIT: File exists in csv format and contains the columns specified.
# We will check your github repo for this file!
#

# In[15]:

outpath = os.path.join("ndvi-automation\outputs\mean_ndvi_both_sites.csv")
ndvi_df_export = ndvi_df_indexed.reset_index()
ndvi_df_export.to_csv(outpath, index=False)
ndvi_df_export


# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""
Auto-scheduling matrix multiplication for CPU
=============================================
**Author**: `Lianmin Zheng <https://github.com/merrymercy>`_, \
            `Chengfan Jia <https://github.com/jcf94/>`_

Different from the existing :ref:`autotvm <tutorials-autotvm-sec>` which relies on
manual templates to define the search space, the auto-scheduler does not require any templates.
The auto-scheduler is template-free, so users only need to write the computation declaration without
any schedule commands or templates.
The auto-scheduler can automatically generate a large
search space and find a good schedule in the space.

We use matrix multiplication as an example in this tutorial.
"""

import numpy as np
import tvm
from tvm import te, testing, auto_scheduler

######################################################################
# Define the computation
# ^^^^^^^^^^^^^^^^^^^^^^
# To begin with, let us define the computation of a matmul with bias add.
# The function should return the list of input/output tensors.
# From these tensors, the auto-scheduler can get the whole computational graph.

@auto_scheduler.register_workload
def matmul_add(N, L, M, dtype):
    A = te.placeholder((N, L), name="A", dtype=dtype)
    B = te.placeholder((L, M), name="B", dtype=dtype)
    C = te.placeholder((N, M), name="C", dtype=dtype)

    k = te.reduce_axis((0, L), name="k")
    matmul = te.compute((N, M), lambda i, j: te.sum(A[i, k] * B[k, j], axis=k), name="matmul")
    out = te.compute((N, M), lambda i, j: matmul[i, j] + C[i, j], name="out")

    return [A, B, C, out]

######################################################################
# Create the search task
# ^^^^^^^^^^^^^^^^^^^^^^
# We then create a search task with N=L=M=128 and dtype="float32"
# If your machine supports avx instructions, you can
# - replace "llvm" below with "llvm -mcpu=core-avx2" to enable AVX2
# - replace "llvm" below with "llvm -mcpu=skylake-avx512" to enable AVX-512

target = tvm.target.Target("llvm")
task = auto_scheduler.create_task(matmul_add, (128, 128, 128, "float32"), target)

# Inspect the computational graph
print(task.compute_dag)

######################################################################
# Next, we set parameters for the auto-scheduler.
#
# * `num_measure_trials` is the number of measurement trials we can use during the search.
#   We only make 10 trials in this tutorial for a fast demonstration. In practice, 1000 is a
#   good value for the search to converge. You can do more trials according to your time budget.
# * In addition, we use `RecordToFile` to dump measurement records into a file `matmul.json`.
#   The measurement records can be used to query the history best, resume the search,
#   and do more analyses later.
# * see :any:`auto_scheduler.auto_schedule.TuningOptions`: for more parameters

tune_option = auto_scheduler.TuningOptions(
    num_measure_trials=10, measure_callbacks=[auto_scheduler.RecordToFile("matmul.json")]
)

######################################################################
# Run the search
# ^^^^^^^^^^^^^^
# Now we get all inputs ready. Pretty simple, isn't it?
# We can kick off the search and let the auto-scheduler do its magic.
# After some measurement trials, it will return the best schedule it found.

sch, args = auto_scheduler.auto_schedule(task, tuning_options=tune_option)

######################################################################
# We can lower the schedule to see the IR after auto-scheduling.
# The auto-scheduler correctly performs optimizations including multi-level tiling,
# parallelization, vectorization, unrolling and operator fusion.

print(tvm.lower(sch, args, simple_mode=True))

######################################################################
# Check correctness and evaluate performance
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# We build the binary and check its correctness and performance.

func = tvm.build(sch, args)
a_np = np.random.uniform(size=(128, 128)).astype(np.float32)
b_np = np.random.uniform(size=(128, 128)).astype(np.float32)
c_np = np.random.uniform(size=(128, 128)).astype(np.float32)
out_np = a_np.dot(b_np) + c_np

ctx = tvm.cpu()
a_tvm = tvm.nd.array(a_np, ctx=ctx)
b_tvm = tvm.nd.array(b_np, ctx=ctx)
c_tvm = tvm.nd.array(c_np, ctx=ctx)
out_tvm = tvm.nd.empty(out_np.shape, ctx=ctx)
func(a_tvm, b_tvm, c_tvm, out_tvm)

# Check results
tvm.testing.assert_allclose(out_np, out_tvm.asnumpy(), rtol=1e-3)

# Evaluate execution time.
evaluator = func.time_evaluator(func.entry_name, ctx, min_repeat_ms=500)
print(
    "Execution time of this operator: %.3f ms"
    % (np.median(evaluator(a_tvm, b_tvm, c_tvm, out_tvm).results) * 1000)
)

######################################################################
# Using the record file
# ^^^^^^^^^^^^^^^^^^^^^
# During the search, all measuremnt records are dumpped into the record
# file "matmul.json". The measurement records can be used to re-apply search results,
# resume the search, and perform other analyses.

######################################################################
# Here is an example where we load the best schedule from a file,
# print the equivalent python schedule API, and build the binary again.

# Load the measuremnt record for the best schedule
inp, res = auto_scheduler.load_best("matmul.json", task.workload_key)

# Print equivalent python schedule API. This can be used for debugging and
# learning the behavior of the auto-scheduler.
print("Equivalent python schedule:")
print(task.compute_dag.print_python_code_from_state(inp.state))

# Rebuild the binary. This shows how you can apply the best schedule from a
# log file without reruning the search again.
sch, args = task.compute_dag.apply_steps_from_state(inp.state)
func = tvm.build(sch, args)

######################################################################
# A more complicated example is to resume the search.
# In this case, we need to create the search policy and cost model by ourselves
# and resume the status of search policy and cost model with the log file.
# In the example below we resume the status and do more 5 trials.

def resume_search(task, log_file):
    cost_model = auto_scheduler.XGBModel()
    cost_model.update_from_file(log_file)
    search_policy = auto_scheduler.SketchPolicy(
        task, cost_model, init_search_callbacks=[auto_scheduler.PreloadMeasuredStates(log_file)]
    )
    tune_option = auto_scheduler.TuningOptions(
        num_measure_trials=5, measure_callbacks=[auto_scheduler.RecordToFile(log_file)]
    )
    sch, args = auto_scheduler.auto_schedule(task, search_policy, tuning_options=tune_option)

# resume_search(task, "matmul.json")

######################################################################
# .. note::
#   We cannot run the line above because of the conflict between
#   python's multiprocessing and tvm's thread pool.
#   After running a tvm generated binary the python's multiprocessing library
#   will hang forever. You have to make sure that you don't run any tvm
#   generated binaries before calling auot-scheduler's search.
#   To run the function above, you should comment out all code in
#   "Check correctness and evaluate performance" section.
#
#   You should be careful about this problem in your applications.
#   There are other workarounds for this problem.
#   For example, you can start a new thread/process (with the builtin python library
#   threading or multiprocessing) and run the tvm binaries in the new thread/process.
#   This provides an isolation and avoids the conflict in the main thread/process.
#   You can also use :any:`auto_scheduler.measure.LocalRPCMeasureContext` for auto-scheduler,
#   as shown in the GPU tutorial (:ref:`auto-scheduler-conv-gpu`).


#!/usr/bin/env python3

#### Software Carpentry Programming with Python ####

#### Before class ####

# ipython notebook located in Dropbox folder, render in nbviewer, share link to latter so students can follow along (and download notebook)
# check software installation: Python v 3.X (Anaconda)

#### Objectives ####

# why python?
#   We're teaching you how to program, and we have to use something
#   free, well documented, and everyone can run it
#   large userbase
#   easy for novices to learn
#   super popular on campus!
# motivation: inflammation in patients who have been given new treatment for arthritis
#   load data into memory, calculate average inflammation per day across all patients, plot to share info with colleagues
#   data: CSV, comma separated values
#   rows contain information for a single patient (observations)
#   columns represent measurements on successive days

#### Setup ####

# many ways to interact with Python
#   python in terminal
#   ipython in terminal
#   save script in text editor
#   IDE like spyder
#   notebook: web application that combines code, graphs, and text
#   interactive mode in terminal, chevrons (>>>) is prompt, waiting for input
#   scripting mode: save commands in file (ends in .py), execute entire file at once
# about our tools
#   Anaconda: distribution (way of obtaining) Python;
#       includes extra packages like ipython, spyder
#   conda: package manager that comes with Anaconda, installs/updates packages
#   jupyter notebook: installed with Anaconda

# setting up jupyter project
#   launch Jupyter Notebook from Anaconda
#   terminal window must stay open, this is kernel (running python)
#   web browser is how you interact with notebook
#   create project directory (new folder), rename, then move into it
#   click "New" in upper right hand, then select "Python3"
#   creates notebook (*.ipynb, or ipython notebook file)
#   autosaves, or can save manually
#   click on title to rename
# executing code in a jupyter notebook:
#   enter code in cell and execute by pressing Shift + Return/enter
#   output is printed directly below cell, prefaced by Out[ ]:
#   add new cell with + button
#   can add Markdown cells with nicely formatted text
#   comments prefaced with # (not read/executed by python)
#   commands and output saved in notebook
#   talk about other menu options and buttons to remove/add/run cells
#   example notebook: https://github.com/rasilab/machkovech_2018/blob/master/scripts/NA43_competition.ipynb

#### Analyzing patient data ####

# Objectives: intro to libraries, read in data, assign values to variables, select data values, operations on arrays, simple graphs

# include human-readable (but not python interpreted) comments following hash signs

# use python as calculator
3 + 5
# shift+enter to execute command

# assign value to variable
weight_kg = 60
# variable names:
#   can include letters, digits, and underscores
#   cannot start with a digit
#   are case sensitive

# data types:
#   integers
#   floating point numbers (decimals)
#   strings (characters)
# weight_kg is integer
# to create as floating point
weight_kg = 60.0
# to create string
weight_kg_text = 'weight in kilograms:'

# using variables
# display value of variable
print(weight_kg)
# print is a function
# can display multiple items at once
print(weight_kg_text, weight_kg)

# perform arithmetic inside print function
print('weight in pounds:', 2.2 * weight_kg)
# note: this doesn't change the value of weight_kg!
print(weight_kg)

# assign new value to weight_kg
weight_kg = 65.0
print('weight in kilograms is now:', weight_kg)
# variable names as sticky notes (analogy)

## Challenge: What values do the variables mass and age have after each statement in the following program? Test your answers by executing the commands.
mass = 47.5
age = 122
mass = mass * 2.0
age = age - 20
print(mass, age)

## Challenge: What does the following program print out?
first, second = 'Grace', 'Hopper'
third, fourth = second, first
print(third, fourth)

# libraries: collections of additional code that provide more functionality to perform specific tasks
# load library
import os
import urllib.request
import zipfile
import numpy

# download data
urllib.request.urlretrieve("http://swcarpentry.github.io/python-novice-inflammation/data/python-novice-inflammation-data.zip", "python-novice-inflammation-data.zip")
# unzip data
zipData = zipfile.ZipFile('python-novice-inflammation-data.zip')
zipData.extractall()

# load data into python (using library)
numpy.loadtxt(fname='data/inflammation-01.csv', delimiter=',')
# numpy.loadtxt(...) is a function call
#   run function loadtxt
#   belongs to numpy library
#   dotted notation in function call means whatever appears before dot contains the thing after the dot
# parameters: specific information that is sent to (passed) to function call
#   name of file
#   delimiter

# assign data to variable (so we can recall it later)
data = numpy.loadtxt(fname='data/inflammation-01.csv', delimiter=',')
# show the variable's value
print(data)
# what type of thing is data?
print(type(data))
# find type of data contained within array (data)
print(data.dtype)
# show shape of data
print(data.shape)
# output is rows, columns; rows are the individual patients, and the columns are their daily inflammation measurements
# arrays have members, or attributes, which use the dot nomenclature because they have the same part-and-whole relationship

# access a specific value
print('first value in data:', data[0, 0])
# python begins indexing (counting) at 0
print('middle value in data:', data[30, 20])

# select sections of data (slicing)
print(data[0:4, 0:10])
# end bound is NOT inclusive (up to but not including)
# can start at indeces besides 0
print(data[5:10, 0:10])
# use empty bound to include the end of axis
small = data[:3, 36:] # assign to value
print('small is:')
print(small)

# perform math on array
doubledata = data * 2.0
# view output
print('original:')
print(data[:3, 36:])
print('doubledata:')
print(doubledata[:3, 36:])

# perform operation involving two arrays
tripledata = doubledata + data
print('tripledata:')
print(tripledata[:3, 36:])

## Challenge: We can slice character strings as well! Given the following:
element = 'oxygen'
print('first three characters:', element[0:3])
print('last three characters:', element[3:6])
# What is the value of element[:4]? What about element[4:]? Or element[:]?
element[4:]
element[:]

## Challenge: What is element[-1]? What is element[-2]?
element[-1]
element[-2]

# Given those answers, explain what element[1:-1] does.
# Creates a substring from index 1 up to (not including) the final index, effectively removing the first and last letters from ‘oxygen’

# perform calculation across entire array
print(numpy.mean(data)) # find mean

# use multiple assignment to obtain descriptive values of data
maxval, minval, stdval = numpy.max(data), numpy.min(data), numpy.std(data)

print('maximum inflammation:', maxval)
print('minimum inflammation:', minval)
print('standard deviation:', stdval)

# to find available functions and information:
#   type name of something, followed by dot, then hit tab (ipython and notebooks)
#   select a function or attribute and add question mark to find help documentation
#   help(thing.attribute) is same as above

# view max inflammation per patient or per day
# create temporary array for data desired
patient_0 = data[0, :] # 0 on the first axis (rows), everything on the second (columns)
print('maximum inflammation for patient 0:', numpy.max(patient_0))

# combine selection and function call (skip temp variable)
print('maximum inflammation for patient 2:', numpy.max(data[2, :]))

# average across all rows (axis 0)
print(numpy.mean(data, axis=0))
# confirm shape of array
print(numpy.mean(data, axis=0).shape)
# average across all columns (axis 1): avg inflammation per day for all patients
print(numpy.mean(data, axis=1))

#### Visualizing data ####

# make pylot available from matplotlib (de facto plotting library)
import matplotlib.pyplot
# allow plots to appear when using show()
%matplotlib inline

# make heatmap from data
image = matplotlib.pyplot.imshow(data)
matplotlib.pyplot.show()

# plot average inflammation over time
ave_inflammation = numpy.mean(data, axis=0)
ave_plot = matplotlib.pyplot.plot(ave_inflammation)
matplotlib.pyplot.show()

# plot max over time
max_plot = matplotlib.pyplot.plot(numpy.max(data, axis=0))
matplotlib.pyplot.show()

# plot min over time
min_plot = matplotlib.pyplot.plot(numpy.min(data, axis=0))
matplotlib.pyplot.show()

## Challenge: Create a plot showing the standard deviation (numpy.std) of the inflammation data for each day across all patients.
std_plot = matplotlib.pyplot.plot(numpy.std(data, axis=0))
matplotlib.pyplot.show()

# grouping plots: complete set of code

# load libraries
import numpy
import matplotlib.pyplot

# load data from file
data = numpy.loadtxt(fname='data/inflammation-01.csv', delimiter=',')

# create space to place plot
fig = matplotlib.pyplot.figure(figsize=(10.0, 3.0)) # state dimensions of figure

# add subplots, parameters are number of subplots, number of columns, which subplot (left-to-right, top-to-bottom)
axes1 = fig.add_subplot(1, 3, 1)
axes2 = fig.add_subplot(1, 3, 2)
axes3 = fig.add_subplot(1, 3, 3)

# title axes
axes1.set_ylabel('average')
axes1.plot(numpy.mean(data, axis=0))

axes2.set_ylabel('max')
axes2.plot(numpy.max(data, axis=0))

axes3.set_ylabel('min')
axes3.plot(numpy.min(data, axis=0))

# spread out graphs
fig.tight_layout()

# show plot
matplotlib.pyplot.show()

## Challenge: Modify the program to display the three plots on top of one another instead of side by side.
import numpy
import matplotlib.pyplot

data = numpy.loadtxt(fname='inflammation-01.csv', delimiter=',')

# change figsize (swap width and height)
fig = matplotlib.pyplot.figure(figsize=(3.0, 10.0))

# change add_subplot (swap first two parameters)
axes1 = fig.add_subplot(3, 1, 1)
axes2 = fig.add_subplot(3, 1, 2)
axes3 = fig.add_subplot(3, 1, 3)

axes1.set_ylabel('average')
axes1.plot(numpy.mean(data, axis=0))

axes2.set_ylabel('max')
axes2.plot(numpy.max(data, axis=0))

axes3.set_ylabel('min')
axes3.plot(numpy.min(data, axis=0))

fig.tight_layout()

matplotlib.pyplot.show()

## Challenge: How would you alter the limits on the x and y axes?
axes3.set_ylim(0,6)
# A more automated approach
min_data = numpy.min(data, axis=0)
axes3.set_ylabel('min')
axes3.plot(min_data)
axes3.set_ylim(numpy.min(min_data), numpy.max(min_data) * 1.1)

## Wrap up:
# some of our data appears suspicious (avg, max, and min has odd pattern)
# we want to re-run for all our datasets

#### Repeating actions with loops ####

# Objectives: write for loop to repeat simple actions, trace changes to variables

# what if we wanted to print each character in a word on a line of its own?
word = 'lead'
print(word[0])
print(word[1])
print(word[2])
print(word[3])
# try a different word
# this doesn't scale well, and is fragile (creates error if word is shorter, doesn't print all for longer word)

# print with for loop
for char in word:
    print(char)
# syntax:
#   for variable in collection:
#       do things using variable
# note on choosing meaningful variable names

# for loop that repeatedly updates variable
length = 0 # define external variable
for vowel in 'aeiou': # initialize for loop for string (not variable)
    length = length + 1 # define continuously updated variable internal to loop
print('There are', length, 'vowels') # report output at end of loop

# loop variables still exist after loop ends!
letter = 'z'
for letter in 'abc':
    print(letter)
print('after the loop, letter is', letter)

# finding length of a string is a built-in function!
print(len('aeiou'))

## Challenge: Exponentiation is built into Python:
print(5 ** 3)
# Write a  loop that calculates the same result as 5 ** 3 using multiplication (and without exponentiation).
result = 1
for i in range(0, 3):
    result = result * 5
print(result)

#### Storing multiple values in lists ####

# Objectives: create and index lists of simple values, change values of elements, append to list, reorder and slice lists, create and manipulate nested lists

# a list if a way to store many values
# create a list
odds = [1, 3, 5, 7]
# recall list
print('odds are:', odds)
# select individual elements via indexing
print('first and last:', odds[0], odds[-1])
# loop over list (loop variable is assigned to elements one at a time)
for number in odds:
    print(number)
# you can change values in a list (but not individual characters in a string)
# list example
names = ['Curie', 'Darwing', 'Turing']  # typo in Darwin's name
print('names is originally:', names)
names[1] = 'Darwin'  # correct the name
print('final value of names:', names)
# string example
name = 'Darwin'
#name[0] = 'd'

# two variables can refer to the same list; modifying one modifies both!
salsa = ['peppers', 'onions', 'cilantro', 'tomatoes']
my_salsa = salsa        # <-- my_salsa and salsa point to the *same* list data in memory
salsa[0] = 'hot peppers'
print('Ingredients in my salsa:', my_salsa)

# a better way is to make a copy of the original list
salsa = ['peppers', 'onions', 'cilantro', 'tomatoes']
my_salsa = list(salsa)        # <-- makes a *copy* of the list
salsa[0] = 'hot peppers'
print('Ingredients in my salsa:', my_salsa)

# lists can contain other lists
x = [['pepper', 'zucchini', 'onion'],
     ['cabbage', 'lettuce', 'garlic'],
     ['apple', 'pear', 'banana']]
# print first row
print([x[0]])
print(x[0])
# print first item in first row
print(x[0][0])

# lists can contain elements of different types
sample_ages = [10, 12.5, 'Unknown']

# append to the list
odds.append(11)
print('odds after adding a value:', odds)

# remove first element
del odds[0]
print('odds after removing the first element:', odds)

# reverse the list
odds.reverse()
print('odds after reversing:', odds)

# demo: making a list and attempting to copy/modify in place is a bad idea!
odds = [1, 3, 5, 7]
primes = odds
primes.append(2)
print('primes:', primes)
print('odds:', odds)
# python stores a list in memory, and then can use multiple names to refer to the same list

# to copy a simple list, use list function
odds = [1, 3, 5, 7]
primes = list(odds)
primes.append(2)
print('primes:', primes)
print('odds:', odds)

## Challenge: Use a for-loop to convert the string “hello” into a list of letters: ["h", "e", "l", "l", "o"]
# Hint: you can create an empty list with: my_list = []
my_list = []
for char in "hello":
    my_list.append(char)
print(my_list)

# slicing lists
binomial_name = "Drosophila melanogaster"
group = binomial_name[0:10]
print("group:", group)

species = binomial_name[11:24]
print("species:", species)

chromosomes = ["X", "Y", "2", "3", "4"]
autosomes = chromosomes[2:5]
print("autosomes:", autosomes)

last = chromosomes[-1]
print("last:", last)

## Challenge: Use slicing to access only the last four characters of a string or entries of a list.
string_for_slicing = "Observation date: 02-Feb-2013"
list_for_slicing = [["fluorine", "F"],
                    ["chlorine", "Cl"],
                    ["bromine", "Br"],
                    ["iodine", "I"],
                    ["astatine", "At"]]
# Expected result:
#   "2013"
#   [["chlorine", "Cl"], ["bromine", "Br"], ["iodine", "I"], ["astatine", "At"]]
string_for_slicing[-4:]
list_for_slicing[-4:]

# take a slice from the beginning of the sequence
# omit the first range to indicate the start
date = "Monday 4 January 2016"
day = date[0:6]
print("Using 0 to begin range:", day)
day = date[:6]
print("Omitting beginning index:", day)

# omit the last range to indicate the end
months = ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"]
sond = months[8:12]
print("With known last position:", sond)
sond = months[8:len(months)]
print("Using len() to get last entry:", sond)
sond = months[8:]
print("Omitting ending index:", sond)

## Challenge: + usually means addition, but when used on strings or lists, it means “concatenate”. Given that, what do you think the multiplication operator * does on lists? In particular, what will be the output of the following code?
counts = [2, 4, 6, 8, 10]
repeats = counts * 2
print(repeats)
# [2, 4, 6, 8, 10, 2, 4, 6, 8, 10]: the multiplication operator * used on a list replicates elements of the list and concatenates them together and is equivalent to: counts + counts

#### Analyzing data from multiple files ####

# import library to find files/directories
import glob

# get names of all csv files in current directory
print(glob.glob('data/inflammation*.csv'))

# combine previous content together and analyze all files

import numpy
import matplotlib.pyplot

filenames = sorted(glob.glob('data/inflammation*.csv'))
filenames = filenames[0:3]
for f in filenames:
    print(f)

    data = numpy.loadtxt(fname=f, delimiter=',')

    fig = matplotlib.pyplot.figure(figsize=(10.0, 3.0))

    axes1 = fig.add_subplot(1, 3, 1)
    axes2 = fig.add_subplot(1, 3, 2)
    axes3 = fig.add_subplot(1, 3, 3)

    axes1.set_ylabel('average')
    axes1.plot(numpy.mean(data, axis=0))

    axes2.set_ylabel('max')
    axes2.plot(numpy.max(data, axis=0))

    axes3.set_ylabel('min')
    axes3.plot(numpy.min(data, axis=0))

    fig.tight_layout()
    matplotlib.pyplot.show()

## Challenge: Plot the difference between the average of the first dataset and the average of the second dataset, i.e., the difference between the leftmost plot of the first two figures.
import glob
import numpy
import matplotlib.pyplot

filenames = sorted(glob.glob('data/inflammation*.csv'))

data0 = numpy.loadtxt(fname=filenames[0], delimiter=',')
data1 = numpy.loadtxt(fname=filenames[1], delimiter=',')

fig = matplotlib.pyplot.figure(figsize=(10.0, 3.0))

matplotlib.pyplot.ylabel('Difference in average')
matplotlib.pyplot.plot(numpy.mean(data0, axis=0) - numpy.mean(data1, axis=0))

fig.tight_layout()
matplotlib.pyplot.show()

#### Making choices ####

# Objectives: write conditional statements including if, elif, else; evaluate expressions containing and and or

# tell python to take different actions with if statements
num = 37
if num > 100:
    print('greater')
else:
    print('not greater')
print('done')

# don't need else; can also do nothing
num = 53
print('before conditional...')
if num > 100:
    print(num,' is greater than 100')
print('...after conditional')

# can have multiple alternatives using elif
num = -3

if num > 0:
    print(num, 'is positive')
elif num == 0: # double equal sign is necessary; single used to assign values
    print(num, 'is zero')
else:
    print(num, 'is negative')

# combine tests using and, when both parts must be true
if (1 > 0) and (-1 > 0):
    print('both parts are true')
else:
    print('at least one part is false')

# combine tests using or if at least one part must be true
if (1 < 0) or (-1 < 0):
    print('at least one test is true')
# true and false are booleans

## Challenge: What do you expect to get from this code?
if 4 > 5:
    print('A')
elif 4 == 5:
    print('B')
elif 4 < 5:
    print('C')
# C gets printed because the first two conditions, 4 > 5 and 4 == 5, are not true, but 4 < 5 is true.

# checking for problems in inflammation data
import numpy # if not already done

# check if max inflammation equals day number (error in data entry)
max_inflammation_0 = numpy.max(data, axis=0)[0]
max_inflammation_20 = numpy.max(data, axis=0)[20]

if max_inflammation_0 == 0 and max_inflammation_20 == 20:
    print('Suspicious looking maxima!')

# check if mins are all zero (healthy patient)
if numpy.sum(numpy.min(data, axis=0)) == 0:
    print('Minima add up to zero!')

# combine together with data
data = numpy.loadtxt(fname='inflammation-01.csv', delimiter=',')

max_inflammation_0 = numpy.max(data, axis=0)[0]
max_inflammation_20 = numpy.max(data, axis=0)[20]

if max_inflammation_0 == 0 and max_inflammation_20 == 20:
    print('Suspicious looking maxima!')
elif numpy.sum(numpy.min(data, axis=0)) == 0:
    print('Minima add up to zero!')
else:
    print('Seems OK!')

## Challenge:
# Write a loop that counts the number of vowels in a character string.
# Test it on a few individual words and full sentences.
# Once you are done, compare your solution to your neighbor’s. Did you make the same decisions about how to handle the letter ‘y’ (which some people think is a vowel, and some do not)?
vowels = 'aeiouAEIOU'
sentence = 'Mary had a little lamb.'
count = 0
for char in sentence:
    if char in vowels:
        count += 1

print("The number of vowels in this string is " + str(count))

#### Creating functions ####

# Objectives: define new functions that takes parameters, return value from function, test and debug, set default values for function parameters

# functions allow us to create a shorthand way to re-execute longer pieces of code
# define function that converts temps from F to C
def fahr_to_celsius(temp):
    return ((temp - 32) * (5/9))

# test function
fahr_to_celsius(32)
# we can use this the same way we use other functions
print('freezing point of water:', fahr_to_celsius(32), 'C')
print('boiling point of water:', fahr_to_celsius(212), 'C')

# composing functions
# write a function to convert C to K
def celsius_to_kelvin(temp_c):
    return temp_c + 273.15

print('freezing point of water in Kelvin:', celsius_to_kelvin(0.))

# convert F to K: compose the two functions we have already created (to apply one function to the result of another)
def fahr_to_kelvin(temp_f):
    temp_c = fahr_to_celsius(temp_f)
    temp_k = celsius_to_kelvin(temp_c)
    return temp_k

print('boiling point of water in Kelvin:', fahr_to_kelvin(212.0))

## Challenge: “Adding” two strings produces their concatenation: 'a' + 'b' is 'ab'. Write a function called fence that takes two parameters called original and wrapper and returns a new string that has the wrapper character at the beginning and end of the original. A call to your function should look like this:
# input: print(fence('name', '*'))
# output: *name*
def fence(original, wrapper):
    return wrapper + original + wrapper

## Challenge: Note that return and print are not interchangeable. print is a Python function that prints data to the screen. It enables us, users, see the data. return statement, on the other hand, makes data visible to the program. Let’s have a look at the following function:
def add(a, b):
    print(a + b)
# What will we see if we execute the following commands?
A = add(7, 3)
print(A)
# Python will first execute the function add with a = 7 and b = 3, and, therefore, print 10. However, because function add does not have a line that starts with return (no return “statement”), it will, by default, return nothing which, in Python world, is called None. Therefore, A will be assigned to None and the last line (print(A)) will print None. As a result, we will see:
#10
#NONE

# make inflammation process easier to read and reuse by defining code as function
def analyze(filename):

    data = numpy.loadtxt(fname=filename, delimiter=',')

    fig = matplotlib.pyplot.figure(figsize=(10.0, 3.0))

    axes1 = fig.add_subplot(1, 3, 1)
    axes2 = fig.add_subplot(1, 3, 2)
    axes3 = fig.add_subplot(1, 3, 3)

    axes1.set_ylabel('average')
    axes1.plot(numpy.mean(data, axis=0))

    axes2.set_ylabel('max')
    axes2.plot(numpy.max(data, axis=0))

    axes3.set_ylabel('min')
    axes3.plot(numpy.min(data, axis=0))

    fig.tight_layout()
    matplotlib.pyplot.show()

# make a function to find the problems we noticed earlier
def detect_problems(filename):

    data = numpy.loadtxt(fname=filename, delimiter=',')

    if numpy.max(data, axis=0)[0] == 0 and numpy.max(data, axis=0)[20] == 20:
        print('Suspicious looking maxima!')
    elif numpy.sum(numpy.min(data, axis=0)) == 0:
        print('Minima add up to zero!')
    else:
        print('Seems OK!')

# we can run both at once across all files in a for loop
filenames = sorted(glob.glob('inflammation*.csv'))

for f in filenames[:3]:
    print(f)
    analyze(f)
    detect_problems(f)

# testing and documenting
# write a function to offset data (allows to test functions)
def offset_mean(data, target_mean_value):
    return (data - numpy.mean(data)) + target_mean_value

# create test matrix of 0s and offset values using new function (to test it)
z = numpy.zeros((2,2))
print(offset_mean(z, 3))

# use offset function on real data
data = numpy.loadtxt(fname='inflammation-01.csv', delimiter=',')
print(offset_mean(data, 0))

# confirm offset has worked
print('original min, mean, and max are:', numpy.min(data), numpy.mean(data), numpy.max(data))
offset_data = offset_mean(data, 0)
print('min, mean, and max of offset data are:',
      numpy.min(offset_data),
      numpy.mean(offset_data),
      numpy.max(offset_data))
# offset isn't exact, but is close

# check standard deviation
print('std dev before and after:', numpy.std(data), numpy.std(offset_data))
# check more precisely
print('difference in standard deviations before and after:',
      numpy.std(data) - numpy.std(offset_data))

# we could add documentation to offset function to describe its purpose using comments
# alternatively, add string to function itself, which embeds in help documentation
def offset_mean(data, target_mean_value):
    '''Return a new array containing the original data
       with its mean offset to match the desired value.'''
    return (data - numpy.mean(data)) + target_mean_value
help(offset_mean)
# docstring; triple quotes allows us to break into separate lines (and add example)
def offset_mean(data, target_mean_value):
    '''Return a new array containing the original data
       with its mean offset to match the desired value.
    Example: offset_mean([1, 2, 3], 0) => [-1, 0, 1]'''
    return (data - numpy.mean(data)) + target_mean_value
help(offset_mean)

# defining defaults
# pass the filename to loadtxt without the fname=
numpy.loadtxt('inflammation-01.csv', delimiter=',')
# delimiter needs to be there!
numpy.loadtxt('inflammation-01.csv', ',')

# redefine offset mean
def offset_mean(data, target_mean_value=0.0):
    '''Return a new array containing the original data with its mean offset to match the
       desired value (0 by default).
    Example: offset_mean([1, 2, 3], 0) => [-1, 0, 1]'''
    return (data - numpy.mean(data)) + target_mean_value

# can still call function with two arguments
test_data = numpy.zeros((2, 2))
print(offset_mean(test_data, 3))

# call it with just one parameter, target_mean_value automatically assigned the default value of 0.0
more_data = 5 + numpy.zeros((2, 2))
print('data before mean offset:')
print(more_data)
print('offset data:')
print(offset_mean(more_data))

# how Python matches values to parameters:
def display(a=1, b=2, c=3):
    print('a:', a, 'b:', b, 'c:', c)

print('no parameters:')
display()
print('one parameter:')
display(55)
print('two parameters:')
display(55, 66)

# override behavior by naming value as it's passed
print('only setting the value of c')
display(c=77)

# readable functions
# show example: http://swcarpentry.github.io/python-novice-inflammation/06-func/index.html

## Challenge: Return vs print

#### Errors and Exceptions ####

# example traceback
def favorite_ice_cream():
    ice_creams = [
        "chocolate",
        "vanilla",
        "strawberry"
    ]
    print(ice_creams[3])

favorite_ice_cream()

#syntax errors
# colon
#indentation

# variable name errors

# index errors

# file errors

#### Defensive programming ####

# are we getting the right answer?
#   Write programs that check their own operation.
#   Write and run tests for widely-used functions.
#   Make sure we know what “correct” actually means.

# assume mistakes will happen and guard against them (defensive programming)

# assertions: statement that something must be true at a certain point in a program
numbers = [1.5, 2.3, 0.7, -0.001, 4.4]
total = 0.0
for n in numbers:
    assert n > 0.0, 'Data should only contain positive values'
    total += n
print('total is:', total)
# types:
#   precondition is something that must be true at the start of a function in order for it to work correctly.
#   postcondition is something that the function guarantees is true when it finishes.
#   invariant is something that is always true at a particular point inside a piece of code.

# test driven development
# normal tendency is to do:
#   Write a function range_overlap.
#   Call it interactively on two or three different inputs.
#   If it produces the wrong answer, fix the function and re-run that test.
# better practice is to:
#   Write a short function for each test.
#   Write a range_overlap function that should pass those tests.
#   If range_overlap produces any wrong answers, fix it and re-run the test functions.

#### Debugging ####

# overview practices

## Challenge: pair up, introduce error, try to debug with applying principles

#### Command-line programs ####

# switch to command line
# download code file: http://swcarpentry.github.io/python-novice-inflammation/data/python-novice-inflammation-code.zip


# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved
import itertools
import logging
import numpy as np
import torch
import torch.nn.functional as F
from fvcore.nn import smooth_l1_loss

from detectron2.layers import batched_nms, cat
from detectron2.structures import Boxes, Instances, pairwise_iou
from detectron2.utils.events import get_event_storage
from detectron2.utils.memory import retry_if_cuda_oom

from detectron2.modeling.sampling import subsample_labels

def rpn_losses(
    gt_objectness_logits,
    gt_anchor_deltas,
    pred_objectness_logits,
    pred_anchor_deltas,
    smooth_l1_beta,
):
    """
    Args:
        gt_objectness_logits (Tensor): shape (N,), each element in {-1, 0, 1} representing
            ground-truth objectness labels with: -1 = ignore; 0 = not object; 1 = object.
        gt_anchor_deltas (Tensor): shape (N, box_dim), row i represents ground-truth
            box2box transform targets (dx, dy, dw, dh) or (dx, dy, dw, dh, da) that map anchor i to
            its matched ground-truth box.
        pred_objectness_logits (Tensor): shape (N,), each element is a predicted objectness
            logit.
        pred_anchor_deltas (Tensor): shape (N, box_dim), each row is a predicted box2box
            transform (dx, dy, dw, dh) or (dx, dy, dw, dh, da        mx = dx.max())
        smooth_l1_beta (float): The transition point between L1 and L2 loss in
            the smooth L1 loss function. When set to 0, the loss becomes L1. When
            set to +inf, the loss becomes constant 0.

    Returns:
        objectness_loss, localization_loss, both unnormalized (summed over samples).
    """
    pos_masks = gt_objectness_logits == 1
    localization_loss = smooth_l1_loss(
        pred_anchor_deltas[pos_masks], gt_anchor_deltas[pos_masks], smooth_l1_beta, reduction="sum"
    )

    low_qual_masks = gt_objectness_logits == -2
    localization_loss = localization_loss + smooth_l1_loss(
        pred_anchor_deltas[low_qual_masks], gt_anchor_deltas[low_qual_masks], smooth_l1_beta, reduction="sum"
    )

    valid_masks = gt_objectness_logits >= 0
    objectness_loss = F.binary_cross_entropy_with_logits(
        pred_objectness_logits[valid_masks],
        gt_objectness_logits[valid_masks].to(torch.float32),
        reduction="sum",
    )
    return objectness_loss, localization_loss

class RPNOutputs(object):
    def __init__(
        self,
        box2box_transform,
        anchor_matcher,
        batch_size_per_image,
        positive_fraction,
        images,
        pred_objectness_logits,
        pred_anchor_deltas,
        anchors,
        boundary_threshold=0,
        gt_boxes=None,
        smooth_l1_beta=0.0,
    ):
        """
        Args:
            box2box_transform (Box2BoxTransform): :class:`Box2BoxTransform` instance for
                anchor-proposal transformations.
            anchor_matcher (Matcher): :class:`Matcher` instance for matching anchors to
                ground-truth boxes; used to determine training labels.
            batch_size_per_image (int): number of proposals to sample when training
            positive_fraction (float): target fraction of sampled proposals that should be positive
            images (ImageList): :class:`ImageList` instance representing N input images
            pred_objectness_logits (list[Tensor]): A list of L elements.
                Element i is a tensor of shape (N, A, Hi, Wi) representing
                the predicted objectness logits for anchors.
            pred_anchor_deltas (list[Tensor]): A list of L elements. Element i is a tensor of shape
                (N, A*4, Hi, Wi) representing the predicted "deltas" used to transform anchors
                to proposals.
            anchors (list[list[Boxes]]): A list of N elements. Each element is a list of L
                Boxes. The Boxes at (n, l) stores the entire anchor array for feature map l in image
                n (i.e. the cell anchors repeated over all locations in feature map (n, l)).
            boundary_threshold (int): if >= 0, then anchors that extend beyond the image
                boundary by more than boundary_thresh are not used in training. Set to a very large
                number or < 0 to disable this behavior. Only needed in training.
            gt_boxes (list[Boxes], optional): A list of N elements. Element i a Boxes storing
                the ground-truth ("gt") boxes for image i.
            smooth_l1_beta (float): The transition point between L1 and L2 loss in
                the smooth L1 loss function. When set to 0, the loss becomes L1. When
                set to +inf, the loss becomes constant 0.
        """
        self.box2box_transform = box2box_transform
        self.anchor_matcher = anchor_matcher
        self.batch_size_per_image = batch_size_per_image
        self.positive_fraction = positive_fraction
        self.pred_objectness_logits = pred_objectness_logits
        self.pred_anchor_deltas = pred_anchor_deltas

        self.anchors = anchors
        self.gt_boxes = gt_boxes
        self.num_feature_maps = len(pred_objectness_logits)
        self.num_images = len(images)
        self.image_sizes = images.image_sizes
        self.boundary_threshold = boundary_threshold
        self.smooth_l1_beta = smooth_l1_beta

    def _get_ground_truth(self):
        """
        Returns:
            gt_objectness_logits: list of N tensors. Tensor i is a vector whose length is the
                total number of anchors in image i (i.e., len(anchors[i])). Label values are
                in {-1, 0, 1}, with meanings: -1 = ignore; 0 = negative class; 1 = positive class.
            gt_anchor_deltas: list of N tensors. Tensor i has shape (len(anchors[i]), 4).
        """
        gt_objectness_logits = []
        gt_anchor_deltas = []
        # Concatenate anchors from all feature maps into a single Boxes per image
        anchors = [Boxes.cat(anchors_i) for anchors_i in self.anchors]
        for image_size_i, anchors_i, gt_boxes_i in zip(self.image_sizes, anchors, self.gt_boxes):
            """
            image_size_i: (h, w) for the i-th image
            anchors_i: anchors for i-th image
            gt_boxes_i: ground-truth boxes for i-th image
            """
            match_quality_matrix = retry_if_cuda_oom(pairwise_iou)(gt_boxes_i, anchors_i)
            matched_idxs, gt_objectness_logits_i = retry_if_cuda_oom(self.anchor_matcher)(
                match_quality_matrix
            )
            # Matching is memory-expensive and may result in CPU tensors. But the result is small
            gt_objectness_logits_i = gt_objectness_logits_i.to(device=gt_boxes_i.device)
            del match_quality_matrix

            if self.boundary_threshold >= 0:
                # Discard anchors that go out of the boundaries of the image
                # NOTE: This is legacy functionality that is turned off by default in Detectron2
                anchors_inside_image = anchors_i.inside_box(image_size_i, self.boundary_threshold)
                gt_objectness_logits_i[~anchors_inside_image] = -1

            if len(gt_boxes_i) == 0:
                # These values won't be used anyway since the anchor is labeled as background
                gt_anchor_deltas_i = torch.zeros_like(anchors_i.tensor)
            else:
                # TODO wasted computation for ignored boxes
                matched_gt_boxes = gt_boxes_i[matched_idxs]
                gt_anchor_deltas_i = self.box2box_transform.get_deltas(
                    anchors_i.tensor, matched_gt_boxes.tensor
                )

            gt_objectness_logits.append(gt_objectness_logits_i)
            gt_anchor_deltas.append(gt_anchor_deltas_i)

        return gt_objectness_logits, gt_anchor_deltas

    def losses(self):
        """
        Return the losses from a set of RPN predictions and their associated ground-truth.

        Returns:
            dict[loss name -> loss value]: A dict mapping from loss name to loss value.
                Loss names are: `loss_rpn_cls` for objectness classification and
                `loss_rpn_loc` for proposal localization.
        """

        def resample(label):
            """
            Randomly sample a subset of positive and negative examples by overwriting
            the label vector to the ignore value (-1) for all elements that are not
            included in the sample.
            """
            pos_idx, neg_idx = subsample_labels(
                label, self.batch_size_per_image, self.positive_fraction, 0
            )
            # Fill with the ignore label (-1), then set positive and negative labels
            label.fill_(-1)
            label.scatter_(0, pos_idx, 1)
            label.scatter_(0, neg_idx, 0)
            return label

        gt_objectness_logits, gt_anchor_deltas = self._get_ground_truth()
        """
        gt_objectness_logits: list of N tensors. Tensor i is a vector whose length is the
            total number of anchors in image i (i.e., len(anchors[i]))
        gt_anchor_deltas: list of N tensors. Tensor i has shape (len(anchors[i]), B),
            where B is the box dimension
        """
        # Collect all objectness labels and delta targets over feature maps and images
        # The final ordering is L, N, H, W, A from slowest to fastest axis.
        num_anchors_per_map = [np.prod(x.shape[1:]) for x in self.pred_objectness_logits]
        num_anchors_per_image = sum(num_anchors_per_map)

        # Stack to: (N, num_anchors_per_image)
        gt_objectness_logits = torch.stack(
            [resample(label) for label in gt_objectness_logits], dim=0
        )

        # Log the number of positive/negative anchors per-image that's used in training
        num_pos_anchors = (gt_objectness_logits == 1).sum().item()
        num_neg_anchors = (gt_objectness_logits == 0).sum().item()
        storage = get_event_storage()
        storage.put_scalar("rpn/num_pos_anchors", num_pos_anchors / self.num_images)
        storage.put_scalar("rpn/num_neg_anchors", num_neg_anchors / self.num_images)

        assert gt_objectness_logits.shape[1] == num_anchors_per_image
        # Split to tuple of L tensors, each with shape (N, num_anchors_per_map)
        gt_objectness_logits = torch.split(gt_objectness_logits, num_anchors_per_map, dim=1)
        # Concat from all feature maps
        gt_objectness_logits = cat([x.flatten() for x in gt_objectness_logits], dim=0)

        # Stack to: (N, num_anchors_per_image, B)
        gt_anchor_deltas = torch.stack(gt_anchor_deltas, dim=0)
        assert gt_anchor_deltas.shape[1] == num_anchors_per_image
        B = gt_anchor_deltas.shape[2]  # box dimension (4 or 5)

        # Split to tuple of L tensors, each with shape (N, num_anchors_per_image)
        gt_anchor_deltas = torch.split(gt_anchor_deltas, num_anchors_per_map, dim=1)
        # Concat from all feature maps
        gt_anchor_deltas = cat([x.reshape(-1, B) for x in gt_anchor_deltas], dim=0)

        # Collect all objectness logits and delta predictions over feature maps
        # and images to arrive at the same shape as the labels and targets
        # The final ordering is L, N, H, W, A from slowest to fastest axis.
        pred_objectness_logits = cat(
            [
                # Reshape: (N, A, Hi, Wi) -> (N, Hi, Wi, A) -> (N*Hi*Wi*A, )
                x.permute(0, 2, 3, 1).flatten()
                for x in self.pred_objectness_logits
            ],
            dim=0,
        )
        pred_anchor_deltas = cat(
            [
                # Reshape: (N, A*B, Hi, Wi) -> (N, A, B, Hi, Wi) -> (N, Hi, Wi, A, B)
                #          -> (N*Hi*Wi*A, B)
                x.view(x.shape[0], -1, B, x.shape[-2], x.shape[-1])
                .permute(0, 3, 4, 1, 2)
                .reshape(-1, B)
                for x in self.pred_anchor_deltas
            ],
            dim=0,
        )

        objectness_loss, localization_loss = rpn_losses(
            gt_objectness_logits,
            gt_anchor_deltas,
            pred_objectness_logits,
            pred_anchor_deltas,
            self.smooth_l1_beta,
        )
        normalizer = 1.0 / (self.batch_size_per_image * self.num_images)
        loss_cls = objectness_loss * normalizer  # cls: classification loss
        loss_loc = localization_loss * normalizer  # loc: localization loss
        losses = {"loss_rpn_cls": loss_cls, "loss_rpn_loc": loss_loc}

        return losses

    def predict_proposals(self):
        """
        Transform anchors into proposals by applying the predicted anchor deltas.

        Returns:
            proposals (list[Tensor]): A list of L tensors. Tensor i has shape
                (N, Hi*Wi*A, B), where B is box dimension (4 or 5).
        """
        proposals = []
        # Transpose anchors from images-by-feature-maps (N, L) to feature-maps-by-images (L, N)
        anchors = list(zip(*self.anchors))
        # For each feature map
        for anchors_i, pred_anchor_deltas_i in zip(anchors, self.pred_anchor_deltas):
            B = anchors_i[0].tensor.size(1)
            N, _, Hi, Wi = pred_anchor_deltas_i.shape
            # Reshape: (N, A*B, Hi, Wi) -> (N, A, B, Hi, Wi) -> (N, Hi, Wi, A, B) -> (N*Hi*Wi*A, B)
            pred_anchor_deltas_i = (
                pred_anchor_deltas_i.view(N, -1, B, Hi, Wi).permute(0, 3, 4, 1, 2).reshape(-1, B)
            )
            # Concatenate all anchors to shape (N*Hi*Wi*A, B)
            # type(anchors_i[0]) is Boxes (B = 4) or RotatedBoxes (B = 5)
            anchors_i = type(anchors_i[0]).cat(anchors_i)
            proposals_i = self.box2box_transform.apply_deltas(
                pred_anchor_deltas_i, anchors_i.tensor
            )
            # Append feature map proposals with shape (N, Hi*Wi*A, B)
            proposals.append(proposals_i.view(N, -1, B))
        return proposals

    def predict_objectness_logits(self):
        """
        Return objectness logits in the same format as the proposals returned by
        :meth:`predict_proposals`.

        Returns:
            pred_objectness_logits (list[Tensor]): A list of L tensors. Tensor i has shape
                (N, Hi*Wi*A).
        """
        pred_objectness_logits = [
            # Reshape: (N, A, Hi, Wi) -> (N, Hi, Wi, A) -> (N, Hi*Wi*A)
            score.permute(0, 2, 3, 1).reshape(self.num_images, -1)
            for score in self.pred_objectness_logits
        ]
        return pred_objectness_logits