import torch
import cupy as cp
import math

from .CustomKernel import CustomKernel
from ..util import get_absolute_path
from time import time

class DistributedIVFPQTop1Cuda(CustomKernel):
  """
    What's new:
      bunch of stuff
  """
  def __init__(
      self,
      m=8,
      k=256,
      tpb=256,
      n_cs=4,
      sm_size=48*256*4,
    ):
    super().__init__()
    assert tpb >= 32
    assert tpb == self.next_power_of_2(tpb), "tpb needs to be a power of 2"
    assert k == 256
    assert m * 1024 <= sm_size
    assert m % n_cs == 0
    self.m = m
    self.k = k
    self.tpb=tpb
    self.n_cs = n_cs
    self.sm_size = sm_size

    with open(get_absolute_path("kernels", "cuda", "distributed_ivfpq_top1.cu"), "r") as f:
      self.kernel = f.read()
    varnames = ", ".join([f"d{i}" for i in range(n_cs)])
    kernel = (self.kernel
      # .replace("_CODEBLOCK_", codeblock)
      .replace("_VARNAMES_", varnames)
      .replace("_M_", str(m))
      .replace("_K_", str(k))
      .replace("_TPB_", str(self.tpb))
      .replace("_NCS_", str(n_cs))
    )
    # print(kernel.split('\n')[60:64])
    self._top1_fn = cp.RawKernel(
      code = kernel,
      name = f'ivfpq_top1',
      options = (
        '--maxrregcount=255',
        '--use_fast_math',
        '-lineinfo'
      ),
      backend='nvrtc',
    )
    self._top1_fn.max_dynamic_shared_size_bytes = sm_size

    self._top1_residual_fn = cp.RawKernel(
      code = kernel,
      name = f'ivfpq_top1_residual',
      options = (
        '--maxrregcount=255',
        '--use_fast_math',
        '-lineinfo'
      ),
      backend='nvrtc',
    )
    self._top1_residual_fn.max_dynamic_shared_size_bytes = sm_size

    self._top1_residual_precomputed_fn = cp.RawKernel(
      code = kernel,
      name = 'ivfpq_top1_residual_precomputed',
      options = (
        '--maxrregcount=255',
        '--use_fast_math',
        '-lineinfo'
      ),
      backend='nvrtc',
    )
    self._top1_residual_precomputed_fn.max_dynamic_shared_size_bytes = sm_size

  @staticmethod
  def next_power_of_2(x):
    return 1 if x == 0 else 2**math.ceil(math.log2(x))

  def topk(
      self,
      address2id_ptr,
      precomputed,
      cell_ptr,
      cell_size,
      cell_capacity,
      n_probe_list,
      n_candidates=1
    ):
    """
      # data: shape=[n_subvectors // n_cs, n_data, n_cs], dtype=uint8
      # is_empty: shape=[n_data], dtype=uint8
      address2id_ptr: shape=[n_query, max_n_probe]
      precomputed: shape=[n_subvectors, n_query, n_clusters], dtype=float32
      cell_ptr: shape=[n_query, max_n_probe], dtype=int64
      cell_size: shape=[n_query, max_n_probe], dtype=int64
      cell_capacity: shape=[n_query, max_n_probe], dtype=int64
      n_probe_list: shape=[n_query], dtype=int64
    """
    n_query = precomputed.shape[1]
    n_probe = cell_ptr.shape[1]
    assert precomputed.shape[0] == self.m
    assert precomputed.shape[2] == self.k
    assert precomputed.dtype == torch.float32
    # n_data = data.shape[1]
    # assert data.shape[0] == self.m // self.n_cs
    # assert data.shape[2] == self.n_cs
    # assert data.dtype == torch.uint8
    # assert is_empty.shape[0] == n_data
    # assert is_empty.dtype == torch.uint8
    assert cell_size.shape[1] == n_probe
    assert cell_ptr.dtype == cell_size.dtype == cell_capacity.dtype == torch.int64
    assert n_probe_list.shape == (n_query, )
    assert n_probe_list.dtype == torch.int64
    assert n_candidates == 1

    cell_info = torch.stack([cell_size, cell_ptr, cell_capacity], dim=-1)
    tot_size = cell_size.sum(dim=1)
    values = torch.empty(n_query, 1, device="cuda:0", dtype=torch.float32)
    values.fill_(float("-inf"))
    address = torch.zeros(n_query, 1, 2, device="cuda:0", dtype=torch.int64)
    ids = torch.zeros(n_query, 1, device="cuda:0", dtype=torch.int64)
    threads_per_block = (self.tpb,)
    blocks_per_grid = (n_query,)

    self._top1_fn(
      grid=blocks_per_grid,
      block=threads_per_block,
      shared_mem = self.sm_size,
      args=[
        # data.data_ptr(),
        address2id_ptr.data_ptr(),
        precomputed.data_ptr(),
        cell_info.data_ptr(),
        tot_size.data_ptr(),
        n_probe_list.data_ptr(),
        values.data_ptr(),
        address.data_ptr(),
        ids.data_ptr(),
        n_query, n_probe
        ],
      stream=self.stream
    )
    return (values, address)

  def topk_residual(
      self,
      address2id_ptr,
      precomputed,
      base_sims,
      cell_ptr,
      cell_size,
      cell_capacity,
      n_probe_list,
      n_candidates=1
    ):
    """
      address2id_ptr: shape=[n_query, max_n_probe], dtype=int64
      precomputed: shape=[n_query, max_n_probe, n_subvectors, n_clusters], dtype=float32
      base_sims: shape=[n_query, max_n_probe], dtype=float32
      cell_ptr: shape=[n_query, max_n_probe], dtype=int64
      cell_size: shape=[n_query, max_n_probe], dtype=int64
      cell_capacity: shape=[n_query, max_n_probe], dtype=int64
      n_probe_list: shape=[n_query], dtype=int64
    """
    # n_data = data.shape[1]
    n_query = cell_ptr.shape[0]
    n_probe = cell_ptr.shape[1]
    assert precomputed.shape == (n_query, n_probe, self.m, self.k)
    assert base_sims.shape == (n_query, n_probe)
    # assert data.shape == (self.m // self.n_cs, n_data, self.n_cs)
    # assert data.dtype == torch.uint8
    # assert is_empty.shape == (n_data, )
    # assert is_empty.dtype == torch.uint8
    assert cell_size.shape == (n_query, n_probe)
    assert cell_ptr.shape == (n_query, n_probe)
    assert cell_capacity.shape == (n_query, n_probe)
    assert precomputed.dtype == torch.float32
    assert cell_ptr.dtype == cell_size.dtype == cell_capacity.dtype == torch.int64
    assert base_sims.dtype == torch.float32
    assert n_candidates == 1
    assert n_probe_list.shape == (n_query, )
    assert n_probe_list.dtype == torch.int64
    base_sims = base_sims.contiguous()

    cell_info = torch.stack([cell_size, cell_ptr, cell_capacity], dim=-1)
    tot_size = cell_size.sum(dim=1)
    values = torch.empty(n_query, 1, device="cuda:0", dtype=torch.float32)
    values.fill_(float("-inf"))
    indices = torch.zeros(n_query, 1, device="cuda:0", dtype=torch.int64)
    threads_per_block = (self.tpb,)
    blocks_per_grid = (n_query,)

    self._top1_residual_fn(
      grid=blocks_per_grid,
      block=threads_per_block,
      shared_mem = self.sm_size,
      args=[
        address2id_ptr.data_ptr(),
        precomputed.data_ptr(),
        base_sims.data_ptr(),
        cell_info.data_ptr(),
        tot_size.data_ptr(),
        n_probe_list.data_ptr(),
        values.data_ptr(),
        indices.data_ptr(),
        n_query, n_probe
        ],
      stream=self.stream
    )
    return (values, indices)

  def topk_residual_precomputed(
      self,
      address2id_ptr,
      part1,
      part2,
      cells,
      base_sims,
      cell_ptr,
      cell_size,
      cell_capacity,
      n_probe_list,
      n_candidates=1
    ):
    """
      address2id_ptr: shape=[n_query, max_n_probe], dtype=int64
      part1: shape=[n_query, n_subvectors, n_pq_clusters], dtype=float32
      part2: shape=[n_cells, n_subvectors, n_pq_clusters], dtype=float32
      cells: shape=[n_query, max_n_probe], dtype=int64
      base_sims: shape=[n_query, max_n_probe], dtype=float32
      cell_ptr: shape=[n_query, max_n_probe], dtype=int64
      cell_size: shape=[n_query, max_n_probe], dtype=int64
      cell_capacity: shape=[n_query, max_n_probe], dtype=int64
      n_probe_list: shape=[n_query], dtype=int64
    """
    # n_data = data.shape[1]
    n_query = cell_ptr.shape[0]
    n_probe = cell_ptr.shape[1]
    assert base_sims.shape == (n_query, n_probe)
    assert cells.shape == (n_query, n_probe)
    assert address2id_ptr.shape == (n_query, n_probe)
    # assert data.shape == (self.m // self.n_cs, n_data, self.n_cs)
    # assert data.dtype == torch.uint8
    # assert is_empty.shape == (n_data, )
    # assert is_empty.dtype == torch.uint8
    assert cell_size.shape == (n_query, n_probe)
    assert cell_ptr.shape == (n_query, n_probe)
    assert cell_capacity.shape == (n_query, n_probe)
    assert cell_ptr.dtype == cell_size.dtype == cell_capacity.dtype == torch.int64
    assert base_sims.dtype == torch.float32
    assert cells.dtype == torch.int64
    assert part1.dtype == part2.dtype == torch.float32
    assert n_candidates == 1
    assert n_probe_list.shape == (n_query, )
    assert n_probe_list.dtype == torch.int64
    part1 = part1.contiguous()
    part2 = part2.contiguous()
    cells = cells.contiguous()
    base_sims = base_sims.contiguous()

    cell_info = torch.stack([cell_size, cell_ptr, cell_capacity], dim=-1)
    tot_size = cell_size.sum(dim=1)
    values = torch.empty(n_query, 1, device="cuda:0", dtype=torch.float32)
    values.fill_(float("-inf"))
    indices = torch.zeros(n_query, 1, device="cuda:0", dtype=torch.int64)
    threads_per_block = (self.tpb,)
    blocks_per_grid = (n_query,)

    self._top1_residual_precomputed_fn(
      grid=blocks_per_grid,
      block=threads_per_block,
      shared_mem = self.sm_size,
      args=[
        address2id_ptr.data_ptr(),
        part1.data_ptr(),
        part2.data_ptr(),
        cells.data_ptr(),
        base_sims.data_ptr(),
        cell_info.data_ptr(),
        tot_size.data_ptr(),
        n_probe_list.data_ptr(),
        values.data_ptr(),
        indices.data_ptr(),
        n_query, n_probe
        ],
      stream=self.stream
    )
    return (values, indices)


#!/usr/bin/env python
# coding: utf-8

#
#
# # Earth Analytics Education - EA  Python Course Spring 2021

# ## Important  - Assignment Guidelines
#
# 1. Before you submit your assignment to GitHub, make sure to run the entire notebook with a fresh kernel. To do this first, **restart the kernel** (in the menubar, select Kernel$\rightarrow$Restart & Run All)
# 2. Always replace the `raise NotImplementedError()` code with your code that addresses the activity challenge. If you don't replace that code, your notebook will not run.
#
# ```
# # YOUR CODE HERE
# raise NotImplementedError()
# ```
#
# 3. Any open ended questions will have a "YOUR ANSWER HERE" within a markdown cell. Replace that text with your answer also formatted using Markdown.
# 4. **DO NOT RENAME THIS NOTEBOOK File!** If the file name changes, the autograder will not grade your assignment properly.
# 6. When you create a figure, comment out `plt.show()` to ensure the autograder can grade your plots. For figure cells, DO NOT DELETE the code that says `DO NOT REMOVE LINE BELOW`.
#
# ```
# ### DO NOT REMOVE LINE BELOW ###
# student_plot1_ax = nb.convert_axes(plt)
# ```
#
# * Only include the package imports, code, and outputs that are required to run your homework assignment.
# * Be sure that your code can be run on any operating system. This means that:
#    1. the data should be downloaded in the notebook to ensure it's reproducible
#    2. all paths should be created dynamically using the `os.path.join`
#
# ## Follow to PEP 8 Syntax Guidelines & Documentation
#
# * Run the `autopep8` tool on all cells prior to submitting (HINT: hit shift + the tool to run it on all cells at once!
# * Use clear and expressive names for variables.
# * Organize your code to support readability.
# * Check for code line length
# * Use comments and white space sparingly where it is needed
# * Make sure all python imports are at the top of your notebook and follow PEP 8 order conventions
# * Spell check your Notebook before submitting it.
#
# For all of the plots below, be sure to do the following:
#
# * Make sure each plot has a clear TITLE and, where appropriate, label the x and y axes. Be sure to include UNITS in your labels.
#

# ### Add Your Name Below
# **Your Name:** Mike Sutherland

#

# ---

# # Week 04 and 05 Homework - Automate NDVI Workflow
#
# For this assignment, you will write code to generate a plot of the mean normalized difference vegetation index (NDVI) for two different sites in the United States across one year of data:
#
# * San Joaquin Experimental Range (SJER) in Southern California, United States
# * Harvard Forest (HARV) in the Northeastern United States
#
# The data that you will use for this week is available from **earthpy** using the following download:
#
# `et.data.get_data('ndvi-automation')`
#
# ## Assignment Goals
#
# Your goal in this assignment is to create the most efficient and concise workflow that you can that allows for:
#
# 1. The code to scale if you added new sites or more time periods to the analysis.
# 2. Someone else to understand your workflow.
# 3. The LEAST and most efficient (i.e. runs fast, minimize repetition) amount of code that completes the task.
#
# ### HINTS
#
# * Remove values outside of the landsat valid range of values as specified in the metadata, as needed.
# * Keep any output files SEPARATE FROM input files. Outputs should be created in an outputs directory that is created in the code (if needed) and/or tested for.
# * Use the functions that we demonstrated during class to make your workflow more efficient.
# * BONUS - if you  chose - you can export your data as a csv file. You will get bonus points for doing this.
#
#
# ## Assignment Requirements
#
# Your submission to the GitHub repository should include:
# * This Jupyter Notebook file (.ipynb) with:
#     * The code to create a plot of mean NDVI across a year for  2 NEON Field Sites:
#         * NDVI on the x axis and formatted dates on the y for both NEON sites on one figure/axis object
#     * The **data should be cleaned to remove the influence of clouds**. See the [earthdatascience website for an example of what your plot might look like with and without removal of clouds](https://www.earthdatascience.org/courses/earth-analytics-python/create-efficient-data-workflows/).
# * BONUS: Create one output `.csv` file that has 3 columns - NDVI, Date and Site Name - with values for SJER and HARV.
#
# Your notebook should:
# * Have *at least* 2 well documented and well named functions with docstrings.
# * Include a Markdown cell at the top of the notebook that outlines the overall workflow using pseudocode (i.e. plain language, not code)
# * Include additional Markdown cells throughout the notebook to describe:
#     * the data that you used - and where it is from
#     * how data are being processing
#     * how the code is optimized to run fast and be more concise

# # Replace this cell with your pseudocode  for this workflow
#
# If you happen to be a diagram person a diagram is ok too
#
#

# # Workflow pseudocode
#
# 1. Download Landsat/vector data
# 2. Iterate through each LANDSAT scene
#     * Obtain date, location information
#     * Iterate through red, NIR bands in each scene directory
#         * convert to xarray
#         * remove "noise" outside valid/expected data range
#         * crop to site boundary location
#     * create NDVI
#     * apply cloud mask to NDVI
#     * query mean value
#     * Aggregate date, location, ndvi infomation
#     * Write to pre-established empty list
# 3. Convert list to pandas dataframe
# 4. Plot dataframe
# 5. Convert dataframe to csv, writing to disk

# In[1]:

# Autograding imports - do not modify this cell
import matplotcheck.autograde as ag
import matplotcheck.notebook as nb
import matplotcheck.timeseries as ts
from datetime import datetime

# In[2]:

# Import required site-packages/modules
import os
from glob import glob

import matplotlib.pyplot as plt
from matplotlib.dates import DateFormatter
import matplotlib.dates as mdates
import seaborn as sns
import pandas as pd
import geopandas as gpd
import numpy as np
from numpy import ma
import xarray as xr
import rioxarray as rxr
import earthpy as et
import earthpy.plot as ep

# Adjust plotting style
sns.set_style('white')
sns.set(font_scale=1.35)

# Download data and set working directory
et.data.get_data('ndvi-automation')
os.chdir(os.path.join(et.io.HOME,
                      'earth-analytics',
                      'data'))

# In[3]:

# DO NOT MODIFY THIS CELL
# Tests that the working directory is set to earth-analytics/data

path = os.path.normpath(os.getcwd())
student_wd_parts = path.split(os.sep)

if student_wd_parts[-2:] == ['earth-analytics', 'data']:
    print("\u2705 Great - it looks like your working directory is set correctly to ~/earth-analytics/data")
else:
    print("\u274C Oops, the autograder will not run unless your working directory is set to earth-analytics/data")

# # Figure 1: Plot 1 - Mean NDVI For Each Site Across the Year (50 points)
#
# Create a plot of the mean normalized difference vegetation index (NDVI) for the two different sites in the United States across the year:
#
# * NDVI on the x axis and formatted dates on the y for both NEON sites on one figure/axis object.
# * Each site should be identified with a different color in the plot and legend.
# * The final plot **data should be cleaned to remove the influence of clouds**.
# * Be sure to include appropriate title and axes labels.
#
# Add additional cells as needed for processing data (e.g. defining functions, etc), but be sure to:
# * follow the instructions in the code cells that have been provided to ensure that you are able to use the sanity check tests that are provided.
# * include only the plot code in the cell identified for the final plot code below

# ## Task 1:
#
# In the cell below, create a single dataframe containing MEAN NDVI, the site name,
# and the date of the data for the HARV site
# scene `HARV/landsat-crop/LC080130302017031701T1-SC20181023151837`.  The column names for the  final
# DataFrame should be`mean_ndvi`, and `site`, and the data should be **indexed on the date**.
#
# Use the functions that we reviewed in class (or create your own versions of them) to implement your code
#
# ### In the Cell below Place  All Functions Needed to Run this Notebook (20 points)

# In[4]:

### DO NOT REMOVE THIS LINE OR EDIT / MOVE THIS CELL ###
start_time = datetime.now()

# In[5]:

# In this cell place all of the functions needed to run your notebook
# You will be graded here on function application, docstrings, efficiency so ensure
# All functions are placed here!

def fetch_site_date(inpath):
    """Retrieve site location identifier and acquisition date from
    a given path.

    Parameters
    -----------
    inpath : string
        A list containing two strings representing the site identifier
        and imagery acquisition date

    Returns
    -----------
    date_site : list
        a cropped and masked xarray object containing NDVI values
    """

    site_id = os.path.normpath(inpath).split(os.sep)[-3]
    date_id = os.path.basename(os.path.normpath(inpath))[10:18]
    date_site = [date_id, site_id]
    return date_site

def create_ndvi(rastpath, clip_shp, cloud_vals,
                valid_drange=None):
    """Iterate through a red, NIR bands in each directory and open as an
    xarray object, concurrently clipping the data in the process and
    constraining the values to a defined range of expected/valid values.
    With the xarray objects, calculate the NDVI then apply a cloud mask to
    eliminate bias in the pixel values. Finally, query and return the mean
    NDVI value.

    Parameters
    ----------
    rastpath: string
        The defined path to the directory containing the spectral bands
        that will be used in NDVI calculation.
    clip_shp: geopandas GeoDataFrame
        The geopandas object representing the boundary used to clip/crop the
        imagery.
    cloud_vals: list
        A list of pixel values representing cloud cover visible in the
        imagery which will be used as a mask to remove these data from
        processing consideration.
    valid_drange: tuple
        The range of valid pixel values in the imagery, used to eliminate
        false/erroneous data (noise) from the imagery.

    Returns
    ------
    ndvi_mean: list
        A list containing the mean NDVI value; this will be combined with
        other data lists to create a final pandas dataframe.
    """

    clip_extent = clip_shp.geometry

    orig_bands = sorted(glob(os.path.join(rastpath, "*band*[4-5].tif")))
    proc_bands = []

    for aband in orig_bands:
        band = rxr.open_rasterio(aband,
                                 masked=True).rio.clip(clip_extent).squeeze()

        if valid_drange:
            mask = ((band < valid_drange[0]) | (band > valid_drange[1]))
            cleaned_band = band.where(~xr.where(mask, True, False))

        proc_bands.append(cleaned_band)

    ndvi_full = (proc_bands[1]-proc_bands[0]) /         (proc_bands[1]+proc_bands[0])

    # ndvi masking
    qa_path = glob(os.path.join(rastpath, "*qa.tif"))
    qa_layer = rxr.open_rasterio(qa_path[0],
                                 masked=True).rio.clip(clip_extent).squeeze()

    ndvi_masked = ndvi_full.where(~qa_layer.isin(cloud_vals))
    ndvi_mean = [np.nanmean(ndvi_masked)]

    return ndvi_mean

def list_to_df(inlist):
    """Convert a list containing site location identifiers, acquisition dates
    and mean NDVI values to a pandas dataframe .

    Parameters
    -----------
    inlist : list
        A list containing the site identifiers and imagery acquisition dates,
        as well as the mean NDVI values.

    Returns
    -----------
    formatted_df : pandas dataframe
        A formatted datafrmae containing requisite column names and indexed
        by acqusition date.
    """

    formatted_df = pd.DataFrame(inlist, columns=["date", "site", "mean_ndvi"])
    formatted_df["date"] = pd.to_datetime(formatted_df["date"])
    formatted_df.set_index("date", inplace=True)
    return formatted_df

# In[6]:

# Path definition to Landsat imagery and field site boundary data
root_dpath = os.path.join("ndvi-automation", "sites",
                          "HARV")
bands_dpath = os.path.join(root_dpath, "landsat-crop",
                           "LC080130302017031701T1-SC20181023151837")
crop_extent = gpd.read_file(os.path.join(root_dpath, "vector",
                                         str(os.path.basename(root_dpath)) +
                                         "-crop.shp"))

# Cloud no data vals for Landsat 8
vals = [328, 392, 840, 904, 1350, 352, 368, 416,
        432, 480, 864, 880, 928, 944, 992, 480, 992]

# Data range representing valid reflectance values to constrain Landsat data
valid_pixels = (0, 10000)

# Empty list creation; needed for subsequent processing steps
df_list = []

# Extract & combine acq date, location and NDVI information into a pandas df
loc_date = fetch_site_date(bands_dpath)
ndvi_calc = create_ndvi(bands_dpath, crop_extent, vals, valid_pixels)
df_list.append(loc_date + ndvi_calc)
final_df = list_to_df(df_list)
final_df

# In[7]:

# This cell  is testing your data output above

student_ndvi_ts_single_site = _

single_scene_points = 0

# Ensure the data is stored in a dataframe.
if isinstance(student_ndvi_ts_single_site, pd.DataFrame):
    print('\u2705 Your data is stored in a DataFrame!')
    single_scene_points += 1
else:
    print('\u274C It appears your data is not stored in a DataFrame. ',
          'To see what type of object your data is stored in, check its type with type(object)')

# Ensure that the date column is the index
if isinstance(student_ndvi_ts_single_site.index, pd.core.indexes.datetimes.DatetimeIndex):
    print('\u2705 You have the index set to the date column!')
    single_scene_points += 2
else:
    print('\u274C You do not have the index set to the date column.')

# Ensure that the date column is datetime
if isinstance(student_ndvi_ts_single_site.index[0], pd._libs.tslibs.timestamps.Timestamp):
    print('\u2705 The data in your date column is datetime!')
    single_scene_points += 2
else:
    print('\u274C The data in your date column is not datetime.')

# Ensure the site name is correct
if student_ndvi_ts_single_site.site.values[0] == 'HARV':
    print('\u2705 You have the correct site name!')
    single_scene_points += 5
else:
    print('\u274C You do not have the correct site name.')

if np.allclose(0.281131628228094, student_ndvi_ts_single_site.mean_ndvi.values[0]):
    print('\u2705 You have the correct mean NDVI value!')
    single_scene_points += 5
else:
    print('\u274C You do not have the correct mean ndvi value.')

print("\n \u27A1 You received {} out of 15 points for creating a dataframe.".format(
    single_scene_points))
single_scene_points

# ## Task 1 Processing Explanation:
#
# During "Task 1 Processing," I've extracted the site name & acquisition date from the string representation of the Landsat scene directory using basic string manipulation methods (indexing/slicing). The raster processing is sequential/methodical in the "create_ndvi" function, and although there is some duplication in terms of cropping the raster bands/qa layer individually (rather than simply cropping the final NDVI layer), I chose to use the guidance given in the examples provided in class -- cropping during the xarray creation step based on the evidence of quicker processing time. Overall, the code is fairly straightforward and "clean", efficient in the sense that all the "heavy lifting" is wrapped up in functions, and the structured in a way that will be easily implemented in iterative loops during subsequent Task 2 processing.

# ## Task 2:
#
# In the cell below, process all of the landsat scenes. Create a DataFrame that contains the following
# information for each scene
#
#
# |   | index  | site  | mean_ndvi  |
# |---|---|---|---|
# | Date  |   |   |   |
# |  2017-01-07  | 0  | SJER  | .4  |
#
# Be sure to call your dataframe at the end of the cell to ensure autograding works.
# HINT: FOR THIS STEP, leave any rows containing missing values (`NAN`).

# In[8]:

# Path definition to the field site directory "level"
sites = sorted(glob(os.path.join("ndvi-automation", "sites", "*")))

# Empty list creation; needed for subsequent processing steps
ndvi_df_list = []

# Iterating through each site directory, processing each contained scene
for site in sites:
    crop_extent = gpd.read_file(os.path.join(site, "vector",
                                             str(os.path.basename(site)) +
                                             "-crop.shp"))
    dirs = sorted(glob(os.path.join(site, "landsat-crop", "*")))
    for dir in dirs:
        loc_date = fetch_site_date(dir)
        ndvi_calc = create_ndvi(dir, crop_extent, vals, valid_pixels)
        ndvi_df_list.append(loc_date + ndvi_calc)

# Final pandas df containing information for all Landsat scenes
final_df = list_to_df(ndvi_df_list)
final_df

# In[9]:

# Last sanity check before creating your plot (10 points)

# Ensure that you call your dataframe at the bottom of the cell above
# and that it has columns called: mean_ndvi and site

# Ensure the data is stored in a dataframe.
student_ndvi_df = _

df_points = 0

if isinstance(student_ndvi_df, pd.DataFrame):
    print('\u2705 Your data is stored in a DataFrame!')
    df_points += 2
else:
    print('\u274C It appears your data is not stored in a DataFrame. ',
          'To see what type of object your data is stored in, check its type with type(object)')

# Check that dataframe contains the appropriate number of NAN values
if student_ndvi_df.mean_ndvi.isna().sum() == 15:
    print('\u2705 Correct number of masked data values!')
    df_points += 2
else:
    print('\u274C The amount of null data in your dataframe is incorrect.')

# Ensure that the date column is the index
if isinstance(student_ndvi_df.index, pd.core.indexes.datetimes.DatetimeIndex):
    print('\u2705 You have the index set to the date column!')
    df_points += 3
else:
    print('\u274C You do not have the index set to the date column.')

# Ensure that the date column is datetime
if isinstance(student_ndvi_df.index[0], pd._libs.tslibs.timestamps.Timestamp):
    print('\u2705 The data in your date column is datetime!')
    df_points += 3
else:
    print('\u274C The data in your date column is not datetime.')

# Output for timer, # DO NOT MODIFY
end_time = datetime.now()
total_time = end_time - start_time
print(
    "Your total run time for processing the data was {0}.".format(total_time))

print("\n \u27A1 You received {} out of 10 points for creating a dataframe.".format(
    df_points))

df_points

# ## Task 2 Processing Explanation
#
# Task 2 builds upon the overall workflow defined during Task 1 processing, with the only difference being it is scaled to process multiple Landsat scenes. Although I wanted to try to avoid using nested loops, doing so enabled me to only open the cropping extent once during each "site level" iteration ("outer loop"), which is somewhat more efficient than what I initially planned to do - simply defining a list at containing all the scenes and iterating through it, which would have forced me to open the crop extent during every iteration.

# In[10]:

# Plotting initialization
f, ax = plt.subplots(figsize=(12, 8))

# Drop nan values in df in order to correctly "connect the dots" in the plot
final_df.dropna(inplace=True)

# Plotting the data according to each respective site location
for title, group in final_df.groupby("site"):
    group.groupby("site").plot(y="mean_ndvi",
                               label=title,
                               ax=ax,
                               alpha=.8)

# Added axes labels and title
ax.set(xlabel="Date",
       ylabel="NDVI Value",
       title="Mean NDVI \n Landsat 8 w/ Clouds Removed \n Jan-Dec 2017")

# Customization of the x-axis tick display
date_form = DateFormatter("%b %Y")
ax.xaxis.set_major_formatter(date_form)
ax.xaxis.set_major_locator(mdates.MonthLocator(interval=1))

### DO NOT REMOVE LINES BELOW ###
final_masked_solution = nb.convert_axes(plt, which_axes="current")

# In[11]:

# Ignore this cell for the autograding tests

# In[12]:

# Ignore this cell for the autograding tests

# # Question 1 (10 points)
#
# Imagine that you are planning NEON’s upcoming flight season to capture remote sensing data in these locations and want to ensure that you fly the area when the vegetation is the most green.
#
# When would you recommend the flights take place for each site?
#
# Answer the question in 2-3 sentences in the Markdown cell below.

# For the HARV field site, I'd suggest flying during the summer months (Jun-Sep) to capture "leaf-on" conditions. With the SJER site, late-winter/early-spring (Feb-May; wetter months) appear to be the most favorable for acquiring the "greenest" vegetation. The predominant control on vegetation "greenness" is likely temperature for the HARV site while for the SJER site it is precipitation.

# # Question 2 (10 points)
#
# How could you modify your workflow to look at vegetation changes over time in each site?
#
# Answer the question in 2-3 sentences in the Markdown cell below.

# Having a longer time series would be another way to look at the change through time. You could calculate the mean NDVI for each scene then aggregate by month/year/season. I'm not sure the code would change tremendously, although there might be another temporal directory level you'd need to loop through. That said, you could potentially bypass significant changes in that you could create a "master" pandas df containing all this information and perform the temporal grouping using pandas functionality (akin to what we did in the time-series and final lesson in the bootcamp during the fall....if memory serves me correct).

# # Do not edit this cell! (10 points)
#
# The notebook includes:
# * additional Markdown cells throughout the notebook to describe:
#     * the data that you used - and where it is from
#     * how data are being processing
#     * how the code is optimized to run fast and be more concise

# # Do not edit this cell! (20 points)
#
# The notebook will also be checked for overall clean code requirements as specified at the **top** of this notebook. Some of these requirements include (review the top cells for more specifics):
#
# * Notebook begins at cell [1] and runs on any machine in its entirety.
# * PEP 8 format is applied throughout (including lengths of comment and code lines).
# * No additional code or imports in the notebook that is not needed for the workflow.
# * Notebook is fully reproducible. This means:
#    * reproducible paths using the os module.
#    * data downloaded using code in the notebook.
#    * all imports at top of notebook.

# ## BONUS - Export a  .CSV File to Share (10 points possible)
#
# This is optional - if you export a **.csv** file with the columns specified above: Site, Date and NDVI Value you can get an additional 10 points.
#
# * FULL CREDIT: File exists in csv format and contains the columns specified.
# We will check your github repo for this file!
#

# In[13]:

# Write pandas df to disk as a csv file
# Note this file represents the df with nan values removed
out_csv = final_df.to_csv(os.path.join("ndvi-automation",
                                       "outputs",
                                       "mean_ndvi.csv"))

# In[ ]:


"""
rocklin_correction.py
A python module for performing Rocklin Correction.

Handles the primary functions
"""

from pkg_resources import resource_filename
from subprocess import call

import quantities as pq
import numpy as np
from gridData import Grid
import MDAnalysis as mda

from . import constants
from .waters import TIP3P

class RocklinCorrection():
    def __init__(self, box, lig_netq, protein_netq, temp=None, water=None):
        '''
        Parameters
        ----------
        box : array
            The unitcell dimensions of the system ``[lx, ly, lz]``.
        lig_netq: float
            The unit charge of the ligand.
        protein_netq : float
            The unit charge of the protein.
        temp : float
            The temperature of the system in K.
        water : water
            The water model being used.
        '''
        self.box = pq.Quantity(box, pq.angstrom)
        self.vol = self.box[0] * self.box[1] * self.box[2]
        self.lig_netq = pq.Quantity(lig_netq, pq.e)
        self.protein_netq = pq.Quantity(protein_netq, pq.e)

        if temp is None:
            self.temp = 298.15 * pq.Kelvin
        else:
            self.temp = pq.Quantity(temp, pq.Kelvin)
        if water is None:
            self.water = TIP3P
        else:
            self.water = water

    def set_APBS_input(self, NS, box=None, qL=None, qP=None,
                       in_prot_only='prot_only.pqr',
                       in_lig_in_prot='lig_in_prot.pqr',
                       in_lig_only='lig_only.pqr',
                       apbs_in='apbs.in'):
        ''' Manually set the input file for the APBS calculations.

        Parameters
        ----------
        NS : int
            The number of solvent molecule in the system.
        box : array, optional
            The unitcell dimensions of the system ``[lx, ly, lz]`` for APBS
            calculations.
        qL: float, optional
            The unit charge of the ligand.
        qP : float, optional
            The unit charge of the protein.
        in_prot_only : str, optional
            The name of the pqr file where the ligand has no partial charge.
            (``prot_only.pqr``)
        in_lig_in_prot : str, optional
            The name of the pqr file where the protein has no partial charge.
            (``lig_in_prot.pqr``)
        in_lig_only : str, optional
            The name of the pqr file of the ligand.
            (``lig_only.pqr``)
        apbs_in: str, optional
            The input file to the APBS program. (``apbs.in``)
        '''
        self.NS = NS

        if box is None:
            self.apbs_box = self.box
        else:
            self.apbs_box = pq.Quantity(box, pq.angstrom)

        if qL is None:
            self.apbs_qL = self.lig_netq
        else:
            self.apbs_qL = pq.Quantity(qL, pq.e)

        if qP is None:
            self.apbs_qP = self.protein_netq
        else:
            self.apbs_qP = pq.Quantity(qP, pq.e)

        self.apbs_vol = np.prod(self.apbs_box)

        self.in_prot_only = in_prot_only
        self.in_lig_in_prot = in_lig_in_prot
        self.in_lig_only = in_lig_only
        self.IP = None

        self._write_APBS_input(apbs_in)

    def make_APBS_input(self, universe, ligand_selection,
                        solvent_selection='resname SOL',
                        in_prot_only='prot_only.pqr',
                        in_lig_in_prot='lig_in_prot.pqr',
                        in_lig_only='lig_only.pqr',
                        apbs_in='apbs.in'):
        ''' Automatically setup the input file for the APBS calculations.

        Parameters
        ----------
        universe : MDAnalysis.Universe
            The Universe object of the system, where the coordinate,
            partial charge, radii and system dimension will be obtained.
        ligand_selection: str
            The selection string for the ligand.
        solvent_selection : str, optional
            The selection string for the solvent. (``resname SOL``)
        in_prot_only : str, optional
            The name of the pqr file where the ligand has no partial charge.
            (``prot_only.pqr``)
        in_lig_in_prot : str, optional
            The name of the pqr file where the protein has no partial charge.
            (``lig_in_prot.pqr``)
        in_lig_only : str, optional
            The name of the pqr file of the ligand.
            (``lig_only.pqr``)
        apbs_in: str, optional
            The input file to the APBS program. (``apbs.in``)

        Attributes
        ----------
        IP : float
            The integrated potential of protein will be set to 0, if no
            protein is found.
        NS : int
            The number of solvent molecules in the system.
        '''
        self.in_prot_only = in_prot_only
        self.in_lig_in_prot = in_lig_in_prot
        self.in_lig_only = in_lig_only

        box = universe.dimensions[:3]
        self.apbs_box = box * pq.angstrom
        charges = universe.atoms.charges

        # Charge only the ligand
        universe.select_atoms('not {}'.format(ligand_selection)).charges = 0
        self.apbs_qL = np.sum(universe.atoms.charges) * pq.e
        universe.select_atoms('not {}'.format(solvent_selection)).write(in_lig_in_prot)
        # Charge only the Rest of the system
        universe.atoms.charges = charges
        universe.select_atoms('{}'.format(ligand_selection)).charges = 0
        self.apbs_qP = np.sum(universe.atoms.charges) * pq.e
        universe.select_atoms('not {}'.format(solvent_selection)).write(in_prot_only)
        # Restore the charge
        universe.atoms.charges = charges

        # Check if there is anything other than ligand and solvent
        if len(universe.select_atoms('not (({}) or ({}))'.format(ligand_selection, solvent_selection))) > 0:
            self.IP = None
        else:
            self.IP = 0

        # Ligand for centering
        universe.select_atoms('{}'.format(ligand_selection)).write(in_lig_only)
        self.NS = len(universe.select_atoms(solvent_selection).residues)

        self._write_APBS_input(apbs_in)

    def _write_APBS_input(self, apbs_in):
        with open(resource_filename(__name__, 'data/apbs.in'), 'r') as f:
            txt = f.read()
        box = self.apbs_box
        with open(apbs_in, 'w') as f:
            f.write(txt.format(prot_only=self.in_prot_only,
                               lig_in_prot=self.in_lig_in_prot,
                               lig_only=self.in_lig_only,
                               x=box[0].magnitude, y=box[1].magnitude, z=box[2].magnitude,
                               e=self.water.epsilon_S,
                               t=self.temp.magnitude,))

    def run_APBS(self, apbs_exe='/opt/local/bin/apbs', apbs_in='apbs.in'):
        ''' Running the APBS calculations, which is the same as ::

          apbs apbs.in

        Parameters
        ----------
        apbs_exe : str, optional
            The path to the APBS program.
            (``/opt/local/bin/apbs``)
        apbs_in: str, optional
            The input file to the APBS program. (``apbs.in``)
        apbs_out : str, optional
            The output file for the APBS calculation. (``apbs.out``)
        '''
        call([apbs_exe, apbs_in])

    def _dx2IP(self, dx):
        g = Grid(dx)
        V = ((np.prod(g.delta*pq.angstrom)) * np.prod(g.grid.shape))
        self.apbs_vol = V
        return np.average(g.grid) * (1/pq.e) * (constants.kB * self.temp) * V

    def read_APBS(self, ligand_RIP_het='ligand_RIP_het.dx',
                  protein_RIP_het='protein_RIP_het.dx',
                  ligand_RIP_hom='ligand_RIP_hom.dx', IP=None, mean_IP=None):
        ''' Read the result from the APBS calculations.

        Parameters
        ----------
        ligand_RIP_het : str, optional
            The APBS program output (``ligand_RIP_het.dx``).
        protein_RIP_het : str, optional
            The APBS program output (``protein_RIP_het.dx``).
        ligand_RIP_hom : str, optional
            The APBS program output (``ligand_RIP_hom.dx``).
        IP : float, optional
            If system only has ligand, set the integrated potential of protein
            to 0.
        '''
        # For robust
        try:
            self.apbs_qL
        except AttributeError:
            self.apbs_qL = self.lig_netq

        try:
            self.apbs_qP
        except AttributeError:
            self.apbs_qP = self.protein_netq

        # Ligand Het
        IL_Bx = self._dx2IP(ligand_RIP_het)
        IL_BQx = (-constants.xi_CB * constants.coulomb_factor / self.water.epsilon_S) * self.apbs_qL * (self.apbs_vol ** (2.0 / 3.0))
        self.IL = IL_Bx - IL_BQx
        # Protein Het
        if IP is not None:
            self.IP = IP
        if mean_IP is not None:
            IP_Bx = mean_IP * (1 / pq.e) * (constants.kB * self.temp) * self.vol
            IP_BQx = (-constants.xi_CB * constants.coulomb_factor / self.water.epsilon_S) * self.apbs_qP * (self.apbs_vol ** (2.0 / 3.0))
            self.IP = IP_Bx - IP_BQx

        if self.IP is None:
            IP_Bx = self._dx2IP(protein_RIP_het)
            IP_BQx = (-constants.xi_CB * constants.coulomb_factor / self.water.epsilon_S) * self.apbs_qP * (self.apbs_vol ** (2.0 / 3.0))
            self.IP = IP_Bx - IP_BQx
        else:
            self.IP = pq.Quantity(self.IP, self.IL.units)
        # Ligand Het
        IL_hom_Bx = self._dx2IP(ligand_RIP_hom)
        IL_hom_BQx = (-constants.xi_CB * constants.coulomb_factor / 1) * self.apbs_qL * (self.apbs_vol ** (2.0 / 3.0))
        IL_hom = IL_hom_Bx - IL_hom_BQx
        self.IL_SLV = self.IL - IL_hom

    def compute(self, NS=None):
        ''' Compute the result.

        Parameters
        ----------
        NS : int, optional
            Rest the number of solvent molecules.

        Returns
        -------
        results : float
            The total correction free energy in cal/mol.
        '''
        if NS:
            self.NS = NS
        delta_DSC = - (self.water.gamma_s * self.lig_netq) / (6 * constants.epsilon_0) * self.NS / self.vol
        delta_NET = - constants.xi_LS / (8 * np.pi * constants.epsilon_0) * (
                    (self.protein_netq + self.lig_netq) ** 2 - self.protein_netq ** 2) / (self.vol ** (1/3))
        delta_NET_delta_USV = delta_NET / self.water.epsilon_S
        delta_RIP = ((self.IP + self.IL) * (self.protein_netq + self.lig_netq) - self.IP * self.protein_netq) / self.apbs_vol
        RL = ((1 / (8 * np.pi * constants.epsilon_0) * (4 * np.pi / 3) * (
                    1 - 1 / self.water.epsilon_S) * self.lig_netq) ** -1 * self.IL_SLV) ** 0.5
        delta_EMP = - 1 / (8 * np.pi * constants.epsilon_0) * (16 * np.pi ** 2 / 45) * (
                    1 - 1 / self.water.epsilon_S) * ((self.protein_netq + self.lig_netq) ** 2 - self.protein_netq ** 2) * \
                    RL ** 5 / self.vol ** 2
        delta_ANA = delta_NET_delta_USV + delta_RIP + delta_EMP
        delta = delta_ANA + delta_DSC

        self.output = []
        self.output.append(
            'The total correction energy is: {:.2f} kJ/mol or {:.2f} kCal/mol'.format(
                delta.rescale(pq.J / pq.mol).item() / 1000,
                delta.rescale(pq.cal / pq.mol).item() / 1000))
        self.output.append(
            '= ΔΔG_ANA(L): {:.2f} kJ/mol + ΔΔG_DSC(L): {:.2f} kJ/mol'.format(
                delta_ANA.rescale(pq.J / pq.mol).item() / 1000,
                delta_DSC.rescale(pq.J / pq.mol).item() / 1000))
        self.output.append(
            'ΔΔG_ANA(L) = ΔΔG_NET(L) + ΔΔG_USV(L) + ΔΔG_RIP(L) + ΔΔG_EMP(L)')
        self.output.append('ΔΔG_NET(L) = {:.2f} kJ/mol'.format(
            delta_NET.rescale(pq.J / pq.mol).item() / 1000))
        self.output.append('ΔΔG_NET(L) + ΔΔG_USV(L) = {:.2f} kJ/mol'.format(
            delta_NET_delta_USV.rescale(pq.J / pq.mol).item() / 1000))
        self.output.append('ΔΔG_RIP(L) = {:.2f} kJ/mol'.format(
            delta_RIP.rescale(pq.J / pq.mol).item() / 1000))
        self.output.append('ΔΔG_EMP(L) = {:.2f} kJ/mol'.format(
            delta_EMP.rescale(pq.J / pq.mol).item() / 1000))
        return delta.rescale(pq.calorie / pq.mol)

    def write(self, outfile):
        '''Write the decomposed results

        Parameters
        ----------
        outfile : str
            The output file name.'''
        with open(outfile, 'w') as f:
            f.write('\n'.join(self.output))


## python src/chapter2/chapter2.py
## python3 src/chapter2/chapter2.py

import sys
import numpy as nm
from numpy import arange
import matplotlib as mat
import matplotlib.pyplot as plt

class Chapter2:
    '''
    CLRS 第二章 2.1 2.2 算法函数和笔记
    '''
    def __init__(self, ok = 1, *args, **kwargs):
        '''
        Summary
        =
        These are notes of Peefy CLRS chapter1

        Parameters
        =
        *args : a tuple like
        **kwargs : a dict like

        Returns
        =
        self

        Example
        =
        >>> chapter2 = Chapter2(ok = 1);
        '''
        self.ok = ok

    def __hello():
        pass

    def insertSortAscending(self, array = []):
        '''
        Summary
        =
        插入排序的升序排列

        Parameter
        =
        array : a list like
        Return
        =
        sortedArray : 排序好的数组
        >>> array = [1, 3, 5, 2, 4, 6]
        >>> Chapter2().insertSortAscending(array)
        >>> [1, 2, 3, 4, 5, 6]
        '''
        A = array
        n = len(A)
        for j in range(1, n):
            ## Insert A[j] into the sorted sequece A[1...j-1] 前n - 1 张牌
            # 下标j指示了待插入到手中的当前牌，所以j的索引从数组的第二个元素开始
            # 后来摸的牌
            key = A[j]
            # 之前手中的已经排序好的牌的最大索引
            i = j - 1
            # 开始寻找插入的位置并且移动牌
            while(i >= 0 and A[i] > key):
                # 向右移动牌
                A[i + 1] = A[i]
                # 遍历之前的牌
                i = i - 1
            # 后来摸的牌插入相应的位置
            A[i + 1] = key
        # 输出升序排序后的牌
        return A

    def insertSortDescending(self, array = []):
        '''
        Summary
        =
        插入排序的降序排列

        Parameter
        =
        array : a list like

        Return
        =
        sortedArray : 排序好的数组
        >>> array = [1, 3, 5, 2, 4, 6]
        >>> Chapter2().insertSortAscending(array)
        >>> [6, 5, 4, 3, 2, 1]
        '''
        A = array
        n = len(A)
        for j in range(1, n):
            ## Insert A[j] into the sorted sequece A[1...j-1] 前n - 1 张牌
            # 下标j指示了待插入到手中的当前牌，所以j的索引从数组的第二个元素开始
            # 后来摸的牌
            key = A[j]
            # 之前手中的已经排序好的牌的最大索引
            i = j - 1
            # 开始寻找插入的位置并且移动牌
            while(i >= 0 and A[i] < key):
                # 向右移动牌
                A[i + 1] = A[i]
                # 遍历之前的牌
                i = i - 1
            # 后来摸的牌插入相应的位置
            A[i + 1] = key
        # 输出降序排序后的牌
        return A

    def arrayContains(self, array = [], v = None):
        '''
        Summary
        =
        * a function
        * *检测一个数组中是否包含一个元素*

        Parameter
        =
        *array* : a list like
        v : a element

        Return
        =
        index:若找到返回找到的索引，没找到返回None

        Example:
        =
        >>> array = [12, 23, 34, 45]
        >>> v = 23
        >>> m = 55
        >>> Chapter2().arrayContains(array, v)
        >>> 1
        >>> Chapter2().arrayContains(array, m)
        >>> None
        '''
        index = None
        length = len(array)
        for i in range(length):
            if v == array[i]:
                index = i
        return index

    def twoNBinNumAdd(self, A = [], B = []):
        '''
        Summary
        =
        两个存放数组A和B中的n位二进制整数相加

        Parameter
        ====
        A : a list like and element of the list must be 0 or 1
        B : a list like and element of the list must be 0 or 1

        Return
        ======
        returnSum : sum of two numbers

        Example:
        =
        >>> A = [1, 1, 0, 0]
        >>> B = [1, 0, 0, 1]
        >>> Chapter2().twoNBinNumAdd(A, B)
        >>> [1, 0, 1, 0, 1]
        '''
        if len(A) != len(B):
            raise Exception('length of A must be equal to length of B')
        length = len(A)
        # 注意：range 函数和 arange 函数都是 左闭右开区间
        '''
        >>> range(0,3)
        >>> [0, 1, 2]
        '''
        returnSum = arange(length + 1)
        bitC = 0
        for i in range(length):
            index = length - 1 - i
            bitSum = A[index] + B[index] + bitC
            if bitSum >= 2:
                bitSum = 0
                bitC = 1
            else:
                bitC = 0
            returnSum[index + 1] = bitSum
            if index == 0:
                returnSum[0] = bitC
        return returnSum

    def selectSortAscending(self, array = []):
        '''
        Summary
        =
        选择排序的升序排列

        Parameter
        =
        array : a list like
        Return
        =
        sortedArray : 排序好的数组
        >>> array = [1, 3, 5, 2, 4, 6]
        >>> Chapter2().selectSortAscending(array)
        >>> [1, 2, 3, 4, 5, 6]
        '''
        A = array
        length = len(A)
        for j in range(length):
            minIndex = j
            # 找出A中第j个到最后一个元素中的最小值
            # 仅需要在头n-1个元素上运行
            for i in range(j, length):
                if A[i] <= A[minIndex]:
                    minIndex = i
            # 最小元素和最前面的元素交换
            min = A[minIndex]
            A[minIndex] = A[j]
            A[j] = min
        return A

    def note(self, *args, **kwargs):
        '''
        Summary
        =
        These are notes of Peefy CLRS chapter1

        Parameters
        =
        *args : a tuple like
        **kwargs : a dict like

        Returns
        =
        self

        Example
        =
        >>> Chapter2().note()
        '''
        print('排序算法有很多，包括插入排序，冒泡排序，堆排序，归并排序，选择排序，计数排序，基数排序，桶排序，快速排序')
        print('2.1 插入排序')
        print('插入排序(INSERTION-SORT):输入n个数，输出n个数的升序或者降序排列')
        print('插入排序是一个对少量元素进行排序的有效算法，工作做原理与打牌摸牌整理手中的牌差不多')
        print('以下是Python的插入排序(升序)算法(模拟打牌)')
        print('书中的伪代码数组索引从1开始，python数组索引从0开始')
        A = [4, 4.5, 2, 5, 1.2, 3.5]
        print("待排序的序列：", A)
        print("插入排序后的序列：", self.insertSortAscending(A))
        print('循环不变式主要用来帮助理解插入算法的正确性。证明循环不变式的三个性质')
        print(' 1.初始化：在循环的第一轮迭代开始前，应该是正确的')
        print(' 2.保持：如果在循环的某一次迭代开始之前它是正确的，那么在下一次迭代开始前，它也应该保持正确')
        print(' 3.终止：当循环结束时，不变式给了我们一个有用的性质，有助于表明算法是正确的')
        print('数学归纳法中，要证明某一性质是成立的，必须首先证明其基本情况和一个归纳步骤都是成立的')
        print('插入排序的循环不变式证明：')
        print(' 1.初始化：插入排序第一步首相将数组中第二个元素当做待插入的元素，被插入的元素只有数组中第一元素，显然一个元素是已经排序好的')
        print(' 2.保持：证明每一轮循环都能时循环不变式保持成立,同时证明外层for循环和内层while循环同时满足循环不变式')
        print(' 3.终止：当j大于n时，外层循环结束，新的数组包含了原来数组中的元素，并且是排序好的，算法正确')
        print('布尔运算符and和or都具有短路运算能力')
        print('练习2.1-1：对于序列[31, 41, 59, 26, 41, 58]首先选出序列中第二个元素41向前插入(升序)，接下来选出59向前插入，依次类推')
        print('练习2.1-2：只要把书中插入排序中的伪代码的不等号方向更换即可')
        A = [31, 41, 59, 26, 21, 58]
        print('  排序好的降序序列为：', self.insertSortDescending(A))
        print('练习2.1-3：结果如下:')
        print('  32在序列A中的索引为(索引从0开始)：', self.arrayContains(A, 32))
        print('  21在序列A中的索引为(索引从0开始)：', self.arrayContains(A, 21))
        print('练习2.1-4：两个n位二进制数相加的算法(适用于FPGA中。参考一位加法器的逻辑表达式或者数学表达式)：')
        print(' 两个n位二进制数的和为：', self.twoNBinNumAdd([1, 1, 0, 1], [1, 0, 0, 0]))
        # range函数和np.arange函数都是左闭右开区间
        print('range(0,4)的值为：', [i for i in range(0,4)])
        print('2.2 算法分析')
        print('算法分析即指对一个算法所需要的资源进行预测')
        print('内存，通信带宽或者计算机硬件等资源是关心的资源, 通常资源指我们希望测度的计算时间')
        print('采用单处理器、随机存取机RAM计算模型')
        print('RAM模型包含了真实计算机中常见的指令：算数指令(加法，减法，除法，取余，向下取整，向上取整指令)，数据移动指令(装入、存储、复制)和控制指令(条件和非条件转移、子程序调用和返回指令)')
        print('RAM模型中的数据类型有整数类型和浮点实数类型')
        print('算法分析所需要的数学工具包括组合数学、概率论、代数')
        print('需要对"运行时间"和"输入规模"更仔细地加以定义')
        print('插入排序算法的分析')
        print('插入排序INSERTION=SORT过程的时间开销与输入有关，排序1000个数的事件比排序三个数的时间要长')
        print('插入算法即使对给定规模的输入，运行时间也有可能依赖于给定的是该规模下的哪种输入')
        print('插入排序当输入是最好情况(即输入的序列已经按顺序排好)，插入排序所需要的时间随输入规模是线性的O(n)')
        print('插入排序当输入是按照逆序排序的(降序排列输入后输出升序排列),就会出现最坏情况,所需要时间是输入规模的二次函数O(n^2)')
        print('一般考察算法的最坏情况运行时间')
        print('当然对于一些"随机化"算法，其行为即使对于固定的输入，运行时间也是可以变化的')
        print('做进一步的抽象：即运行时间的增长率，只考虑算法运行时间公式中的最高次项，并且忽略最高次项的常数系数,时间复杂度')
        print('练习题2.2-1: n^3/1000 - 100n^2 - 100n + 3 的时间复杂度：O(n^3)')
        print('练习题2.2-2:选择排序如下：')
        A = [21, 11, 9, 66, 51, 48]
        print(' 选择排序排列前的元素：', A)
        print(' 选择排序排列后的元素：', self.selectSortAscending(A))
        print(' 选择排序最好情况o(n^2),最坏情况o(n^2)')
        print(' 因为在第n-1次比较选择的时候已经比较出了最大元素和次大元素并选择，所以这时选择完之后第n个元素已经是最大值，没有必要再比较下去了')
        print('练习题2.2-3:线性查找最好情况是o(1),最坏情况是o(n)，平均情况是(n)')
        print('要使算法具有较好的最佳情况运行时间就一定要对输入进行控制，使之偏向能够使得算法具有最佳运行情况的排列。')

        #python src/chapter2/chapter2.py
        #python3 src/chapter2/chapter2.py
        return self

if __name__ == '__main__':
    print('Run main : single chapter two!')
    Chapter2().note()
else:
    pass

## python src/chapter2/chapter2.py
## python3 src/chapter2/chapter2.py


#
# anim_svg.py:  render/animate PhysiCell .svg files, using left/right arrows on keyboard
#
# Usage:
#  python anim_svg.py <show_nucleus start_index axes_min axes_max>
#    i.e., the arguments <...> are optional and have defaults.
#
# Keyboard arrows: right/left arrows will single step forward/backward; up/down will increment/decrement step size
#
# Dependencies include matplotlib and numpy. We recommend installing the Anaconda Python3 distribution.
#
# Examples (run from directory containing the .svg files):
#  python anim_svg.py
#  python anim_svg.py 0 5 700 1300
#
# Author: Randy Heiland (except for the circles() function)
#
#
__author__ = "Randy Heiland"

import sys
import glob
import os
import xml.etree.ElementTree as ET
import math
join_our_list = "(Join/ask questions at https://groups.google.com/forum/#!forum/physicell-users)\n"
try:
  import matplotlib
  import matplotlib.colors as mplc
  from matplotlib.patches import Circle, Ellipse, Rectangle
  from matplotlib.collections import PatchCollection
except:
  print("\n---Error: cannot import matplotlib")
  print("---Try: python -m pip install matplotlib")
  print(join_our_list)
#  print("---Consider installing Anaconda's Python 3 distribution.\n")
  raise
try:
  import numpy as np  # if mpl was installed, numpy should have been too.
except:
  print("\n---Error: cannot import numpy")
  print("---Try: python -m pip install numpy\n")
  print(join_our_list)
  raise
from collections import deque
try:
  # apparently we need mpl's Qt backend to do keypresses
#  matplotlib.use("Qt5Agg")
  matplotlib.use("TkAgg")
  import matplotlib.pyplot as plt
except:
  print("\n---Error: cannot use matplotlib's TkAgg backend")
  print(join_our_list)
#  print("Consider installing Anaconda's Python 3 distribution.")
  raise

current_idx = 0
print("# args=",len(sys.argv)-1)

#for idx in range(len(sys.argv)):
use_defaults = True
show_nucleus = 0
current_idx = 0
axes_min = 0.0
axes_max = 1000  # but overridden by "width" attribute in .svg
if (len(sys.argv) == 5):
  use_defaults = False
  kdx = 1
  show_nucleus = int(sys.argv[kdx])
  kdx += 1
  current_idx = int(sys.argv[kdx])
  kdx += 1
  axes_min = float(sys.argv[kdx])
  kdx += 1
  axes_max = float(sys.argv[kdx])
elif (len(sys.argv) != 1):
  print("Please provide either no args or 4 args:")
  usage_str = "show_nucleus start_index axes_min axes_max"
  print(usage_str)
  print("e.g.,")
  eg_str = "%s 0 0 0 2000" % (sys.argv[0])
  print(eg_str)
  sys.exit(1)

#"""
print("show_nucleus=",show_nucleus)
print("current_idx=",current_idx)
print("axes_min=",axes_min)
print("axes_max=",axes_max)
#"""

"""
if (len(sys.argv) > 1):
   current_idx = int(sys.argv[1])
if (len(sys.argv) > 2):
   axes_min = float(sys.argv[2])
   axes_max = float(sys.argv[3])

if (len(sys.argv) > 4):
  usage_str = "[<start_index> [<axes_min axes_max>]]"
  print(usage_str)
  print("e.g.,")
  eg_str = "%s 1 10 700 1300" % (sys.argv[0])
  print(eg_str)
  sys.exit(1)
"""

print("current_idx=",current_idx)

#d={}   # dictionary to hold all (x,y) positions of cells

"""
--- for example ---
In [141]: d['cell1599'][0:3]
Out[141]:
array([[ 4900.  ,  4900.  ],
       [ 4934.17,  4487.91],
       [ 4960.75,  4148.02]])
"""

fig = plt.figure(figsize=(7,7))
ax = fig.gca()
#ax.set_aspect("equal")

#plt.ion()

time_delay = 0.1

count = -1
#while True:

#-----------------------------------------------------
def circles(x, y, s, c='b', vmin=None, vmax=None, **kwargs):
    """
    See https://gist.github.com/syrte/592a062c562cd2a98a83

    Make a scatter plot of circles.
    Similar to plt.scatter, but the size of circles are in data scale.
    Parameters
    ----------
    x, y : scalar or array_like, shape (n, )
        Input data
    s : scalar or array_like, shape (n, )
        Radius of circles.
    c : color or sequence of color, optional, default : 'b'
        `c` can be a single color format string, or a sequence of color
        specifications of length `N`, or a sequence of `N` numbers to be
        mapped to colors using the `cmap` and `norm` specified via kwargs.
        Note that `c` should not be a single numeric RGB or RGBA sequence
        because that is indistinguishable from an array of values
        to be colormapped. (If you insist, use `color` instead.)
        `c` can be a 2-D array in which the rows are RGB or RGBA, however.
    vmin, vmax : scalar, optional, default: None
        `vmin` and `vmax` are used in conjunction with `norm` to normalize
        luminance data.  If either are `None`, the min and max of the
        color array is used.
    kwargs : `~matplotlib.collections.Collection` properties
        Eg. alpha, edgecolor(ec), facecolor(fc), linewidth(lw), linestyle(ls),
        norm, cmap, transform, etc.
    Returns
    -------
    paths : `~matplotlib.collections.PathCollection`
    Examples
    --------
    a = np.arange(11)
    circles(a, a, s=a*0.2, c=a, alpha=0.5, ec='none')
    plt.colorbar()
    License
    --------
    This code is under [The BSD 3-Clause License]
    (http://opensource.org/licenses/BSD-3-Clause)
    """

    if np.isscalar(c):
        kwargs.setdefault('color', c)
        c = None

    if 'fc' in kwargs:
        kwargs.setdefault('facecolor', kwargs.pop('fc'))
    if 'ec' in kwargs:
        kwargs.setdefault('edgecolor', kwargs.pop('ec'))
    if 'ls' in kwargs:
        kwargs.setdefault('linestyle', kwargs.pop('ls'))
    if 'lw' in kwargs:
        kwargs.setdefault('linewidth', kwargs.pop('lw'))
    # You can set `facecolor` with an array for each patch,
    # while you can only set `facecolors` with a value for all.

    zipped = np.broadcast(x, y, s)
    patches = [Circle((x_, y_), s_)
               for x_, y_, s_ in zipped]
    collection = PatchCollection(patches, **kwargs)
    if c is not None:
        c = np.broadcast_to(c, zipped.shape).ravel()
        collection.set_array(c)
        collection.set_clim(vmin, vmax)

    ax = plt.gca()
    ax.add_collection(collection)
    ax.autoscale_view()
    plt.draw_if_interactive()
    if c is not None:
        plt.sci(collection)
    return collection

#-----------------------------------------------------
def plot_svg():
  global current_idx, axes_max
  fname = "snapshot%08d.svg" % current_idx
  if (os.path.isfile(fname) == False):
    print("File does not exist: ",fname)
    return

  xlist = deque()
  ylist = deque()
  rlist = deque()
  rgb_list = deque()

#  print('\n---- ' + fname + ':')
  tree = ET.parse(fname)
  root = tree.getroot()
#  print('--- root.tag ---')
#  print(root.tag)
#  print('--- root.attrib ---')
#  print(root.attrib)

#  print('--- child.tag, child.attrib ---')
  numChildren = 0
  for child in root:
#    print(child.tag, child.attrib)
#    print("keys=",child.attrib.keys())
    if use_defaults and ('width' in child.attrib.keys()):
      axes_max = float(child.attrib['width'])
#      print("--- found width --> axes_max =", axes_max)
    if child.text and "Current time" in child.text:
      svals = child.text.split()
      title_str = "(" + str(current_idx) + ") Current time: " + svals[2] + "d, " + svals[4] + "h, " + svals[7] + "m"

#    print("width ",child.attrib['width'])
#    print('attrib=',child.attrib)
#    if (child.attrib['id'] == 'tissue'):
    if ('id' in child.attrib.keys()):
#      print('-------- found tissue!!')
      tissue_parent = child
      break

#  print('------ search tissue')
  cells_parent = None

  for child in tissue_parent:
#    print('attrib=',child.attrib)
    if (child.attrib['id'] == 'cells'):
#      print('-------- found cells, setting cells_parent')
      cells_parent = child
      break
    numChildren += 1

  num_cells = 0
#  print('------ search cells')
  for child in cells_parent:
#    print(child.tag, child.attrib)
#    print('attrib=',child.attrib)
    for circle in child:  # two circles in each child: outer + nucleus
    #  circle.attrib={'cx': '1085.59','cy': '1225.24','fill': 'rgb(159,159,96)','r': '6.67717','stroke': 'rgb(159,159,96)','stroke-width': '0.5'}
#      print('  --- cx,cy=',circle.attrib['cx'],circle.attrib['cy'])
      xval = float(circle.attrib['cx'])

      s = circle.attrib['fill']
#      print("s=",s)
#      print("type(s)=",type(s))
      if (s[0:3] == "rgb"):  # if an rgb string, e.g. "rgb(175,175,80)"
        #  circle.attrib={'cx': '1085.59','cy': '1225.24','fill': 'rgb(159,159,96)','r': '6.67717','stroke': 'rgb(159,159,96)','stroke-width': '0.5'}
        rgb = list(map(int, s[4:-1].split(",")))
        rgb[:]=[x/255. for x in rgb]
      else:     # otherwise, must be a color name
        rgb_tuple = mplc.to_rgb(mplc.cnames[s])  # a tuple
        rgb = [x for x in rgb_tuple]

      # test for bogus x,y locations (rwh TODO: use max of domain?)
      too_large_val = 10000.
      if (math.fabs(xval) > too_large_val):
        print("bogus xval=",xval)
        break
      yval = float(circle.attrib['cy'])
      if (math.fabs(yval) > too_large_val):
        print("bogus xval=",xval)
        break

      rval = float(circle.attrib['r'])
#      print('rval=',rval)

      xlist.append(xval)
      ylist.append(yval)
      rlist.append(rval)
      rgb_list.append(rgb)
#      print('rgb_list = ',rgb_list)

#     For .svg files with cells that *have* a nucleus, there will be a 2nd
      if (show_nucleus == 0):
        break

    num_cells += 1

#    if num_cells > 3:   # for debugging
#      print(fname,':  num_cells= ',num_cells," --- debug exit.")
#      sys.exit(1)
#      break

  print(fname,':  num_cells= ',num_cells)

  xvals = np.array(xlist)
  yvals = np.array(ylist)
  rvals = np.array(rlist)
  rgbs =  np.array(rgb_list)
#  print('type(rgbs) = ',type(rgbs))
#  print('rgbs = ',rgbs)
#print("xvals[0:5]=",xvals[0:5])
#print("rvals[0:5]=",rvals[0:5])
#  print("rvals.min, max=",rvals.min(),rvals.max())

  plt.cla()
  title_str += " (" + str(num_cells) + " agents)"
  plt.title(title_str)
  plt.xlim(axes_min,axes_max)
  plt.ylim(axes_min,axes_max)
#  plt.scatter(xvals,yvals, s=rvals*scale_radius, c=rgbs)
#  plt.scatter(xvals,yvals, s=rvals*scale_radius, c=rgbs, alpha=0.5, edgecolor='black')
#  plt.scatter(xvals,yvals, s=rvals*scale_radius, c=rgbs, alpha=1.0, edgecolor='black')
#  circles(xvals,yvals, s=rvals, c=rgbs, alpha=1.0, edgecolor='black')
#  circles(xvals,yvals, s=rvals)
#  circles(xvals,yvals, s=rvals, c=rgbs)
  circles(xvals,yvals, s=rvals, color=rgbs)
#plt.xlim(0,2000)  # TODO - get these values from width,height in .svg at top
#plt.ylim(0,2000)
  plt.pause(time_delay)

step_value = 1
def press(event):
  global current_idx, step_value
#    print('press', event.key)
  sys.stdout.flush()
  if event.key == 'escape':
    sys.exit(1)
  elif event.key == 'h':  # help
    print('esc: quit')
    print('right arrow: increment by step_value')
    print('left arrow:  decrement by step_value')
    print('up arrow:   increment step_value by 1')
    print('down arrow: decrement step_value by 1')
    print('0: reset to 0th frame')
    print('h: help')
  elif event.key == 'left':  # left arrow key
#    print('go backwards')
#    fig.canvas.draw()
    current_idx -= step_value
    if (current_idx < 0):
      current_idx = 0
    plot_svg()
  elif event.key == 'right':  # right arrow key
#        print('go forwards')
#        fig.canvas.draw()
    current_idx += step_value
    plot_svg()
  elif event.key == 'up':  # up arrow key
    step_value += 1
    print('step_value=',step_value)
  elif event.key == 'down':  # down arrow key
    step_value -= 1
    if (step_value <= 0):
      step_value = 1
    print('step_value=',step_value)
  elif event.key == '0':  # reset to 0th frame/file
    current_idx = 0
    plot_svg()
  else:
    print('press', event.key)

#for current_idx in range(40):
#  fname = "snapshot%08d.svg" % current_idx
#  plot_svg(fname)
plot_svg()
print("\nNOTE: click in plot window to give it focus before using keys.")

fig.canvas.mpl_connect('key_press_event', press)

# keep last plot displayed
#plt.ioff()
plt.show()


#!/usr/bin/env python
# coding: utf-8

# # ETHZ: 227-0966-00L
# # Quantitative Big Imaging
# # March 14, 2019
#
# ## Basic Segmentation and Discrete Binary Structures
# ### Part 1

# # Lesson Outline
# - Motivation
# - Qualitative Approaches
# - Thresholding
#  - Other types of images
#  - Selecting a good threshold
# - Implementation
# - Morphology
# - Contouring / Mask Creation

# ### Applications
#
# - Simple two-phase materials (bone, cells, etc)
#   - Beyond 1 channel of depth
# - Multiple phase materials
# - Filling holes in materials
# - Segmenting Fossils
# - Attempting to segment the cortex in brain imaging
# ![Cortex Image](ext-figures/cortex.png)

# # Literature / Useful References
#
# - John C. Russ, “The Image Processing Handbook”,(Boca Raton, CRC Press)
#  - Available [online](http://dx.doi.org/10.1201/9780203881095) within domain ethz.ch (or proxy.ethz.ch / public VPN)
#
# ### Models / ROC Curves
#
# - [Julia Evans - Recalling with Precision](https://www.youtube.com/watch?v=ryZL4XNUmwo)
# - [Stripe's Next Top Model](https://github.com/stripe/topmodel)

# # Motivation:  Why do we do imaging experiments?
#
#
# - Exploratory
#  - To visually, qualitatively examine samples and differences between them
#  - No prior knowledge or expectations
# - To test a hypothesis
#  - Quantitative assessment coupled with statistical analysis
#  - Does temperature affect bubble size?
#  - Is this gene important for cell shape and thus mechanosensation in bone?
#  - Does higher canal volume make bones weaker?
#  - Does the granule shape affect battery life expectancy?
#

# - What we are looking at
# ![Standard Cell, http://en.wikipedia.org/wiki/File:Average_prokaryote_cell-_en.svg](ext-figures/Average_prokaryote_cell.svg)
# - What we get from the imaging modality

# In[1]:

from skimage.io import imread
from skimage.color import rgb2gray
import matplotlib.pyplot as plt

dkimg = imread("../common/figures/Average_prokaryote_cell.jpg")
plt.matshow(rgb2gray(dkimg), cmap="bone")

# # To test a hypothesis
# - We perform an experiment bone to see how big the cells are inside the tissue
# $$\downarrow$$ ![Bone Measurement](ext-figures/tomoimage.png)
#
# ### 2560 x 2560 x 2160 x 32 bit = 56GB / sample
# - Filtering and Preprocessing!
# $$\downarrow$$
# - 20h of computer time later ...
# - 56GB of less noisy data
# - Way too much data, we need to reduce
#

# # What did we want in the first place
#
#
# ### _Single number_:
# * volume fraction,
# * cell count,
# * average cell stretch,
# * cell volume variability

# # Why do we perform segmentation?
#
# - In model-based analysis every step we peform, simple or complicated is related to an underlying model of the system we are dealing with
# - [_Occam's Razor_](http://en.wikipedia.org/wiki/Occams_Razor) is very important here : The simplest solution is usually the right one
#  - Bayesian, neural networks optimized using genetic algorithms with Fuzzy logic has a much larger parameter space to explore, establish sensitivity in, and must perform much better and be tested much more thoroughly than thresholding to be justified.
#  - We will cover some of these techniques in the next 2 lectures since they can be very powerful particularly with unknown data

# # Review: Filtering and Image Enhancement
#
#
# - This was a noise process which was added to otherwise clean imaging data
# - $$ I_{measured}(x,y) = I_{sample}(x,y) + \text{Noise}(x,y) $$
# - What would the perfect filter be
#  - $$ \textit{Filter} \ast I_{sample}(x,y) = I_{sample}(x,y) $$
#  - $$ \textit{Filter} \ast \text{Noise}(x,y) = 0 $$
#  - $$ \textit{Filter} \ast I_{measured}(x,y) = \textit{Filter} \ast I_{real}(x,y) + \textit{Filter}\ast \text{Noise}(x,y) \rightarrow \bf I_{sample}(x,y) $$
# - What most filters end up doing
# $$ \textit{Filter} \ast I_{measured}(x,y) = 90\%  I_{real}(x,y) + 10\% \text{Noise}(x,y) $$
# - What bad filters do
# $$ \textit{Filter} \ast I_{measured}(x,y) = 10\% I_{real}(x,y) + 90\% \text{Noise}(x,y) $$

# # Qualitative Metrics: What did people used to do?
#
# - What comes out of our detector / enhancement process

# In[2]:

from skimage.io import imread
from skimage.color import rgb2gray
import matplotlib.pyplot as plt

dkimg = rgb2gray(imread("../common/figures/Average_prokaryote_cell.jpg"))
fig, (ax_hist, ax_img) = plt.subplots(1, 2, figsize=(12, 6))

ax_hist.hist(dkimg.ravel())
ax_hist.set_xlabel("Absorption Coefficient")
ax_hist.set_ylabel("Pixel Count")

m_show_obj = ax_img.matshow(dkimg, cmap="bone")
cb_obj = plt.colorbar(m_show_obj)
cb_obj.set_label("Absorption Coefficient")

# - Identify objects by eye
#  - Count, describe qualitatively: "many little cilia on surface", "long curly flaggelum", "elongated nuclear structure"
# - Morphometrics
#  - Trace the outline of the object (or sub-structures)
#  - Can calculate the area by using equal-weight-paper
#  - Employing the "[cut-and-weigh](http://ion.chem.usu.edu/~sbialkow/Classes/361/GC/GC.html)" method
#

# # Segmentation Approaches
#
#
# They match up well to the world view / perspective
#
# ![Approaches](../common/figures/approaches.png)

# ### Model-Based
#
# - $\rightarrow$ Experimentalist
# - Problem-driven
#  - Top-down
#  - _Reality_ Model-based
#
#
# ### Machine Learning Approach
#
# - $\rightarrow$ Computer Vision / Deep Learning
# - Results-driven

# # Model-based Analysis
#
# ![Traditional Imaging](../common/figures/image-formation.png)
#
# - Many different imaging modalities ( $\mu \textrm{CT}$ to MRI to Confocal to Light-field to AFM).
# - Similarities in underlying equations
#  - different coefficients, units, and mechanism
#
# $$ I_{measured}(\vec{x}) = F_{system}(I_{stimulus}(\vec{x}),S_{sample}(\vec{x})) $$

# # Direct Imaging (simple)
#
# In many setups there is un-even illumination caused by incorrectly adjusted equipment and fluctations in power and setups
#
# - $F_{system}(a,b) = a*b$
# - $I_{stimulus} = \textrm{Beam}_{profile}$
# - $S_{system} = \alpha(\vec{x})$
#
# $\longrightarrow \alpha(\vec{x})=\frac{I_{measured}(\vec{x})}{\textrm{Beam}_{profile}(\vec{x})}$
#
#

# In[3]:

from skimage.io import imread
from skimage.color import rgb2gray
import matplotlib.pyplot as plt
from skimage.morphology import disk
from scipy.ndimage import zoom
import numpy as np

cell_img = 1 - rgb2gray(imread("../common/figures/Average_prokaryote_cell.jpg"))
s_beam_img = np.pad(
    disk(2) / 1.0, [[1, 1], [1, 1]], mode="constant", constant_values=0.2
)
beam_img = zoom(s_beam_img, [cell_img.shape[0] / 7.0, cell_img.shape[1] / 7.0])

fig, (ax_beam, ax_img, ax_det) = plt.subplots(1, 3, figsize=(12, 4))

ax_beam.imshow(beam_img, cmap="hot")
ax_beam.set_title("Beam Profile")

ax_img.imshow(cell_img, cmap="hot")
ax_img.set_title("Sample Profile")

ax_det.imshow(cell_img * beam_img, cmap="hot")
ax_det.set_title("Detector")

# In[4]:

fig, (ax_prof) = plt.subplots(1, 1, figsize=(12, 4))

ax_prof.plot(beam_img[beam_img.shape[0] // 2], label="Beam Profile")
ax_prof.plot(cell_img[beam_img.shape[0] // 2], label="Sample Image")
ax_prof.plot((cell_img * beam_img)[beam_img.shape[0] // 2], label="Detector")
ax_prof.set_ylabel("Intensity")
ax_prof.set_xlabel("Pixel Position")
# make an interactive plot
import plotly.offline as py
import plotly.tools as tls

py.init_notebook_mode()
py.iplot(tls.mpl_to_plotly(fig))

# Frequently there is a fall-off of the beam away from the center (as is the case of a Gaussian beam which frequently shows up for laser systems). This can make extracting detail away from the center challenging

# In[5]:

fig, ax1 = plt.subplots(1, 1, figsize=(8, 8))
ax1.matshow(cell_img * beam_img, cmap="hot")

# # Absorption Imaging (X-ray, Ultrasound, Optical)
#
# - For absorption/attenuation imaging $\rightarrow$ [Beer-Lambert Law](http://en.wikipedia.org/wiki/Attenuation_coefficient)
#  $$ I_{detector} = \underbrace{I_{source}}_{I_{stimulus}}\underbrace{\exp(-\alpha d)}_{S_{sample}} $$
#  - Different components have a different $\alpha$ based on the strength of the interaction between the light and the chemical / nuclear structure of the material
# $$ I_{sample}(x,y) = I_{source}\exp(-\alpha(x,y) d) $$
# $$ \alpha = f(N,Z,\sigma,\cdots) $$
#
# - For segmentation this model is:
#  - there are 2 (or more) distinct components that make up the image
#  - these components are distinguishable by their values (or vectors, colors, tensors, ...)
#

# In[6]:

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

I_source = 1.0
d = 1.0
alpha_1 = np.random.normal(1, 0.25, size=100)
alpha_2 = np.random.normal(2, 0.25, size=100)
alpha_3 = np.random.normal(3, 0.5, size=100)

abs_df = pd.DataFrame(
    [
        dict(alpha=c_x, material=c_mat)
        for c_vec, c_mat in zip(
            [alpha_1, alpha_2, alpha_3], ["material 1", "material 2", "material 3"]
        )
        for c_x in c_vec
    ]
)
abs_df["I_detector"] = I_source * np.exp(-abs_df["alpha"] * d)

fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(12, 8))
for c_mat, c_df in abs_df.groupby("material"):
    ax1.scatter(x=c_df["alpha"], y=c_df["I_detector"], label=c_mat)
    ax3.hist(c_df["alpha"], alpha=0.5, label=c_mat)
    ax2.hist(c_df["I_detector"], alpha=0.5, label=c_mat, orientation="horizontal")
ax1.set_xlabel("$\\alpha(x,y)$", fontsize=15)
ax1.set_ylabel("$I_{detector}$", fontsize=18)
ax1.legend()
ax2.legend()
ax3.legend(loc=0)

ax4.axis("off")

# # Example Mammography
# Mammographic imaging is an area where model-based absorption imaging is problematic. Even if we assume a constant illumination (_rarely_ the case),
#
# $$ I_{detector} = \underbrace{I_{source}}_{I_{stimulus}}\underbrace{\exp(-\alpha d)}_{S_{sample}} $$
# $$ \downarrow $$
# $$ I_{detector} = \exp(-\alpha(x,y) d(x,y)) $$
# $$ \downarrow $$
# $$ I_{detector} = \exp(-\int_{0}^{l}\alpha(x,y, z) dz) $$
#

# Specifically the problem is related to the inability to separate the $\alpha$ and $d$ terms. We model a basic breast volume as a half sphere with a constant absorption factor.
#
# $$\alpha(x,y,z) = 1e-2$$
#
# The $\int$ then turns into a $\Sigma$ in discrete space

# In[7]:

import matplotlib.pyplot as plt
import numpy as np
from skimage.morphology import ball

breast_mask = ball(50)[:, 50:]

# just for 3d rendering, don't worry about it
import plotly.offline as py
from plotly.figure_factory import create_trisurf

py.init_notebook_mode()
from skimage.measure import marching_cubes_lewiner

vertices, simplices, _, _ = marching_cubes_lewiner(breast_mask > 0)
x, y, z = zip(*vertices)
fig = create_trisurf(
    x=x, y=y, z=z, plot_edges=False, simplices=simplices, title="Breast Phantom"
)
py.iplot(fig)

# In[8]:

breast_alpha = 1e-2
breast_vol = breast_alpha * breast_mask
i_detector = np.exp(-np.sum(breast_vol, 2))

fig, (ax_hist, ax_breast) = plt.subplots(1, 2, figsize=(20, 8))

b_img_obj = ax_breast.imshow(i_detector, cmap="bone_r")
plt.colorbar(b_img_obj)

ax_hist.hist(i_detector.flatten())
ax_hist.set_xlabel("$I_{detector}$")
ax_hist.set_ylabel("Pixel Count")

# If we know that $\alpha$ is constant we can reconstruct d from the image

# In[9]:

breast_thickness = -np.log(i_detector) / breast_alpha
fig, (ax_hist, ax_breast) = plt.subplots(1, 2, figsize=(12, 5))

b_img_obj = ax_breast.imshow(breast_thickness, cmap="bone")
plt.colorbar(b_img_obj)

ax_hist.hist(breast_thickness.flatten())
ax_hist.set_xlabel("Breast Thickness ($d$)\nIn Pixels")
ax_hist.set_ylabel("Pixel Count")

# In[10]:

from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure(figsize=(8, 4), dpi=200)
ax = fig.gca(projection="3d")
# Plot the surface.
yy, xx = np.meshgrid(
    np.linspace(0, 1, breast_thickness.shape[1]),
    np.linspace(0, 1, breast_thickness.shape[0]),
)
surf = ax.plot_surface(
    xx, yy, breast_thickness, cmap=plt.cm.jet, linewidth=0, antialiased=False
)
ax.view_init(elev=30, azim=45)
ax.set_zlabel("Breast Thickness")

# We run into problems when the $\alpha$ is no longer constant. For example if we place a dark lump in the center of the breast. It is impossible to tell if the breast if thicker or if the lump inside is denser. For the lump below we can see on the individual slices of the sample that the lesion appears quite clearly and is very strangely shaped.

# In[11]:

breast_vol = breast_alpha * breast_mask
renorm_slice = np.sum(breast_mask[10:40, 0:25], 2) / np.sum(breast_mask[30, 10])
breast_vol[10:40, 0:25] /= np.stack([renorm_slice] * breast_vol.shape[2], -1)

from skimage.util import montage as montage2d

fig, ax1 = plt.subplots(1, 1, figsize=(12, 12))
ax1.imshow(
    montage2d(breast_vol.swapaxes(0, 2).swapaxes(1, 2)[::3]),
    cmap="bone",
    vmin=breast_alpha * 0.8,
    vmax=breast_alpha * 1.2,
)

# When we make the projection and apply Beer's Law we see that it appears as a relatively constant region in the image

# In[12]:

i_detector = np.exp(-np.sum(breast_vol, 2))

fig, (ax_hist, ax_breast) = plt.subplots(1, 2, figsize=(20, 8))

b_img_obj = ax_breast.imshow(i_detector, cmap="bone_r")
plt.colorbar(b_img_obj)

ax_hist.hist(i_detector.flatten())
ax_hist.set_xlabel("$I_{detector}$")
ax_hist.set_ylabel("Pixel Count")

# And as a flat constant region in the thickness reconstruction. So we fundamentally from this single image cannot answer is the breast oddly shaped or does it have an possible tumor inside of it

# In[13]:

breast_thickness = -np.log(i_detector) / 1e-2
fig, (ax_hist, ax_breast) = plt.subplots(1, 2, figsize=(12, 5))

b_img_obj = ax_breast.imshow(breast_thickness, cmap="bone")
plt.colorbar(b_img_obj)

ax_hist.hist(breast_thickness.flatten())
ax_hist.set_xlabel("Breast Thickness ($d$)\nIn Pixels")
ax_hist.set_ylabel("Pixel Count")

# In[14]:

from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure(figsize=(8, 4), dpi=200)
ax = fig.gca(projection="3d")
# Plot the surface.
yy, xx = np.meshgrid(
    np.linspace(0, 1, breast_thickness.shape[1]),
    np.linspace(0, 1, breast_thickness.shape[0]),
)
surf = ax.plot_surface(
    xx, yy, breast_thickness, cmap=plt.cm.jet, linewidth=0, antialiased=False
)
ax.view_init(elev=30, azim=130)
ax.set_zlabel("Breast Thickness")

# # Where does segmentation get us?
#
#
# - We convert a decimal value (or something even more complicated like 3 values for RGB images, a spectrum for hyperspectral imaging, or a vector / tensor in a mechanical stress field)
# - to a single, discrete value (usually true or false, but for images with phases it would be each phase, e.g. bone, air, cellular tissue)
#
# - __2560 x 2560 x 2160 x 32 bit = 56GB / sample__
# $$\downarrow$$
# - 2560 x 2560 x 2160 x **1 bit** = 1.75GB / sample
#

# # Applying a threshold to an image
# Start out with a simple image of a cross with added noise
# $$ I(x,y) = f(x,y) $$

# In[15]:

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

nx = 5
ny = 5
xx, yy = np.meshgrid(
    np.arange(-nx, nx + 1) / nx * 2 * np.pi, np.arange(-ny, ny + 1) / ny * 2 * np.pi
)
cross_im = 1.5 * np.abs(np.cos(xx * yy)) / (
    np.abs(xx * yy) + (3 * np.pi / nx)
) + np.random.uniform(-0.25, 0.25, size=xx.shape)
plt.matshow(cross_im, cmap="hot")
plt.colorbar()

# The intensity can be described with a probability density function
# $$ P_f(x,y) $$

# In[16]:

fig, ax1 = plt.subplots(1, 1)
ax1.hist(cross_im.ravel(), 20)
ax1.set_title("$P_f(x,y)$")
ax1.set_xlabel("Intensity")
ax1.set_ylabel("Pixel Count")

# # Applying a threshold to an image
#
# By examining the image and probability distribution function, we can _deduce_ that the underyling model is a whitish phase that makes up the cross and the darkish background
#
# Applying the threshold is a deceptively simple operation
#
# $$ I(x,y) =
# \begin{cases}
# 1, & f(x,y)\geq0.40 \\
# 0, & f(x,y)<0.40
# \end{cases}$$

# In[17]:

fig, ax1 = plt.subplots(1, 1)
ax1.imshow(cross_im, cmap="hot", extent=[xx.min(), xx.max(), yy.min(), yy.max()])
thresh_img = cross_im > 0.4

ax1.plot(
    xx[np.where(thresh_img)],
    yy[np.where(thresh_img)],
    "ks",
    markerfacecolor="green",
    alpha=0.5,
    label="threshold",
    markersize=20,
)
ax1.legend()

# # Various Thresholds
# We can see the effect of choosing various thresholds
#

# In[18]:

fig, m_axs = plt.subplots(3, 3, figsize=(15, 15))
for c_thresh, ax1 in zip(np.linspace(0.1, 0.9, 9), m_axs.flatten()):

    ax1.imshow(cross_im, cmap="bone", extent=[xx.min(), xx.max(), yy.min(), yy.max()])
    thresh_img = cross_im > c_thresh

    ax1.plot(
        xx[np.where(thresh_img)],
        yy[np.where(thresh_img)],
        "rs",
        alpha=0.5,
        label="img>%2.2f" % c_thresh,
        markersize=20,
    )
    ax1.legend(loc=1)

# # Segmenting Cells
#
# - We can peform the same sort of analysis with this image of cells
# - This time we can derive the model from the basic physics of the system
#  - The field is illuminated by white light of nearly uniform brightness
#  - Cells absorb light causing darker regions to appear in the image
#  - _Lighter_ regions have no cells
#  - __Darker__ regions have cells

# In[19]:

from skimage.io import imread
from skimage.color import rgb2gray
import matplotlib.pyplot as plt
import numpy as np

cell_img = rgb2gray(imread("../common/figures/Cell_Colony.jpg"))
fig, (ax_hist, ax_img) = plt.subplots(1, 2, figsize=(12, 6))
ax_hist.hist(cell_img.ravel(), np.arange(255))
ax_obj = ax_img.matshow(cell_img, cmap="bone")
plt.colorbar(ax_obj)

# In[20]:

from skimage.color import label2rgb

fig, m_axs = plt.subplots(3, 3, figsize=(15, 15), dpi=200)
for c_thresh, ax1 in zip(np.linspace(100, 200, 9), m_axs.flatten()):
    thresh_img = cell_img < c_thresh

    ax1.imshow(label2rgb(thresh_img, image=1 - cell_img, bg_label=0, alpha=0.4))

    ax1.set_title("img<%2.2f" % c_thresh)

# # Other Image Types
#
# While scalar images are easiest, it is possible for any type of image
# $$ I(x,y) = \vec{f}(x,y) $$

# In[21]:

import pandas as pd
import matplotlib.pyplot as plt
import numpy as np

nx = 10
ny = 10
xx, yy = np.meshgrid(
    np.linspace(-2 * np.pi, 2 * np.pi, nx), np.linspace(-2 * np.pi, 2 * np.pi, ny)
)

intensity_img = 1.5 * np.abs(np.cos(xx * yy)) / (
    np.abs(xx * yy) + (3 * np.pi / nx)
) + np.random.uniform(-0.25, 0.25, size=xx.shape)

base_df = pd.DataFrame(
    dict(x=xx.ravel(), y=yy.ravel(), I_detector=intensity_img.ravel())
)

base_df["x_vec"] = base_df.apply(
    lambda c_row: c_row["x"]
    / np.sqrt(1e-2 + np.square(c_row["x"]) + np.square(c_row["y"])),
    1,
)
base_df["y_vec"] = base_df.apply(
    lambda c_row: c_row["y"]
    / np.sqrt(1e-2 + np.square(c_row["x"]) + np.square(c_row["y"])),
    1,
)

base_df.head(5)

# In[22]:

import seaborn as sns

sns.pairplot(base_df)

# In[23]:

fig, ax1 = plt.subplots(1, 1, figsize=(8, 8))
ax1.quiver(
    base_df["x"],
    base_df["y"],
    base_df["x_vec"],
    base_df["y_vec"],
    base_df["I_detector"],
    cmap="hot",
)

# # Applying a threshold
#
# A threshold is now more difficult to apply since there are now two distinct variables to deal with. The standard approach can be applied to both
# $$ I(x,y) =
# \begin{cases}
# 1, & \vec{f}_x(x,y) \geq0.25 \text{ and}\\
# & \vec{f}_y(x,y) \geq0.25 \\
# 0, & \text{otherwise}
# \end{cases}$$

# In[24]:

thresh_df = base_df.copy()
thresh_df["thresh"] = thresh_df.apply(
    lambda c_row: c_row["x_vec"] > 0.25 and c_row["y_vec"] > 0.25, 1
)

fig, ax1 = plt.subplots(1, 1, figsize=(8, 8))
ax1.quiver(
    thresh_df["x"],
    thresh_df["y"],
    thresh_df["x_vec"],
    thresh_df["y_vec"],
    thresh_df["thresh"],
    cmap="hot",
)

# This can also be shown on the joint probability distribution as

# In[25]:

fig, ax1 = plt.subplots(1, 1, figsize=(4, 4), dpi=200)
ax1.hist2d(thresh_df["x_vec"], thresh_df["y_vec"], cmap="hot")
ax1.set_xlabel("$\\vec{f}_x(x,y)$")
ax1.set_ylabel("$\\vec{f}_y(x,y)$")

# # Applying a threshold
# Given the presence of two variables; however, more advanced approaches can also be investigated. For example we can keep only components parallel to the x axis by using the dot product.
# $$ I(x,y) =
# \begin{cases}
# 1, & |\vec{f}(x,y)\cdot \vec{i}| = 1 \\
# 0, & \text{otherwise}
# \end{cases}$$

# # Looking at Orientations
# We can tune the angular acceptance by using the fact $$\vec{x}\cdot\vec{y}=|\vec{x}| |\vec{y}| \cos(\theta_{x\rightarrow y}) $$
# $$ I(x,y) =
# \begin{cases}
# 1, & \cos^{-1}(\vec{f}(x,y)\cdot \vec{i}) \leq \theta^{\circ} \\
# 0, & \text{otherwise}
# \end{cases}$$


#!/usr/bin/env python
# coding: utf-8

# # Sentiment Analysis end-to-end example
#
# This example is brought to you by Udacity - consider doing the great Udacity Deep Learning course. Find out more [here](https://www.udacity.com/course/deep-learning-nanodegree--nd101).
#
# > These are my own personal notes
#
# ----

# In this notebook, the aium is to build `TODO`
#
# We begin by looking at the dataset we have:
# - reviews.txt: reviews of a movie
# - labels.txt: positive/negative label associated with the movie
#
# We will use the python `open()` function to open the file, with the parameter `'r'` to read the file. Using `readlines` will return a list made up of each line in the file, returned as a list item. Hence, each character will be an item in the list.

# In[1]:

review_file = open('sentiment_data/reviews.txt', 'r')
reviews = list(map(lambda x : x[:-1], review_file.readlines()))
review_file.close()

label_file = open('sentiment_data/labels.txt', 'r')
labels = list(map(lambda x : x[:-1].upper(), label_file.readlines()))
label_file.close()

# Now lets find some information out about our data.

# In[2]:

print(f'Size of our data: {len(reviews)}')
print(f'No of labels: {len(labels)}')
print('\nNow, lets see one row of our data. First feature in our data:')
print(reviews[0])
print('\nPrediction:')
print(labels[0])

# ---

# ## Now, lets build up a hypothesis
#
# We will begin by looking at our data, and trying to see what conclusions we can draw. This is ofter called the `exploratory` phase. We will begin by looking at some random predictions...

# In[3]:

def print_review_with_label(ith_row):
    print(labels[ith_row] + '\t:\t' + reviews[ith_row][:80] + '...')

# Using the function above, we can beautifully print our data; feature along with its prediction.

# In[4]:

print("labels.txt \t : \t reviews.txt\n")
print_review_with_label(2137)
print_review_with_label(12816)
print_review_with_label(6267)
print_review_with_label(21934)

# We will be using the `Counter` python class throughout this section, as it provides a nice way to count the occurances of words.

# In[5]:

from collections import Counter
import numpy as np

# In[6]:

positive_words_counter = Counter()
negative_words_counter = Counter()
total_words_counter = Counter()
example_counter_with_stuff = Counter([1,2,3,4,4,4])

def counter_pretty_print():
    print('positive counter: ', positive_words_counter)
    print('negative counter: ', negative_words_counter)
    print('total words counter: ', total_words_counter)

print('At this stage, our counters are empty...')
counter_pretty_print()
print('Here is a test counter: ', example_counter_with_stuff)

# Now, lets fill out our three counters.

# In[7]:

# for each row in our dataset
for sentence_no in range(len(reviews)):
   # for each word in our sentence
   for word in reviews[sentence_no].split(' '):
       # if it is positive - add a positive counter
       if labels[sentence_no] == 'POSITIVE':
           positive_words_counter[word] +=1
       # if it is negative - add to negative counter
       if labels[sentence_no] == 'NEGATIVE':
           negative_words_counter[word] +=1
       # regardless, add to total word counter
       total_words_counter[word] +=1

# In[8]:

# lets take a look at the most common words.
print('Most common positive words:\n')
positive_words_counter.most_common()

# In[9]:

# lets take a look at the most common words.
print('\nMost common negative words:\n')
negative_words_counter.most_common()

# Instead of looking at the counts of the words, lets now instead look at the ratios between words. Looking at how often words occur, either positive or negative, does not really give us what we are looking for. e.g. you can see there are a lot of common words between both the positive and negative counters. Instead, by looking at a raio, we will be looking at the words that are found in positive reviews over negative, and vice versa.
#
# This will basically tell us how many more times a word is seen in positive reviews than in the negatives. e.g. we can imagine that positive reviews use the word "love" more, hence the ratio should be larger. Hence:
# - Positive words will have a large ratio - bigger than 1
# - Negative words will have a smaller ratio - less than 1
# - words that are neither positive or negative, but neutral, will be centered around 0

# In[10]:

positive_to_negative_ratio = Counter()

for word, count, in list(total_words_counter.most_common()):
    if count > 100:
        positive_to_negative_ratio[word] = positive_words_counter[word] / (negative_words_counter[word] + 1) # +1 so we dont divide by 0

# Now, lets take a look at some words...

# In[11]:

print(f'positive:negative ratio for the word and: {round(positive_to_negative_ratio["and"],2)}')
print(f'positive:negative ratio for the word good: {round(positive_to_negative_ratio["best"],2)}')
print(f'positive:negative ratio for the word bad: {round(positive_to_negative_ratio["bad"],2)}')

# Okay, but is a score of 2 twice as good as other scores? With the ratios as they are now, it will be difficult to actually compare the scores. So instead, we will do what every computer scientists loves to do, which is to log the numbers.
#
# To find out more about why computer scientists love log, feel free to watch the series by Killian Weiberger on Machine Learning [here](https://www.youtube.com/watch?v=MrLPzBxG95I&list=PLl8OlHZGYOQ7bkVbuRthEsaLr7bONzbXS)

# In[12]:

for word, count in positive_to_negative_ratio.most_common():
    positive_to_negative_ratio[word] = np.log(count)

# Now, lets take a look at the log(words)...

# In[13]:

print(f'positive:negative ratio for the word and: {round(positive_to_negative_ratio["and"],2)}')
print(f'positive:negative ratio for the word good: {round(positive_to_negative_ratio["best"],2)}')
print(f'positive:negative ratio for the word bad: {round(positive_to_negative_ratio["bad"],2)}')

# You can see now that:
# - positive words are close to +1
# - negative words are close to -1
# - neutral words are centered around 0
#
# Now, to close our hypothesis section where we wanted to draw a hypothesis from the data, we will take a peek at our ratio data.

# In[14]:

positive_to_negative_ratio.most_common()[0:20]

# As we expected, some positive words like `flawless` and `perfection` have high scores....but also `lincoln`. Interesting.

# In[15]:

list(reversed(positive_to_negative_ratio.most_common()))[0:20]

# There are some funny negative words, including `lousy` and `unwatcheable`. But again, some interesting words like `prom`.

# ## Transforming words into numbers
#
# we now need to prerpare our words so that we can feed them into our neural network. In order to do that, we want to transform them so we can do the maths of neural networks.
#
# What we want to do for our network, is build a dictionary. With this dictionary, we will count each word in our input review, and feed that into the network.
#
# As we have already built a Count object that has every word possible from our training data, we are able to now compare each single review from our dataset, and see how often each word occurs per review. This will allow us to feed our reviews into the network whilst maintaining consistency between inputs.
#
# We will begin by building a `vocab`, a set that contains all the words.

# In[16]:

vocab = set(total_words_counter.keys())

# Vocab is s Set, similar to the mathematical set. This means that it only has each word appearing only once.
#
# Now, lets take a look at how our Neural network will look.
#
# ![image of our neural network](sentiment_network.png)
#
# You can see that our NN will have:
# - one input layer:
#     - This will be the Vocab
#     - we will represent this as a np array
# - one hidden layer
# - one output layer that has one output neuron

# In[17]:

layer_0 = np.zeros(shape=(1,len(vocab)))

# lets take a look at the first layer...

# In[18]:

layer_0.shape

# This first layer now has a neuron/input per word from our vocab. With the input being a count of how many times the word occurs in the review. However, to pass words from a review into this first layer, we need to be able to build a way that will allow us to feed a new review in with the words organised the same way as the first layer in our network.

# In[19]:

word_to_index_translator = {}
# lets map each word in our vocab to an index, and capture that as a dictionary
for index, word in enumerate(vocab):
    word_to_index_translator[word] = index

# lets temporarily use a Counter object to look at the first few rows in our dictionary
Counter(word_to_index_translator).most_common(5)

# now, lets build a function that can take a new review, and spit out a vector that matches the input layer.

# In[20]:

def input_for_input_layer(review):
    ''' New input layer, layer_0, for our network to train on.

    layer_0 represents how many times a word occurs in a review.

    Args:
        review (str) : a review for a movie
    Returns:
        None
    '''
    global layer_0
    # clear out previous layer 0
    layer_0 *=0
    for word in review.split(' '):
        # find index location of the word from our vocab
        index_of_word = word_to_index_translator[word]
        # add it to our layer 0
        layer_0[:, index_of_word] += 1

# Lets test this by feeding it a review.
# Before we test it, lets look at layer_0

# In[21]:

layer_0

# In[22]:

input_for_input_layer(reviews[200])
layer_0

# Great, it has updated layer_0.
#
# Now, we will build a function that can take a label (e.g. POSITIVE or NEGATIVE), and return either 1 or 0. This is needed as our network needs to be built ontop of numbers, and not strings.

# In[23]:

def translate_label(label):
    '''Converts label to 0 or 1.

    Args:
        label (str) : POSITIVE or NEGATIVE label for a review
    RETURNS:
        0 : if negative
        1 : if positive
    '''
    if label == 'POSITIVE':
        return 1
    else:
        return 0

# again, lets test this by running a label into our function.

# In[24]:

print(f'testing +ve label: {labels[200]}')
print(f'This is the output from our function: {translate_label(labels[200])}')
print(f'\ntesting -ve label: {labels[1]}')
print(f'This is the output from our function: {translate_label(labels[1])}')

# Great, so it works.
#
# Now it is finally time to build our Neural Network!
#
# We will:
# - build a basic neural network that has an input layer, hidden layer and an output layer
# - we will not be adding non-linearity in our hidden layer
# - we will use the same functions we defined above, to build up our training data set
# - we will create a vocab from our training data
# - we will train over the entire corpus

# In[25]:

import sentiment_network
import importlib
importlib.reload(sentiment_network)

# In[113]:

mlp = sentiment_network.SentimentNetwork(reviews[:-1000],labels[:-1000], learning_rate=0.1)

# In[114]:

mlp.test(reviews[-1000:],labels[-1000:])

# In[115]:

mlp.train(reviews[:-1000],labels[:-1000])

# In[130]:

mlp2 = sentiment_network.SentimentNetwork(reviews[:-1000],labels[:-1000], learning_rate=0.001)
mlp2.train(reviews[:-1000],labels[:-1000])


from PIL import Image, ImageOps
from src.models.cc0 import patcher
import numpy as np
import skimage.io as io
from src.utils.imgproc import *
from skimage.color import rgb2hsv, hsv2rgb

class patcher(patcher):
    def __init__(self, body='./body/body_orion.png', **options):
        try:
            options = options['options']
        except:
            pass
        options['is_4k'] = True
        options['with_bra'] = True
        super().__init__(options=options)
        self.name = 'オリオン'
        self.body = Image.open(body)
        self.body_size = self.body.size
        self.pantie_position = [407, 838]

        self.frill_position = [361, 1412]
        self.frill_shade = np.float32(io.imread('./material/orion_frill.png')[:, :, -1] / 255)

    def pick_color(self, arr):
        return np.mean(np.mean(arr, axis=0), axis=0)

    def gen_frill(self, image):
        pantie = np.array(image)
        front = pantie[20:100, 30:80, :3] / 255.0
        front_shade = pantie[130:150, 0:40, :3] / 255.0
        front_color = self.pick_color(front)
        front_shade_color = self.pick_color(front_shade)
        r, c = self.frill_shade.shape
        frill = np.ones((r, c, 3), dtype=np.float32) * front_color
        frill_shade = self.frill_shade[:, :, None] * front_shade_color
        frill = alpha_brend(frill_shade, frill, self.frill_shade)
        frill = np.dstack((frill, frill[:, :, 0] > 0))
        return Image.fromarray(np.uint8(np.clip(frill, 0, 1) * 255))

    def patch(self, image, transparent=False):
        pantie = self.convert(image)
        if transparent:
            patched = Image.new("RGBA", (2048, 2048))
        else:
            patched = self.body.copy()
        patched = self.paste(patched, pantie, self.pantie_position)
        bra = self.gen_bra(image)
        patched = self.paste(patched, bra, self.bra_position)
        patched = self.paste(patched, ImageOps.mirror(bra), [self.bra_position[0] - bra.width, self.bra_position[1]])
        patched = self.paste(patched, self.gen_frill(image), self.frill_position)
        return patched