""" Final Project – Draft Report
ALY 6110 Data Management and Big Data, Fall 2021
Module 5 Group Assignment 5

Ankit Yadav, Neil Mascarenhas, Sai Anila Sushma Malladhi

College of Professional Studies, Northeastern University
Prof: Daya Rudhramoorthi
Oct 20th, 2021

"""

#%%
# import modules
from IPython.core.display import display_png
from IPython.display import display
import numpy as np
import pandas as pd
import re, os, glob
import matplotlib.pyplot as plt
from matplotlib import rcParams, cycler
import seaborn as sns
from scipy import interpolate

# scikit-learn machine learning
# from sklearn.preprocessing import Normalizer, StandardScaler, normalize, scale
from sklearn import metrics
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.neighbors import KNeighborsClassifier

# statsmodels
from statsmodels.formula.api import ols
from statsmodels.stats.anova import anova_lm
from statsmodels.stats.multicomp import pairwise_tukeyhsd

# map functions (https://scitools.org.uk/cartopy/docs/v0.15/)
import cartopy.feature as cfeature
import cartopy.crs as ccrs
from cartopy.io.img_tiles import OSM
osm_tiles = OSM()

# ordinary least squares
from statsmodels.formula.api import ols

plt.style.use('seaborn-colorblind')

# THis is a special Jupyter Notebook command to prepare the notebook for matplotlib and other libraries
#%matplotlib inline

# Setting up Pandas paramenters so that we see all the results
pd.set_option('display.width', 500)
pd.set_option('display.max_columns', 100)
pd.set_option('display.notebook_repr_html', True)

# Setup for Seaborn
sns.set_style("darkgrid")
#sns.set_context("notebook")

## Read the data, and doing basic fixes.

#%%
# read file(s)
path = os.getcwd()
all_files = glob.glob(os.path.join(path+'\\Data\\', "allemployees_*.csv"))

#%%

# look at the first ten thousand bytes to guess the character encoding
import chardet
with open("Data/allemployees_2020.csv", 'rb') as rawdata:
    result = chardet.detect(rawdata.read(10000))

# check what the character encoding might be
display(result)

#%%
col_names = ['name', 'department', 'title', 'regular', 'retro', 'other', 'overtime', 'injured',\
                'detail', 'quinn', 'total', 'zip']

#enocode encoding = "ISO-8859-5",

df_from_each_file = (pd.read_csv(f, delimiter=',', encoding='utf-8', \
                                 header=0, names=col_names, \
                                 index_col=None).assign(year=f) for f in all_files)  # read year from filename
EmpEarn   = pd.concat(df_from_each_file, ignore_index=True)
display(EmpEarn.info())

#%%
## Glimpese of the data.
display(EmpEarn.head(5))
display(EmpEarn.isna().sum())

#%%
## The basic cleaning steps we will be performing are:

#%%
# 1. We will extract year from path (filename of earnings report) and assign to new column "year"

## Before
display(EmpEarn.year.head(5))

# extract year from filename
EmpEarn['year'] = EmpEarn['year'].replace({'\D':''}, regex=True)
EmpEarn['year'] = EmpEarn['year'].str[-4:]  # remove any numbers from file path

## After
display(EmpEarn.year.head(5))

#%%
# 2. Columns "department" and "title" are in reverse order for 2013 and 2014

#EmpEarn.where(EmpEarn['year']=='2021').isin(["department","title"])

# Before
print("Read Carefully")
display(EmpEarn.loc[EmpEarn['year'].isin(['2013'])].filter(items=["department","title"]).head(5))
display(EmpEarn.loc[EmpEarn['year'].isin(['2014'])].filter(items=["department","title"]).head(5))

# switch "department" and "title" columns for 2013 and 2014
EmpEarn.loc[EmpEarn.year.isin(['2013', '2014']),['department','title']] = EmpEarn.loc[EmpEarn.year.isin(['2013', '2014']),['title','department']].values

## After

#EmpEarn.where(EmpEarn['year']=='2021').isin(["department","title"])
print("Read Carefully")
display(EmpEarn.loc[EmpEarn['year'].isin(['2013'])].filter(items=["department","title"]).head(5))
display(EmpEarn.loc[EmpEarn['year'].isin(['2014'])].filter(items=["department","title"]).head(5))

#%%
# 3. Make all Zip code, upto 5 digits Trim set of 5+ digit zipcodes

## Before
display(EmpEarn['zip'].sample(n = 5, random_state=1234))

# ignore "+4" zip codes
EmpEarn['zip'] = EmpEarn['zip'].str[:5]

##
display(EmpEarn['zip'].sample(n = 5, random_state=1234))

#%%
# 4. Zipcodes are 5 digits for some years, 4 for other years and only 4 digits in 2017-20 where the leading "0" has been dropped, Adding the leading zero.

## Before
EmpEarn['zip'] = np.where(EmpEarn['zip'].str.len() == 4, '0' + EmpEarn['zip'], EmpEarn['zip'])

display(EmpEarn['zip'].sample(n = 5, random_state=1234))

#%%
# 5. Missing zipcodes can be filled by comparing to previous year's employee entry

## Before
display(EmpEarn.zip.isna().sum())

# Asding missing zip codes from the previous year data.

EmpEarn.loc[EmpEarn['zip'].str.len() < 4, 'zip'] = np.NaN
EmpEarn['zip'] = EmpEarn.sort_values(by='name')['zip'].fillna(method='ffill')

## After
display(EmpEarn.zip.isna().sum())

#%%
# 6. Convert all numbers to numeric dtype

EmpEarn.dtypes

# converting number strings to numeric dtype
num_cols = ['regular', 'retro', 'other', 'overtime', 'injured', 'detail', 'quinn', 'total']
EmpEarn[num_cols] = EmpEarn[num_cols].replace({'\$': '', ',': ''}, regex=True)\
                                .apply(pd.to_numeric, errors='coerce').fillna(0, axis=1)

## After
EmpEarn.dtypes
#%%
# 7. Replacing , with a <space> in the names

## Before
display("Before",EmpEarn.name.head(3))

EmpEarn.name=EmpEarn.name.apply(lambda x: x.replace(',',' '))
EmpEarn.department=EmpEarn.department.apply(lambda x: x.replace(',',' '))
EmpEarn.title=EmpEarn.title.apply(lambda x: x.replace(',',' '))

## After
display("After",EmpEarn.name.head(3))

#%%
# 8. Consolidating Depaerment names

# unique job titles and departments by year
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(10,6))
ax1.bar(EmpEarn['year'].unique(), EmpEarn.groupby('year')['title'].nunique())
ax1.set_title('unique job titles')
ax1.set_ylabel('count')
ax1.set_xlabel('year')

ax2.bar(EmpEarn['year'].unique(), EmpEarn.groupby('year')['department'].nunique())
ax2.set_title('unique department names')
ax2.set_ylabel('count')
ax2.set_xlabel('year')

plt.tight_layout()
plt.show()

#%%

# combine all school departments into a single "Boston Public Schools" department

EmpEarn['dept_clean'] = EmpEarn['department'] # all others to stay the same
EmpEarn['dept_clean'] = np.where(EmpEarn.department.astype(str).str[:3] == \
                                  'BPS', 'Boston Public Schools', EmpEarn.dept_clean)
EmpEarn['dept_clean'] = np.where(EmpEarn.department.astype(str).str[:10] == \
                                  'Asst Super', 'Boston Public Schools', EmpEarn.dept_clean)

bps = ['K-8', 'EEC', 'ELC', 'Middle', 'School', 'Academy', 'Elementary', 'Greenwood',
       'E Leadership Acad', 'UP Academy Dorchester', 'UP "Unlocking Potential" Acad',
       'Lyon Pilot High 9-12', 'Ellison/Parks EES', 'Chief Academic Officer',
       'UP Academy Holland', 'Achievement Gap', 'English Language Learn', 'Haley Pilot',
       'Greater Egleston High', 'Early Learning Services', 'Career & Technical Ed',
       'Teaching & Learning', 'Unified Student Svc', 'Superintendent',
       'Student Support Svc', 'Harbor High', 'Fam & Student Engagemt',
       'Enrollment Services', 'Food & Nutrition Svc', 'HPEC: Com Acd Science & Health',
       'Institutional Advancemt', 'Legal Advisor', 'Professional Developmnt',
       'Chief Operating Officer', 'Research Assess & Eval', 'Info & Instr Technology',
       'BTU Pilot', 'Boston Collaborative High Sch', 'Diplomas Plus', 'Chief Financial Officer']
for school in bps:
    EmpEarn['dept_clean'] = np.where(EmpEarn.department.astype(str).str[-len(school):] == \
                                      school, 'Boston Public Schools', EmpEarn.dept_clean)

#%%
print(set(EmpEarn.dept_clean.loc[EmpEarn.year == '2020'].unique()) - \
      set(EmpEarn.dept_clean.loc[EmpEarn.year == '2019'].unique()))

#%%

"""
Rename various departments to 2020 name:

"""

dept_names = {'Transportation Department': 'Traffic Division',
              'Dept of Voter Mobilization': 'Election Division',
              'State Boston Retirement Syst': 'Boston Retirement System',
              'Youth Fund': 'Youth Engagement & Employment',
              'Administration and Finance': 'Office of Admin & Finance',
              'Office of Finance & Budget': 'Office of Admin & Finance',
              'Office Of Civil Rights': 'Fair Housing & Equity',
              'Small & Local Business': 'Office of Economic Development',
              'Ofc Chf Public Works Transport': 'Office of Streets',
              'Ofc of Strts, Trnsp & Sani': 'Office of Streets',
              'Property Management': 'Public Facilities Department',
              'Mayor\'s Office-Public Info': 'Mayor\'s Office',
              'Arts & Cultural Development': 'Office of Arts & Culture',
              'Accountability': 'Licensing Board',
              'Women\'s Commission': 'Women\'s Advancement'}
for d in dept_names:
    EmpEarn['dept_clean'] = np.where(EmpEarn['department'].str[-len(d):] == \
                                 d, dept_names[d], EmpEarn.dept_clean)

#%%

# pivot to obtain number of employees in each department by year
departments = pd.pivot_table(EmpEarn, values='name', index='dept_clean',  columns='year', aggfunc='count')\
                    .sort_values('2020', ascending=False).fillna(0)

# spread over 3 sub plots for better visibility
fig, (ax1, ax2, ax3) = plt.subplots(nrows=3, figsize=(8, 8), sharex=True)
ax1.set_title('Department Size by Year')
departments.iloc[:8].T.plot(ax=ax1, cmap='tab10') # large departments with log scale
ax1.set_yscale('log')
departments.iloc[8:16].T.plot(ax=ax2, cmap='tab20') # medium departments
departments.iloc[16:24].T.plot(ax=ax3, cmap='tab20b') # small departments
ax2.set_ylabel('Number of Employees')
ax1.legend(bbox_to_anchor=(1.01, 1.0))
ax2.legend(bbox_to_anchor=(1.01, 1.0))
ax3.legend(bbox_to_anchor=(1.01, 1.0))
ax3.set_xticklabels(['none', '2011', '2012', '2013', '2014', '2015', '2016'])

plt.tight_layout()
plt.show()

#%%
## Consolidate Job Titles

# police titles:
police_titles = EmpEarn.loc[(EmpEarn.department == 'Boston Police Department')\
                            & (EmpEarn.title.str.startswith('Police O'))\
                            & (EmpEarn.year == '2020')]\
                            ['title'].value_counts()
print(police_titles)

#%%

# consolidate police department titles

EmpEarn['title_clean'] = EmpEarn['title'] # all others to stay the same
pol = {'Police Of': 'Police Officer',
#         'Police De': 'Police Officer',
        'Police Se': 'Police Sergeant',
        'PoliceSer': 'Police Sergeant',
        'Police Ca': 'Police Captain',
        'Police Li': 'Police Lieutenant'}
for p in pol:
    EmpEarn['title_clean'] = np.where(EmpEarn.title.str[:9] == p, pol[p], EmpEarn.title_clean)

#%%

# Display before and after stats

# most common police department titles before adjustment:
before = EmpEarn['title'][(EmpEarn.department == 'Boston Police Department') \
                                        & (EmpEarn.year == '2020')] \
                                        .value_counts() \
                                        .nlargest(10) \
                                        .reset_index()
# after adjustment
after = EmpEarn['title_clean'][(EmpEarn.department == 'Boston Police Department') \
                                        & (EmpEarn.year == '2020')] \
                                        .value_counts() \
                                        .nlargest(10) \
                                        .reset_index()
# combine and sort
joined = pd.merge(before, after, how='left') \
                        .fillna(0) \
                        .sort_values(by=['title'], ascending=False) \
                        .rename(index=str, columns={'index': 'title', 'title': 'Before', 'title_clean': 'After'})

# data by year
titles_by_year = pd.pivot_table(EmpEarn[EmpEarn['dept_clean'] == 'Boston Police Department'],\
                            values=['title', 'title_clean'], columns='year', aggfunc=pd.Series.nunique) \
                        .sort_values(by='2020', ascending=False)[:10] \
                        .transpose() \
                        .reset_index()

# plots
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8,4))
fig.suptitle('Police titles before and after consolidation')

joined.plot.barh('title', 'Before', ax=ax1, width=0.4, color='#2f79b4', position=1).invert_yaxis()
joined.plot.barh('title', 'After', ax=ax1, width=0.4, color='#1adf5a', position=0).invert_yaxis()
ax1.set_title('Police titles consolidation')
ax1.set_xlabel('Count')
ax1.set_ylim(10, -0.8)

titles_by_year.plot('year', 'title', kind='barh', ax=ax2, width=0.4, color='#2f79b4', position=1, label='Before', legend=False)
titles_by_year.plot('year', 'title_clean', kind='barh', ax=ax2, width=0.4, color='#1adf5a', position=0, label='After', legend=False)
ax2.set_title('Police titles by year')
ax2.set_xlabel('year')
ax2.set_ylabel('count unique')

#plt.tight_layout(rect=[0, 0.03, 1, 0.95])
plt.show()

#%%

# consolidate fire department titles

fd = {'FF': 'Fire Fighter',
      'Fire Fi': 'Fire Fighter',
      'FireFig': 'Fire Fighter',
      'Fire Ca': 'Fire Captain',
      'Fire L': 'Fire Lieutenant',
      'FireLi': 'Fire Lieutenant',
      'Distric': 'District Fire Chief',
      'Dist Fi': 'District Fire Chief',
      'DistFCh': 'District Fire Chief',
      'Dep Fir': 'Dep Fire Chief',
      'DepFire': 'Dep Fire Chief'}

for k in fd:
    EmpEarn['title_clean'] = np.where(EmpEarn.title.astype(str).str[:len(k)] == k, fd[k], EmpEarn.title_clean)

EmpEarn['title_clean'] = np.where((EmpEarn.title.str[:10] == 'Sr Admin A') \
                                   & (EmpEarn.dept_clean == 'Boston Fire Department'),\
                                   'Sr Admin (Fire)', EmpEarn.title_clean)

fd_titles = EmpEarn.title_clean.loc[(EmpEarn.dept_clean == 'Boston Fire Department')]
print('Number of unique job titles in fire department:', len(set(fd_titles)))
print(fd_titles.value_counts().nlargest(10))

#%%

# Display before and after stats

# most common police department titles before adjustment:
before = EmpEarn['title'][(EmpEarn.department == 'Boston Fire Department') \
                                        & (EmpEarn.year == '2020')] \
                                        .value_counts() \
                                        .nlargest(10) \
                                        .reset_index()
# after adjustment
after = EmpEarn['title_clean'][(EmpEarn.department == 'Boston Fire Department') \
                                        & (EmpEarn.year == '2020')] \
                                        .value_counts() \
                                        .nlargest(10) \
                                        .reset_index()
# combine and sort
joined = pd.merge(before, after, how='left') \
                        .fillna(0) \
                        .sort_values(by=['title'], ascending=False) \
                        .rename(index=str, columns={'index': 'title', 'title': 'before', 'title_clean': 'after'})

# data by year
titles_by_year = pd.pivot_table(EmpEarn[EmpEarn['dept_clean'] == 'Boston Fire Department'],\
                            values=['title', 'title_clean'], columns='year', aggfunc=pd.Series.nunique) \
                        .sort_values(by='2020', ascending=False)[:10] \
                        .transpose() \
                        .reset_index()

# plots
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8,4))
fig.suptitle('Fire department titles before and after consolidation')

joined.plot.barh('title', 'before', ax=ax1, width=0.4, color='#1f78b4', position=1).invert_yaxis()
joined.plot.barh('title', 'after', ax=ax1, width=0.4, color='#b2df8a', position=0).invert_yaxis()
ax1.set_title('Fire department titles consolidation')
ax1.set_xlabel('Count')
ax1.set_ylim(10, -0.8)

titles_by_year.plot('year', 'title', kind='barh', ax=ax2, width=0.4, color='#1f78b4', position=1, label='before', legend=False)
titles_by_year.plot('year', 'title_clean', kind='barh', ax=ax2, width=0.4, color='#b2df8a', position=0, label='after', legend=False)
ax2.set_title('Fire department titles by year')
ax2.set_xlabel('year')
ax2.set_ylabel('count unique')

plt.tight_layout(rect=[0, 0.03, 1, 0.95])
plt.show()

#%%

library = {'Spec Library Asst': 'Spec Library Asst',
       'Sr Library Asst': 'Sr Library Asst',
       'Librarian': 'Librarian',
       'Special Library': 'Spec Library Asst',
       'Spec Collection L': 'Librarian',
       'Collection Libr': 'Librarian'}
for k in library:
    EmpEarn['title_clean'] = np.where(EmpEarn.title.str.contains(k), library[k], EmpEarn.title_clean)

bpl_titles = EmpEarn.title_clean.loc[(EmpEarn.dept_clean == 'Boston Public Library')]
print('Number of unique job titles in Boston Public Library:', len(set(bpl_titles)))
print(bpl_titles.value_counts().nlargest(20))

#%%

# Display before and after stats

# most common police department titles before adjustment:
before = EmpEarn['title'][(EmpEarn.department == 'Boston Public Library') \
                                        & (EmpEarn.year == '2020')] \
                                        .value_counts() \
                                        .nlargest(10) \
                                        .reset_index()
# after adjustment
after = EmpEarn['title_clean'][(EmpEarn.department == 'Boston Public Library') \
                                        & (EmpEarn.year == '2020')] \
                                        .value_counts() \
                                        .nlargest(10) \
                                        .reset_index()
# combine and sort
joined = pd.merge(before, after, how='left') \
                        .fillna(0) \
                        .sort_values(by=['title'], ascending=False) \
                        .rename(index=str, columns={'index': 'title', 'title': 'before', 'title_clean': 'after'})

# data by year
titles_by_year = pd.pivot_table(EmpEarn[EmpEarn['dept_clean'] == 'Boston Public Library'],\
                            values=['title', 'title_clean'], columns='year', aggfunc=pd.Series.nunique) \
                        .sort_values(by='2020', ascending=False)[:10] \
                        .transpose() \
                        .reset_index()

# plots
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8,4))
fig.suptitle('Library department titles before and after consolidation')

joined.plot.barh('title', 'before', ax=ax1, width=0.4, color='#1f78b4', position=1).invert_yaxis()
joined.plot.barh('title', 'after', ax=ax1, width=0.4, color='#b2df8a', position=0).invert_yaxis()
ax1.set_title('Library department titles consolidation')
ax1.set_xlabel('Count')
ax1.set_ylim(10, -0.8)

titles_by_year.plot('year', 'title', kind='barh', ax=ax2, width=0.4, color='#1f78b4', position=1, label='before', legend=False)
titles_by_year.plot('year', 'title_clean', kind='barh', ax=ax2, width=0.4, color='#b2df8a', position=0, label='after', legend=False)
ax2.set_title('Library department titles by year')
ax2.set_xlabel('Year')
ax2.set_ylabel('count unique')

plt.tight_layout(rect=[0, 0.03, 1, 0.95])
plt.show()

#%%

# unique job titles and departments by year
title_gb = EmpEarn.groupby('year')[['title', 'title_clean']].nunique()
dept_gb = EmpEarn.groupby('year')[['department', 'dept_clean']].nunique()

# plots
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12,6))
title_gb.plot.bar(ax=ax1, cmap='Dark2', rot=0)
ax1.set_title('Unique job titles')
ax1.set_ylabel('count')
ax1.set_xlabel('year')
ax1.legend(loc=4, frameon=True, framealpha=0.8)

dept_gb.plot.bar(ax=ax2, cmap='Dark2', rot=0)
ax2.set_title('Unique department names')
ax2.set_ylabel('count')
ax2.set_xlabel('year')

#plt.tight_layout()
plt.show()

#%%

#%%
## Exporting CSV
##We are exporting the dataset to CSV so that we can import it to ProsgresSQL using PgAdmin 4

#BosEmpEarn

EmpEarn.to_csv("Data/EmpEarn.csv", sep=",", header=False, index=False, na_rep='', index_label=None, encoding='utf-8')

#%%

# %%
# unique job titles and departments by year
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(8,4))
ax1.bar(EmpEarn['year'].unique(), EmpEarn.groupby('year')['title'].nunique())
ax1.set_title('unique job titles')
ax1.set_ylabel('count')
ax1.set_xlabel('year')

ax2.bar(EmpEarn['year'].unique(), EmpEarn.groupby('year')['department'].nunique())
ax2.set_title('unique department names')
ax2.set_ylabel('count')
ax2.set_xlabel('year')

plt.tight_layout()
plt.show()
# %%

import numpy,from scipy,from statsmodels


# -*- coding: utf-8 -*-
# written by mark zeng 2018-11-14
# modified by Yao Zhao 2019-10-30

import multiprocessing as mp
import time
import sys
import argparse
import os
import numpy as np

class Worker(mp.Process):
    def __init__ (self, inQ, outQ, random_seed):
        super(Worker, self).__init__(target=self.start)
        self.inQ = inQ
        self.outQ = outQ
        np.random.seed(random_seed)  #  如果子进程的任务是有随机性的，一定要给每个子进程不同的随机数种子，否则就在重复相同的结果了

    def run (self):
        while True:
            task = self.inQ.get()  # 取出任务， 如果队列为空， 这一步会阻塞直到队列有元素
            x, y = task     # 解析任务
            sum, product = sum_and_product(x, y)   # 执行任务
            self.outQ.put((sum, product))  # 返回结果

def create_worker (num):
    '''
    创建子进程备用
    :param num: 多线程数量
    '''
    for i in range(num):
        worker.append(Worker(mp.Queue(), mp.Queue(), np.random.randint(0, 10 ** 9)))
        worker[i].start()

def finish_worker ():
    '''
    关闭所有子线程
    '''
    for w in worker:
        w.terminate()

def sum_and_product(x, y):
    '''
    计算两个数的和与积
    '''
    return x + y, x * y

if __name__ == '__main__':
    '''
    从命令行读参数示例
    '''
    print("从命令行读参数示例")
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--file_name', type=str, default='network.txt')
    parser.add_argument('-s', '--seed', type=str, default='seeds.txt')
    parser.add_argument('-m', '--model', type=str, default='IC')
    parser.add_argument('-t', '--time_limit', type=int, default=60)

    args = parser.parse_args()
    file_name = args.file_name
    seed = args.seed
    model = args.model
    time_limit = args.time_limit

    print(file_name, seed, model, time_limit)

    '''
    多进程示例
    '''
    print("多进程示例")
    np.random.seed(0)
    worker = []
    worker_num = 8
    create_worker(worker_num)
    Task = [np.random.randint(0, 10, 2) for i in range(16)]  # 生成16个随机任务， 每个任务是2个整数， 需要计算两数之和与积
    print('Task', Task)
    for i, t in enumerate(Task):
        worker[i % worker_num].inQ.put(t)  # 根据编号取模， 将任务平均分配到子进程上
    result = []
    for i, t in enumerate(Task):
        result.append(worker[i % worker_num].outQ.get())  # 用同样的规则取回结果， 如果任务尚未完成，此处会阻塞等待子进程完成任务
    print('result', result)
    finish_worker()

    '''
    程序结束后强制退出，跳过垃圾回收时间, 如果没有这个操作会额外需要几秒程序才能完全退出
    '''
    sys.stdout.flush()


import re
import types

import numpy as np
import warnings

from numba.cuda.testing import unittest, skip_on_cudasim, CUDATestCase
from numba import cuda, jit, int32
from numba.core.errors import TypingError

class TestDeviceFunc(CUDATestCase):

    def test_use_add2f(self):

        @cuda.jit("float32(float32, float32)", device=True)
        def add2f(a, b):
            return a + b

        def use_add2f(ary):
            i = cuda.grid(1)
            ary[i] = add2f(ary[i], ary[i])

        compiled = cuda.jit("void(float32[:])")(use_add2f)

        nelem = 10
        ary = np.arange(nelem, dtype=np.float32)
        exp = ary + ary
        compiled[1, nelem](ary)

        self.assertTrue(np.all(ary == exp), (ary, exp))

    def test_indirect_add2f(self):

        @cuda.jit("float32(float32, float32)", device=True)
        def add2f(a, b):
            return a + b

        @cuda.jit("float32(float32, float32)", device=True)
        def indirect(a, b):
            return add2f(a, b)

        def indirect_add2f(ary):
            i = cuda.grid(1)
            ary[i] = indirect(ary[i], ary[i])

        compiled = cuda.jit("void(float32[:])")(indirect_add2f)

        nelem = 10
        ary = np.arange(nelem, dtype=np.float32)
        exp = ary + ary
        compiled[1, nelem](ary)

        self.assertTrue(np.all(ary == exp), (ary, exp))

    def _check_cpu_dispatcher(self, add):
        @cuda.jit
        def add_kernel(ary):
            i = cuda.grid(1)
            ary[i] = add(ary[i], 1)

        ary = np.arange(10)
        expect = ary + 1
        add_kernel[1, ary.size](ary)
        np.testing.assert_equal(expect, ary)

    def test_cpu_dispatcher(self):
        # Test correct usage
        @jit
        def add(a, b):
            return a + b

        self._check_cpu_dispatcher(add)

    @skip_on_cudasim('not supported in cudasim')
    def test_cpu_dispatcher_invalid(self):
        # Test invalid usage
        # Explicit signature disables compilation, which also disable
        # compiling on CUDA.
        @jit('(i4, i4)')
        def add(a, b):
            return a + b

        # Check that the right error message is provided.
        with self.assertRaises(TypingError) as raises:
            self._check_cpu_dispatcher(add)
        msg = "Untyped global name 'add':.*using cpu function on device"
        expected = re.compile(msg)
        self.assertTrue(expected.search(str(raises.exception)) is not None)

    def test_cpu_dispatcher_other_module(self):
        @jit
        def add(a, b):
            return a + b

        mymod = types.ModuleType(name='mymod')
        mymod.add = add
        del add

        @cuda.jit
        def add_kernel(ary):
            i = cuda.grid(1)
            ary[i] = mymod.add(ary[i], 1)

        ary = np.arange(10)
        expect = ary + 1
        add_kernel[1, ary.size](ary)
        np.testing.assert_equal(expect, ary)

    @skip_on_cudasim('not supported in cudasim')
    def test_inspect_ptx(self):
        @cuda.jit(device=True)
        def foo(x, y):
            return x + y

        args = (int32, int32)
        cres = foo.compile(args)

        fname = cres.fndesc.mangled_name
        # Verify that the function name has "foo" in it as in the python name
        self.assertIn('foo', fname)

        ptx = foo.inspect_ptx(args)
        # Check that the compiled function name is in the PTX.
        self.assertIn(fname, ptx.decode('ascii'))

    @skip_on_cudasim('not supported in cudasim')
    def test_inspect_llvm(self):
        @cuda.jit(device=True)
        def foo(x, y):
            return x + y

        args = (int32, int32)
        cres = foo.compile(args)

        fname = cres.fndesc.mangled_name
        # Verify that the function name has "foo" in it as in the python name
        self.assertIn('foo', fname)

        llvm = foo.inspect_llvm(args)
        # Check that the compiled function name is in the LLVM.
        self.assertIn(fname, llvm)

    @skip_on_cudasim('not supported in cudasim')
    def test_deprecated_eager_device(self):
        with warnings.catch_warnings(record=True) as w:
            cuda.jit('int32(int32)', device=True)

        self.assertEqual(len(w), 1)
        self.assertIn('Eager compilation of device functions is deprecated',
                      str(w[0].message))

if __name__ == '__main__':
    unittest.main()


"""
Tests Tensor/Operation implementation that uses topological sorting against the naive implementation.

Here, a rule-based state machine constructs computational graphs via 'rules' by which nodes (tensors)
are 'fused' via addition and multiplication operations. The state machine create and fuse nodes in
any patterns, invoke `null_gradients` and `clear_graph` arbitrarily as well.

The values and gradients of the nodes in the mygrad and naive graphs must match as an invariant to
any permutation of test states (i.e. permutations of the aforementioned rules)"""

from typing import List, Tuple

import hypothesis.strategies as st
from hypothesis import assume, settings
from hypothesis.stateful import Bundle, RuleBasedStateMachine, invariant, rule
from numpy.testing import assert_allclose, assert_equal
from pytest import raises

from mygrad import Tensor, add, multiply
from mygrad.errors import InvalidBackprop
from tests.utils.stateful import clear_all_mem_locking_state

from .simple_graph import Node, _add, _multiply

def _node_ID_str(num):
    return "v{}".format(num + 1)

@settings(max_examples=125, stateful_step_count=100, deadline=None)
class GraphCompare(RuleBasedStateMachine):
    def __init__(self):
        super().__init__()
        # stores the corresponding node/tensor v1, v2, ... as they are
        # created via the unit test (through `create_node` or `fuse_nodes`)
        # `Node` is the naive implementation of `Tensor` that we are checking
        # against
        self.node_list = []  # type: List[Tuple[Node, Tensor]]
        self.str_to_tensor_op = {"add": add, "multiply": multiply}
        self.str_to_node_op = {"add": _add, "multiply": _multiply}
        self.raised = False

    nodes = Bundle("nodes")

    @rule(target=nodes, value=st.floats(-10, 10), constant=st.booleans())
    def create_node(self, value, constant):
        n = Node(value, constant=constant)
        t = Tensor(value, constant=constant)
        self.node_list.append((n, t))
        return n, t

    @rule(
        target=nodes,
        a=nodes,
        b=nodes,
        op=st.sampled_from(["add", "multiply"]),
        constant=st.sampled_from([True, None]),
    )
    def fuse_nodes(self, a, b, op, constant):
        """
        Combine any pair of nodes (tensors) using either addition or multiplication, producing
        a new node (tensor)"""
        n_a, t_a = a  # type: Node, Tensor
        n_b, t_b = b  # type: Node, Tensor
        n_op = self.str_to_node_op[op]
        t_op = self.str_to_tensor_op[op]
        out = (
            n_op(n_a, n_b, constant=bool(constant)),
            t_op(t_a, t_b, constant=constant),
        )
        self.node_list.append(out)
        return out

    @rule(items=nodes)
    def clear_graph(self, items):
        """
        Invoke `clear_graph` on the computational graph (naive and mygrad)
        """
        n, t = items  # type: Node, Tensor
        n.clear_graph()
        t.clear_graph()

    @rule(items=nodes, grad=st.floats(-10, 10))
    def backprop(self, items, grad):
        """
        Invoke `backward(grad)` on the computational graph (naive and mygrad) from a randomly-selected
        node in the computational graph and using a randomly-generated gradient value.

        An exception should be raised if `clear_graph` is invoked anywhere prior to the invoking node.
        """
        n, t = items  # type: Node, Tensor
        try:
            n.backward(grad, terminal_node=True)
        except InvalidBackprop:
            with raises(InvalidBackprop):
                t.backward(grad)
            self.raised = True
            assume(False)
        else:
            t.backward(grad)
            assert not t._accum_ops
            assert not t._ops
            assert not t.creator

    @invariant()
    def all_agree(self):
        """
        Ensure that all corresponding nodes/tensors have matching data and gradients
        across the respective graphs.
        """
        assert not self.raised, (
            "there is a problem with the state machine: "
            "invalid-backprop should be marked as 'assumed false' "
            "by Hypothesis"
        )
        for num, (n, t) in enumerate(self.node_list):
            assert bool(n._ops) is bool(t._ops), _node_ID_str(num)
            assert_equal(n.data, t.data, err_msg=_node_ID_str(num))
            if n.grad is None or t.grad is None:
                assert n.grad is t.grad, _node_ID_str(num)
            else:
                assert_allclose(
                    actual=t.grad,
                    desired=n.grad,
                    atol=1e-5,
                    rtol=1e-5,
                    err_msg=_node_ID_str(num),
                )
            assert not t._accum_ops, _node_ID_str(num)

    def teardown(self):
        clear_all_mem_locking_state()

TestGraphComparison = GraphCompare.TestCase


import numpy as np

def ensure_random_state(random_state):
    """Derive PRNG from given random state.

    Parameters
    ----------
    random_state : int or numpy.random.Generator
        Either an integer used to seed numpy's default PRNG, or an already
        seeded PRNG.

    Returns
    -------
    random_state : numpy.random.Generator
        The derived PRNG.
    """
    if isinstance(random_state, np.random.Generator):
        return random_state
    return np.random.default_rng(random_state)


#
# Zero dimensional submesh
#
import pybamm
from .meshes import SubMesh

import numpy as np

class SubMesh0D(SubMesh):
    """
    0D submesh class.
    Contains the position of the node.

    Parameters
    ----------
    position : dict
        A dictionary that contains the position of the 0D submesh (a signle point)
        in space
    npts : dict, optional
        Number of points to be used. Included for compatibility with other meshes,
        but ignored by this mesh class

    **Extends:"": :class:`pybamm.SubMesh`
    """

    def __init__(self, position, npts=None):
        # Remove tabs
        position.pop("tabs", None)

        # check that only one variable passed in
        if len(position) != 1:
            raise pybamm.GeometryError("position should only contain a single variable")

        # extract the position
        position = list(position.values())[0]
        spatial_position = position["position"]
        self.nodes = np.array([spatial_position])
        self.edges = np.array([spatial_position])
        self.coord_sys = None
        self.npts = 1

    def add_ghost_meshes(self):
        # No ghost meshes to be added to this class
        pass


import myutil as mu

################################################################################
# - 자연어 처리 전처리 이해하기
#   - 자연어 처리는 일반적으로 토큰화, 단어 집합 생성, 정수 인코딩, 패딩, 벡터화의 과정을 거칩니다.
#   - 이번 챕터에서는 이러한 전반적인 과정에 대해서 이해합니다.

################################################################################
# - spaCy 사용하기
#
# ```
# pip install spacy
# python3 -m spacy download en
# ```

import spacy

en_text = "A Dog Run back corner near spare bedrooms"
spacy_en = spacy.load("en")
mu.log("spacy_en", spacy_en)

def tokenize(en_text):
    return [
        tok.text
        for tok in spacy_en.tokenizer(en_text)
    ]

mu.log("tokenize(en_text)", tokenize(en_text))

################################################################################
# - NLTK 사용하기
#
# ```
# pip install nltk
# ```
#

import nltk
from nltk.tokenize import word_tokenize

nltk.download("punkt")
mu.log("word_tokenize(en_text)", word_tokenize(en_text))

################################################################################
# - 띄어쓰기로 토큰화

mu.log("en_text.split()", en_text.split())

################################################################################
# - 띄어쓰기로 토큰화
#   - 위의 예제에서는 '사과'란 단어가 총 4번 등장했는데
#   - 모두 '의', '를', '가', '랑' 등이 붙어있어
#   - 이를 제거해주지 않으면 기계는 전부 다른 단어로 인식하게 됩니다.

kor_text = "사과의 놀라운 효능이라는 글을 봤어. 그래서 오늘 사과를 먹으려고 했는데 사과가 썩어서 슈퍼에 가서 사과랑 오렌지 사왔어"
mu.log("kor_text.split()", kor_text.split())

################################################################################
# - 형태소 토큰화
#   - 위와 같은 상황을 방지하기 위해서 한국어는 보편적으로 '형태소 분석기'로 토큰화를 합니다.
#   - 여기서는 형태소 분석기 중에서 mecab을 사용해보겠습니다.
#   - 아래의 커맨드로 colab에서 mecab을 설치합니다.
#   - 앞선 예와 다르게 '의', '를', '가', '랑' 등이 전부 분리되어 기계는 '사과'라는 단어를 하나의 단어로 처리할 수 있습니다.
#
# ```
# git clone https://github.com/SOMJANG/Mecab-ko-for-Google-Colab.git
# cd Mecab-ko-for-Google-Colab
# chmod u+x install_mecab-ko_on_colab190912.sh
# ./install_mecab-ko_on_colab190912.sh
# ```

from konlpy.tag import Mecab

tokenizer = Mecab()
mu.log("tokenizer.morphs(kor_text)", tokenizer.morphs(kor_text))

################################################################################
# - 단어 집합(Vocabulary) 생성
#   - 단어 집합(vocabuary)이란 중복을 제거한 텍스트의 총 단어의 집합(set)을 의미합니다.
#   - 우선, 실습을 위해서 깃허브에서 '네이버 영화 리뷰 분류하기' 데이터를 다운로드하겠습니다.
#   - 네이버 영화 리뷰 데이터는 총 20만 개의 영화 리뷰를 긍정 1, 부정 0으로 레이블링한 데이터입니다.
#
# ```
# pip3 install pandas
# ```

import urllib.request
import pandas as pd
from konlpy.tag import Mecab
from nltk import FreqDist
import numpy as np
import matplotlib.pyplot as plt

urllib.request.urlretrieve(
    url="https://raw.githubusercontent.com/e9t/nsmc/master/ratings.txt",
    filename=".ratings.txt"
)

data = pd.read_table(".ratings.txt")
mu.log("len(data)", len(data))
mu.log("data[:10]", data[:10])
sample_data = data[:100]

# 불용어 정의
stopwords = ['의', '가', '이', '은', '들', '는', '좀', '잘', '걍', '과', '도', '를', '으로', '자', '에', '와', '한', '하다']
tokenized = []

for sentence in sample_data["document"]:
    temp = []
    temp = tokenizer.morphs(sentence)

    temp = [
        word
        for word in temp
        if not word in stopwords
    ]

    tokenized.append(temp)

mu.log("tokenized[:10]", tokenized[:10])

vocab = FreqDist(np.hstack(tokenized))
mu.log("len(vocab)", len(vocab))
mu.log("vocab['재밌']", vocab['재밌'])

################################################################################
# - 각 단어에 고유한 정수 부여
#   - enumerate()는 순서가 있는 자료형(list, set, tuple, dictionary, string)을 입력으로 받아
#   - 인덱스를 순차적으로 함께 리턴한다는 특징이 있습니다.
#   - 인덱스 0과 1은 다른 용도로 남겨두고 나머지 단어들은 2부터 501까지 순차적으로 인덱스를 부여해봅시다.

word_to_index = {
    word[0]: index + 2
    for index, word in enumerate(vocab)
}

word_to_index["pad"] = 0
word_to_index["unk"] = 1
mu.log("len(word_to_index)", len(word_to_index))

encoded = []

for line in tokenized:
    temp = []
    for w in line:
        try:
            temp.append(word_to_index[w])
        except:
            temp.append(word_to_index["unk"])
    encoded.append(temp)

mu.log("len(encoded)", len(encoded))
mu.log("encoded[:10]", encoded[:10])

################################################################################
# - 길이가 다른 문장들을 모두 동일한 길이로 바꿔주는 패딩(padding)
#   - 이제 길이가 다른 리뷰들을 모두 동일한 길이로 바꿔주는 패딩 작업을 진행해보겠습니다.
#   - 앞서 단어 집합에 패딩을 위한 토큰인 'pad'를 추가했었습니다.
#   - 패딩 작업은 정해준 길이로 모든 샘플들의 길이를 맞춰주되,
#   - 길이가 정해준 길이보다 짧은 샘플들에는 'pad' 토큰을 추가하여 길이를 맞춰주는 작업입니다.

len_array = []
for line in encoded:
    len_array.append(len(line))

max_len = max(len_array)
mu.log("max_len", max_len)
min_len = min(len_array)
mu.log("min_len", min_len)

plt.hist(len_array, bins=50, label="count of sample")
plt.xlabel("length of sample")
plt.legend()
plt.show()

mu.log("encoded[:3]", encoded[:3])


import numpy as np
import torch
import random

def fixseed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)

SEED = 10
EVALSEED = 0
# Provoc warning: not fully functionnal yet
# torch.set_deterministic(True)
torch.backends.cudnn.benchmark = False

fixseed(SEED)


# height and weight are available as a regular lists

# Import numpy
import numpy as np

# Store weight and height lists as numpy arrays
np_weight = np.array(weight)
np_height = np.array(height)

# Print out the weight at index 50
print(np_weight[50])

# Print out sub-array of np_height: index 100 up to and including index 110
print(np_height[100:111])


import pandas as pd
import numpy as np
from collections import defaultdict
import RemovingValuesSolns as s

## Removing Values
# Question 1
def all_drop_test(all_drop):
    '''
    INPUT all_drop - a pandas dataframe with all rows with missing values dropped.

    Prints statement related to the correctness of the solution of the dataframe
    '''
    if all_drop.equals(s.all_drop):
        print("Nice job! That looks right!")
    else:
        print("That wasn't quite as expected.  Try again, or take a look at the solution notebook if you get stuck.")

# Question 2
def all_row_test(all_row):
    '''
    INPUT all_row - a pandas dataframe with all rows that have every value as a missing value dropped.

    Prints statement related to the correctness of the solution of the dataframe
    '''
    if all_row.equals(s.all_row):
        print("Nice job! That looks right!")
    else:
        print("That wasn't quite as expected.  Try again, or take a look at the solution notebook if you get stuck.")

# Question 3
def only3_drop_test(only3_drop):
    '''
    INPUT all_row - a pandas dataframe with all rows that are missing a value in col3

    Prints statement related to the correctness of the solution of the dataframe
    '''
    if only3_drop.equals(s.only3_drop):
        print("Nice job! That looks right!")
    else:
        print("That wasn't quite as expected.  Try again, or take a look at the solution notebook if you get stuck.")

# Question 4
def only3or1_drop_test(only3or1_drop):
    '''
    INPUT all_row - a pandas dataframe with all rows that are missing a value in col1 or col3

    Prints statement related to the correctness of the solution of the dataframe
    '''
    if only3or1_drop.equals(s.only3or1_drop):
        print("Nice job! That looks right!")
    else:
        print("That wasn't quite as expected.  Try again, or take a look at the solution notebook if you get stuck.")


"""Tools for setting up interactive sessions. """

from sympy.interactive.printing import init_printing

preexec_source = """\
from __future__ import division
from sympy import *
x, y, z, t = symbols('x y z t')
k, m, n = symbols('k m n', integer=True)
f, g, h = symbols('f g h', cls=Function)
"""

verbose_message = """\
These commands were executed:
%(source)s
Documentation can be found at http://www.sympy.org
"""

no_ipython = """\
Couldn't locate IPython. Having IPython installed is greatly recommended.
See http://ipython.scipy.org for more details. If you use Debian/Ubuntu,
just install the 'ipython' package and start isympy again.
"""

def _make_message(ipython=True, quiet=False, source=None):
    """Create a banner for an interactive session. """
    from sympy import __version__ as sympy_version
    from sympy.polys.domains import GROUND_TYPES
    from sympy.utilities.misc import ARCH
    from sympy import SYMPY_DEBUG

    import sys
    import os

    python_version = "%d.%d.%d" % sys.version_info[:3]

    if ipython:
        shell_name = "IPython"
    else:
        shell_name = "Python"

    info = ['ground types: %s' % GROUND_TYPES]

    cache = os.getenv('SYMPY_USE_CACHE')

    if cache is not None and cache.lower() == 'no':
        info.append('cache: off')

    if SYMPY_DEBUG:
        info.append('debugging: on')

    args = shell_name, sympy_version, python_version, ARCH, ', '.join(info)
    message = "%s console for SymPy %s (Python %s-%s) (%s)\n" % args

    if not quiet:
        if source is None:
            source = preexec_source

        _source = ""

        for line in source.split('\n')[:-1]:
            if not line:
                _source += '\n'
            else:
                _source += '>>> ' + line + '\n'

        message += '\n' + verbose_message % {'source': _source}

    return message

def int_to_Integer(s):
    """
    Wrap integer literals with Integer.

    This is based on the decistmt example from
    http://docs.python.org/library/tokenize.html.

    Only integer literals are converted.  Float literals are left alone.
    Example
    =======

    >>> from sympy.interactive.session import int_to_Integer
    >>> from sympy import Integer
    >>> s = '1.2 + 1/2 - 0x12 + a1'
    >>> int_to_Integer(s)
    '1.2 +Integer (1 )/Integer (2 )-Integer (0x12 )+a1 '
    >>> s = 'print (1/2)'
    >>> int_to_Integer(s)
    'print (Integer (1 )/Integer (2 ))'
    >>> exec(s) #doctest: +SKIP
    0.5
    >>> exec(int_to_Integer(s))
    1/2
    """
    from tokenize import generate_tokens, untokenize, NUMBER, NAME, OP
    from StringIO import StringIO

    def _is_int(num):
        """
        Returns true if string value num (with token NUMBER) represents an integer.
        """
        # XXX: Is there something in the standard library that will do this?
        if '.' in num or 'j' in num.lower() or 'e' in num.lower():
            return False
        return True

    result = []
    g = generate_tokens(StringIO(s).readline)   # tokenize the string
    for toknum, tokval, _, _, _  in g:
        if toknum == NUMBER and _is_int(tokval):  # replace NUMBER tokens
            result.extend([
                (NAME, 'Integer'),
                (OP, '('),
                (NUMBER, tokval),
                (OP, ')')
            ])
        else:
            result.append((toknum, tokval))
    return untokenize(result)

# XXX: Something like this might be used, but it only works on single line
# inputs.  See
# http://mail.scipy.org/pipermail/ipython-user/2012-August/010846.html and
# https://github.com/ipython/ipython/issues/1491.  So instead we are forced to
# just monkey-patch run_cell until IPython builds a better API.
#
# class IntTransformer(object):
#     """
#     IPython command line transformer that recognizes and replaces int
#     literals.
#
#     Based on
#     https://bitbucket.org/birkenfeld/ipython-physics/src/71b2d850da00/physics.py.
#
#     """
#     priority = 99
#     enabled = True
#     def transform(self, line, continue_prompt):
#         import re
#         from tokenize import TokenError
#         leading_space = re.compile(' *')
#         spaces = re.match(leading_space, line).span()[1]
#         try:
#             return ' '*spaces + int_to_Integer(line)
#         except TokenError:
#             return line
#
# int_transformer = IntTransformer()
#
# def enable_automatic_int_sympification(app):
#     """
#     Allow IPython to automatically convert integer literals to Integer.
#
#     This lets things like 1/2 be executed as (essentially) Rational(1, 2).
#     """
#     app.shell.prefilter_manager.register_transformer(int_transformer)

def enable_automatic_int_sympification(app):
    """
    Allow IPython to automatically convert integer literals to Integer.
    """
    hasshell = hasattr(app, 'shell')

    import ast
    if hasshell:
        old_run_cell = app.shell.run_cell
    else:
        old_run_cell = app.run_cell
    def my_run_cell(cell, *args, **kwargs):
        try:
            # Check the cell for syntax errors.  This way, the syntax error
            # will show the original input, not the transformed input.  The
            # downside here is that IPython magic like %timeit will not work
            # with transformed input (but on the other hand, IPython magic
            # that doesn't expect transformed input will continue to work).
            ast.parse(cell)
        except SyntaxError:
            pass
        else:
            cell = int_to_Integer(cell)
        old_run_cell(cell, *args, **kwargs)

    if hasshell:
        app.shell.run_cell = my_run_cell
    else:
        app.run_cell = my_run_cell

def enable_automatic_symbols(app):
    """Allow IPython to automatially create symbols (``isympy -a``). """
    # XXX: This should perhaps use tokenize, like int_to_Integer() above.
    # This would avoid re-executing the code, which can lead to subtle
    # issues.  For example:
    #
    # In [1]: a = 1
    #
    # In [2]: for i in range(10):
    #    ...:     a += 1
    #    ...:
    #
    # In [3]: a
    # Out[3]: 11
    #
    # In [4]: a = 1
    #
    # In [5]: for i in range(10):
    #    ...:     a += 1
    #    ...:     print b
    #    ...:
    # b
    # b
    # b
    # b
    # b
    # b
    # b
    # b
    # b
    # b
    #
    # In [6]: a
    # Out[6]: 12
    #
    # Note how the for loop is executed again because `b` was not defined, but `a`
    # was already incremented once, so the result is that it is incremented
    # multiple times.

    import re
    re_nameerror = re.compile("name '(?P<symbol>[A-Za-z_][A-Za-z0-9_]*)' is not defined")

    def _handler(self, etype, value, tb, tb_offset=None):
        """Handle :exc:`NameError` exception and allow injection of missing symbols. """
        if etype is NameError and tb.tb_next and not tb.tb_next.tb_next:
            match = re_nameerror.match(str(value))

            if match is not None:
                # XXX: Make sure Symbol is in scope. Otherwise you'll get infinite recursion.
                self.run_cell("%(symbol)s = Symbol('%(symbol)s')" %
                    {'symbol': match.group("symbol")}, store_history=False)

                try:
                    code = self.user_ns['In'][-1]
                except (KeyError, IndexError):
                    pass
                else:
                    self.run_cell(code, store_history=False)
                    return None
                finally:
                    self.run_cell("del %s" % match.group("symbol"),
                                  store_history=False)

        stb = self.InteractiveTB.structured_traceback(etype, value, tb, tb_offset=tb_offset)
        self._showtraceback(etype, value, stb)

    if hasattr(app, 'shell'):
        app.shell.set_custom_exc((NameError,), _handler)
    else:
        # This was restructured in IPython 0.13
        app.set_custom_exc((NameError,), _handler)

def init_ipython_session(argv=[], auto_symbols=False, auto_int_to_Integer=False):
    """Construct new IPython session. """
    import IPython

    if IPython.__version__ >= '0.11':
        # use an app to parse the command line, and init config
        from IPython.frontend.terminal import ipapp
        app = ipapp.TerminalIPythonApp()

        # don't draw IPython banner during initialization:
        app.display_banner = False
        app.initialize(argv)

        if auto_symbols:
            enable_automatic_symbols(app)
        if auto_int_to_Integer:
            enable_automatic_int_sympification(app)

        return app.shell
    else:
        from IPython.Shell import make_IPython
        return make_IPython(argv)

def init_python_session():
    """Construct new Python session. """
    from code import InteractiveConsole

    class SymPyConsole(InteractiveConsole):
        """An interactive console with readline support. """

        def __init__(self):
            InteractiveConsole.__init__(self)

            try:
                import readline
            except ImportError:
                pass
            else:
                import os
                import atexit

                readline.parse_and_bind('tab: complete')

                if hasattr(readline, 'read_history_file'):
                    history = os.path.expanduser('~/.sympy-history')

                    try:
                        readline.read_history_file(history)
                    except IOError:
                        pass

                    atexit.register(readline.write_history_file, history)

    return SymPyConsole()

def init_session(ipython=None, pretty_print=True, order=None,
        use_unicode=None, quiet=False, auto_symbols=False, auto_int_to_Integer=False, argv=[]):
    """
    Initialize an embedded IPython or Python session. The IPython session is
    initiated with the --pylab option, without the numpy imports, so that
    matplotlib plotting can be interactive.

    Parameters
    ==========

    pretty_print: boolean
        If True, use pretty_print to stringify;
        if False, use sstrrepr to stringify.
    order: string or None
        There are a few different settings for this parameter:
        lex (default), which is lexographic order;
        grlex, which is graded lexographic order;
        grevlex, which is reversed graded lexographic order;
        old, which is used for compatibility reasons and for long expressions;
        None, which sets it to lex.
    use_unicode: boolean or None
        If True, use unicode characters;
        if False, do not use unicode characters.
    quiet: boolean
        If True, init_session will not print messages regarding its status;
        if False, init_session will print messages regarding its status.
    auto_symbols: boolean
        If True, IPython will automatically create symbols for you.
        If False, it will not.
        The default is False.
    auto_int_to_Integer: boolean
        If True, IPython will automatically wrap int literals with Integer, so
        that things like 1/2 give Rational(1, 2).
        If False, it will not.
        The default is False.
    ipython: boolean or None
        If True, printing will initialize for an IPython console;
        if False, printing will initialize for a normal console;
        The default is None, which does what False does.
    argv: list of arguments for IPython
        See sympy.bin.isympy for options that can be used to initialize IPython.

    See Also
    ========

    sympy.interactive.printing.init_printing: for examples and the rest of the parameters.

    Examples
    ========

    >>> from sympy import init_session, Symbol, sin, sqrt
    >>> sin(x) #doctest: +SKIP
    NameError: name 'x' is not defined
    >>> init_session() #doctest: +SKIP
    >>> sin(x) #doctest: +SKIP
    sin(x)
    >>> sqrt(5) #doctest: +SKIP
      ___
    \/ 5
    >>> init_session(pretty_print=False) #doctest: +SKIP
    >>> sqrt(5) #doctest: +SKIP
    sqrt(5)
    >>> y + x + y**2 + x**2 #doctest: +SKIP
    x**2 + x + y**2 + y
    >>> init_session(order='grlex') #doctest: +SKIP
    >>> y + x + y**2 + x**2 #doctest: +SKIP
    x**2 + y**2 + x + y
    >>> init_session(order='grevlex') #doctest: +SKIP
    >>> y * x**2 + x * y**2 #doctest: +SKIP
    x**2*y + x*y**2
    >>> init_session(order='old') #doctest: +SKIP
    >>> x**2 + y**2 + x + y #doctest: +SKIP
    x + y + x**2 + y**2
    >>> theta = Symbol('theta') #doctest: +SKIP
    >>> theta #doctest: +SKIP
    theta
    >>> init_session(use_unicode=True) #doctest: +SKIP
    >>> theta # doctest: +SKIP
    \u03b8
    """
    import sys

    in_ipython = False

    if ipython is False:
        ip = init_python_session()
        mainloop = ip.interact
    else:
        try:
            import IPython
        except ImportError:
            if ipython is not True:
                if not quiet:
                    print no_ipython
                ip = init_python_session()
                mainloop = ip.interact
            else:
                raise RuntimeError("IPython is not available on this system")
        else:
            ipython = True

            if IPython.__version__ >= '0.11':
                try:
                    ip = get_ipython()
                except NameError:
                    ip = None
            else:
                ip = IPython.ipapi.get()
                if ip:
                    ip = ip.IP

            if ip is not None:
                in_ipython = True
            else:
                ip = init_ipython_session(argv=argv,
                    auto_symbols=auto_symbols, auto_int_to_Integer=auto_int_to_Integer)

            if IPython.__version__ >= '0.11':
                # runsource is gone, use run_cell instead, which doesn't
                # take a symbol arg.  The second arg is `store_history`,
                # and False means don't add the line to IPython's history.
                ip.runsource = lambda src, symbol='exec': ip.run_cell(src, False)

                #Enable interactive plotting using pylab.
                try:
                    ip.enable_pylab(import_all=False)
                except ImportError:
                    #Causes an import error if matplotlib is not installed.
                    pass
            if not in_ipython:
                mainloop = ip.mainloop

    if auto_symbols and (not ipython or IPython.__version__ < '0.11'):
        raise RuntimeError("automatic construction of symbols is possible only in IPython 0.11 or above")
    if auto_int_to_Integer and (not ipython or IPython.__version__ < '0.11'):
        raise RuntimeError("automatic int to Integer transformation is possible only in IPython 0.11 or above")

    _preexec_source = preexec_source

    ip.runsource(_preexec_source, symbol='exec')
    init_printing(pretty_print=pretty_print, order=order, use_unicode=use_unicode, ip=ip)

    message = _make_message(ipython, quiet, _preexec_source)

    if not in_ipython:
        mainloop(message)
        sys.exit('Exiting ...')
    else:
        ip.write(message)
        ip.set_hook('shutdown_hook', lambda ip: ip.write("Exiting ...\n"))


#!/usr/bin/env python
# coding: utf-8

#
#
# #  Gravitational Wave Open Data Workshop #3
#
#
# #### Tutorial 1.2: Introduction to GWpy
#
# This tutorial will briefly describe GWpy, a python package for gravitational astrophysics, and walk-through how you can use this to speed up access to, and processing of, GWOSC data.
#
# [Click this link to view this tutorial in Google Colaboratory](https://colab.research.google.com/github/gw-odw/odw-2019/blob/master/Day_1/Tuto%201.2%20Open%20Data%20access%20with%20GWpy.ipynb)
#
#
This notebook were generated using python 3.7, but should work on python 2.7, 3.6, or 3.7.

# ##  Installation  (execute only if running on a cloud platform or if you haven't done the installation already!)
#
# Note: we use [`pip`](https://docs.python.org/3.6/installing/), but **it is recommended** to use [conda](https://docs.ligo.org/lscsoft/conda/) on your own machine, as explained in the [installation instructions](https://github.com/gw-odw/odw-2019/blob/master/setup.md). This usage might look a little different than normal, simply because we want to do this directly from the notebook.

# In[2]:

# -- Uncomment following line if running in Google Colab
#! pip install -q 'gwpy==1.0.1'

# **Important:** With Google Colab, you may need to restart the runtime after running the cell above.

# ## Initialization

# In[1]:

import gwpy
print(gwpy.__version__)

# ## A note on object-oriented programming
#
# Before we dive too deeply, its worth a quick aside on object-oriented programming (OOP).
# GWpy is heavily object-oriented, meaning almost all of the code you run using GWpy is based around an object of some type, e.g. `TimeSeries`.
# Most of the methods (functions) we will use are attached to an object, rather than standing alone, meaning you should have a pretty good idea of what sort of data you are dealing with (without having to read the documentation!).
#
# For a quick overview of object-oriented programming in Python, see [this blog post by Jeff Knupp](https://jeffknupp.com/blog/2014/06/18/improve-your-python-python-classes-and-object-oriented-programming/).

# ## Handling data in the time domain

# #### Finding open data
#
# We have seen already that the `gwosc` module can be used to query for what data are available on GWOSC.
# The next thing to do is to actually read some open data. Let's try to get some for GW150914, the first direct detection of an astrophysical gravitational-wave signal from a BBH (binary black hole system).
#
# We can use the [`TimeSeries.fetch_open_data`](https://gwpy.github.io/docs/stable/api/gwpy.timeseries.TimeSeries.html#gwpy.timeseries.TimeSeries.fetch_open_data) method to download data directly from https://www.gw-openscience.org, but we need to know the GPS times.
# We can query for the GPS time of an event as follows:

# In[2]:

from gwosc.datasets import event_gps
gps = event_gps('GW150914')
print(gps)

# Now we can build a `[start, end)` GPS segment to 10 seconds around this time, using integers for convenience:

# In[3]:

segment = (int(gps)-5, int(gps)+5)
print(segment)

# and can now query for the full data.
# For this example we choose to retrieve data for the LIGO-Livingston interferometer, using the identifier `'L1'`.
# We could have chosen any of
#
# - `'G1`' - GEO600
# - `'H1'` - LIGO-Hanford
# - `'L1'` - LIGO-Livingston
# - `'V1'` - (Advanced) Virgo
#
# In the future, the Japanese observatory KAGRA will come online, with the identifier `'K1'`.

# In[4]:

from gwpy.timeseries import TimeSeries
ldata = TimeSeries.fetch_open_data('L1', *segment, verbose=True)
print(ldata)

# ##### The `verbose=True` flag lets us see that GWpy has discovered two files that provides the data for the given interval, downloaded them, and loaded the data.
# The files are not stored permanently, so next time you do the same call, it will be downloaded again, however, if you know you might repeat the same call many times, you can use `cache=True` to store the file on your computer.
#
# Notes:
#
# * To read data from a local file instead of from the GWOSC server, we can use [`TimeSeries.read`](https://gwpy.github.io/docs/stable/api/gwpy.timeseries.TimeSeries.html#gwpy.timeseries.TimeSeries.read) method.
#
# We have now downloaded real LIGO data for GW150914! These are the actual data used in the analysis that discovered the first binary black hole merger.

# To sanity check things, we can easily make a plot, using the [`plot()`](https://gwpy.github.io/docs/stable/timeseries/plot.html) method of the `data` `TimeSeries`.

#

# Since this is the first time we are plotting something in this notebook, we need to make configure `matplotlib` (the plotting library) to work within the notebook properly:
#

# Matplotlib documentation can be found [`here`](https://matplotlib.org/contents.html).

# In[5]:

get_ipython().run_line_magic('matplotlib', 'inline')
plot = ldata.plot()

# Notes: There are alternatives ways to access the GWOSC data.
#
# * [`readligo`](https://losc.ligo.org/s/sample_code/readligo.py) is a light-weight Python module that returns the time series into a Numpy array.
# * The [PyCBC](http://github.com/ligo-cbc/pycbc) package has the `pycbc.frame.query_and_read_frame` and `pycbc.frame.read_frame` methods. We use [PyCBC](http://github.com/ligo-cbc/pycbc) in Tutorial 2.1, 2.2 and 2.3.

# ## Handling data in the frequency domain using the Fourier transform
#
# The [Fourier transform](https://en.wikipedia.org/wiki/Fourier_transform) is a widely-used mathematical tool to expose the frequency-domain content of a time-domain signal, meaning we can see which frequencies contian lots of power, and which have less.
#
# We can calculate the Fourier transform of our `TimeSeries` using the [`fft()`](https://gwpy.github.io/docs/stable/api/gwpy.timeseries.TimeSeries.html#gwpy.timeseries.TimeSeries.fft) method:

# In[6]:

fft = ldata.fft()
print(fft)

# The result is a [`FrequencySeries`](https://gwpy.github.io/docs/stable/frequencyseries/), with complex amplitude, representing the amplitude and phase of each frequency in our data.
# We can use `abs()` to extract the amplitude and plot that:

# In[7]:

plot = fft.abs().plot(xscale="log", yscale="log")
plot.show(warn=False)

# This doesn't look correct at all!
# The problem is that the FFT works under the assumption that our data are periodic, which means that the edges of our data look like discontinuities when transformed.
# We need to apply a window function to our time-domain data before transforming, which we can do using the [`scipy.signal`](https://docs.scipy.org/doc/scipy/reference/signal.html) module:

# In[8]:

from scipy.signal import get_window
window = get_window('hann', ldata.size)
lwin = ldata * window

# Let's try our transform again and see what we get

# In[9]:

fftamp = lwin.fft().abs()
plot = fftamp.plot(xscale="log", yscale="log")
plot.show(warn=False)

# This looks a little more like what we expect for the amplitude spectral density of a gravitational-wave detector.

# ## Calculating the power spectral density
#
# In practice, we typically use a large number of FFTs to estimate an averages power spectral density over a long period of data.
# We can do this using the [`asd()`](https://gwpy.github.io/docs/stable/api/gwpy.timeseries.TimeSeries.html#gwpy.timeseries.TimeSeries.asd) method, which uses [Welch's method](https://en.wikipedia.org/wiki/Welch%27s_method) to combine FFTs of overlapping, windowed chunks of data.

# In[10]:

asd = ldata.asd(fftlength=4, method="median")
plot = asd.plot()
plot.show(warn=False)

# In[11]:

ax = plot.gca()
ax.set_xlim(10, 1400)
ax.set_ylim(2e-24, 1e-20)
plot

# The ASD is a standard tool used to study the frequency-domain sensitivity of a gravitational-wave detector.
# For the LIGO-Livingston data we loaded, we can see large spikes at certain frequencies, including
#
# - ~300 Hz
# - ~500 Hz
# - ~1000 Hz
#
# The [O2 spectral lines](https://www.gw-openscience.org/o2speclines/) page on GWOSC describes a number of these spectral features for O2, with some of them being forced upon us, and some being deliberately introduced to help with interferometer control.
#
# Loading more data allows for more FFTs to be averaged during the ASD calculation, meaning random variations get averaged out, and we can see more detail:

# In[12]:

ldata2 = TimeSeries.fetch_open_data('L1', int(gps)-512, int(gps)+512, cache=True)
lasd2 = ldata2.asd(fftlength=4, method="median")
plot = lasd2.plot()
ax = plot.gca()
ax.set_xlim(10, 1400)
ax.set_ylim(5e-24, 1e-20)
plot.show(warn=False)

# Now we can see some more features, including sets of lines around ~30 Hz and ~65 Hz, and some more isolate lines through the more sensitive region.
#
# For comparison, we can load the LIGO-Hanford data and plot that as well:

# In[13]:

# get Hanford data
hdata2 = TimeSeries.fetch_open_data('H1', int(gps)-512, int(gps)+512, cache=True)
hasd2 = hdata2.asd(fftlength=4, method="median")

# and plot using standard colours
ax.plot(hasd2, label='LIGO-Hanford', color='gwpy:ligo-hanford')

# update the Livingston line to use standard colour, and have a label
lline = ax.lines[0]
lline.set_color('gwpy:ligo-livingston')  # change colour of Livingston data
lline.set_label('LIGO-Livingston')

ax.set_ylabel(r'Strain noise [$1/\sqrt{\mathrm{Hz}}$]')
ax.legend()
plot

# Now we can see clearly the relative sensitivity of each LIGO instrument, the common features between both, and those unique to each observatory.

# # Challenges:

# ##### Quiz Question 1:
#
# The peak amplitude in the LIGO-Livingston data occurs at approximately 5 seconds into the plot above and is undetectable above the background noise by the eye. Plot the data for the LIGO-Hanford detector around GW150914. Looking at your new LIGO-Handford plot, can your eye identify a signal peak?

# In[14]:

from gwosc.datasets import event_gps
gps1 = event_gps('GW150914')
print(gps1)
segment1 = (int(gps)-7, int(gps)+7)
print(segment1)

from gwpy.timeseries import TimeSeries
ldata1 = TimeSeries.fetch_open_data('L1', *segment1, verbose=True)
print(ldata1)

get_ipython().run_line_magic('matplotlib', 'inline')
plot1 = ldata1.plot()

# In[24]:

import matplotlib.pyplot as plt

ldata2 = TimeSeries.fetch_open_data('L1', int(gps)-512, int(gps)+512, cache=True)
lasd2 = ldata2.asd(fftlength=4, method="median")

# get Hanford data
hdata2 = TimeSeries.fetch_open_data('H1', int(gps)-512, int(gps)+512, cache=True)
hasd2 = hdata2.asd(fftlength=4, method="median")

fig, ax = plt.subplots(figsize=(13,7))
# and plot using standard colours
ax.plot(hasd2, label='LIGO-Hanford', color='gwpy:ligo-hanford')
ax.plot(lasd2, label='LIGO-Livingston', color='gwpy:ligo-livingston')

ax.set_xlim(2, 1400)
ax.set_ylim(5e-24, 2e-19)
# update the Livingston line to use standard colour, and have a label

ax.set_yscale('log')
ax.set_xscale('log')
ax.set_ylabel(r'Strain noise [$1/\sqrt{\mathrm{Hz}}$]')
ax.set_xlabel(r'Frequency [Hz]')

ax.legend()
plt.show()

# # Quiz Question 2 :
#
# Make an ASD around the time of an O3 event, GW190412 for L1 detector .  Compare this with the ASDs around GW150914 for L1 detector.  Which data have lower noise - and so are more sensitive - around 100 Hz?
#

# In[25]:

from gwosc.datasets import event_gps
gps_GW190412 = event_gps('GW190412')
print(gps_GW190412)
segment_GW190412 = (int(gps_GW190412)-7, int(gps_GW190412)+7)
print(segment_GW190412)

from gwpy.timeseries import TimeSeries
ldata_GW190412 = TimeSeries.fetch_open_data('L1', *segment_GW190412, verbose=True)
print(ldata_GW190412)

get_ipython().run_line_magic('matplotlib', 'inline')
plot_GW190412 = ldata_GW190412.plot()

# In[32]:

import matplotlib.pyplot as plt

ldata_GW190412 = TimeSeries.fetch_open_data('L1', int(gps_GW190412)-512, int(gps_GW190412)+512, cache=True)
lasd_GW190412 = ldata_GW190412.asd(fftlength=4, method="median")

# get Hanford data
hdata_GW190412 = TimeSeries.fetch_open_data('H1', int(gps_GW190412)-512, int(gps_GW190412)+512, cache=True)
hasd_GW190412 = hdata_GW190412.asd(fftlength=4, method="median")

fig, ax = plt.subplots(figsize=(13,7))
# and plot using standard colours
ax.plot(hasd_GW190412, label='LIGO-Hanford', color='gwpy:ligo-hanford')
ax.plot(lasd_GW190412, label='LIGO-Livingston', color='gwpy:ligo-livingston')

ax.set_xlim(2, 1400)
ax.set_ylim(4e-24, 7e-19)
# update the Livingston line to use standard colour, and have a label

ax.set_yscale('log')
ax.set_xscale('log')
ax.set_ylabel(r'Strain noise [$1/\sqrt{\mathrm{Hz}}$]')
ax.set_xlabel(r'Frequency [Hz]')

ax.legend()
plt.show()

# In[33]:

import matplotlib.pyplot as plt

ldata_GW190412 = TimeSeries.fetch_open_data('L1', int(gps_GW190412)-512, int(gps_GW190412)+512, cache=True)
lasd_GW190412 = ldata_GW190412.asd(fftlength=4, method="median")

ldata2 = TimeSeries.fetch_open_data('L1', int(gps)-512, int(gps)+512, cache=True)
lasd2 = ldata2.asd(fftlength=4, method="median")

fig, ax = plt.subplots(figsize=(13,7))
# and plot using standard colours
ax.plot(lasd2, label='LIGO-L1-GW150914', color='blue')
ax.plot(lasd_GW190412, label='LIGO-L1-GW190412', color='green')

ax.set_xlim(2, 1400)
ax.set_ylim(4e-24, 7e-19)
# update the Livingston line to use standard colour, and have a label

ax.set_yscale('log')
ax.set_xscale('log')
ax.set_ylabel(r'Strain noise [$1/\sqrt{\mathrm{Hz}}$]')
ax.set_xlabel(r'Frequency [Hz]')

ax.legend()
plt.show()

# # The GW190412 data has lesser noise around 100 Hz

# In[ ]:

gw-odw_Day1_with_Solns/Tuto 1.2 Open Data access with GWpy with solutions.py


# -*- coding: utf-8 -*-
# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: percent
#       format_version: '1.3'
#       jupytext_version: 1.11.1
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# %% [markdown]
# # Setup Notebook: Import, Cleanup, Normalize & Split Data

# %% [markdown]
# ### Lib imports & Options

# %%
import sys
from typing import Dict, OrderedDict, Tuple
import warnings
from collections import namedtuple

# ML libs
import numpy as np
import pandas as pd

# ASSERTS
# Python ≥3.5 is required
assert sys.version_info >= (3, 5)

# Pandas options
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 25)

# Ignore useless warnings (see SciPy issue #5998)
warnings.filterwarnings(action="ignore", message="^internal gelsd")

# %% [markdown]
# ### Import my libs

# %% [markdown]
# - The `loader` module is a py module located in the same dir of the notebooks.
# It sets up the `PYTHONPATH` in order to import py modules from other dirs within the project root.
# This way you can import the generated scripts (in `/src`) from within a notebook.
# - The `reloader` module let you reload the imported py modules (deep reload) that were modified since last time they were imported.
# By calling `reloader.clear()` it invalidated the cache of imported modules that were modified.

# %%
import loader # set PYTHONPATH for imports
import reloader # Reload local modified files with

# %% [markdown]
# Verify that the syspath was indeed modified by the `loader`.
# You should see a list of path here where python module are looked up, the last one should be your project root.

# %%
print(sys.path)

# %% [markdown]
# Once `import loader` has been executed you can now import other modules located in different directories under your project root.

# %%
from lib.pd import load_df, drop_na_cols, print_na_cols

# %% [markdown]
# If any of your external modules was modified then execute this cell but don't forget to comment it again once reloaded the modules or it will created import troubles when this notebook will be imported as module.

# %%
# reloader.clear() # ⚠ Uncomment and execute this to reload modules that were modified

# %% [markdown]
# ## Load the Data & Arrange the data structure

# %% [markdown]
# `load_df` with no arguments takes the `data_file_abs_path` OR `data_dir` + `data_file_name` defined in `config.toml`

# %%
df = load_df()
df

# %%
print_na_cols(df)

# %% [markdown]
# Example of imported function from the `lib` dir

# %%
# Drop all columns having more than 60% of missing values
df = drop_na_cols(df, perc=0.6)
df.head() # Content and Headquarters were dropped

# %% [markdown]
# we can pass multiple arguments to `load_df`, they will combine with what's defined in `config.toml`

# %%
df_future50 = load_df(data_file_name='Future50.csv')
df_future50.head()

# %%
print_na_cols(df_future50)

# %% [markdown]
# ## Other data Manipulation
#
# Usually you need to cleanup and re-arrange the data structure

# %% [markdown]
# ## Exported Vars
#
# Since we want to import this notebooks as if it was (it will be) a python module from another notebook, it would be nice not to pollute that notebooks with all the variables of this one.
# A solution would be to have a function to return only the needed variables, for example:

# %%
def get_export():
    return df, df_future50

# %% [markdown]
# So we can simply import `from src.load_data import get_export` and get the vars by executing `get_export`

# %% [markdown]
# # Transform this notebook into a python module
#
# Automatically whenever you save this notebook then **Jupytext** will export a new script in `src/load_data.py`

# %%


from __future__ import print_function, division

import io
from io import BytesIO
import os
from os.path import join
import shutil
import tempfile

try:
    from subprocess import STDOUT, CalledProcessError, check_output
except ImportError:
    pass

from sympy.core.compatibility import unicode, u_decode
from sympy.utilities.decorator import doctest_depends_on
from sympy.utilities.misc import find_executable
from .latex import latex

__doctest_requires__ = {('preview',): ['pyglet']}

@doctest_depends_on(exe=('latex', 'dvipng'), modules=('pyglet',),
            disable_viewers=('evince', 'gimp', 'superior-dvi-viewer'))
def preview(expr, output='png', viewer=None, euler=True, packages=(),
            filename=None, outputbuffer=None, preamble=None, dvioptions=None,
            outputTexFile=None, **latex_settings):
    r"""
    View expression or LaTeX markup in PNG, DVI, PostScript or PDF form.

    If the expr argument is an expression, it will be exported to LaTeX and
    then compiled using the available TeX distribution.  The first argument,
    'expr', may also be a LaTeX string.  The function will then run the
    appropriate viewer for the given output format or use the user defined
    one. By default png output is generated.

    By default pretty Euler fonts are used for typesetting (they were used to
    typeset the well known "Concrete Mathematics" book). For that to work, you
    need the 'eulervm.sty' LaTeX style (in Debian/Ubuntu, install the
    texlive-fonts-extra package). If you prefer default AMS fonts or your
    system lacks 'eulervm' LaTeX package then unset the 'euler' keyword
    argument.

    To use viewer auto-detection, lets say for 'png' output, issue

    >>> from sympy import symbols, preview, Symbol
    >>> x, y = symbols("x,y")

    >>> preview(x + y, output='png')

    This will choose 'pyglet' by default. To select a different one, do

    >>> preview(x + y, output='png', viewer='gimp')

    The 'png' format is considered special. For all other formats the rules
    are slightly different. As an example we will take 'dvi' output format. If
    you would run

    >>> preview(x + y, output='dvi')

    then 'view' will look for available 'dvi' viewers on your system
    (predefined in the function, so it will try evince, first, then kdvi and
    xdvi). If nothing is found you will need to set the viewer explicitly.

    >>> preview(x + y, output='dvi', viewer='superior-dvi-viewer')

    This will skip auto-detection and will run user specified
    'superior-dvi-viewer'. If 'view' fails to find it on your system it will
    gracefully raise an exception.

    You may also enter 'file' for the viewer argument. Doing so will cause
    this function to return a file object in read-only mode, if 'filename'
    is unset. However, if it was set, then 'preview' writes the genereted
    file to this filename instead.

    There is also support for writing to a BytesIO like object, which needs
    to be passed to the 'outputbuffer' argument.

    >>> from io import BytesIO
    >>> obj = BytesIO()
    >>> preview(x + y, output='png', viewer='BytesIO',
    ...         outputbuffer=obj)

    The LaTeX preamble can be customized by setting the 'preamble' keyword
    argument. This can be used, e.g., to set a different font size, use a
    custom documentclass or import certain set of LaTeX packages.

    >>> preamble = "\\documentclass[10pt]{article}\n" \
    ...            "\\usepackage{amsmath,amsfonts}\\begin{document}"
    >>> preview(x + y, output='png', preamble=preamble)

    If the value of 'output' is different from 'dvi' then command line
    options can be set ('dvioptions' argument) for the execution of the
    'dvi'+output conversion tool. These options have to be in the form of a
    list of strings (see subprocess.Popen).

    Additional keyword args will be passed to the latex call, e.g., the
    symbol_names flag.

    >>> phidd = Symbol('phidd')
    >>> preview(phidd, symbol_names={phidd:r'\ddot{\varphi}'})

    For post-processing the generated TeX File can be written to a file by
    passing the desired filename to the 'outputTexFile' keyword
    argument. To write the TeX code to a file named
    "sample.tex" and run the default png viewer to display the resulting
    bitmap, do

    >>> preview(x + y, outputTexFile="sample.tex")

    """
    special = [ 'pyglet' ]

    if viewer is None:
        if output == "png":
            viewer = "pyglet"
        else:
            # sorted in order from most pretty to most ugly
            # very discussable, but indeed 'gv' looks awful :)
            # TODO add candidates for windows to list
            candidates = {
                "dvi": [ "evince", "okular", "kdvi", "xdvi" ],
                "ps": [ "evince", "okular", "gsview", "gv" ],
                "pdf": [ "evince", "okular", "kpdf", "acroread", "xpdf", "gv" ],
            }

            try:
                for candidate in candidates[output]:
                    path = find_executable(candidate)
                    if path is not None:
                        viewer = path
                        break
                else:
                    raise SystemError(
                        "No viewers found for '%s' output format." % output)
            except KeyError:
                raise SystemError("Invalid output format: %s" % output)
    else:
        if viewer == "StringIO":
            viewer = "BytesIO"
            if outputbuffer is None:
                raise ValueError("outputbuffer has to be a BytesIO "
                                 "compatible object if viewer=\"StringIO\"")
        elif viewer == "BytesIO":
            if outputbuffer is None:
                raise ValueError("outputbuffer has to be a BytesIO "
                                 "compatible object if viewer=\"BytesIO\"")
        elif viewer not in special and not find_executable(viewer):
            raise SystemError("Unrecognized viewer: %s" % viewer)

    if preamble is None:
        actual_packages = packages + ("amsmath", "amsfonts")
        if euler:
            actual_packages += ("euler",)
        package_includes = "\n" + "\n".join(["\\usepackage{%s}" % p
                                             for p in actual_packages])

        preamble = r"""\documentclass[varwidth,12pt]{standalone}
%s

\begin{document}
""" % (package_includes)
    else:
        if packages:
            raise ValueError("The \"packages\" keyword must not be set if a "
                             "custom LaTeX preamble was specified")
    latex_main = preamble + '\n%s\n\n' + r"\end{document}"

    if isinstance(expr, str):
        latex_string = expr
    else:
        latex_string = ('$\\displaystyle ' +
                        latex(expr, mode='plain', **latex_settings) +
                        '$')

    try:
        workdir = tempfile.mkdtemp()

        with io.open(join(workdir, 'texput.tex'), 'w', encoding='utf-8') as fh:
            fh.write(unicode(latex_main) % u_decode(latex_string))

        if outputTexFile is not None:
            shutil.copyfile(join(workdir, 'texput.tex'), outputTexFile)

        if not find_executable('latex'):
            raise RuntimeError("latex program is not installed")

        try:
            # Avoid showing a cmd.exe window when running this
            # on Windows
            if os.name == 'nt':
                creation_flag = 0x08000000 # CREATE_NO_WINDOW
            else:
                creation_flag = 0 # Default value
            check_output(['latex', '-halt-on-error', '-interaction=nonstopmode',
                          'texput.tex'],
                         cwd=workdir,
                         stderr=STDOUT,
                         creationflags=creation_flag)
        except CalledProcessError as e:
            raise RuntimeError(
                "'latex' exited abnormally with the following output:\n%s" %
                e.output)

        if output != "dvi":
            defaultoptions = {
                "ps": [],
                "pdf": [],
                "png": ["-T", "tight", "-z", "9", "--truecolor"],
                "svg": ["--no-fonts"],
            }

            commandend = {
                "ps": ["-o", "texput.ps", "texput.dvi"],
                "pdf": ["texput.dvi", "texput.pdf"],
                "png": ["-o", "texput.png", "texput.dvi"],
                "svg": ["-o", "texput.svg", "texput.dvi"],
            }

            if output == "svg":
                cmd = ["dvisvgm"]
            else:
                cmd = ["dvi" + output]
            if not find_executable(cmd[0]):
                raise RuntimeError("%s is not installed" % cmd[0])
            try:
                if dvioptions is not None:
                    cmd.extend(dvioptions)
                else:
                    cmd.extend(defaultoptions[output])
                cmd.extend(commandend[output])
            except KeyError:
                raise SystemError("Invalid output format: %s" % output)

            try:
                # Avoid showing a cmd.exe window when running this
                # on Windows
                if os.name == 'nt':
                    creation_flag = 0x08000000 # CREATE_NO_WINDOW
                else:
                    creation_flag = 0 # Default value
                check_output(cmd, cwd=workdir, stderr=STDOUT,
                             creationflags=creation_flag)
            except CalledProcessError as e:
                raise RuntimeError(
                    "'%s' exited abnormally with the following output:\n%s" %
                    (' '.join(cmd), e.output))

        src = "texput.%s" % (output)

        if viewer == "file":
            if filename is None:
                buffer = BytesIO()
                with open(join(workdir, src), 'rb') as fh:
                    buffer.write(fh.read())
                return buffer
            else:
                shutil.move(join(workdir,src), filename)
        elif viewer == "BytesIO":
            with open(join(workdir, src), 'rb') as fh:
                outputbuffer.write(fh.read())
        elif viewer == "pyglet":
            try:
                from pyglet import window, image, gl
                from pyglet.window import key
            except ImportError:
                raise ImportError("pyglet is required for preview.\n visit http://www.pyglet.org/")

            if output == "png":
                from pyglet.image.codecs.png import PNGImageDecoder
                img = image.load(join(workdir, src), decoder=PNGImageDecoder())
            else:
                raise SystemError("pyglet preview works only for 'png' files.")

            offset = 25

            config = gl.Config(double_buffer=False)
            win = window.Window(
                width=img.width + 2*offset,
                height=img.height + 2*offset,
                caption="sympy",
                resizable=False,
                config=config
            )

            win.set_vsync(False)

            try:
                def on_close():
                    win.has_exit = True

                win.on_close = on_close

                def on_key_press(symbol, modifiers):
                    if symbol in [key.Q, key.ESCAPE]:
                        on_close()

                win.on_key_press = on_key_press

                def on_expose():
                    gl.glClearColor(1.0, 1.0, 1.0, 1.0)
                    gl.glClear(gl.GL_COLOR_BUFFER_BIT)

                    img.blit(
                        (win.width - img.width) / 2,
                        (win.height - img.height) / 2
                    )

                win.on_expose = on_expose

                while not win.has_exit:
                    win.dispatch_events()
                    win.flip()
            except KeyboardInterrupt:
                pass

            win.close()
        else:
            try:
                # Avoid showing a cmd.exe window when running this
                # on Windows
                if os.name == 'nt':
                    creation_flag = 0x08000000 # CREATE_NO_WINDOW
                else:
                    creation_flag = 0 # Default value
                check_output([viewer, src], cwd=workdir, stderr=STDOUT,
                             creationflags=creation_flag)
            except CalledProcessError as e:
                raise RuntimeError(
                    "'%s %s' exited abnormally with the following output:\n%s" %
                    (viewer, src, e.output))
    finally:
        try:
            shutil.rmtree(workdir) # delete directory
        except OSError as e:
            if e.errno != 2: # code 2 - no such file or directory
                raise


# -*- coding: utf-8 -*-
"""
@date Created on Tue Jan 12 13:54:56 2016
@copyright (C) 2015-2016 EOMYS ENGINEERING.
@author pierre_b
"""

from os.path import join
from unittest import TestCase

import matplotlib.pyplot as plt
from numpy import array, pi, zeros

from pyleecan.Classes.Frame import Frame
from pyleecan.Classes.LamSlotWind import LamSlotWind
from pyleecan.Classes.LamSquirrelCage import LamSquirrelCage
from pyleecan.Classes.Machine import Machine
from pyleecan.Classes.Shaft import Shaft
from pyleecan.Classes.VentilationCirc import VentilationCirc
from pyleecan.Classes.VentilationPolar import VentilationPolar
from pyleecan.Classes.VentilationTrap import VentilationTrap
from pyleecan.Classes.Winding import Winding
from pyleecan.Classes.WindingUD import WindingUD
from pyleecan.Classes.WindingCW2LT import WindingCW2LT
from pyleecan.Classes.WindingDW2L import WindingDW2L
from pyleecan.Classes.BHCurveMat import BHCurveMat
from pyleecan.Classes.BHCurveParam import BHCurveParam
from pyleecan.Classes.MatLamination import MatLamination
from pyleecan.Classes.SlotW23 import SlotW23

from pyleecan.Tests.Plot import save_path
from pyleecan.Tests.Plot.LamWind import wind_mat

class test_Lam_Wind_23_plot(TestCase):
    """unittest for Lamination with winding plot"""

    def test_Lam_Wind_23_wind_22(self):
        """Test machine plot with Slot 23 and winding rad=2, tan=2
        """
        print("\nTest plot Slot 23")
        plt.close("all")
        test_obj = Machine()
        test_obj.rotor = LamSlotWind(
            Rint=0,
            Rext=0.5,
            is_internal=True,
            is_stator=False,
            L1=0.9,
            Nrvd=5,
            Wrvd=0.02,
        )
        test_obj.rotor.slot = SlotW23(
            Zs=6, W0=50e-3, W1=90e-3, W2=100e-3, H0=20e-3, H1=35e-3, H2=130e-3
        )
        test_obj.rotor.winding = WindingUD(
            user_wind_mat=wind_mat, qs=4, p=4, Lewout=60e-3
        )
        test_obj.rotor.mat_type.magnetics = MatLamination(Wlam=0.5e-3)
        test_obj.shaft = Shaft(Drsh=test_obj.rotor.Rint * 2, Lshaft=1)

        test_obj.stator = LamSlotWind(
            Rint=0.51,
            Rext=0.8,
            is_internal=False,
            is_stator=True,
            L1=0.9,
            Nrvd=5,
            Wrvd=0.02,
        )
        test_obj.stator.slot = SlotW23(
            Zs=18, W0=50e-3, W1=80e-3, W2=50e-3, H0=15e-3, H1=0, H2=170e-3
        )
        test_obj.stator.winding = WindingDW2L(qs=3, p=3)
        test_obj.stator.mat_type.magnetics = MatLamination(Wlam=0.5e-3)
        test_obj.stator.winding.Lewout = 60e-3
        test_obj.frame = Frame(Rint=0.8, Rext=1, Lfra=1)

        test_obj.plot()
        fig = plt.gcf()
        fig.savefig(join(save_path, "test_Lam_Wind_s23_1-Machine.png"))
        # Rotor + Stator + 2 for frame + 0 shaft
        self.assertEqual(len(fig.axes[0].patches), 65)

        test_obj.rotor.plot()
        fig = plt.gcf()
        fig.savefig(join(save_path, "test_Lam_Wind_s23_2-Rotor.png"))
        # 1 for lam + Zs*4 for wind
        self.assertEqual(len(fig.axes[0].patches), 25)

        test_obj.stator.plot()
        fig = plt.gcf()
        fig.savefig(join(save_path, "test_Lam_Wind_s23_3-Stator.png"))
        # 2 for lam + Zs *2 for wind
        self.assertEqual(len(fig.axes[0].patches), 38)


import warnings  # python运行代码的时候，经常会碰到代码可以正常运行但是会提出警告，不想看到这些不重要的警告，所以使用控制警告输出

warnings.filterwarnings("ignore")  # 使用警告过滤器来控制忽略发出的警告

import pandas as pd
import numpy as np
import matplotlib  # python中类似于MATLAB的绘图工具，是一个2D绘图库
import matplotlib.pyplot as plt
import datetime  # datetime模块提供了各种类,用于操作日期和时间

# %matplotlib inline 表示内嵌绘图，有了这个命令就可以省略掉plt.show()命令了
from finrl.config import config  # 引入finrl包的配置
from finrl.marketdata.yahoodownloader import YahooDownloader
from finrl.preprocessing.preprocessors import FeatureEngineer
from finrl.preprocessing.data import data_split
from finrl.env.env_stocktrading import StockTradingEnv
from finrl.model.models import DRLAgent, DRLEnsembleAgent

from finrl.trade.backtest import (
    backtest_stats,
    get_daily_return,
    get_baseline,
    backtest_plot,
)
from pprint import pprint  # 用于打印 Python 数据结构. 使输出数据格式整齐, 便于阅读

import sys  # 该语句告诉Python，我们想要使用sys，此模块包含了与Python解释器和它的环境有关的函数

sys.path.append("../FinRL-Library")
# 在Python执行import sys语句的时候，python会根据sys.path的路径来寻找sys.py模块。
# 添加自己的模块路径， Sys.path.append(“mine module path”)

import itertools  # itertools模块中的函数可以用来对数据进行循环操作

"""
os.path.exists(path)，如果path是一个存在的路径，返回True，否则返回 False
os.path.exists(path)的应用：判断路径是否存在，不存在则创建
举例子
log_dir = "logs/"
if not os.path.exists(log_dir):
    os.makedirs(log_dir)
"""

import os

if not os.path.exists("./" + config.DATA_SAVE_DIR):  # "./"代表当前目录
    os.makedirs("./" + config.DATA_SAVE_DIR)
if not os.path.exists("./" + config.TRAINED_MODEL_DIR):
    os.makedirs("./" + config.TRAINED_MODEL_DIR)
if not os.path.exists("./" + config.TENSORBOARD_LOG_DIR):
    os.makedirs("./" + config.TENSORBOARD_LOG_DIR)
if not os.path.exists("./" + config.RESULTS_DIR):
    os.makedirs("./" + config.RESULTS_DIR)

# 下载数据
# Attributes
# ----------
#   start_date : str
#       start date of the data (modified from config.py)
#   end_date : str
#       end date of the data (modified from config.py)
#    ticker_list : list
#       a list of stock tickers (modified from config.py)

# Methods
# -------
# fetch_data()
#   Fetches data from yahoo API
# from config.py start_date is a string

config.START_DATE
config.END_DATE
print(config.DOW_30_TICKER)

# 缓存数据，如果日期或者股票列表发生变化，需要删除该缓存文件重新下载
SAVE_PATH = "./datasets/20210616-12h19.csv"
if os.path.exists(SAVE_PATH):
    df = pd.read_csv(SAVE_PATH)
else:
    df = YahooDownloader(
        config.START_DATE,  #'2000-01-01',
        config.END_DATE,  # 2021-01-01，预计将改日期改为'2021-06-20'（今日日期）
        ticker_list=config.DOW_30_TICKER,
    ).fetch_data()  # DOW_30_TICKER)道琼斯30只股票
    df.to_csv(SAVE_PATH)

df.head()  # 最开始5条
df.tail()  # tail仅展示了最后五条数据
df.shape
df.sort_values(["date", "tic"]).head()  # ticker表示股票代码，e.g.AAPL是苹果的股票
"""
pandas中的sort_values()函数可以根据指定行、列的数据进行排序
#DataFrame.sort_values(by=‘##’-按照指定列、行排序,ascending=True-默认升序排列, inplace=False-默认不替换原来数据集, na_position=‘last’-默认缺失值位置为last最后一个)
#按照df数据集的['date','tic']两列排序，其余参数取默认值
"""
# 数据预处理
"""
features：4+2+1+1(turb)
1.1. Add technical indicators: MACD  RSI  cci  adx
1.2. user_defined_feature: stock prices, current holding shares
1.3. current balance(当前账户所有额)
2.   turbulence index.
 FinRL employs the financial turbulence index that measures extreme asset price fluctuation.

Attributes
    ----------
        use_technical_indicator : boolean   注意：boolean(布尔值)取值为true or false,默认值是false
        we technical indicator or not
        tech_indicator_list : list
            a list of technical indicator names (modified from config.py)
        use_turbulence : boolean
            use turbulence index or not
        user_defined_feature:boolean
            user user defined features or not
注意：本文创新点是引入了一个turbulence提高模型的抗风险能力。文章通过定义turbulence的指数来反应股市状况，在崩盘的时候，则强制抛售所有资产以抵御股市崩盘的金融风险。
但是该特征比较trick，eg，崩盘事件概率太小，一般模型使用该指标合理么？且turbulence的阈值是超参数
事实上，当崩盘发生的时候，根本没时间反应

Methods
    -------
    preprocess_data()
        main method to do the feature engineering
"""

tech_indicators = ["macd", "rsi_30", "cci_30", "dx_30"]

fe = FeatureEngineer(
    use_technical_indicator=True,
    tech_indicator_list=tech_indicators,
    use_turbulence=True,
    user_defined_feature=False,
)
##使用finrl.preprocessing.preprocessors中的FeatureEngineer来对股价数据进行预处理

# 缓存数据，如果日期或者股票列表发生变化，需要删除该缓存文件重新下载
SAVE_PATH = "./datasets/20210616-12h19.preprocess.csv"
if os.path.exists(SAVE_PATH):
    processed = pd.read_csv(SAVE_PATH)
else:
    processed = fe.preprocess_data(df)
    processed.to_csv(SAVE_PATH)

list_ticker = processed["tic"].unique().tolist()  # 按照processed的"tic"列去重
list_date = list(
    pd.date_range(processed["date"].min(), processed["date"].max()).astype(str)
)  # 成一个固定频率的时间索引
combination = list(itertools.product(list_date, list_ticker))
"""
1.pandas.date_range(start=None, end=None, periods=None, freq='D', tz=None, normalize=False, name=None, closed=None, **kwargs)
由于import pandas as pd,所以也可以写成pd.date_range（start=None, end=None）
该函数主要用于生成一个固定频率的时间索引，使用时必须指定start、end、periods中的两个参数值，否则报错。
2.df.astype('str') #改变整个df变成str数据类型
3.itertools.product(*iterables[, repeat]) # 对应有序的重复抽样过程
  itertools.product(a,b),将a,b元组中的每个分量依次乘开。
"""

processed_full = pd.DataFrame(combination, columns=["date", "tic"]).merge(
    processed, on=["date", "tic"], how="left"
)
"""1.  pd.DataFrame( 某数据集 ，index  ，columns ),给某数据集加上行名index和列名columns
       此处只有pd.DataFrame( 某数据集 ，columns )，第一列加列名date，第二列加列名tic.
   2.  merge(df1,df2,on='key',how)
   按照["date","tic"]为关键字链接，以左边的dataframe为主导，左侧dataframe取全部数据，右侧dataframe配合左边
"""

processed_full = processed_full[processed_full["date"].isin(processed["date"])]
# isin函数，清洗数据，删选过滤掉processed_full中一些行，processed_full新加一列['date']若和processed_full中的['date']不相符合，则被剔除
processed_full = processed_full.sort_values(["date", "tic"])

processed_full = processed_full.fillna(0)
# 对于processed_full数据集中的缺失值使用 0 来填充.
processed_full.sample(5)  # sample（）是random模块中的一个函数，即随机取五个样本展示

# 设计强化学习实验环境
# trading environments, based on OpenAI Gym framework, simulate live stock markets with real market data
# according to the principle of time-driven simulation.
# action space:{-1,0,1}-{selling,holding,buying};
# {k,...,-1,0,1,...,k}-{number of shares to sell,number of shares to hold,number of shares to buy}

# The continuous action space needs to be normalized to [-1, 1],
# since the policy is defined on a Gaussian distribution,
# which needs to be normalized and symmetric.

stock_dimension = len(processed_full.tic.unique())
state_space = 1 + 2 * stock_dimension + len(tech_indicators) * stock_dimension
print(f"Stock Dimension: {stock_dimension}, State Space: {state_space}")

"""
1.按照processed_full的"tic"列去重并计算个数
2.计算状态空间的维数
"""
env_kwargs = {
    "hmax": 100,
    "initial_amount": 1000000,  # Since in Indonesia the minimum number of shares per trx is 100, then we scaled the initial amount by dividing it with 100
    "buy_cost_pct": 0.001,  # IPOT has 0.1% buy cost
    "sell_cost_pct": 0.001,  # IPOT has 0.1% sell cost
    "state_space": state_space,
    "stock_dim": stock_dimension,
    "tech_indicator_list": tech_indicators,
    "action_space": stock_dimension,
    "reward_scaling": 1e-4,
    "print_verbosity": 5,
}

# 使用DRL算法（validating 3 agents:A2C、PPO、DDPG）
rebalance_window = 25  # rebalance_window is the number of days to retrain the model
validation_window = (
    25  # validation_window is the number of days to do validation and trading
)
# e.g. if validation_window=63, then both validation and trading period will be 63 days
train_start = "2017-01-01"
train_end = "2020-07-01"
val_test_start = "2020-07-01"
val_test_end = "2021-01-01"

ensemble_agent = DRLEnsembleAgent(
    df=processed_full,
    train_period=(train_start, train_end),
    val_test_period=(val_test_start, val_test_end),
    rebalance_window=rebalance_window,
    validation_window=validation_window,
    **env_kwargs,
)

A2C_model_kwargs = {"n_steps": 5, "ent_coef": 0.01, "learning_rate": 0.0005}

PPO_model_kwargs = {
    "ent_coef": 0.01,
    "n_steps": 2048,
    "learning_rate": 0.00025,
    "batch_size": 128,
}

DDPG_model_kwargs = {
    "action_noise": "ornstein_uhlenbeck",
    "buffer_size": 50_000,
    "learning_rate": 0.000005,
    "batch_size": 128,
}

timesteps_dict = {"a2c": 30_000, "ppo": 100_000, "ddpg": 10_000}
# 疑问，为什么这里两个变量一样，赋值不一样呢？
timesteps_dict = {"a2c": 1_000, "ppo": 1_000, "ddpg": 1_000}

df_summary,model_ppo,model_a2c,model_ddpg = ensemble_agent.run_ensemble_strategy(
    A2C_model_kwargs, PPO_model_kwargs, DDPG_model_kwargs, timesteps_dict
)

models = [model_ppo,model_a2c,model_ddpg]

# r(s_t,a_t,s_(t+1) = (b_(t+1)+p_(t+1)*(h_(t+1)))-((b_t)+p_t*h_t)-ct
# 1.未使用model的累计收益
# 2.使用A2C得到的累计收益：A2C_model_kwargs
# 3.使用ppo得到的累计收益：PPO_model_kwargs
# 4.使用DDPG得到的累计收益：DDPG_model_kwargs
# 5.使用 集成策略 得到的累计收益：df_summary

def stat_result():
    # Backtest of Ensemble Strategy
    unique_trade_date = processed_full[
        (processed_full.date > val_test_start) & (processed_full.date <= val_test_end)
    ].date.unique()  # 使用划分好的验证集作为trade数据

    df_trade_date = pd.DataFrame(
        {"datadate": unique_trade_date}
    )  # 建立一个新的表，列名是datadate：内容是trade_date

    # 结果数据缓存在./result中，拼接所有的结果数据进行画图
    # ensemble, A2C, PPO, DDPG, (不使用策略)
    df_ensemble = pd.DataFrame()
    for i in range(
        rebalance_window + validation_window,
        len(unique_trade_date) + 1,
        rebalance_window,
    ):
        temp = pd.read_csv(
            "results/account_value_trade_{}_{}.csv".format("ensemble", i)
        )
        df_ensemble = df_ensemble.append(temp, ignore_index=True)

    sharpe = (
        (252 ** 0.5)
        * dfs[0].account_value.pct_change(1).mean()
        / dfs[0].account_value.pct_change(1).std()
    )
    print("Ensemble Sharpe Ratio: ", sharpe)
    dfs[0] = dfs[0].join(df_trade_date[validation_window:].reset_index(drop=True))

    #用3种模型去交易验证集中的数据
    ensemble_agent.DRL_validation(model=model_ddpg,test_data=validation,test_env=val_env_ddpg,test_obs=val_obs_ddpg)

    return dfs

dfs = stat_result()
# 将所有数据画到同一个图中
for i in [1,2,3]:
    x = dfs[i]["date"]
    y = dfs[i]["account_value"]
    #backtest_plot(dfs[i], '2020-07-02', '2020-11-20')
    plt.plot(x, y)

plt.savefig('account_value_plot.png')