#!/usr/bin/env python
# coding: utf-8

# In[ ]:

# 安装pandas
# pip install Pandas

# 运行测试套件
# 运行前需要安装: hypothesis和pytest
import pandas as pd
# pd.test()

# In[5]:

# 对象创建
# 传入一些值的列表来创建一个Series,pandas会自动创建一个默认的整数索引.
import pandas as pd
import numpy as np
import pprint

s = pd.Series([1,3,5,np.nan,6,8])
print(s)
print('-'*30)
# 传递带有日期时间索引和带标签列的NumPy数组来创建DataFrame
dates = pd.date_range('20190815',periods=6)
pprint.pprint(dates)

# In[8]:

df = pd.DataFrame(np.random.randn(6,4),index=dates,columns=list('ABCD'))
# df.to_excel('./output.xlsx')
pprint.pprint(df)

# In[9]:

# 转化为类似Series的dict对象来创建DataFrame
df2 = pd.DataFrame({'A': 1.,
                   'B': pd.Timestamp('20190820'),
                   'C': pd.Series(1,index=list(range(4)),dtype='float32'),
                   'D': np.array([3] * 4,dtype='int32'),
                   'E': pd.Categorical(["test","train","test1","train2"]),
                   'F': 'foo'})
print(df2)
# DataFrame的列具有不同的数据类型
print(df2.dtypes)

# In[10]:

### 查看数据
# 查看DataFrame顶部数据
print(df.head(3))
print('+='*30)
# 查看DataFrame尾部数据
print(df.tail(3))
print('--+'*30)
# 显示索引,列和底层NumPy数据.
print(df.index)
print('-='*30)
print(df.columns)
print('-|'*30)
# DataFrame.to_numpy() 会给出Numpy对象. 输出时不包含行索引和列索引.
print(df.to_numpy())

# In[11]:

# describe()方法显示数据的快速统计摘要
print(df.describe())
print('--'*30)
# 转置数据
print(df.T)
print('=='*30)
# 按轴排序
print(df.sort_index(axis=1,ascending=False))
print('-='*30)
# 按值排序
print(df.sort_values(by='B'))

# In[12]:

### 获取
print(df['A'])
# 对行进行切片
print(df[0:])
print(df[0:2])
print('-=='*30)
print(df['20190816':'20190818'])

# In[13]:

### 按标签选择
# 通过标签获取一行数据
print(df.loc[dates[0]])
print(df.loc[dates[1]])
print('=='*30)
# 通过标签在多个轴上选择数据
print('通过标签在多个轴上选择数据')
print(df.loc[:,['A','B']])
print('--'*30)
print(df.loc[:,['C']])

# In[14]:

# 通过标签同时在两个轴上切片
print('通过标签同时在两个轴上切片')
print(df.loc['20190817':'20190819',['A','B']])

# In[15]:

# 减小返回对象的大小
print(df.loc['20190820',['A','B']])

# In[16]:

# 获取标量值
print(df.loc[dates[0],'A'])

# In[17]:

# 快速访问标量
print(df.at[dates[0],'A'])

# In[18]:

### 布尔索引
# 使用单个列的值来选择数据
print(df[df.A > 0])   # 会输出为True的内容.
print(df.A > 0) # 将True和False都打印出来.

# In[19]:

# 从满足布尔条件的DataFrame中选择值:
print(df[df > 0])

# In[20]:

# 使用isin()方法过滤
df3 = df.copy()
# print(df3)
# df3['E'] = ['one','one','two','three','four','three']
df3['E'] = ['one','two','three','four','five','six']

# print(df3)
print('-='*30)
print(df3[df3['E'].isin(['two','four'])])

# In[21]:

### 赋值
# 添加新列将自动根据索引对齐数据.
s1 = pd.Series([1,2,3,4,5,6],index=pd.date_range('20190818',periods=6))
print(s1)
df3['F'] = s1
print(df3['F'])

# In[22]:

# 通过标签赋值
df3.at[dates[0],'A'] = 0
print(df3)

# In[27]:

### 通过位置赋值
df.iat[0,1] = 0
print(df)

# In[29]:

# 使用NumPy数组赋值
df3.loc[:,'D'] = np.array([5] * len(df))
print(df3) # 前面一系列赋值操作的结果.

# In[41]:

# 带有where条件的赋值操作.
df2 = df.copy()
df2[df2 > 0] = -df2
print(df2)

# In[42]:

### 缺失值
# pandas主要使用值np.nan来表示缺失的数据.
# 重建索引允许更改/添加/删除指定轴上的索引.这个操作会返回一个副本.
df5 = df.reindex(index=dates[0:4],columns=list(df.columns) + ['E'])
df5.loc[dates[0]:dates[1],'E'] = 1
print(df5)

# In[45]:

# 删除任何带有缺失值的行
print(df5.dropna(how='any'))

# In[46]:

# 填充缺失值
print(df5.fillna(value=5))

# In[47]:

# 获取值为nan的掩码
print(pd.isna(df5))

# In[49]:

### 统计
# 进行描述性统计
print(df5.mean())
print('-='*30)
# 在其它轴上进行同样的操作:
print(df5.mean(1))

# In[51]:

# 使用具有不同维度且需要对齐的对象进行操作. pandas会自动沿指定维度进行广播.
s = pd.Series([1,3,5,np.nan,6,8],index=dates).shift(2)
print(s)

print(df5.sub(s,axis='index'))

# In[53]:

### 应用
# 将函数应用于数据
print(df5.apply(np.cumsum))

print(df5.apply(lambda x: x.max() - x.min()))

# In[56]:

### 直方图化
s1 = pd.Series(np.random.randint(0,7,size=10))
print(s1)
print('=+'*30)
print(s1.value_counts())

# In[57]:

### 字符串方法
# Series在str属性中有一组字符串处理方法,可对数组的每个元素进行操作.
s2 = pd.Series(['A','B','C','Aaba','Baca',np.nan,'CABA','dog','cat'])
print(s2.str.lower())

# In[60]:

## 合并
### 连接
# 使用concat()连接pandas对象.
df6 = pd.DataFrame(np.random.randn(10,4))
print(df6)
print('---'*30)
pieces = [df6[:3],df6[3:7],df6[7:]]
print(pd.concat(pieces))

# In[62]:

### Join
# SQL风格的合并
left = pd.DataFrame({'key': ['foo','foo'],'lval':[1,2]})
right = pd.DataFrame({'key': ['foo','foo'],'rval': [4,5]})
print(left)
print('='*30)
print(right)
print('='*30)
print(pd.merge(left,right,on='key'))

# In[63]:

# 另一个例子
left = pd.DataFrame({'key': ['foo','bar'],'lval': [1,2]})
right = pd.DataFrame({'key': ['foo','bar'],'rval':[4,5]})
print(left)
print('-'*30)
print(right)
print(pd.merge(left,right,on='key'))

# In[66]:

### 追加
df7 = pd.DataFrame(np.random.randn(8,4),columns=['A','B','C','D'])
print(df7)
print('=='*30)
s3 = df7.iloc[3]
print(df7.append(s3,ignore_index=True))

# In[68]:

### 分组
"""
group by包括:
分割: 根据一些标准将数据分解成组.
应用: 将函数独立地应用于每个组.
组合: 将结果组合成数据结构.
"""
df8 = pd.DataFrame({'A': ['foo','bar','foo','bar',
                         'foo','bar','foo','foo'],
                   'B': ['one','one','two','three',
                        'two','two','one','three'],
                   'C': np.random.randn(8),
                   'D': np.random.randn(8)})

print(df8)
# 分组,然后将sum()函数应用于分组结果.
print(df8.groupby('A').sum())
print('=-'*30)
# 按多列分组形成层次索引,用sum函数
print(df8.groupby(['A','B']).sum())

# In[69]:

### 堆叠(Stack)
tuples = list(zip(*[['bar','bar','baz','baz',
                    'foo','foo','qux','qux'],
                   ['one','two','one','two',
                   'one','two','one','two']]))

index = pd.MultiIndex.from_tuples(tuples,names=['first','second'])
df = pd.DataFrame(np.random.randn(8,2),index=index,columns=['A','B'])
df9 = df[:4]
print(df9)

# In[70]:

### stack()方法压缩DataFrame的列
stacked = df9.stack()
print(stacked)

# In[72]:

# stack()的逆操作是unstack(),默认情况下取消最后压缩的哪个级别.
print(stacked.unstack())
print('=='*30)
print(stacked.unstack(1))
print('-='*30)
print(stacked.unstack(0))

# In[75]:

### 数据透视表
df10 = pd.DataFrame({'A': ['one','one','two','three'] * 3,
                    'B': ['A','B','C'] * 4,
                    'C': ['foo','foo','foo','bar','bar','bar'] * 2,
                    'D': np.random.randn(12),
                    'E': np.random.randn(12)})

print(df10)
print('-='*30)
# 从这些数据生成数据透视表
pd.pivot_table(df10,values='D',index=['A','B'],columns=['C'])

# In[76]:

### 时间序列(TimeSeries)
# 用于在频率转换期间执行重采样操作.
rng = pd.date_range('22/08/2019',periods=100,freq='S')
ts = pd.Series(np.random.randint(0,500,len(rng)),index=rng)
print(ts.resample('5Min').sum())

# In[79]:

# 时区代表
rng = pd.date_range('21/08/2019 21:29:30',periods=5,freq='D')
ts = pd.Series(np.random.randn(len(rng)),rng)
print(ts)

print('-='*30)
ts_utc = ts.tz_localize('UTC')
print(ts_utc)

print('-='*30)
# 转换为另一个时区
print(ts_utc.tz_convert('US/Eastern'))

# In[82]:

# 在时间跨度表示之间转换
rng = pd.date_range('22/08/2019',periods=5,freq='M')
ts = pd.Series(np.random.randn(len(rng)),index=rng)
print(ts)
print('-='*30)
ps = ts.to_period()
print(ps)
print('-='*30)
print(ps.to_timestamp())

# In[83]:

# 周期和时间戳之间的转换可以用算术函数.
# 示例: 以11月为结束年份的季度频率转换为季度结束后一个月末的上午9点.
prng = pd.period_range('2010Q1','2019Q4',freq='Q-NOV')
ts = pd.Series(np.random.randn(len(prng)),prng)
ts.index = (prng.asfreq('M','e') + 1).asfreq('H','s') + 9
print(ts.head())

# In[86]:

### 分类(Categoricals)
# pandas可以在DataFrame中包含分类数据.
df11 = pd.DataFrame({"id": [1,2,3,4,5,6],
                    "raw_grade": ['a','b','b','a','a','e']})
# 将原始成绩转换为category数据类型
df11["grade"] = df11["raw_grade"].astype("category")
print(df11["grade"])
print('-='*30)

# In[87]:

# 将类别重命名为更有意义的名称(通过调用Series.cat.categories来替换)
df11["grade"].cat.categories = ["very good","good","very bad"]
print(df11["grade"].cat.categories)

# In[88]:

# 对categories重新排序并同时添加缺少的category(Series.cat下的方法默认返回一个新的Series)
df11["grade"] = df11["grade"].cat.set_categories(["very bad","bad","medium",
                                                 "good","very good"])
print(df11["grade"])

# In[90]:

# 排序时按categories中的顺序排序,不是按照词汇顺序排序.
print(df11.sort_values(by="grade"))

# In[91]:

# 按分好类的列分组(groupby)可以显示空categories。
print(df11.groupby("grade").size())

# In[92]:

### 绘图
ts = pd.Series(np.random.randn(1000),
              index=pd.date_range('22/08/2019',periods=1000))
ts = ts.cumsum()
ts.plot()

# In[94]:

import matplotlib.pyplot as plt
# 在一个DataFrame中,plot方法绘制带有label的所有列.
df12 = pd.DataFrame(np.random.randn(1000,4),index=ts.index,
                   columns=['A','B','C','D'])

df13 = df12.cumsum()
plt.figure()
df13.plot()
plt.legend(loc='best')

# In[96]:

### 数据输入/输出
# 写入csv文件
df13.to_csv('./best.csv')
# 从csv文件读数据
pd.read_csv('./best.csv')

# In[102]:

### HDF5
# pip install tables
# 写入HDF5
df13.to_hdf('./best.h5','df')
# 从HDF5读数据
pd.read_hdf('./best.h5','df')

# In[103]:

### Excel
# 写入excel文件
df13.to_excel('./best.xlsx',sheet_name='best')
# 从excel文件读取数据
pd.read_excel('./best.xlsx','best',index_col=None,na_values=['NA'])

# In[104]:

# Gotchas 坑
# 异常
if pd.Series([False,True,False]):
    print("I was true")


# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np
from op_test import OpTest
import paddle
import paddle.fluid.core as core

paddle.enable_static()
np.random.seed(0)

class TestLerp(OpTest):

    def setUp(self):
        self.op_type = "lerp"
        self.python_api = paddle.lerp
        self.init_dtype()
        self.init_shape()
        x = np.arange(1., 101.).astype(self.dtype).reshape(self.shape)
        y = np.full(100, 10.).astype(self.dtype).reshape(self.shape)
        w = np.asarray([0.5]).astype(self.dtype)
        self.inputs = {'X': x, 'Y': y, 'Weight': w}
        self.outputs = {'Out': x + w * (y - x)}

    def init_dtype(self):
        self.dtype = np.float64

    def init_shape(self):
        self.shape = [100]

    def test_check_output(self):
        self.check_output(check_eager=True)

    def test_check_grad(self):
        self.check_grad(['X', 'Y'], 'Out', check_eager=True)

class TestLerpWithDim2(TestLerp):

    def init_shape(self):
        self.shape = [2, 50]

class TestLerpWithDim3(TestLerp):

    def init_shape(self):
        self.shape = [2, 2, 25]

class TestLerpWithDim4(TestLerp):

    def init_shape(self):
        self.shape = [2, 2, 5, 5]

class TestLerpWithDim5(TestLerp):

    def init_shape(self):
        self.shape = [2, 1, 2, 5, 5]

class TestLerpWithDim6(TestLerp):

    def init_shape(self):
        self.shape = [2, 1, 2, 5, 1, 5]

class TestLerpAPI(unittest.TestCase):

    def init_dtype(self):
        self.dtype = 'float32'

    def setUp(self):
        self.init_dtype()
        self.x = np.arange(1., 5.).astype(self.dtype)
        self.y = np.full(4, 10.).astype(self.dtype)
        self.w = np.asarray([0.75]).astype(self.dtype)
        self.res_ref = self.x + self.w * (self.y - self.x)
        self.place = [paddle.CPUPlace()]
        if core.is_compiled_with_cuda():
            self.place.append(paddle.CUDAPlace(0))

    def test_static_api(self):
        paddle.enable_static()

        def run(place):
            with paddle.static.program_guard(paddle.static.Program()):
                x = paddle.fluid.data('x', [1, 4], dtype=self.dtype)
                y = paddle.fluid.data('y', [1, 4], dtype=self.dtype)
                out = paddle.lerp(x, y, 0.5)
                exe = paddle.static.Executor(place)
                res = exe.run(feed={
                    'x': self.x.reshape([1, 4]),
                    'y': self.y.reshape([1, 4]),
                })
            for r in res:
                self.assertEqual(np.allclose(self.res_ref, r), True)

        for place in self.place:
            run(place)

    def test_dygraph_api(self):

        def run(place):
            paddle.disable_static(place)
            x = paddle.to_tensor(self.x)
            y = paddle.to_tensor(self.y)
            w = paddle.to_tensor(np.full(4, 0.75).astype(self.dtype))
            out = paddle.lerp(x, y, w)
            self.assertEqual(np.allclose(self.res_ref, out.numpy()), True)
            paddle.enable_static()

        for place in self.place:
            run(place)

    def test_inplace_api(self):

        def run(place):
            paddle.disable_static(place)
            x = paddle.to_tensor(self.x)
            y = paddle.to_tensor(self.y)
            x.lerp_(y, 0.75)
            self.assertEqual(np.allclose(self.res_ref, x.numpy()), True)
            paddle.enable_static()

        for place in self.place:
            run(place)

    def test_inplace_api_exception(self):

        def run(place):
            paddle.disable_static(place)
            x = paddle.to_tensor(self.x)
            y = paddle.to_tensor(self.y)
            w = paddle.to_tensor([0.75, 0.75], dtype=self.dtype)
            with self.assertRaises(ValueError):
                x.lerp_(y, w)
            paddle.enable_static()

        for place in self.place:
            run(place)

    def test_x_broadcast_y(self):
        paddle.disable_static()
        x = np.arange(1., 21.).astype(self.dtype).reshape([2, 2, 5])
        y = np.full(30, 10.).astype(self.dtype).reshape([3, 2, 1, 5])
        out = paddle.lerp(paddle.to_tensor(x), paddle.to_tensor(y), 0.5)
        res_ref = x + 0.5 * (y - x)
        self.assertEqual(np.allclose(res_ref, out.numpy()), True)
        paddle.enable_static()

    def test_x_y_broadcast_w(self):
        paddle.disable_static()
        x = np.arange(11., 21.).astype(self.dtype).reshape([2, 5])
        y = np.full(20, 7.5).astype(self.dtype).reshape([2, 2, 5])
        w = np.full(40, 0.225).astype(self.dtype).reshape([2, 2, 2, 5])
        out = paddle.lerp(paddle.to_tensor(x), paddle.to_tensor(y),
                          paddle.to_tensor(w))
        res_ref = x + w * (y - x)
        self.assertEqual(np.allclose(res_ref, out.numpy()), True)
        paddle.enable_static()

if __name__ == "__main__":
    unittest.main()


#!/usr/bin/env python
# coding: utf-8

# # Classifying Pulsars from the High Time Resolution Universe Survey (HTRU2) - Exploratory Data Analysis

# ## Overview & Citation

# The purpose of this project is to analyze telescope data to correctly separate actual pulsar data from radio frequency noise. The dataset was retrieved from the UC Irvine Machine Learning Repository at the following link: <https://archive.ics.uci.edu/ml/datasets/HTRU2#>
#
# The data comes from the High Time Resolution Universe Survey (South), known as HTRU2. The dataset was donated to the UCI Repository by Dr. Robert Lyon of The University of Manchester, United Kingdom. The two papers requested for citation in the description are listed below:
# * R. J. Lyon, B. W. Stappers, S. Cooper, J. M. Brooke, J. D. Knowles, Fifty Years of Pulsar Candidate Selection: From simple filters to a new principled real-time classification approach, Monthly Notices of the Royal Astronomical Society 459 (1), 1104-1123, DOI: 10.1093/mnras/stw656
# * R. J. Lyon, HTRU2, DOI: 10.6084/m9.figshare.3080389.v1.
#
# Below is an edited description of the dataset, taken from the website. The citation numbers refer to the publications listed on the website:
#
# *HTRU2 is a data set which describes a sample of pulsar candidates collected during the High Time Resolution Universe Survey (South) [1]. Pulsars are a rare type of Neutron star that produce radio emission detectable here on Earth. They are of considerable scientific interest as probes of space-time, the inter-stellar medium, and states of matter (see [2] for more uses).*
#
# *As pulsars rotate, their emission beam sweeps across the sky, and when this crosses our line of sight, produces a detectable pattern of broadband radio emission. As pulsars
# rotate rapidly, this pattern repeats periodically. Thus pulsar search involves looking for periodic radio signals with large radio telescopes. Each pulsar produces a slightly different emission pattern, which varies slightly with each rotation (see [2] for an introduction to pulsar astrophysics to find out why). Thus a potential signal detection known as a 'candidate', is averaged over many rotations of the pulsar, as determined by the length of an observation. In the absence of additional info, each candidate could potentially describe a real pulsar. However in practice almost all detections are caused by radio frequency interference (RFI) and noise, making legitimate signals hard to find.*
#
# *Machine learning tools are now being used to automatically label pulsar candidates to facilitate rapid analysis. Classification systems in particular are being widely adopted,
# (see [4,5,6,7,8,9]) which treat the candidate data sets as binary classification problems. Here the legitimate pulsar examples are a minority positive class, and spurious examples the majority negative class. At present multi-class labels are unavailable, given the costs associated with data annotation. The data set shared here contains 16,259 spurious examples caused by RFI/noise, and 1,639 real pulsar examples. These examples have all been checked by human annotators.*

# ## Feature Names

# The dataset was downloaded from the UC Irvine Repository as a CSV file. Column names were not provided in the original dataset. The features are listed below according to the feature names listed on the website, with the annotated column names used in this project listed in parentheses:
#
# 1. Mean of the integrated profile. (IP_Mean)
# 2. Standard deviation of the integrated profile. (IP_StdDev)
# 3. Excess kurtosis of the integrated profile. (IP_Kurtosis)
# 4. Skewness of the integrated profile. (IP_Skewness)
# 5. Mean of the DM-SNR curve. (DM_Mean)
# 6. Standard deviation of the DM-SNR curve. (DM_StdDev)
# 7. Excess kurtosis of the DM-SNR curve. (DM_Kurtosis)
# 8. Skewness of the DM-SNR curve. (DM_Skewness)
# 9. Class (Class)
#
# The XLSX and CSV files with the added column names are provided in the GitHub repository. We will import and explore the data in the following sections.

# ## Exploratory Data Analysis

# ### Import the Relevant Libraries

# In[1]:

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
sns.set()

# ### Import & Check the Data

# In[3]:

df = pd.read_csv('2020_1125_Pulsar_Data.csv')
pulsar_data = df.copy()

# In[4]:

pulsar_data.head()

# In[9]:

# Showing only data for input variables, since the statistics for binary variables in Class are meaningless
pulsar_data.drop('Class',axis=1,inplace=False).describe()

# ### Data Visualization

# The visualization below shows paired plots for every input and outut variable.

# In[10]:

sns.pairplot(pulsar_data)

# ## Modeling Recommendations

# This dataset poses a binary classification problem. All input values are numerical, and the output variable is categorical. Since there are no null values in the dataset, no additional preprocessing is requred prior to modeling. The following classification models should be explored and compared for performance:
# * Multiple Logistic Regression
# * Decision Tree Classification
# * Random Forest Classification
# * Support Vector Machines (SVM)
# * Deep Artificial Neural Networks (DNN)


# %% [markdown]
'''
# Jupyter notebooks
'''
# %% [markdown]
'''
Jupyter Notebook <cite> kluyver2016jupyter </cite> (with the `ipynb` extension) are documents that can contain code (including Python) and rich text elements (links, latex equations, figures).
They are powerfull and highly customizable for sharing interactive documents (through python widgets), and also easy to use (through your browser).

That is why I use it for my blog.
'''
# %% [markdown]
'''
## tl;dr
1. Jupyter notebooks has lot of benefits for data science works (interactive, customizable, easy of use).
2. Use magic commands for system specific or other language.
'''
#%% [markdown]
'''
## An example ?
'''
#%% [markdown]
'''
Yeah sure !

You will see below an example of a python code inside a jupyter notebook.
We will extract the histogram of randomly sampled data from a gaussian distribution, and plot it using matplotlib.
'''
# %% [markdown]
'''
First, let's import the modules:
'''
# %%
import numpy as np
import matplotlib.pyplot as plt
# %% [markdown]
'''
Then, we initialize some values:
'''
# %%
# Fixing random state for reproducibility
np.random.seed(1)

mu = 500
sigma = 10
x = mu + sigma * np.random.randn(1000)
# %% [markdown]
'''
Now, we calculate and plot the histogram
'''
# %%
# the histogram of the data
plt.figure()
n, bins, patches = plt.hist(x, 50, density=1, facecolor='g', alpha=0.8)
plt.xlabel('x')
plt.ylabel('p')
plt.title('Histogram')
plt.grid(True)
plt.show()
# %% [markdown]
'''
That's it!

>**Note**
>There is native support for matplotlib, so the figures are directly rendered in a cell.
If you want to render them in a new window, you should add in the cell:
```python
%matplotlib qt
```
'''
#%% [markdown]
'''
## To go further
'''
#%% [markdown]
'''
By default, all code cells run the language from the kernel you selected. Here I used the python kernel, so all the cells run python code.
If you want to run specific system command or code from another language, Jupyter notebooks supports [magic commands](https://ipython.readthedocs.io/en/stable/interactive/magics.html).

For example to run a bash command you would do,
'''
#%% [code]
%%bash
echo "Hello jupyter world!"
#%% [markdown]
'''
>**Note**
>Specifically for shell commands, you can also use this simpler syntax:
```shell
!echo "Hello jupyter world!"
```
'''
# %% [markdown]
'''
## Tags
'''
# %% [markdown]
'''
Software-Development; Interactivity; Open-Science
'''


# Copyright 2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================

"""create data for LM task"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import collections
import json
import logging
from typing import List, Optional
import numpy as np
import regex as re

logger = logging.getLogger(__name__)

def bytes_to_unicode():
    """
    bytes to unicode
    """
    bs = list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(range(ord("®"), ord("ÿ") + 1))
    cs = bs[:]
    n = 0
    for b in range(2 ** 8):
        if b not in bs:
            bs.append(b)
            cs.append(2 ** 8 + n)
            n += 1
    cs = [chr(i) for i in cs]
    return dict(zip(bs, cs))

def get_pairs(word):
    """
    Return set of symbol pairs in a word.
    Word is represented as tuple of symbols (symbols being variable-length strings).
    """
    pairs = set()
    prev_char = word[0]
    for char in word[1:]:
        pairs.add((prev_char, char))
        prev_char = char
    return pairs

class GPT2Tokenizer():
    """
    GPT2Tokenizer
    """
    def __init__(
            self,
            vocab_file,
            merge_file,
            add_prefix_space=False,
    ):
        with open(vocab_file, 'r', encoding="utf-8") as vocab_handle:
            self.encoder = json.load(vocab_handle)
        self.decoder = {v: k for k, v in self.encoder.items()}
        self.vocab_size = len(self.decoder)
        with open(merge_file, 'r', encoding="utf-8") as merge_handle:
            bpe_merges = merge_handle.read().split('\n')[1:-1]

        bpe_merges = [tuple(merge.split()) for merge in bpe_merges]

        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
        self.byte_encoder = bytes_to_unicode()
        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}

        self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
        self.add_prefix_space = add_prefix_space
        self.cache = {}

        self.unk_token = "<|endoftext|>"
        self.unk_token_id = 50256
        self.bos_token = "<|endoftext|>"
        self.bos_token_id = 50256
        self.eos_token = "<|endoftext|>"
        self.eos_token_id = 50256
        self.pad_token = "<|endoftext|>"
        self.pad_token_id = 50256

    def bpe(self, token):
        """
        bpe encode
        """

        if token in self.cache:
            return self.cache[token]

        word = tuple(token)
        pairs = get_pairs(token)
        if not pairs:
            return token

        while True:
            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
            if bigram not in self.bpe_ranks:
                break
            first, second = bigram
            new_word = []
            i = 0
            while i < len(word):
                try:
                    j = word.index(first, i)
                except ValueError:
                    new_word.extend(word[i:])
                    break
                else:
                    new_word.extend(word[i:j])
                    i = j

                if word[i] == first and i + 1 < len(word) and word[i + 1] == second:
                    new_word.append(first + second)
                    i += 2
                else:
                    new_word.append(word[i])
                    i += 1
            new_word = tuple(new_word)
            word = new_word
            if len(word) == 1:
                break
            else:
                pairs = get_pairs(word)
        word = " ".join(word)
        self.cache[token] = word
        return word

    def _tokenize(self, text):
        """ Tokenize a string using bpe encode. """
        text = self.prepare_for_tokenization(text, is_pretokenized=False)
        # print(text)
        bpe_tokens = []
        for token in re.findall(self.pat, text):
            token = "".join(
                self.byte_encoder[b] for b in token.encode("utf-8")
            )
            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
        return bpe_tokens

    def _convert_token_to_id(self, token):
        """ the index of the token in the vocabulary. """
        return self.encoder.get(token, self.encoder.get(self.unk_token))

    def _convert_id_to_token(self, _id):
        """ return the origin bpe token according to id"""
        return self.decoder.get(_id)

    def _convert_tokens_to_string(self, tokens):
        """ return a string according to the list of tokens"""
        text = "".join(tokens)
        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors='ignore')
        return text

    def encode(self, text):
        """ get the index list of text"""
        text_id = []
        bpe_tokens = self._tokenize(text)
        for token in bpe_tokens:
            text_id.append(self._convert_token_to_id(token))
        return text_id

    def decode(self, ids):
        """ return a string according to the index list of tokens"""
        tokens = []
        for id_ in ids:
            tokens.append(self._convert_id_to_token(id_))
        return self._convert_tokens_to_string(tokens)

    def prepare_for_tokenization(self, text, is_pretokenized=False, **kwargs):
        """ whether to add a whitespace in the front of text """
        add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space)
        if is_pretokenized or add_prefix_space:
            text = " " + text
        return text

    def num_special_tokens_to_add(self, pair: bool = False):
        token_ids_0 = []
        token_ids_1 = []
        return len(self.build_inputs_with_special_tokens(token_ids_0, token_ids_1 if pair else None))

    def build_inputs_with_special_tokens(self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None):
        """
        Build model inputs from a sequence or a pair of sequence by concatenating and adding special tokens.

        A GPT2 sequence has the following format:
        - single sequence: ``<bos> X <eos>``
        - pair of sequences: ``<bos> A <eos> B <eos>``

        Args:
            token_ids_0 (List[int]): List of IDs to which the special tokens will be added
            token_ids_1 (List[int], `optional`, defaults to `None`): Optional second list of IDs for sequence pairs.
        """
        bos = [self.bos_token_id]
        eos = [self.eos_token_id]
        if token_ids_1 is None:
            return bos + token_ids_0 + eos
        return bos + token_ids_0 + eos + token_ids_1 + eos

    def truncate_sequences(self, ids, num_tokens_to_remove, truncation_strategy="ONLY_FIRST", direction="RIGHT"):
        """
        truncate sequences
        Args:
            ids: Any
            num_tokens_to_remove:
            truncation_strategy: str
            direction: str

        Returns:
            (ids, overflowing_tokens): (Any, list)

        """
        if num_tokens_to_remove <= 0:
            return ids, []

        overflowing_tokens = []
        if truncation_strategy == "ONLY_FIRST":
            if len(ids) > num_tokens_to_remove:
                if direction == "RIGHT":
                    overflowing_tokens = ids[-num_tokens_to_remove:]
                    ids = ids[:-num_tokens_to_remove]
                if direction == "LEFT":
                    overflowing_tokens = ids[:num_tokens_to_remove]
                    ids = ids[num_tokens_to_remove:]
            else:
                logger.error("The first sequence length is smaller than removed tokens. ")
        else:
            logger.error("Please select correct truncation strategy, for instance 'ONLY_FIRST'")
        return (ids, overflowing_tokens)

    def _pad(self, encoded_inputs, max_length=None, padding_strategy=None,
             return_attention_mask: Optional[bool] = None):
        """
        _pad
        Args:
            encoded_inputs:
            max_length: Any
            padding_strategy: Any
            return_attention_mask: Optional[bool]

        Returns:
            encoded_inputs:

        """
        needs_to_be_padded = (len(encoded_inputs["input_ids"]) != max_length)
        if needs_to_be_padded:
            if padding_strategy == "MAX_LENGTH":
                difference = max_length - len(encoded_inputs["input_ids"])
                if return_attention_mask:
                    encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"]) + [0] * difference
                    encoded_inputs["input_ids"] = encoded_inputs["input_ids"] + [self.pad_token_id] * difference
            else:
                raise ValueError("Invalid padding strategy")
        else:
            if return_attention_mask:
                encoded_inputs["attention_mask"] = [1] * len(encoded_inputs["input_ids"])

        return encoded_inputs

    def pad(self, encoded_inputs, max_length: Optional[int] = None, padding_strategy="MAX_LENGTH",
            return_attention_mask=True):
        """
        pad
        Args:
            encoded_inputs:
            max_length: Optional[int]
            padding_strategy: str
            return_attention_mask: bool

        Returns:
            batch_outputs: Dict[Any, list]

        """
        # no batch encoded_inputs["input_ids"]--->[98, 67, 32388, 318, 1912, 287, 170, 8496, 318, 905, 2667, 32]
        if encoded_inputs["input_ids"] and not isinstance(encoded_inputs["input_ids"][0], (list, tuple)):
            encoded_inputs = self._pad(
                encoded_inputs,
                max_length=max_length,
                padding_strategy=padding_strategy,
                return_attention_mask=return_attention_mask
            )
            return encoded_inputs

        # encoded_inputs with batch_size
        batch_size = len(encoded_inputs["input_ids"])
        assert all(
            len(v) == batch_size for v in encoded_inputs.values()
        ), "Some items in the output dictionary have a different batch size than others."

        if padding_strategy == "LONGEST":
            max_length = max(len(inputs) for inputs in encoded_inputs["input_ids"])
            padding_strategy = "MAX_LENGTH"

        batch_outputs = {}
        for i in range(batch_size):
            inputs = dict((k, v[i]) for k, v in encoded_inputs.items())
            outputs = self._pad(
                encoded_inputs=inputs,
                max_length=max_length,
                padding_strategy=padding_strategy,
                return_attention_mask=return_attention_mask
            )
            for key, value in outputs.items():
                if key not in batch_outputs:
                    batch_outputs[key] = []
                batch_outputs[key].append(value)

        return batch_outputs

    def prepare_for_model(self,
                          ids,
                          pair_ids=None,
                          add_special_tokens=True,
                          max_length=None,
                          padding=None,
                          truncate_direction="RIGHT",
                          return_overflowing_tokens=False,
                          return_attention_mask=True):
        """
        prepare for model
        Args:
            ids:
            pair_ids:
            add_special_tokens: bool
            max_length: Any
            padding: Any
            truncate_direction: str
            return_overflowing_tokens: bool
            return_attention_mask: bool

        Returns:
            encoded_inputs:Dict

        """

        pair = bool(pair_ids is not None)
        len_ids = len(ids)
        len_pair_ids = len(pair_ids) if pair else 0

        encoded_inputs = {}
        # Compute the total size of the returned encodings
        total_len = len_ids + len_pair_ids + (self.num_special_tokens_to_add(pair=pair) if add_special_tokens else 0)

        # Truncation: Handle max sequence length
        if max_length and total_len > max_length:

            ids, overflowing_tokens = self.truncate_sequences(ids=ids,
                                                              num_tokens_to_remove=total_len - max_length,
                                                              truncation_strategy="ONLY_FIRST",
                                                              direction=truncate_direction)
            if return_overflowing_tokens:
                encoded_inputs["overflowing_tokens"] = overflowing_tokens
                encoded_inputs["num_truncated_tokens"] = total_len - max_length

        if add_special_tokens:
            sequence = self.build_inputs_with_special_tokens(ids, pair_ids)
        else:
            sequence = ids + pair_ids if pair else ids

        # build output dictionary
        encoded_inputs["input_ids"] = sequence
        # check lengths
        if max_length is None or len(encoded_inputs["input_ids"]) > max_length:
            logger.warning(
                "Token indices sequence length is longer than the specified maximum sequence length "
                "for this model (%ids > %length). Running this sequence through the model will result in "
                "indexing errors", len(ids), max_length
            )
        # padding
        if padding or return_attention_mask:
            encoded_inputs = self.pad(encoded_inputs=encoded_inputs,
                                      max_length=max_length,
                                      padding_strategy="MAX_LENGTH",
                                      return_attention_mask=return_attention_mask)

        return encoded_inputs

def create_instance(tokenizer, text, max_length=None):
    """A single sample instance for LM task."""
    sentence = text.strip().split("\t")

    ids = tokenizer.encode(sentence[0])
    pair_ids = None
    if len(sentence) == 2:
        pair_ids = tokenizer.encode(sentence[1])

    output = tokenizer.prepare_for_model(ids=ids,
                                         pair_ids=pair_ids,
                                         add_special_tokens=True,
                                         max_length=max_length,
                                         padding=True,
                                         truncate_direction="LEFT",
                                         return_overflowing_tokens=False,
                                         return_attention_mask=True)
    return output

def write_instance_to_file(instance):
    """write the instance to file"""
    input_ids = instance["input_ids"]
    input_mask = instance["attention_mask"]
    label_ids = instance["input_ids"]
    assert len(input_ids) == len(label_ids)

    features = collections.OrderedDict()
    features["input_ids"] = np.asarray(input_ids)
    features["input_mask"] = np.asarray(input_mask)
    features["label_ids"] = np.asarray(label_ids)

    return features

def dataset_preprocess_seq(input_file, tokenizer, max_length):
    """ infer dataset preprocess for PTB or 1BW"""
    print("***** Reading from  %s *****", input_file)
    input_ids = []
    input_mask = []
    label_ids = []
    total_read = 0
    total_written = 0
    with open(input_file, "r") as f:
        while True:
            line = f.readline()
            if not line:
                break
            total_read += 1
            if total_read % 500 == 0:
                print("%d ...", total_read)

            output = create_instance(tokenizer, line, max_length)
            features = write_instance_to_file(instance=output)
            total_written += 1

            if total_written <= 20:
                print("***** Example *****")
                print("input tokens: %s", tokenizer.decode(output["input_ids"][:-1]))
                print("label tokens: %s", tokenizer.decode(output["input_ids"][1:]))

                input_ids.append(np.array(features['input_ids'], dtype=np.int64))
                input_mask.append(np.array(features['input_mask'], dtype=np.int64))
                label_ids.append(np.array(features['label_ids'], dtype=np.int64))

                for feature_name in features.keys():
                    feature = features[feature_name]
                    print("%s: %s", feature_name, feature)
    print("Wrote %d total instances", total_written)
    return input_ids, input_mask, label_ids

def wikitext_dataset_preprocess_seq(input_file, tokenizer, max_length):
    """ infer dataset preprocess for wikitext"""
    print("***** Reading from  %s *****", input_file)
    passage = []
    input_ids = []
    input_mask = []
    label_ids = []
    total_read = 0
    total_written = 0
    with open(input_file, 'r', encoding='utf-8') as f:
        for line in f:
            line = line.strip()
            if line:
                if line.startswith('=') and line.endswith('=') and passage:
                    passage = []
                elif line.startswith('=') and line.endswith('='):
                    continue
                else:
                    passage.append(line)
                    total_read += 1
                    if total_read % 500 == 0:
                        print("%d ...", total_read)

                    output = create_instance(tokenizer, line, max_length)
                    features = write_instance_to_file(instance=output)
                    total_written += 1

                    if total_written <= 20:
                        print("***** Example *****")
                        print("input tokens: %s", tokenizer.decode(output["input_ids"][:-1]))
                        print("label tokens: %s", tokenizer.decode(output["input_ids"][1:]))

                        input_ids.append(np.array(features['input_ids'], dtype=np.int64))
                        input_mask.append(np.array(features['input_mask'], dtype=np.int64))
                        label_ids.append(np.array(features['label_ids'], dtype=np.int64))

                        for feature_name in features.keys():
                            feature = features[feature_name]
                            print("%s: %s", feature_name, feature)
    print("Wrote %d total instances", total_written)
    return input_ids, input_mask, label_ids

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--input_file", type=str, required=True, help='Input raw text file. ')
    parser.add_argument("--output_file", type=str, required=True, help='Output MindRecord file. ')
    parser.add_argument("--num_splits", type=int, default=1,
                        help='The MindRecord file will be split into the number of partition. ')
    parser.add_argument("--max_length", type=int, required=True, help='Maximum sequence length. ')
    parser.add_argument("--dataset", type=str, default="ptb",
                        help="The name of dataset which should be processed, only for LanguageModeling task.")
    parser.add_argument("--vocab_file", type=str, required=True, default='', help='url of gpt2-vocab.json ')
    parser.add_argument("--merge_file", type=str, required=True, default='', help='url of gpt2-merges.txt ')
    args = parser.parse_args()

    tokenizer = GPT2Tokenizer(vocab_file=args.vocab_file, merge_file=args.merge_file)

    input_file = args.input_file
    print("***** Reading from input files *****")
    print("Input File: %s", input_file)

    output_file = args.output_file
    print("***** Writing to output files *****")
    print("Output File: %s", output_file)

    if args.dataset == "ptb" or args.dataset == "onebw":
        input_ids, input_mask, label_ids = dataset_preprocess_seq(input_file,
                                                                  tokenizer, args.max_length)
    elif args.dataset == "wikitext2" or args.dataset == "wikitext103":
        input_ids, input_mask, label_ids = wikitext_dataset_preprocess_seq(input_file,
                                                                           tokenizer, args.max_length)
    else:
        raise Exception("Dataset not supported. support: [ptb, onebw, wikitext2, wikitext103]")

    np.savetxt(output_file + '/input_ids.txt', np.array(input_ids), fmt='%i', delimiter=' ')
    np.savetxt(output_file + '/input_mask.txt', np.array(input_mask), fmt='%i', delimiter=' ')
    np.savetxt(output_file + '/label_ids.txt', np.array(label_ids), fmt='%i', delimiter=' ')

if __name__ == "__main__":
    main()


# coding: utf-8

# # Analisi del problema di Quora
# ---
#
# Il problema è proposto da http://www.quora.com. Il sito propone un sistema per condividere le proprie conoscenze in vari ambiti e per espandere queste stesse conoscenze. Questo grazie ad un sistema di semplici domande e risposte, che sono però organizzate in vari topic (interessi) che le persone possono seguire per personalizzare la loro esperienza. E’ presente un profilo personale che mostra domande e risposte e le conoscenze principali di ogni persona.
# Il problema posto dall’azienda è il riconoscimento di domande che hanno lo stesso significato: sono molte infatti le domande che vengono ripetute (per pigrizia degli utenti o anche a causa della ricerca). Diventerebbe quindi importante ottimizzare il numero di domande unendo quelle uguali (con stesso scopo), ma anche (e soprattutto) unire le risposte, per rendere più completa possibile l’esperienza dell’utente riguardo lo specifico argomento della domanda.
# Attualmente il sistema utilizza un modello RandomForest per identificare le domande uguali: la sfida lanciata è quella di trovare degli algoritmi migliori di rilevamento usando tecniche avanzate di Machine Learning e di Natural Language Processing.
#
# Il problema si presenta come una classificazione binaria con l'uso di addestramento supervisionato
#
# # Analisi  Training Set
# ---
#
# Abbiamo iniziato ad analizzare il dataset partendo dal Training Set (`train.csv`).
# L'header indica la denominazione delle 5 features e del label:
# 1. `id` = id della coppia di domande
# 2. `qid1` = id della prima domanda
# 3. `qid2` = id della seconda domanda
# 4. `question1` = testo della prima domanda
# 5. `question2` = testo della seconda domanda
# 6. `is_duplicate` = indica se le due domande sono uguali (è il label)

# In[1]:

get_ipython().run_cell_magic('time', '', "\nimport pandas as pd\nimport numpy as np\nimport time\n\n#Read dataset\ntrain = pd.read_csv('/usr/local/share/kaggle/Quora Question Pairs/uncompressed/train.csv')\n\n#Print dataframe columns types infos\nprint(train.dtypes)\n\n#Print some examples\ntrain.head()\nprint(len(train))")

# # Elaborazione dei dati
# ---
#
# Inizialmente, seguendo i suggerimenti trovati in alcuni notebook presenti su kaggle, si era deciso di ripulire il dataset, eliminando tutto ciò che non fosse lettere o numeri.
# Questa decisione si è rivelata poi controproducente in quanto l'algoritmo risulta meno performante su di un dataset "pulito".
#
# Si noti la differenza tra una frase "sporca" e una "pulita"

# ### FASE 0A: Creazione dizionari
#
# Inizialmente è utile costruire delle strutture dati che poi saranno impiegate nel corso dell'elaborazione dei dati; anche se esse vengono utilizzate in fasi diverse dell'elaborazione risulta utile crearle tutte insime in un unica operazione.
# Le strutture dati sono le seguenti
#
# * `dictionary`: Dizionario che associa a ciascuna parola un id intero
# * `f_list`: lista che associa a ciascuna parola la propria frequenza, nello specifico contiene coppie (`id_parola`, `frequenza_parola`); la lista è ordinata in ordine decrescente di frequenza
# * `n_data`: dataset formato da due colonne contenente le coppie di domande, dove ciascuna domanda è rappresentata da un array di `id_parola`
# * `id_train`: `n_data` con l'aggiunta di una prima colonna contenente l'id della coppia di domande
#

# In[2]:

get_ipython().run_cell_magic('time', '', "\ndef build_dictionary(data):\n    #Initialization\n    dictionary = {} #dizionario: stringa -> id\n    c = 0\n    frequency_list = [] #lista frequenze: list[id] = (id_parola, frequenza_parola)\n    converted_data = [] #dataset che contiene numeri al posto delle parole \n    #Takes every word in every couple and creates a dictionary\n    for col in ['question1', 'question2']:\n        new_col = []\n        for sentence in data[col]:\n            sentence_converted = []\n            sentence = str(sentence).split(' ')\n            for p in sentence:\n                #Checks if word is already in dictionary\n                if str(p) not in dictionary.keys():\n                    #Checks length of word and if it's a space, then add\n                    if not (len(str(p)) == 1 and ord(str(p)) == 32):\n                        dictionary[str(p)] = c\n                        frequency_list.append((c,1))\n                        sentence_converted.append(c)\n                        c = c+1\n\n                else:\n                    c_old = dictionary[str(p)]\n                    (n, freq) = frequency_list[c_old]\n                    frequency_list[c_old] = (n, freq+1)\n                    sentence_converted.append(c_old)\n\n            new_col.append(sentence_converted)\n\n        converted_data.append(new_col)            \n\n    sorted_frequency_list = sorted(frequency_list, key=lambda pair: pair[1], reverse=True)                \n\n    return dictionary, sorted_frequency_list, converted_data\n\ndictionary, f_list, n_data = build_dictionary(train)\n\nid_train = pd.DataFrame({\n        'id': train['id'],\n        'question1': n_data[0],\n        'question2': n_data[1]\n        })")

# ### Fase 0B: Inizializzazione rete di test
# ---
#
# La rete prescelta per la soluzione di questo problema è `MLPClassifier` presente all'interno del package `sklearn`.
# Come funzione di attivazione è stata scelta `logistic` in quanto la probabilità che le due domande siano un duplicato è rappresentata da un numero reale compreso tra 0 e 1 (come da istruzioni presenti sul sito)
# Come algoritmo solver è stato scelto `adam` perchè, dopo aver effettuato dei test di confronto con gli altri algoritmi, è risultato essere quello che dava il risultato migliore
# Come metrica di valutazione è stata scelta la funzione `LogLoss` in quanto è la stessa utilizzata da Kaggle
#
# In questa fase seleziono anche il numero di elementi da usare come insieme di train e quanti invece come insieme di test

# In[3]:

from sklearn.neural_network import MLPClassifier
from sklearn.metrics import log_loss

from keras.models import Sequential
from keras.layers import Dropout, Dense, BatchNormalization
from keras.losses import binary_crossentropy
import keras.utils as kUtils

#labels
duplicate = train['is_duplicate']

#initialize algorithm
alg = MLPClassifier(solver='adam', activation='logistic')

# split dataset train/test
split = np.random.rand(len(train)) < 0.8

#funzione di test
def test(features, k=0):

    features_train = features[split]
    features_test = features[~split]

    labels_train = duplicate[split]
    labels_test = duplicate[~split]

    alg.fit(features_train, labels_train)
    predict = alg.predict_proba(features_test)
    logLoss = log_loss(labels_test, predict)

    print('Sklearn Log Loss: {}'.format(logLoss))

    plot(predict[:,1], labels_test)

    ###KERAS NETWORK###
    features_train = np.asmatrix(features_train)
    features_test = np.asmatrix(features_test)

    labels_train = np.asmatrix(labels_train)
    labels_test = np.asmatrix(labels_test)

    labels_train = kUtils.to_categorical(labels_train, 2)
    labels_test = kUtils.to_categorical(labels_test, 2)

    model = Sequential()
    model.add(Dense(200, activation='relu', input_dim = features_train.shape[1]))
    model.add(Dropout(0.1))
    model.add(BatchNormalization())
    model.add(Dense(100, activation='relu'))
    model.add(Dropout(0.1))
    model.add(BatchNormalization())
    model.add(Dense(2, activation='sigmoid'))

    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

    model.fit(features_train, labels_train,
          batch_size=128,
          epochs=20,
          verbose=0,
          validation_data=(features_test, labels_test))

    score = model.evaluate(features_test, labels_test, verbose=0)
    keras_predict = model.predict_proba(features_test)

    print('Keras Log loss: {}'.format(score[0]))
    #plot(keras_predict[:,1], labels_test)

    ####################

    if (k == 1):
        return predict[:,1]

import matplotlib.pyplot as plt

#funzione per mostrare la distribuzione delle predizioni rispetto alle label di test
def plot(p, labels_test):

    t = []
    f = []

    for p, l in zip(p, labels_test):
        if l == True:
            t.append(p)
        else:
            f.append(p)

    plt.hist(f, bins=20, normed=True, label='Not Duplicate')
    plt.hist(t, bins=20, normed=True, alpha=0.7, label='Duplicate')
    plt.show()

# ### FASE 0C: Multithreading
# ---
#
# Scrivo una funzione che suddivida il lavoro su 4 core in modo da velocizzare l'esecuzione dell'algoritmo

# In[4]:

from threading import Thread
import numpy as np

def split_work(fun, train):
    dataframes = np.array_split(train, 4)

    threads = []
    results = {}
    i = 0

    for data in dataframes:
        t = Thread(target=fun, args=(data, results, i))
        threads.append(t)
        t.start()
        i = i+1

    for t in threads:
        t.join()

    features = []
    for key in sorted(results):
        features.extend(results[key])

    return features

# ### FASE 0D: Salvataggio e caricamento delle strutture dati
# ---

# In[5]:

def save(s, o):
    obj = np.asarray(o)
    np.save(s, obj)

def load(s):
    return np.load(s)

def compute(fun, data, file):
    if not os.path.exists(file):
        res = split_work(fun, data)
        save(file, res)
    else:
        res = load(file)

    return res

# ### FASE 1: Estrazione Features di base
# ---
#
# Come prima cosa risulta utile estrarre le feature più semplici che possono caratterizzare una frase:
#
# * `q1_words`: numero di parole contenute nella prima domanda
# * `q1_letters`: numero di caratteri contenuti nella prima domanda
# * `q2_words`: numero di parole contenute nella seconda domanda
# * `q2_letters`: numero di lettere contenute nella seconda domanda
#
# Infatti risulta piuttosto intuitivo che frasi di simile lunghezza hanno più probabilità di avere lo stesso significato

# In[6]:

get_ipython().run_cell_magic('time', '', "import os\n\ndef base_extraction(data):\n\n    if not os.path.exists('base_features.csv'):\n        #Count number of words and letters for each 'question1'\n        q1_words = data['question1'].apply(lambda x: len(str(x).split(' ')))\n        q1_letters = data['question1'].apply(lambda x: len(str(x)))\n\n        #Count number of words and letters for each 'question2'\n        q2_words = data['question2'].apply(lambda x: len(str(x).split(' ')))\n        q2_letters = data['question2'].apply(lambda x: len(str(x)))\n\n        features = pd.DataFrame({\n                    'q1_words': q1_words,\n                    'q1_letters': q1_letters,\n                    'q2_words': q2_words,\n                    'q2_letters': q2_letters,\n                    })\n        \n        features.to_csv('base_features.csv', index=False, header=list(features))\n    else:\n        features = pd.read_csv('base_features.csv')\n    \n    return features\n\nfeatures = base_extraction(train)\n\n#TEST\n\n#test(features)")

# ### Fase 2: LCS
# ---
#
# Dovendo stabilire la similarità tra due frasi un buon indicatore può essere la presenza di sottosequenze comuni, ho quindi deciso di prendere come parametro la lunghezza della più lunga sottostringa comune tra le due domande.

# In[7]:

get_ipython().run_cell_magic('time', '', "\ndef LCSLength(data, results, k):\n    res =[]\n    \n    for row in data.itertuples():\n        X = getattr(row, 'question1')\n        Y = getattr(row, 'question2')\n        m = len(X)\n        n = len(Y)\n        C = np.zeros((m,n), dtype=np.int8);\n        for i in range(m):\n            for j in range(n):\n                if X[i] == Y[j]:\n                    C[i,j] = C[i-1,j-1] + 1\n                else:\n                    C[i,j] = max(C[i,j-1], C[i-1,j])    \n        res.append(C[m-1,n-1])\n    results[k] = res\n\nLCSs = compute(LCSLength, id_train, 'LCS.npy')\nfeatures = features.assign(**{'LCS' : LCSs})\n\n#TEST\n\n#test(features)")

# ### Fase 3: Word Similar Share
# ---
#
# Appoggiandomi alla libreria `Spacy`, libreria specifica per il Natural Language Processing, ho calcolato un indice di similarità tra le due domande come somma degli indici di similarità tra ciascuna coppia di parole (`w1`,`w2`) dove `w1` appartiene alla prima domanda e `w2` alla seconda
#
# Inizialmente avevo normalizzato l'indice rispetto alla lunghezza delle due domande ma la predizione risultava più precisa senza normalizzazione

# In[8]:

import spacy
nlp = spacy.load('en_core_web_md')

# In[9]:

def word_similar_share(data, results, i):
    res = []
    for row in data.itertuples():
        q1 = getattr(row, 'question1')
        q2 = getattr(row, 'question2')

        q1words = nlp(str(q1))
        q2words = nlp(str(q2))

        jac = 0

        for w1 in q1words:
            s_w1 = str(w1)
            if not (len(s_w1) == 1 and ord(s_w1) == 32):
                for w2 in q2words:
                    s_w2 = str(w2)
                    if not (len(s_w2) == 1 and ord(s_w2) == 32):
                        jac = jac + w1.similarity(w2)

        res.append(jac)
    results[i] = res
############################################

#word_match = split_work(word_similar_share, train)
#features_tmp = features.assign(**{'Word_Match' : word_match})

#TEST

#test(features_tmp)

# ### Fase 4: Common/Rare Words
# ---
#
# In miglioramento dell'algoritmo visto alla fase 3 è l'introduzione del concetto di parola comune e parola rara
# Una parola è considerata rara se la sua frequenza è bassa, e similmente è considerata comune se la sua frequenza è alta
#
# Si ottengono quindi due algoritmi distinti, entrambi ignorano le parole comuni
# * `word_match_rare_function`: prendendo in considerazione le parole rare si ottiene un indice di similarità che viene incrementato di 1 per ogni parola rara che le due domande hanno in comune e decrementato di 1 per ogni parola rara che compare in una sola delle due domande
# * `word_similar_share_cr`: come il `word_similar_share` visto al punto 3
#
# Anche in questo caso normalizzare gli indici ne riduce la capacità predittiva

# In[10]:

def word_match_rare_function(data, results, i):
    res = []
    for row in data.itertuples():
        q1words = getattr(row, 'question1')
        q2words = getattr(row, 'question2')

        rare_jac = 0
        #n_rare = 0
        q1_rares = set()
        q2_rares = set()

        for w1 in q1words:
            if w1 in rare_words:
                q1_rares.add(w1)

        for w2 in q2words:
            if w2 in rare_words:
                q2_rares.add(w2)

        for w1r in q1_rares:
            if w1r in q2_rares:
                rare_jac = rare_jac + 1
            else:
                rare_jac = rare_jac - 1

        for w2r in q2_rares:
            if not w2r in q1_rares:
                rare_jac = rare_jac - 1
        '''
        try:
            rare_jac = rare_jac/n_rare
        except:
            rare_jac = 0
        '''
        res.append(rare_jac)
    results[i] = res

def word_similar_share_cr(data, results, i):
    res = []
    for row in data.itertuples():
        q1 = getattr(row, 'question1')
        q2 = getattr(row, 'question2')

        q1words = nlp(str(q1))
        q2words = nlp(str(q2))

        jac = 0

        for w1 in q1words:
            s_w1 = str(w1)
            if not (len(s_w1) == 1 and ord(s_w1) == 32 and s_w1 not in common_words):
                for w2 in q2words:
                    s_w2 = str(w2)
                    if not (len(s_w2) == 1 and ord(s_w2) == 32 and s_w2 not in common_words):
                        jac = jac + w1.similarity(w2)
                try:
                    jac = jac/(len(q1words)+len(q2words))
                except:
                    jac = 0
        res.append(jac)
    results[i] = res

common_words = set()
rare_words = set()

#build rare words set
for (p,f) in f_list:
    if f<=20:
        rare_words.add(p)

#Build common words set
for (p,f) in f_list[:10]:
    common_words.add(p)

#Find similarity with Jaccard for each question pair
#word_match_cr = split_work(word_similar_share_cr, train)
#word_match_rare = split_work(word_match_rare_function, id_train)

#features_tmp = features.assign(**{'Word_Match_cr' : word_match_cr})
#features_tmp = features.assign(**{'Word_Match_rare' : word_match_rare})

#TEST

#test(features_tmp)

# ### Fase 4B: vari valori per Common e Rare
# ---
#
# I valori di frequenza per le parole comuni e rare al punto 4 erano stati scelti arbitrariamente, quindi in questa fase si testeranno vari valori per tali frequenze per determinare le migliori

# In[11]:

# start = time.time()

# for F in range(20, 110, 10):
#     for n in range(4, 11):
#         data_tmp = features.copy()
#         C = pow(2,n)

#         common_words = set()
#         rare_words = set()

#         #build rare words list
#         for (p,f) in f_list:
#             if f<=F:
#                 rare_words.add(p)

#         #Build common words list
#         for (p,f) in f_list[:C]:
#             common_words.add(p)

#         #Find similarity with Jaccard for each question pair
#         word_match_cr = train.apply(word_similar_share_cr, axis=1, raw=True)
#         word_match_rare = id_train.apply(word_match_rare_function, axis=1, raw=True)

#         data_tmp = data_tmp.assign(**{'Word_Match_cr' : word_match_cr})
#         data_tmp = data_tmp.assign(**{'Word_Match_rare' : word_match_rare})

#         #TEST

#         features_train = features_tmp[split]
#         features_test = features_tmp[~split]

#         labels_train = duplicate[split]
#         labels_test = duplicate[~split]

#         alg.fit(features_train, labels_train)
#         predict = alg.predict_proba(features_test)
#         logLoss = log_loss(labels_test, predict)

#         results = []
#         results.append((C,F,logLoss))
#         print("Log loss C: {}, F: {} = {}".format(C,F,logLoss))

# sorted_results = sorted(results, key=lambda pair: pair[2])
# print(sorted_results[:5])
# end = time.time()
# print('time: {}'.format(end - start))

# ### Fase 4C: deduzioni
# ---
#
# Dall'esecuzione della fase 5 si deduce che i parametri che danno il risultato migliore sono `C = 64 ` e `F = 50`. Non è detto che si rivelino buoni parametri anche per il set di test ma dato che comunque l'algoritmo basato sulle parole rare si ha dimostrato di dare buoni risultati terrò questi parametri validi anche per il caso di test.
# In ogni caso il valore dei parametri incide poco sul risultato finale, in quanto il valore migliore risulta essere `0.4996397287043904` mentre il peggiore è `0.5068634596789862` per una differenza di `0.00722373097`

# In[12]:

get_ipython().run_cell_magic('time', '', "\nF = 50\nC = 64\n\ndef word_match_rare_function(data, results, i):\n    res = []\n    for row in data.itertuples():   \n        q1words = getattr(row, 'question1')\n        q2words = getattr(row, 'question2')\n\n        rare_jac = 0\n        #n_rare = 0\n        q1_rares = set()\n        q2_rares = set()\n\n        for w1 in q1words:\n            if w1 in rare_words:\n                q1_rares.add(w1)\n\n        for w2 in q2words:\n            if w2 in rare_words:\n                q2_rares.add(w2)\n\n        for w1r in q1_rares:\n            if w1r in q2_rares:\n                rare_jac = rare_jac + 1\n            else:\n                rare_jac = rare_jac - 1\n\n        for w2r in q2_rares:\n            if not w2r in q1_rares:\n                rare_jac = rare_jac - 1\n        '''                \n        try:\n            rare_jac = rare_jac/n_rare\n        except:\n            rare_jac = 0                            \n        '''\n        res.append(rare_jac)\n    results[i] = res\n\ndef word_similar_share_cr(data, results, i):\n    res = []\n    for row in data.itertuples():   \n        q1 = getattr(row, 'question1')\n        q2 = getattr(row, 'question2')\n    \n        q1words = nlp(str(q1))\n        q2words = nlp(str(q2))\n\n        jac = 0\n\n        for w1 in q1words:\n            s_w1 = str(w1)\n            if not (len(s_w1) == 1 and ord(s_w1) == 32 and s_w1 not in common_words):\n                for w2 in q2words:\n                    s_w2 = str(w2)\n                    if not (len(s_w2) == 1 and ord(s_w2) == 32 and s_w2 not in common_words):\n                        jac = jac + w1.similarity(w2)\n                '''\n                try:\n                    jac = jac/(len(q1words)+len(q2words))\n                except:\n                    jac = 0\n                '''    \n        res.append(jac)\n    results[i] = res\n\ncommon_words = set()\nrare_words = set()\n\n#build rare words set\nfor (p,f) in f_list:\n    if f<=F:\n        rare_words.add(p)\n\n#Build common words set\nfor (p,f) in f_list[:C]:\n    common_words.add(p)\n\n#Find similarity with Jaccard for each question pair\nword_match_cr = compute(word_similar_share_cr, train, 'word_match_cr.npy')\nword_match_rare = compute(word_match_rare_function, id_train, 'word_match_rare.npy')\n    \nfeatures = features.assign(**{'Word_Match_cr' : word_match_cr})\nfeatures = features.assign(**{'Word_Match_rare' : word_match_rare})\n\n#TEST\n\n#test(features)")

# ### Fase 5: Cosine Similarity
# ---
#
# Un altro utile parametro per la similarità tra frasi è la Cosine similarity: una tecnica euristica per la misurazione della similitudine tra due vettori effettuata calcolando il coseno tra di loro.
#
# Per poterla applicare è quindi necessario convertire ciascuna frase in una `Bag of words`, cioè un'array di dimensione pari al numero di parole distinte presenti all'interno del dataset. In questo array la posizione i-esima è settata a 1 se la parola comprare nella frase, 0 altrimenti.
# In pratica ciascuna frase è rappresentata come un vettore in uno spazio n-dimensionale (con n pari alla dimensione di ciascuna `Bag of words`)
# A questo punto si applica semplicemente la cosine similarity tra i due vettori ottenuti

# In[13]:

# start = time.time()

# #bag words size
# dim = len(dictionary.keys())
# print("bag words size: {}".format(dim))

# #converte ogni frase in una bag of words da usare per la cosine similarity
# def cosineConversion(sentence):
#     s = np.zeros(dim)
#     for w in sentence:
#         s[w] = 1
#     return s

# #simple wrap for cosineConversion
# def cosineConversion_wrap(data):
#     c1 = []
#     c2 = []
#     for sent1 in data['question1']:
#         s1 = cosineConversion(sent1)
#         c1.append(s1)

#     for sent2 in data['question2']:
#         s2 = cosineConversion(sent2)
#         c2.append(s2)

#     return c1, c2

# #calculate Cosine Similarity
# def cosineSimilarity(row):
#     a = row['question1']
#     b = row['question2']

#     cos_sim = dot(a, b)/(norm(a)*norm(b))

#     return cos_sim

# cosine1, cosine2 = cosineConversion_wrap(id_train)

# cosine_data = pd.DataFrame({
#             'id': train['id'],
#             'question1': cosine1,
#             'question2': cosine2
#             })

# mid = time.time()

# print('Cosine Similarity dataset: {}'.format(mid - start))

# cosine_similarity = cosine_data.apply(cosineSimilarity, axis=1, raw=True)

# features_tmp = features.copy().assign(**{'Cosine Similarity' : cosine_similarity})

# end = time.time()
# print('Cosine Similarity Algorithm: {}'.format(end - mid))

# #TEST

# test(features_tmp)

# Sfortunatamente questa prima versione dell'algoritmo occupa troppa memoria

# ### Fase 5B: cosine similarity migliorata
# ---
# Questa versione dell'algoritmo è più efficiente rispetto alla precedente e occupa molta meno memoria. Inoltre invece che far assumere a ciascun elemento della `Bag of words` i valori 0/1, `Sentence[w]` varrà  `n` cioè il numero di apparizioni della data parola all'interno della frase

# In[14]:

from numpy.linalg import norm

#bag words size
dim = len(dictionary.keys())

def cosineConversion(sentence):
    s = np.zeros(dim, dtype=np.int8)
    for w in sentence:
        s[w] = s[w] + 1
    return s

def cosineSimilarity(data, results, i):
    res = []
    for row in data.itertuples():
        q1 = getattr(row, 'question1')
        q2 = getattr(row, 'question2')

        a = cosineConversion(q1)
        b = cosineConversion(q2)

        cos_sim = np.dot(a, b)/(norm(a)*norm(b))

        res.append(cos_sim)
    results[i] = res

#cosine_similarity = split_work(cosineSimilarity, id_train)
#features_tmp = features.assign(**{'Cosine Similarity' : cosine_similarity})

#TEST

#test(features_tmp)

# ### Fase 5C: Cosine Similarity Sklearn
# ---
#
# La `Cosine Similarity` sembrava un buon parametro ma l'algoritmo precedentemente utilizzato non dava il risultato sperato, così mi sono appoggiato all'algoritmo presente all'interno del pacchetto `Sklearn`

# In[15]:

get_ipython().run_cell_magic('time', '', "from sklearn.feature_extraction.text import TfidfVectorizer as TF\nfrom sklearn.metrics.pairwise import cosine_similarity\n\ndef cosineSimilarity(data, results, i):\n    res = []\n    for row in data.itertuples():   \n        q1 = str(getattr(row, 'question1'))\n        q2 = str(getattr(row, 'question2'))\n\n        vect = TF(min_df=1)\n        tfidf = vect.fit_transform([q1, q2])\n\n        res.append((tfidf * tfidf.T).A[0,1])\n    results[i] = res\n    \ndef cosineSimilarity2(data):\n    vect = TF(min_df=1)\n    l1 = data['question1']\n    l2 = data['question2']\n    res = []\n    \n    vect.fit(data['question1'] + data['question2'])\n    for row in data.itertuples():   \n        q1 = str(getattr(row, 'question1'))\n        q2 = str(getattr(row, 'question2'))\n\n        tf1 = vect.transform(q1)\n        tf2 = vect.transform(q2)\n\n        cs = cosine_similarity(tf1,tf2)\n        res.append(cs)\n\n#cos_sim = compute(cosineSimilarity, train, 'cosine_similarity.npy')\ncos_sim = cosineSimilarity2(train)\nfeatures = features.assign(**{'cos_sim': cos_sim})\n\n#TEST\n#test(features)")

# La cosine similarity calcolata da Sklearn risulta essere migliore rispetto a quella calcolata precedente

# ### Fase 6: Spacy Sentence similarity
# ---
#
# La libreria Spacy prevede la possibilità di calcolare direttamente un indice di similarità tra frasi, questo si va ad aggiungere ai parametri precedenti

# In[16]:

get_ipython().run_cell_magic('time', '', "\ndef sentenceSimilarity(data, results, i):\n    res = []\n    for row in data.itertuples():   \n        q1s = getattr(row, 'question1')\n        q2s = getattr(row, 'question2')\n    \n        q1 = nlp(str(q1s))\n        q2 = nlp(str(q2s))\n\n        res.append(q1.similarity(q2))\n    results[i] = res\n\nspacy_sim = compute(sentenceSimilarity, train, 'spacy_similarity.npy')\nfeatures = features.assign(**{'spacy_sim': spacy_sim})     \n\n#TEST\n\n#test(features)\n\nfeatures_set = features")

# ## Recap
# ---
#
# I parametri finora individuati sono:
# * features di base
# * LCS
# * jaccard common/rare
# * cosine similarity (Sklearn)
# * Spacy sentence similarity
#
# essi risultano ottimi per l'individuazione di frasi non duplicate, risulta invece più difficile individuare i doppioni

# ## GridSearch

# In[18]:

from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer
'''
X_train = features[split]
X_test = features[~split]

y_train = duplicate[split]
y_test = duplicate[~split]

possible_parameters = {
    'activation': ['identity', 'logistic', 'tanh', 'relu'],
    'solver': ['lbfgs', 'sgd', 'adam']
}

alg = MLPClassifier()
loss_f = make_scorer(log_loss, greater_is_better=False, needs_proba=True)

# The GridSearchCV is itself a classifier
# we fit the GridSearchCV with the training data
# and then we use it to predict on the test set
clf = GridSearchCV(alg, possible_parameters, n_jobs=4, scoring=loss_f) # n_jobs=4 means we parallelize the search over 4 threads
clf.fit(X_train, y_train)

y_pred = clf.predict_proba(X_test)
logLoss = log_loss(y_test, y_pred)

logLoss
'''

# In[ ]:

'''
cv = clf.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv['mean_fit_time'],
    'mean_score_time': cv['mean_score_time'],
    'mean_test_score': cv['mean_test_score'],
    'mean_train_score': cv['mean_train_score'],
    'param_activation': cv['param_activation'],
    'param_solver': cv['param_solver'],
    'rank_test_score': cv['rank_test_score']
})

tab.sort_values(['rank_test_score']).head(3)
'''

# In[26]:

get_ipython().run_cell_magic('time', '', "\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import make_scorer\n\nalg = MLPClassifier(solver='adam', activation='logistic')\nloss_f = make_scorer(log_loss, greater_is_better=False, needs_proba=True)\n\nX_train = features[split]\nX_test = features[~split]\n\ny_train = duplicate[split]\ny_test = duplicate[~split]\n\npossible_parameters = {\n    'alpha': [0.0001, 0.00005],\n    'batch_size': [100, 50],\n    'learning_rate_init': [0.001, 0.0005],\n    'beta_1': [0.9, 0.99],\n    'beta_2': [0.999, 0.5],\n    'epsilon': [5e-9, 2e-9]\n}\n\nclf2 = GridSearchCV(alg, possible_parameters, n_jobs=4, scoring=loss_f) # n_jobs=4 means we parallelize the search over 4 threads\nclf2.fit(X_train, y_train)\n\ny_pred = clf2.predict_proba(X_test)\nlogLoss = log_loss(y_test, y_pred)\n\nlogLoss")

# In[28]:

cv = clf2.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv['mean_fit_time'],
    'mean_score_time': cv['mean_score_time'],
    'mean_test_score': cv['mean_test_score'],
    'mean_train_score': cv['mean_train_score'],
    'param_alpha': cv['param_alpha'],
    'param_batch_size': cv['param_batch_size'],
    'param_learning_rate_init': cv['param_learning_rate_init'],
    'rank_test_score': cv['rank_test_score'],
    'param_beta_1': cv['param_beta_1'],
    'param_epsilon': cv['param_epsilon']
})

tab.sort_values(['rank_test_score']).head(3)

# ## Random Forest Classifier

# In[39]:

get_ipython().run_cell_magic('time', '', "from sklearn.ensemble import RandomForestClassifier\n\nrf = RandomForestClassifier(random_state=0, n_jobs=4)\n\npossible_parameters = {\n    'max_depth': [x for x in range(1, 20, 1)],\n    'criterion': ['gini', 'entropy'],\n    'max_features': ['auto', 'log2', None]\n}\n\nclf = GridSearchCV(rf, possible_parameters, n_jobs=4, scoring=loss_f)\n\nclf.fit(X_train, y_train)")

# In[42]:

y_pred = clf.predict_proba(X_test)
logLoss = log_loss(y_test, y_pred)

logLoss

# Max_depth = 2, gini: 0.5456

# In[41]:

cv_rf = clf.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv_rf['mean_fit_time'],
    'mean_score_time': cv_rf['mean_score_time'],
    'mean_test_score': cv_rf['mean_test_score'],
    'mean_train_score': cv_rf['mean_train_score'],
    'param_max_depth': cv_rf['param_max_depth'],
    'param_criterion': cv_rf['param_criterion'],
    'param_max_features': cv_rf['param_max_features'],
    'rank_test_score': cv_rf['rank_test_score'],
})

tab.sort_values(['rank_test_score']).head(3)

# ## Normalize Data

# In[22]:

from sklearn.preprocessing import MinMaxScaler
import warnings
warnings.filterwarnings('ignore')

features_scaled = pd.DataFrame(columns = list(features))

letters_scaler = MinMaxScaler()
words_scaler = MinMaxScaler()

letters_scaler.fit(features['q1_letters'] + features['q2_letters'])
features_scaled['q1_letters'] = letters_scaler.transform(features['q1_letters'])
features_scaled['q2_letters'] = letters_scaler.transform(features['q2_letters'])

words_scaler.fit(features['q1_words'] + features['q2_words'])
features_scaled['q1_words'] = words_scaler.transform(features['q1_words'])
features_scaled['q2_words'] = words_scaler.transform(features['q2_words'])

for feat in  ['LCS', 'Word_Match_cr', 'Word_Match_rare', 'cos_sim', 'spacy_sim']:
    features_scaled[feat] = MinMaxScaler().fit_transform(features[feat])

# ### MLPClassifier

# In[24]:

from sklearn.model_selection import GridSearchCV
from sklearn.metrics import make_scorer

alg = MLPClassifier(solver='adam', activation='logistic')
loss_f = make_scorer(log_loss, greater_is_better=False, needs_proba=True)

X_train = features_scaled[split]
X_test = features_scaled[~split]

y_train = duplicate[split]
y_test = duplicate[~split]

possible_parameters = {
    'alpha': [0.0001, 0.00005],
    'batch_size': [100, 50],
    'learning_rate_init': [0.001, 0.0005],
    'beta_1': [0.9, 0.99],
    'beta_2': [0.999, 0.5],
    'epsilon': [5e-9, 2e-9]
}

mlp = GridSearchCV(alg, possible_parameters, n_jobs=4, scoring=loss_f) # n_jobs=4 means we parallelize the search over 4 threads
mlp.fit(X_train, y_train)

# In[25]:

cv = mlp.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv['mean_fit_time'],
    'mean_score_time': cv['mean_score_time'],
    'mean_test_score': cv['mean_test_score'],
    'mean_train_score': cv['mean_train_score'],
    'param_alpha': cv['param_alpha'],
    'param_batch_size': cv['param_batch_size'],
    'param_learning_rate_init': cv['param_learning_rate_init'],
    'rank_test_score': cv['rank_test_score'],
    'param_beta_1': cv['param_beta_1'],
    'param_epsilon': cv['param_epsilon']
})

tab.sort_values(['rank_test_score']).head(3)

# ### Random Forest

# In[29]:

from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(random_state=0, n_jobs=4)

possible_parameters = {
    'max_depth': [x for x in range(1, 20, 1)],
    'criterion': ['gini', 'entropy'],
    'max_features': ['auto', 'log2', None]
}

possible_parameters['max_depth'].append(None)

clf = GridSearchCV(rf, possible_parameters, n_jobs=4, scoring=loss_f)

clf.fit(X_train, y_train)

# In[30]:

cv_rf = clf.cv_results_
tab = pd.DataFrame({
    'mean_fit_time': cv_rf['mean_fit_time'],
    'mean_score_time': cv_rf['mean_score_time'],
    'mean_test_score': cv_rf['mean_test_score'],
    'mean_train_score': cv_rf['mean_train_score'],
    'param_max_depth': cv_rf['param_max_depth'],
    'param_criterion': cv_rf['param_criterion'],
    'param_max_features': cv_rf['param_max_features'],
    'rank_test_score': cv_rf['rank_test_score'],
})

tab.sort_values(['rank_test_score']).head(3)

# In[21]:

X_train = features[split]
X_test = features[~split]

y_train = duplicate[split]
y_test = duplicate[~split]

# ### Naive Bayes Gaussian

# In[33]:

from sklearn.naive_bayes import GaussianNB

gnb = GaussianNB()
gnb.fit(X_train, y_train)

y_pred = gnb.predict_proba(X_test)
logLoss = log_loss(y_test, y_pred)

logLoss

# ### Quadratic Discriminant

# In[34]:

from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis

qda = QuadraticDiscriminantAnalysis()
qda.fit(X_train, y_train)

y_pred = qda.predict_proba(X_test)
logLoss = log_loss(y_test, y_pred)

logLoss

# ### Support Vector Classifier

# In[ ]:

get_ipython().run_cell_magic('time', '', '\nfrom sklearn import svm\n\nsvc = svm.SVC()\nsvc.fit(X_train, y_train)\n\ny_pred = svc.predict_proba(X_test)\nlogLoss = log_loss(y_test, y_pred)\n\nlogLoss')

# ## FC Network

# In[17]:

get_ipython().run_cell_magic('time', '', "\nfeatures_train = features[split]\nfeatures_test = features[~split]\n\nlabels_train = duplicate[split]\nlabels_test = duplicate[~split]\n\nfeatures_train = np.asmatrix(features_train)\nfeatures_test = np.asmatrix(features_test)\n\nlabels_train = np.asmatrix(labels_train)\nlabels_test = np.asmatrix(labels_test)\n\nlabels_train = kUtils.to_categorical(labels_train, 2)\nlabels_test = kUtils.to_categorical(labels_test, 2)\n\n#architecture\nmodel = Sequential()\nmodel.add(Dense(300, activation= 'elu', input_dim = features_train.shape[1]))\n#model.add(Dropout(0.2))\nmodel.add(BatchNormalization())\nfor i in range(8):\n    model.add(Dense(200, activation= 'elu'))\n    #model.add(Dropout(0.2))\n    #model.add(BatchNormalization())\nmodel.add(Dense(2, activation= 'softmax'))\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nhistory= model.fit(features_train, labels_train, batch_size=128, epochs=20, verbose=2, validation_data=(features_test, labels_test))\n\nscore = model.evaluate(features_test, labels_test, verbose=0)\n\nprint('Log loss: {}'.format(score[0]))\nprint('Accuracy: {}'.format(score[1]))")

# ### Add Dropout and Normalization

# In[19]:

get_ipython().run_cell_magic('time', '', "\nfeatures_train = features[split]\nfeatures_test = features[~split]\n\nlabels_train = duplicate[split]\nlabels_test = duplicate[~split]\n\nfeatures_train = np.asmatrix(features_train)\nfeatures_test = np.asmatrix(features_test)\n\nlabels_train = np.asmatrix(labels_train)\nlabels_test = np.asmatrix(labels_test)\n\nlabels_train = kUtils.to_categorical(labels_train, 2)\nlabels_test = kUtils.to_categorical(labels_test, 2)\n\n#architecture\nmodel = Sequential()\nmodel.add(Dense(300, activation= 'elu', input_dim = features_train.shape[1]))\nmodel.add(Dropout(0.2))\nmodel.add(BatchNormalization())\nfor i in range(8):\n    model.add(Dense(200, activation= 'elu'))\n    model.add(Dropout(0.2))\n    model.add(BatchNormalization())\nmodel.add(Dense(2, activation= 'softmax'))\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nhistory= model.fit(features_train, labels_train, batch_size=128, epochs=20, verbose=2, validation_data=(features_test, labels_test))\n\nscore = model.evaluate(features_test, labels_test, verbose=0)\n\nprint('Log loss: {}'.format(score[0]))\nprint('Accuracy: {}'.format(score[1]))")

# ### Esperimenti

# In[ ]:

get_ipython().run_cell_magic('time', '', "\nfeatures_train = features[split]\nfeatures_test = features[~split]\n\nlabels_train = duplicate[split]\nlabels_test = duplicate[~split]\n\nfeatures_train = np.asmatrix(features_train)\nfeatures_test = np.asmatrix(features_test)\n\nlabels_train = np.asmatrix(labels_train)\nlabels_test = np.asmatrix(labels_test)\n\nlabels_train = kUtils.to_categorical(labels_train, 2)\nlabels_test = kUtils.to_categorical(labels_test, 2)\n\n#architecture\nmodel = Sequential()\nmodel.add(Dense(400, activation= 'elu', input_dim = features_train.shape[1]))\nfor i in range(13):\n    model.add(Dense(400, activation= 'elu'))\nmodel.add(Dense(2, activation= 'softmax'))\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nhistory= model.fit(features_train, labels_train, batch_size=128, epochs=20, verbose=2, validation_data=(features_test, labels_test))\n\nscore = model.evaluate(features_test, labels_test, verbose=0)\n\nprint('Log loss: {}'.format(score[0]))\nprint('Accuracy: {}'.format(score[1]))")


import glob
import numpy as np
import matplotlib.pyplot as plt

import analysis_tools

filenames = sorted(glob.glob('../03-fundamentals-of-python/inflammation*.csv')) # is equivalent to
#glob.glob(filname) -> filename.sort()

for f in filenames[:3]:
    print(f)
    analysis_tools.analyse(f)
    analysis_tools.detect_problems(f)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
Some example's of good and bad plotting practices

Resources:
	For all the great color paletes for python, got to
	https://jiffyclub.github.io/palettable/

	For more infomation about which colormaps are safe, see
	http://colorbrewer2.org/
	Note: all colorbrewer colormaps are in the palettable package

	For general infomation about map making see:
	http://gisgeography.com/map-elements-how-to-guide-map-making/
	http://www.esri.com/news/arcuser/0911/making-a-map-meaningful.html

	For a detailed guide to mapmaking see the Mapbox guide pdf in the repo
"""
__title__ = "Plotting and map making"
__author__ = "Arden Burrell"
__version__ = "1.0 (19.01.2018)"
__email__ = "arden.burrell@unsw.edu.au"

#==============================================================================

# general modules
# import os, sys # check for files and so on
import numpy as np # array manipulations

# plotting modules
import matplotlib.pyplot as plt
#import matplotlib.cm as cm
import matplotlib.colors as mpc
from matplotlib import ticker
# import matplotlib as mpl
import pdb
# for mapping
from mpl_toolkits.basemap import Basemap

# for all the awesome color palets!!!!!!!!!!
import palettable

#==============================================================================
# =========== Main function ==========

def main():
	"""

	Call the demonstration functions in order

	"""
	# Show the mapping of years
	yearmapper()

	# Map a variable that diverges
	divergingmap()

	# Map a variable that is in groups
	groupmap()

	# Map a variable that is sequental
	sequentialmap()

#==============================================================================
# =========== plotting functions ==========
def mapper(
	image, cmap, vmin, vmax, region="global", title=None,
	tks=False, tckstr=None, norm=False, nancolour='dimgrey'
	):
	"""
	A general mapping function that can be used to generate many different
	map types
	"""

	# ===== set the spatial cordinates of the grid, mask the ocean and draw coastlines =====
	# Note:
	# 	because all of the imput datastes are numpy arrays, the lon and lat bounds
	# 	and the spatial projection (epsg) are hard coded. This infomation can be
	# 	can be pulled automitically from nectdf, geoTif's and other raster data
	if region == "global":
		# Set the map frame
		map = Basemap(llcrnrlon=-180.0,llcrnrlat=-90.0,urcrnrlon=180,urcrnrlat=90.0, resolution = 'h', epsg=4326)
		#adds national borders
		map.drawcountries()
	elif region == "australia":
		# Set the map frame
		map = Basemap(llcrnrlon=112.0,llcrnrlat=-44.5,urcrnrlon=156.25,urcrnrlat=-10, resolution = 'h', epsg=4326)
		map.drawlsmask(land_color='none', )
	elif region == "mongolia":
		# Set the map frame
		map = Basemap(llcrnrlon=85.0,llcrnrlat=40,urcrnrlon=120,urcrnrlat=52, resolution = 'h', epsg=4326)
		map.drawlsmask(land_color='none', )
		# Draw national borders with a thicker line
		map.drawcountries(linewidth = 1.25)
	else:
		sys.exit("Unknown Grid region")
	# Set the color or NaN's in the data
	map.drawmapboundary(fill_color=nancolour) #used to be grey
	# For land data, it is usefull to mask lakes etc
	map.fillcontinents(color= 'none', lake_color='white')
	# Add all of the coastlines to the map
	map.drawcoastlines()
	# Add the US and Australian state lines (Doesn't yet support other counteries)
	map.drawstates()

	# ====== Add mpc.BoundaryNorm made custom cmaps =====
	if norm == False:
		map.imshow(np.flipud(image), cmap=cmap, vmin=vmin, vmax=vmax)
	else:
		map.imshow(np.flipud(image), cmap=cmap, vmin=vmin, vmax=vmax, norm=norm)

	cb = map.colorbar()

	# ===== Move and adjust the number and location of colorbar ticks =====
	if tks == False:
		tick_locator = ticker.MaxNLocator(nbins=20)
	elif type(tks)==int:
		tick_locator = ticker.MaxNLocator(nbins=tks)
	else:
		tick_locator = ticker.MaxNLocator(nbins=vmax-vmin)
	# Add the changes to the colorbar
	cb.locator = tick_locator
	cb.update_ticks()

	# ===== Overwrite the tick labels and replace them with user defined =====
	if tckstr is not None:
		cb.ax.set_yticklabels(tckstr)

	# ===== add in the Graticules (lon lat grid lines) ======
	if region == "global":
		map.drawparallels(np.arange(-90, 90, 10),labels=[1,0,0,0], dashes=[1,2])
		map.drawmeridians(np.arange(-180, 180, 20),labels=[0,0,0,1], dashes=[1,2])
	elif region == "australia":
		map.drawparallels(np.arange(-50, -10, 10),labels=[1,0,0,0], dashes=[1,2])
		map.drawmeridians(np.arange(110, 156.25, 10),labels=[0,0,0,1], dashes=[1,2])
	elif region == "mongolia":
		map.drawparallels(np.arange(40, 60, 5),labels=[1,0,0,0], dashes=[1,2])
		map.drawmeridians(np.arange(80, 120, 5),labels=[0,0,0,1], dashes=[1,2])

	# ===== Show the plot =====
	plt.show()

def yearmapper():
	"""
	Demonstrate how to map a variable where there a many subdevisions,
	in this case the years of a breakpoint detected by the TSS.RESTREND method
	applied to GIMMSv3.1g NDVI data over mongolia.

	In this data:
		- Nan values mean the method failed
		- 0 value means that the method detected no breakpoints
		- else: the year in which a ecosystem breakpoint occured
	"""

	# Load the data from a numpy array
	mong_bp = np.load("./Data/Mongolia_breakpoint_years.npy")

	# ===== Make a naive plot =====
	# Comments: this plot is meant to be terrible,
	# 	All the values that are relevant are hidden
	# 	There is no colorbar

	# Set the figure number
	#	This isnt strictly nessary, but usefull if you
	# 	want to generate multiple plots then show them together
	plt.figure(1)
	# Make the plot
	plt.imshow(mong_bp)
	plt.show()

	# pdb.set_trace()

	# ===== Make a slightly less terrible one =====
	# Comments: this plot easier to read but not a map
	# 	Added bounds, and a much better colormap
	# 	The colormap is still continuous (hard to read)

	#Hard code the start and end years, (these can be determined automatically)
	vmin = 1987
	vmax = 2011
	# Get a colormap thats a bit better and tweak it
	cmap = plt.cm.jet
	cmap.set_under('w') #sets values <vmin to white in the plot
	cmap.set_bad('dimgrey')

	# Create the image

	plt.imshow(mong_bp, vmin = vmin, vmax = vmax, cmap=cmap)
	plt.colorbar()
	plt.show()

	# ===== Turn this into a proper map with a segmented colorbar =====
	plt.figure(2)
	# Comments: Now its an actual map
	# 	Segmented colorbars are easier to read
	# 	The colormap is still continuous (hard to read)

	# +++++ turn a continuous colormap into a segmented one ++++
	# The continuous color map
	cmap = plt.cm.jet
	# extract all colors from the cmap
	cmaplist = [cmap(i) for i in range(cmap.N)]
	cmap = cmap.from_list('Custom cmap', cmaplist, cmap.N)
	# Define the coverage and segments (max values, min values, number of segments)
	bounds = np.linspace(vmin, vmax, vmax-(vmin-1))
	# Select the colors for each bound using BoundaryNorm function
	norm = mpc.BoundaryNorm(bounds, cmap.N)
	# set pixels below the vmin to color to be white
	cmap.set_under('w')
	# cmap.set_above('w') # set colors above the vmax value to a color
	tks = (vmax-vmin)/2+1

	# +++++ Pass the data and infomation the mapper function +++++
	mapper(mong_bp, cmap, vmin, vmax, region="mongolia", tks=tks, norm=norm)

	# Note: There is almost no reason to use Jet in science,
	# this is one of very few exceptions. Use palettable ones instead!!!!!

def divergingmap():
	"""
	Demonstrate how to map a variable where the values diverge,
	in this case,  the total change in NDVI between 1982-2015 as
	detected by the TSS.RESTREND method	applied to GIMMSv3.1g NDVI
	data.

	In this data:
		- Nan values mean the method failed
		- -1 values have no statistically significant change in vegetation
		- else: total change in NDVI (note:NDVI is normalised between 0 and 1)
	"""

	# ===== Load the data from a numpy array =====
	TC = np.load("./Data/Global_total_change.npy")

	# ===== setup the mapping variables =====
	# set the min and max
	vmax = 0.3
	vmin = -0.3
	# Grab a diverging colormap from palettable
	cmap = mpc.ListedColormap(palettable.colorbrewer.diverging.PiYG_10.mpl_colors)
	# set the color of values of pixels with no significant change (have been set to -1)
	cmap.set_under('w')
	# This number needs to be adjusted to get the ticks tp line up
	tks=11

	# Generate the map
	mapper(TC, cmap, vmin, vmax, region="global", tks=tks)

def sequentialmap():
	"""
	Demonstrate how to map a variable where the values are sequential.
	in this case,  the p between NDVI and rainfall detected by the
	TSS.RESTREND method	applied to GIMMSv3.1g NDVI data.

	In this data:
		- NaN values mean the method failed
		- else: and Rsquared value between 0 and 1
	"""

	# ===== Load the data from a numpy array =====
	pval = np.load("./Data/Global_pval.npy")

	# ===== setup the mapping variables =====
	# set the min and max
	vmax = 1
	vmin = 0
	# Grab a sequential colormap from palettable
	cmap=mpc.ListedColormap(palettable.cmocean.sequential.Thermal_20_r.mpl_colors)
	# set the color of values of pixels with no significant change (have been set to -1)
	# cmap.set_under('w')
	# set the number of ticks
	tks=21

	# ===== Generate the map =====
	mapper(pval, cmap, vmin, vmax, region="global", tks=tks)

	# Note:
	# 	This is a terrible plot. Hopefully MS will
	# 	update the script with a function that allows
	#	colormas to be scalled.

def groupmap():
	"""
	Demonstrate how to map a variable where the values are in
	classified groups. in this case,  the comparison of TSS.RESTREND
	direction between GIMMSv3.1g NDVI and VOD data.

	In this data:
		- The numbers refer to groups (0 - 8)
	"""

	# ===== Load the data from a numpy array =====
	groups = np.load("./Data/Global_classes.npy")
	# show what the data looks like
	plt.imshow(groups)
	plt.colorbar()
	plt.show()

	# ===== setup the mapping variables =====
	# set the min and max
	vmin = -0.5
	vmax = 8.5
	# Create the labels
	label2=['No Change', 'Both Dec', 'Both Inc','NDVI Inc, VOD Dec','NDVI Inc, VOD stable', 'NDVI stable, VOD Dec','VOD Inc, NDVI Dec', 'VOD Inc, NDVI stable', 'VOD stable, NDVI Dec']
	# Create a custom discrete colourmap from hex values
	cmaphex = ['#d1cdca','#AF0627', '#ffffff','#1a9850', '#91cf60','#d9ef8b','#6a51a3','#9e9ac8', '#cbc9e2']
	cmap = mpc.ListedColormap(cmaphex)
	# Set the color under
	cmap.set_under('w')

	# ===== Generate the map =====
	mapper(groups, cmap, vmin, vmax, region="global", tks=True, tckstr = label2)

if __name__ == '__main__':
	main()


import numpy as np

# object is the base class
class Regressor(object):
    """
    Base class for regressors
    """

    def fit(self, X, t, **kwargs):
        """
        estimates parameters given training dateset

        Parameters
        ----------
        X: (sample_size, n_fatures) np.ndarray
            training data input

        t: (sample_size, ) np.ndarray
            training data target
        """
        self.__check_input(X)
        self.__check_target(t)
        if hasattr(self, "__fit"):
            self.__fit(X, t, **kwargs)
        else:
            raise NotImplementedError

    def __check_input(self, X):
        if not isinstance(X, np.ndarray):
            raise TypeError("X(input) is not a np.ndarray")
        if X.ndim != 2:
            raise ValueError("X(input) is not two dimentional array")
        # The hasattr() method returns true if an object has the given named attribute
        # and false if it does not.
        if hasattr(self, "n_features") and self.n_features != np.size(X, 1):
            raise ValueError(
                "mismatch in dimension 1 of X(input) "
                "(size {} is different from {})"
                .format(np.size(X, axis = 1), self.n_features))

    def __check_target(self, t):
        if not isinstance(t, np.ndarray):
            raise TypeError("t(target) must be np.ndarray")
        if t.ndim != 1:
            raise ValueError("t(target) mst be one dimensional array")