import numpy as np
from matplotlib import pyplot as plt

fruit = ['apple','orange','mango','guava']
quatity = [100,67,34,29]

# plt.pie(quatity,labels = fruit)

# Adding Colour and percentage in pie plot.
plt.pie(quatity,labels = fruit,colors = ['yellow','grey','blue','black'],startangle = 90)

#pull the apple "wedge 0.2 from the center of the pie".
myexplode = [0.2,0,0,0]
plt.pie(quatity,labels = fruit,explode = myexplode,shadow = True)

# Adding list of explanation of each wedge.
plt.legend(title = "Four Fruit")
# plt.legend()

plt.show()


#!/usr/bin/env python
# coding: utf-8

# ## This solution is described about the Data Scientist Blog Post. CRISP-DM process will be applied.

# In[2]:

# import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.metrics import r2_score, mean_squared_error, median_absolute_error
from sklearn.ensemble import RandomForestRegressor

# # 1. Business Understanding

# It would be quite interesting to apply data analysis skills here as a football fan. I chose FIFA 19 complete player dataset. I will fouc on mentioned below questions:
#
# Q1. What is the ratio of total wages/ total potential for clubs. Which clubs are the most economical？
#
# Q2. How is nation team player in total market value distributed?
#
# Q3. Which player skils set influence potential/wage? Can we predict player/player's potential based on his skills set?

# # 2. Data Understanding and Exploration

# In[3]:

# Load dataset
fifa19_player_data_frame = pd.read_csv('data.csv')
fifa19_player_data_frame.head()

# In[4]:

# Number of players
fifa19_player_data_frame.shape[0]

# In[5]:

# Data format for each column
fifa19_player_data_frame.info()

# In[6]:

# Types of informations in data set
fifa19_player_data_frame.columns

# In[7]:

# Missing values
fifa19_player_data_frame.isnull().sum()

# # 3. Prepare Data

# As per data exploration in above section, tthere are some necessary steps to be applied before preparing data:
#
# 1. Unused column to be dropted
#
# 2. String to be converted to number
#
# 3. Handle missing values, if necessary drop them

# In[8]:

# Drop unused columns
columns_to_drop = ['Unnamed: 0', 'ID', 'Photo', 'Flag','Club Logo', 'Preferred Foot',
                   'Body Type', 'Real Face', 'Jersey Number', 'Joined', 'Loaned From',
                   'Contract Valid Until', 'Height', 'Weight','LS', 'ST', 'RS', 'LW', 'LF', 'CF', 'RF', 'RW',
                   'LAM', 'CAM', 'RAM', 'LM', 'LCM', 'CM', 'RCM', 'RM', 'LWB', 'LDM',
                   'CDM', 'RDM', 'RWB', 'LB', 'LCB', 'CB', 'RCB', 'RB', 'Release Clause']

fifa19_player_data_frame.drop(columns_to_drop, axis=1, inplace=True)

# In[9]:

# Display data after dropped column
fifa19_player_data_frame.head()

# In[10]:

# Convert value and wage columns string to number
# Example: €110.5M = 110.5 * 1000000
def string2number(amount_str):
    """
    This function convert value and wage string to floating point number

    Parameter:
    amount(str): Amount string with M & K as Abbreviation for Million and Thousands

    Returns:
    float: A float number represents the numerical value of the input parameter amount(str)
    """
    if amount_str[-1] == 'M':
        return float(amount_str[1:-1])*1000000
    elif amount_str[-1] == 'K':
        return float(amount_str[1:-1])*1000
    else:
        return float(amount_str[1:])

# In[11]:

# First convert value, wage string to actual amount, then divide by 1 million and 1 thousand.
# Assigned to new columns "Value_M and Wage_K respectively"
fifa19_player_data_frame['Value_M'] = fifa19_player_data_frame['Value'].apply(lambda x: string2number(x) / 1000000)
fifa19_player_data_frame['Wage_K'] = fifa19_player_data_frame['Wage'].apply(lambda x: string2number(x) / 1000)

# Drop original value & wage column
fifa19_player_data_frame.drop(['Value', 'Wage'], axis=1, inplace=True)

# In[12]:

# Display data set ater string to number conversion
fifa19_player_data_frame.describe()

# In[13]:

# Find player Name who's Value is highest
fifa19_player_data_frame.loc[fifa19_player_data_frame['Value_M'].idxmax()]

# In[14]:

# Find player Name who's Wage is highest
fifa19_player_data_frame.loc[fifa19_player_data_frame['Wage_K'].idxmax()]

# In[15]:

# Missing value handling
missing_player_data_frame = fifa19_player_data_frame[fifa19_player_data_frame['Agility'].isnull()]

# In[16]:

missing_player_data_frame.describe()

# ## 3.1 Fifa19 data observation

# From above analysis result there are 48 missing values that quite a few columns which are related to player's skills.
#
# So there were 48 players that simply missing those values. But, to answer Question-1 and Question-2 we will reserve those players since there were no missing value in Value_M and Wage_K column.
#
# To explain Question-3, we will drop those player rows since there are many missing values.

# # 4. Answer Questions base on dataset

# ### Q1. What is the ratio of total wages/ total potential for clubs. Which clubs are the most economical？

# In[17]:

club_wages = fifa19_player_data_frame.groupby('Club').sum()

# In[18]:

club_player_count = fifa19_player_data_frame.groupby('Club').count()

# In[19]:

# Total Number of clubs and average number of players in each club
print('Total Number of clubs is {}'.format(club_player_count.shape[0]))
print('Players average in each club is {}'.format(round(club_player_count['Age'].mean(),2)))
print('Total Average wage(K) potential ratio is {}'
      .format(round(club_wages['Wage_K'].sum() / club_wages['Potential'].sum(), 2)))

# In[20]:

club_wages['Wage/Potential'] = club_wages['Wage_K'] / club_wages['Potential']
club_wages['Player Number'] = club_player_count['Age']
club_wages['Player Average Age'] = club_wages['Age'] / club_wages['Player Number']

# In[21]:

club_wages.sort_values('Wage/Potential', ascending=False, inplace=True)

# In[22]:

club_wages.head()

# In[23]:

club_wages['Wage/Potential'].head(10).plot(kind='bar', color='Blue')
plt.title('Top 10 clubs spending wage on players potential')

# In[24]:

club_wages['Wage/Potential'].tail(10).plot(kind='bar', color='Blue')
plt.title('Top 10 economical clubs ')

# From the above analysis and plot, the Real Madrid, Barcelona, and Juventus club are willing to spend more wage for high potential players than other clubs.
#
# The economical clubs are not famous and from nowhere that we heard about. Few of them are quite famous like AEK Athens, Dynamo Kyiv may be more. This conclude that those club's players are potiential but underpayed. It would be good approach for 'Giant' clubs to bring more econimical players to reduce their overall wage spent.

# ### Q2. How is nation team player in total market value distributed?

# In[25]:

# Age count
age_count = fifa19_player_data_frame['Age'].value_counts()
age_count.sort_index(ascending=True, inplace=True)

# In[26]:

# Calculate average overall rating
age_mean = fifa19_player_data_frame.groupby('Age').mean()

# In[27]:

# Collect age distribuion and overall rating together
age_count_list = age_count.values.tolist()
age_overall_rating_list = age_mean['Overall'].values.tolist()

# In[30]:

# Plot age distribution and overall rating together
age = age_count.index.values.tolist()
figure = plt.figure()
axis_1 = figure.add_subplot(111)
axis_1.plot(age,age_overall_rating_list, color = 'red', label='Average Rating')
axis_1.legend(loc=1)
axis_1.set_ylabel('Average Rating')

axis_2 = axis_1.twinx()
plt.bar(age, age_count_list, label='Age Count')
axis_2.legend(loc=2)
axis_2.set_ylabel('Age Count')
plt.show()

# In figure above, we can see that most of the players are between 20-26 years age. The number of player's start decreases after 26 years age and much more decreases after 30. The main reasons could be that, many young player didn't get enough opportunities to prove themselves as a football player.
#
# In ideal scenario, When a football player reaches their age of 20 years, they must have gain enough experience and reaches peak of their rating. The golden era for most of the football player starts 20 years of there age and ends when age reaches 35 years. Most of the football playes physical body condition drops quickly after their 35 years of age and rating quite low.
#
# But there are also set of player's rating can remain quite high with age over 37, 38 years.

# ### Q3. Which player skils set influence potential/wage? Can we predict player/player's potential based on his skills set?

# In[31]:

# Drop unused columns to answers Question-3
columns_to_drop_q3 = ['Name', 'Nationality', 'Club']
fifa19_player_data_frame.drop(columns_to_drop_q3, axis=1, inplace=True)

# In[32]:

# Drop players whose skill set is missing.
fifa19_player_data_frame.dropna(axis=0, how='any', inplace=True)

# In[33]:

# Split Work Rate is in format of attack work rate and defence work rate
# Create two new columns here.
fifa19_player_data_frame['Work Rate Attack'] = fifa19_player_data_frame['Work Rate'].map(lambda x: x.split('/')[0])
fifa19_player_data_frame['Work Rate Defence'] = fifa19_player_data_frame['Work Rate'].map(lambda x: x.split('/')[1])

# In[34]:

#Drop origin Work Rate column
fifa19_player_data_frame.drop('Work Rate', axis=1, inplace=True)

# In[35]:

fifa19_player_data_frame.head()

# In[36]:

# One Hot Encoding for Position, Work Rate Attack and Work Rate Defence
one_hot_columns = ['Position', 'Work Rate Attack', 'Work Rate Defence']
fifa19_player_data_frame = pd.get_dummies(fifa19_player_data_frame, columns=one_hot_columns, prefix = one_hot_columns)

# In[37]:

fifa19_player_data_frame.shape

# # 5. Train model and Performance Evaluation

# In[38]:

y = fifa19_player_data_frame['Potential']
X = fifa19_player_data_frame.drop(['Value_M', 'Wage_K', 'Potential', 'Overall'], axis=1)

# In[39]:

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.3, random_state=42)

# In[40]:

forest_regressor = RandomForestRegressor(n_estimators=500)
forest_regressor.fit(X_train, y_train)
y_test_preds = forest_regressor.predict(X_test)
print(r2_score(y_test, y_test_preds))
print(mean_squared_error(y_test, y_test_preds))

# In[41]:

coefs_df = pd.DataFrame()

coefs_df['Features'] = X_train.columns
coefs_df['Coefs'] = forest_regressor.feature_importances_
coefs_df.sort_values('Coefs', ascending=False).head(10)

# As a football fan we know, ball control, reactions, and age are the main three features that describe player's potential and performance. In this analysis our perception is also same.
#
# Players with excellent ball control and fast reactions tends to give us an outstanding performance in football match.

# In[42]:

coefs_df.set_index('Features', inplace=True)
coefs_df.sort_values('Coefs', ascending=False).head(5).plot(kind='bar', color='Blue')

# In[ ]:


"""
Some examples of how to annotate points in figures.  You specify an
annotation point xy=(x,y) and a text point xytext=(x,y) for the
annotated points and text location, respectively.  Optionally, you can
specify the coordinate system of xy and xytext with one of the
following strings for xycoords and textcoords (default is 'data')

  'figure points'   : points from the lower left corner of the figure
  'figure pixels'   : pixels from the lower left corner of the figure
  'figure fraction' : 0,0 is lower left of figure and 1,1 is upper, right
  'axes points'     : points from lower left corner of axes
  'axes pixels'     : pixels from lower left corner of axes
  'axes fraction'   : 0,1 is lower left of axes and 1,1 is upper right
  'offset points'   : Specify an offset (in points) from the xy value
  'data'            : use the axes data coordinate system

Optionally, you can specify arrow properties which draws and arrow
from the text to the annotated point by giving a dictionary of arrow
properties

Valid keys are

          width : the width of the arrow in points
          frac  : the fraction of the arrow length occupied by the head
          headwidth : the width of the base of the arrow head in points
          shrink : move the tip and base some percent away from the
                   annotated point and text
          any key for matplotlib.patches.polygon  (eg facecolor)

For physical coordinate systems (points or pixels) the origin is the
(bottom, left) of the figure or axes.  If the value is negative,
however, the origin is from the (right, top) of the figure or axes,
analogous to negative indexing of sequences.
"""

from matplotlib.pyplot import figure, show
from matplotlib.patches import Ellipse
import numpy as np

if 1:
    # if only one location is given, the text and xypoint being
    # annotated are assumed to be the same
    fig = figure()
    ax = fig.add_subplot(111, autoscale_on=False, xlim=(-1,5), ylim=(-3,5))

    t = np.arange(0.0, 5.0, 0.01)
    s = np.cos(2*np.pi*t)
    line, = ax.plot(t, s, lw=3, color='purple')

    ax.annotate('axes center', xy=(.5, .5),  xycoords='axes fraction',
                horizontalalignment='center', verticalalignment='center')

    ax.annotate('pixels', xy=(20, 20),  xycoords='figure pixels')

    ax.annotate('points', xy=(100, 300),  xycoords='figure points')

    ax.annotate('offset', xy=(1, 1),  xycoords='data',
                xytext=(-15, 10), textcoords='offset points',
                arrowprops=dict(facecolor='black', shrink=0.05),
                horizontalalignment='right', verticalalignment='bottom',
                )

    ax.annotate('local max', xy=(3, 1),  xycoords='data',
                xytext=(0.8, 0.95), textcoords='axes fraction',
                arrowprops=dict(facecolor='black', shrink=0.05),
                horizontalalignment='right', verticalalignment='top',
                )

    ax.annotate('a fractional title', xy=(.025, .975),
                xycoords='figure fraction',
                horizontalalignment='left', verticalalignment='top',
                fontsize=20)

    # use negative points or pixels to specify from right, top -10, 10
    # is 10 points to the left of the right side of the axes and 10
    # points above the bottom
    ax.annotate('bottom right (points)', xy=(-10, 10),
                xycoords='axes points',
                horizontalalignment='right', verticalalignment='bottom',
                fontsize=20)

if 1:
    # you can specify the xypoint and the xytext in different
    # positions and coordinate systems, and optionally turn on a
    # connecting line and mark the point with a marker.  Annotations
    # work on polar axes too.  In the example below, the xy point is
    # in native coordinates (xycoords defaults to 'data').  For a
    # polar axes, this is in (theta, radius) space.  The text in this
    # example is placed in the fractional figure coordinate system.
    # Text keyword args like horizontal and vertical alignment are
    # respected
    fig = figure()
    ax = fig.add_subplot(111, polar=True)
    r = np.arange(0,1,0.001)
    theta = 2*2*np.pi*r
    line, = ax.plot(theta, r, color='#ee8d18', lw=3)

    ind = 800
    thisr, thistheta = r[ind], theta[ind]
    ax.plot([thistheta], [thisr], 'o')
    ax.annotate('a polar annotation',
                xy=(thistheta, thisr),  # theta, radius
                xytext=(0.05, 0.05),    # fraction, fraction
                textcoords='figure fraction',
                arrowprops=dict(facecolor='black', shrink=0.05),
                horizontalalignment='left',
                verticalalignment='bottom',
                )

if 1:
    # You can also use polar notation on a cartesian axes.  Here the
    # native coordinate system ('data') is cartesian, so you need to
    # specify the xycoords and textcoords as 'polar' if you want to
    # use (theta, radius)

    el = Ellipse((0,0), 10, 20, facecolor='r', alpha=0.5)

    fig = figure()
    ax = fig.add_subplot(111, aspect='equal')
    ax.add_artist(el)
    el.set_clip_box(ax.bbox)
    ax.annotate('the top',
                xy=(np.pi/2., 10.),      # theta, radius
                xytext=(np.pi/3, 20.),   # theta, radius
                xycoords='polar',
                textcoords='polar',
                arrowprops=dict(facecolor='black', shrink=0.05),
                horizontalalignment='left',
                verticalalignment='bottom',
                clip_on=True, # clip to the axes bounding box
     )

    ax.set_xlim(-20, 20)
    ax.set_ylim(-20, 20)

show()


# -*- coding: utf-8 -*-
"""
Created on Tue Jul 13 11:07:00 2021

@author: Joanna Brown, Github : jbrown888

Basics of Matplotlib Graph formatting settings

By no means is this a definitive guide, or the best way of doing things, and it's definitely not complete. But hopefully it gives some basic ideas for formatting, and serves as a starting point for plotting.

Extra tips: stackoverflow and the matplotlib and numpy websites are your friends!
90% of the time your question has already been asked by somebody else on stack overflow, and the documentation for the python websites is actually quite good.

https://matplotlib.org/
https://numpy.org/

"""

import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt
import matplotlib.cm as cm #colormap library
from scipy.stats import norm

#%% axes  - the OOP way to make plots

fig, ax = plt.subplots() # creates a figure (fig) and axes (ax) object.
"""
By creating objects, you can access and alter the properties more easily than if you use plt.plot(...), but it is a little difficult to wrap your head around at first.
It saves massive amounts of time, as you can create default settings for these objects, and apply functions to them.
You can do this because Python is an object oriented programming language, and has objects and classes - this is the fun bit! You can even create your own classes.

Aside: confusingly, the axes object is not actually the axes of the graph - it's kind of a canvas on which you add stuff like plot lines and the x and y axis.
"""

def standard_axes_settings(ax):
        """
        A function to apply standard settings to a graph for label sizes, gridlines, frame etc.

        input argument ax is an axes object of matplotlib

        """
        ax.set_frame_on # adds frame around plot
        ax.grid(b=True, which = 'major', axis = 'both', c = 'grey', ls='--', lw = 1) #adds major grid lines
        # ax.grid(b=True, which='minor', axis='both', c='darkgrey', ls = '--', linewidth =2) # adds minor grid lines
        ax.tick_params(axis = 'both', which = 'major', direction ='in', labelsize = 22) # adds major ticks (the little marks on axes like on a ruler)
        ax.tick_params(axis ='both', which = 'minor', direction ='in') # adds minor ticks
        ax.xaxis.label.set_size(28) # sets size of axis labels (the numbers on the axis)
        ax.yaxis.label.set_size(28)
        # ax.minorticks_on() # turns on minor ticks
        ax.yaxis.get_offset_text().set_fontsize(20) # sets fontsize of the axis label i.e. the "Time [s]" bit
        ax.xaxis.get_offset_text().set_fontsize(20)
        ax.ticklabel_format(axis = 'x', style = 'sci', scilimits = (-3,3), useOffset = True) # formats the numbering of the axes if you have really big or really small numbers - if >1e3 or <1e-3, will use scientific notation automatically
        ax.ticklabel_format(axis = 'y', style = 'sci', scilimits = (-3,3), useOffset=True)

# these settings can be copy and pasted out of the function and applied individually to each graph - but putting it in a function like this saves you having to rewrite it all everytime you make a new graph

standard_axes_settings(ax) # applies formatting to the graph (axes object)

red = 'indianred' # a nice red
blue = 'royalblue' # a nice blue

plot_arguments = {'marker':'o', #marker style
                  'lw':3, #line width
                  'ms':10, #marker size
                  'ls':'--', #line style
                  'c': 'k', #line color
                  'mew':2, #marker edge width
                  'mec': 'red', #marker edge color
                  'mfc':'None',#marker face color
                  } # this is a dictionary of arguments to set the style of the line/points you draw

plot_arguments_y = {'marker':'x', #marker style
                  'lw':3, #line width
                  'ms':7, #marker size
                  'ls':':', #line style - none gives no line
                  'c': 'darkslategray', #line color
                  'mew':2, #marker edge width
                  'mec': 'blue', #marker edge color
                  'mfc':'blue',#marker face color
                  } # same thing but for the y data, to distinguish from the x data

#Create some data
def example_function(t):
    return (t-6.5)**2

t = np.linspace(0, 4*np.pi, 25) # generates evenly distributed points
y = 30*np.sin(t)
x = example_function(t)

ax.plot(t, x, **plot_arguments, label = 'x data') # plot x data
ax.plot(t, y, **plot_arguments_y, label = 'y data') # plot y data

"""
the **plot_arguments adds all the items in the dictionary plot_arguments into the plot as if you had written them directly as ax.plot(t, x, lw = 3, marker = 'o',... etc). Just makes the code a little bit neater
"""

# Set x and y axis labels
ax.set_xlabel('Time [s]')
ax.set_ylabel('Dependent Variable')

### LEGENDS ###
"""
There are a few ways of creating legends. Uncomment one to use it. Notice both produce the same result!
"""

# Method A
"""
Since I labelled my data when I plotted it (label = 'x data' etc in the ax.plot() line), I can simply call legend directly and Python will automatically create the legend with the correct key. Nice!
"""
leg = ax.legend(fontsize = 26, loc='best', markerfirst = True, frameon = True) # creates a legend for the axes object ax that I created earlier

# Method B
"""
If displaying lots of data on one plot, it may be more useful to customise the legend and create your own entries
"""

# handles = [mpl.lines.Line2D([], [], **plot_arguments), mpl.lines.Line2D([], [], **plot_arguments_y)] # creates the symbols for the legend
# labels = ['x data', 'y data'] #labels for legend
# leg = ax.legend(handles, labels, fontsize = 26, loc='best', markerfirst = True, frameon = True) # creates custom legend

#general formatting of legend for either method
leg.get_frame().set_edgecolor('k')
leg.get_frame().set_facecolor('w')

plt.show()

#%% COLOR RANGE EXAMPLE
fig, ax = plt.subplots()
standard_axes_settings(ax)

t = np.linspace(-5, 5, 100)
mean = 0.0
Z = np.linspace(0.5, 5, 6) # values for standard deviation (sigma)

cs = [cm.magma(i/len(Z), 1) for i in range(len(Z))] # creates list of colours according to the colour map magma, evenly distributed over the range of Z. Colour maps available at https://matplotlib.org/stable/tutorials/colors/colormaps.html

# here for the labels I use f-string formatting. This was new with Python 3 and I much prefer it the Python 2 method. Google

for i, stddev in enumerate(Z): # iterate over the different values of standard deviation
    x = norm.pdf(t, loc = mean, scale = stddev)
    ax.plot(t, x, marker = 'None', ls = '-', lw = 2.5, color = cs[i], label = fr'$\sigma$ = {stddev:.1f}')

ax.set_xlabel(r'Distance [$\mu$m]')
ax.set_ylabel('Intensity')

leg = ax.legend(fontsize = 26, loc='best', markerfirst = True, frameon = True) # creates a legend for the axes object ax that I created earlier

leg.get_frame().set_edgecolor('k')
leg.get_frame().set_facecolor('w')

#%% 4 plots together
t = np.linspace(-5, 5, 100)
x = np.cos(t)
y = np.sin(t)
z = t**3
v = t*t
q = 3*t**4 -t**3 +2

fig, AXES = plt.subplots(1, 2, sharex = False, sharey = False, num =3) # create 2 subplots - allows shared axes like this
fig.frameon = False
[AXL, AXR] = AXES # left and right subplot
ax1 = fig.add_subplot(221)
ax2 = fig.add_subplot(223)
ax3 = fig.add_subplot(222)
ax4 = fig.add_subplot(224)
axes = [ax1, ax2, ax3, ax4]
fig.subplots_adjust(wspace =0.3, hspace=0.3) # adjust spacing

for AX in AXES:
    AX.spines['top'].set_color('None') # remove visible frame from the the left and right underlying subplots
    AX.spines['bottom'].set_color('None')
    AX.spines['left'].set_color('None')
    AX.spines['right'].set_color('None')
    AX.set_facecolor('None')
    # AX.set_xticklabels([])
    # AX.set_yticklabels([])
    AX.tick_params(labelcolor='None', top='off', bottom='off', left='off', right='off')

for ax in axes: #format the 4 axe
    standard_axes_settings(ax)

axes[2].plot(t, x, '-', label='x(t)',linestyle= '-', color = red)
axes[2].plot(t, y, '-', label='y(t)',linestyle= '-', color = blue)
# axes[2].set_xlabel('t [s]')
axes[2].set_ylabel('Position [m]')

axes[3].plot(t, z, '-', label='z(t)', linestyle= '-', color = 'purple')
axes[3].set_xlabel('t [s]')
axes[3].set_ylabel(r'$z$ [${ms}^{-1}$]')

axes[1].plot(t, v, '-', label=r'$v$(t)',linestyle= '-', color = 'orange')
axes[1].set_xlabel('t [s]')
axes[1].set_ylabel(r'$v$ [${ms}^{-1}$]')

axes[0].plot(t, q, '-', label=r'$v$(t)',linestyle= '-', color = 'k')
# axes[0].set_xlabel('t [s]')
axes[0].set_ylabel(r'$q$ [${s}^{-1}$]')

handles = [mpl.lines.Line2D([0], [0], ls = '-', color = red), mpl.lines.Line2D([0], [0], ls = '-', color = blue)]
leg1 = axes[2].legend(handles = handles, labels = ['x', 'y'], fontsize = 20, loc='best', markerfirst = False, frameon = True)
leg1.get_frame().set_edgecolor('k')
leg1.get_frame().set_facecolor('w')

#add annotating text
bbox_props = dict(boxstyle = "square,pad=0.2", fc = "w", ec ="k", lw = 1)
axes[0].text(-1, 1.25e3, 'Some useful text', fontsize = 24, bbox = bbox_props)

plt.show()

#%% f-string and latex

#These expressions won't work printing in e.g. spyder console, but will work in jupyter notebooks or in matplotlib.
#Using LaTeX with strings:
print(r'$\theta$')

theta = 35.4282711
#Using LaTeX and f-string formatting
print(fr'$\theta$ = {theta:.2f}')

#Using f-string and latex together but you need {} in Latex:
print(fr'$\Delta_{{\mu}}$ = {np.pi:.2f}')


import pandas as pd
import numpy as np
import unittest

class Mytest(unittest.TestCase):
    '''
    checks array o scalar values for not missing values and returns boolean
    '''
    def setUp(self):
        self.str='Hira'
        self.testNone=None
        self.testNan=' '
        self.testarray=[1,2,3]
        self.test1=[None,6,'h']

    def test_notnull(self):
        self.assertEqual(pd.notnull(self.str),True)
        self.assertEqual(pd.notnull(self.testNone),False)
        self.assertEqual(pd.notnull(self.testNan),True)
        self.assertEqual(pd.notnull(self.testarray[2]),True)
        self.assertEqual(pd.notnull(self.testarray).all(), (True))
        self.assertEqual(pd.notnull(self.test1).all(), (False))

if __name__  == '__main__':
    unittest.main()


r"""
LaTeX macros

AUTHORS:

- John H. Palmieri (2009-03)

The code here sets up LaTeX macro definitions for use in the
documentation. To add a macro, modify the list ``macros``, near the
end of this file, and then run 'sage -b'. The entries in this list are
used to produce ``sage_latex_macros``, a list of strings of the form
'\\newcommand...', and ``sage_mathjax_macros``, a list of strings
suitable for parsing by MathJax.  The LaTeX macros are produced using
the ``_latex_`` method for each Sage object listed in ``macros``, and
the MathJax macros are produced from the LaTeX macros.  The list of
LaTeX macros is used in the file
``sage_docbuild.conf`` to add to the preambles of
both the LaTeX file used to build the PDF version of the documentation
and the LaTeX file used to build the HTML version.

Any macro defined here may be used in docstrings or in the tutorial
(or other pieces of documentation).  In a docstring, for example,
"\ZZ" in backquotes (demarking math mode) will appear as "ZZ" in
interactive help, but will be typeset as "\\Bold{Z}" in the
reference manual.

More details on the list ``macros``: the entries are lists or tuples
of the form ``[name]`` or ``[name, arguments]``, where ``name`` is a
string and ``arguments`` consists of valid arguments for the Sage
object named ``name``.  For example, ``["ZZ"]`` and ``["GF", 2]``
produce the LaTeX macros '\\newcommand{\\ZZ}{\\Bold{Z}}' and
'\\newcommand{\\GF}[1]{\\Bold{F}_{#1}}', respectively.  (For the
second of these, ``latex(GF(2))`` is called and the string '2' gets
replaced by '#1', so ``["GF", 17]`` would have worked just as well.
``["GF", p]`` would have raised an error, though, because ``p`` is not
defined, and ``["GF", 4]`` would have raised an error, because to
define the field with four elements in Sage, you also need to specify
the name of a generator.)

To see evidence of the results of the code here, run ``sage --docbuild
tutorial latex`` (for example), and look at the resulting LaTeX file in
``SAGE_DOC/latex/en/tutorial/``.  The preamble should
contain '\newcommand' lines for each of the entries in ``macros``.
"""

def produce_latex_macro(name, *sample_args):
    r"""
    Produce a string defining a LaTeX macro.

    INPUT:

    -  ``name`` -- name of macro to be defined, also name of corresponding Sage object

    -  ``sample_args`` -- (optional) sample arguments for this Sage object

    EXAMPLES::

        sage: from sage.misc.latex_macros import produce_latex_macro
        sage: produce_latex_macro('ZZ')
        '\\newcommand{\\ZZ}{\\Bold{Z}}'

    If the Sage object takes arguments, then the LaTeX macro will
    accept arguments as well. You must pass valid arguments, which
    will then be converted to #1, #2, etc. in the macro
    definition. The following allows the use of "\GF{p^n}", for
    example::

         sage: produce_latex_macro('GF', 37)
         '\\newcommand{\\GF}[1]{\\Bold{F}_{#1}}'

    If the Sage object is not in the global name space, describe it
    like so::

         sage: produce_latex_macro('sage.rings.finite_rings.finite_field_constructor.FiniteField', 3)
         '\\newcommand{\\FiniteField}[1]{\\Bold{F}_{#1}}'
    """
    from sage.misc.latex import LatexCall  # type: ignore
    # this import is used inside a string below
    names_split = name.rsplit('.', 1)
    if len(names_split) == 1:
        module = 'sage.all'
        real_name = names_split[0]
    else:
        module, real_name = names_split
    newcommand = '\\newcommand{\\' + real_name + '}'
    count = 0
    args = "("
    for x in sample_args:
        count += 1
        args += str(x) + ','
    args += ')'
    exec('from ' + module + ' import ' + real_name)
    if count:
        defn = '[' + str(count) + ']{'
        defn += eval('str(LatexCall()(' + real_name + args + '))') + '}'
    else:
        defn = '{' + eval('str(LatexCall()(' + real_name + '))') + '}'
    count = 0
    for x in sample_args:
        count += 1
        defn = defn.replace(str(x), "#" + str(count))
    return newcommand + defn

def convert_latex_macro_to_mathjax(macro):
    r"""
    This converts a LaTeX macro definition (\newcommand...) to a
    MathJax macro definition (MathJax.Macro...).

    INPUT:

    -  ``macro`` -- LaTeX macro definition

    See the web page
    https://docs.mathjax.org/en/latest/input/tex/macros.html for a
    description of the format for MathJax macros.

    EXAMPLES::

        sage: from sage.misc.latex_macros import convert_latex_macro_to_mathjax
        sage: convert_latex_macro_to_mathjax('\\newcommand{\\ZZ}{\\Bold{Z}}')
        ('ZZ', '\\Bold{Z}')
        sage: convert_latex_macro_to_mathjax('\\newcommand{\\GF}[1]{\\Bold{F}_{#1}}')
        ('GF', ['\\Bold{F}_{#1}', 1])
    """
    left_bracket = macro.find('[')
    right_bracket = macro.find('[')
    if left_bracket >= 0:
        right_bracket = macro.find(']')
        num_args = int(macro[left_bracket + 1 : right_bracket])
    else:
        num_args = 0
    start_name = macro.find('{') + 1  # add one to go past the backslash
    end_name = macro.find('}')
    name = macro[start_name + 1 : end_name]
    start_defn = macro.find('{', end_name)
    end_defn = macro.rfind('}')
    defn = macro[start_defn + 1 : end_defn]
    if num_args == 0:
        return name, defn
    else:
        return name, [defn, num_args]

# To add a new macro for use in the Sage documentation, add a list or
# tuple to the following list.  Each list (or tuple) should have the
# form [name, arguments], which will be passed to the function
# produce_latex_macro: see that for more documentation.
#
# To see the results of this, run 'sage --docbuild tutorial latex' (for
# example -- you could replace 'tutorial' with your favorite piece of
# documentation), and look at the resulting tex file in
# SAGE_DOC/latex/en/tutorial.  The preamble should contain
# \newcommand's for each of the entries here.
macros = [["ZZ"],
          ["NN"],
          ["RR"],
          ["CC"],
          ["QQ"],
          ["QQbar"],
          ["GF", 2],
          ["Zp", 2],
          ["Qp", 2],
          ["Zmod", 2],
          ["CDF"],
          ["CIF"],
          ["CLF"],
          ["RDF"],
          ["RIF"],
          ["RLF"],
          ]

# Use this list to define additional latex macros for sage documentation
latex_macros = [r"\newcommand{\SL}{\mathrm{SL}}",
                r"\newcommand{\PSL}{\mathrm{PSL}}"]

# The following is to allow customization of typesetting of rings:
# mathbf vs mathbb.  See latex.py for more information.
sage_configurable_latex_macros = [r"\newcommand{\Bold}[1]{\mathbf{#1}}"]

def sage_latex_macros():
    r"""
    Return list of LaTeX macros for Sage. This just runs the function
    :func:`produce_latex_macro` on the list ``macros`` defined in this
    file, and appends ``sage_configurable_latex_macros``. To add a new
    macro for permanent use in Sage, modify ``macros``.

    EXAMPLES::

        sage: from sage.misc.latex_macros import sage_latex_macros
        sage: sage_latex_macros()
        ['\\newcommand{\\ZZ}{\\Bold{Z}}', '\\newcommand{\\NN}{\\Bold{N}}', ...
    """
    return [produce_latex_macro(*x) for x in macros] + latex_macros + sage_configurable_latex_macros

def sage_mathjax_macros():
    r"""
    Return Sage's macro definitions for usage with MathJax.

    This feeds each item output by :func:`sage_latex_macros` to
    :func:`convert_latex_macro_to_mathjax`.

    EXAMPLES::

        sage: from sage.misc.latex_macros import sage_mathjax_macros
        sage: sage_mathjax_macros()
        {'Bold': ['\\mathbf{#1}', 1], 'CC': '\\Bold{C}', ...
    """
    return dict(convert_latex_macro_to_mathjax(m) for m in sage_latex_macros())


import multiprocessing as mp
import pyastar
import numpy as np

from . import scorecard

def fast_astar(queue : mp.Queue, array: np.ndarray, start : tuple, end: tuple):
    with open("./exports/costs31-12-2020.npy", "rb") as f:
        arr = np.load(f)
        new_arr = np.zeros(arr.shape)
        scorecard.flip_scorecard(arr, new_arr)
        arr = new_arr
        arr = arr.astype(np.float32)
        queue.put(
            pyastar.astar_path(arr, start, end, allow_diagonal=True)
        )

def astar(array : np.ndarray, start: tuple, end :tuple):
    try:
        mp.set_start_method('spawn')
    except:
        pass

    queue = mp.Queue()
    process = mp.Process(target = fast_astar, args=(queue, array, start, end))
    process.start()
    path = queue.get()
    process.join()
    return path


# coding: utf-8

# In[111]:

#########################################################
#Program: Grader for PHY161L
#Author: Aaron Mahler
#
#Dependencies: All standard python libraries
#
#Inputs: Requires the sakai directory structure for
#   downloaded submission of Student/Submission
#   Directory/files.  Change the hardcoded file
#   names below to match submission folder name,
#   csv file of grades(graCSV), and grader file
#   name(cfil).
#
#Outputs: Prints messages to stdout, recommend redirect
#   to desired log file.  Will save grades in csv file
#   with  name graCSV+graded.  The grading script
#   output is redirected to 2>errlog
#
#Version:   Date:      Comments:
#0.0        24FEB17
#########################################################

from __future__ import print_function #force python3 print()
import pandas as pd
import os
import numpy as np
import subprocess as sp

#Hard-Code File names, extensions, etc
graCSV = "gradebook_submit10.csv" #Empty (of grades) csv file with studentIDs
subF = str("Submission attachment(s)") #submission folder name from sakai
cfil = "alm_hw1check" #Name of check file, to concatenate with submitted file
ext1 = ".ipynb" #extension of submissions
ext2 = ".py" #python extension
dummyFile = "dum1.csv" #dummy file to pass info from grading scipt back to this program

#gradebook csv input
arr = pd.read_csv(graCSV, quotechar='"', skipinitialspace=True)
gkey = arr.keys()[2] #Grade key
ckey = arr.keys()[3] #Comment Key
snum=int(arr.shape[0])                      #number of students
arr2 = np.empty((2, snum), dtype=object)    #array to key studentIDs to dir names
arr2[0]=arr.as_matrix().T[0]                #StudentIDs from csv

#Directories in the top folder
currDir = '.'
dirs = [ name for name in os.listdir(currDir) if os.path.isdir(os.path.join(currDir, name)) ]

if (len(dirs) != snum):
    print("Warning, ",len(dirs)," directories and", snum," students in the csv!")

#Match studentIDs to directory names
for i in range(0,snum):
    for j in range(0,len(dirs)):
        if(dirs[j].find(arr2[0][i]) > -1):
            arr2[1][i]=dirs[j]

# In[106]:

#Set Grade & Comment
#   grade1: float of grade
#   comment1: comment of score
#   array: DataFrame of grades
#   index: student index in array (row #)
#   gkey: key in array for grade column
#   ckey: key in array for comment column
def setGrade(grade1, comment1, array, index, gkey, ckey):
    array.set_value(index,gkey,grade1)
    array.set_value(index,ckey,comment1)

#bash execute command
def bexec(cmd1):
     return( sp.call(["bash","-c",cmd1]) )

#Find and Replace a string throughout a file with perl thru bash
def bashRep(strFind,strRep,file1):
    cmd1 = str("perl -p -i -e 's/")
    cmd1 += strFind
    cmd1 += str("/")
    cmd1 += strRep
    cmd1 += str("/g' ")
    cmd1 += file1
    return(bexec(cmd1)) #return if errors

#grade a submitted file with a grading file that will be concatenated
#   with the submitted file and store result in dummyFile
def grade(subFile,gradFile):
    dfil=dummyFile #dummy file to pass grade and comment back to this program

    #surround files with quotes to avoid whitespace errors
    sf1 = "\"" + subFile + ext1 + "\"" #submission file with .ipynb
    sf2 = "\"" + subFile + ext2 + "\"" #submission file with .py
    gf2 = "\"" + gradFile + ext2 + "\"" #grading file with .py
    catFile = "\""+subFile+"+grader"+ext2+"\""#sub file cat'd with grad file

    bexec("jupyter nbconvert --to python "+sf1) #Convert from .ipynb to .py
    bexec("cat " + sf2 + " " + gf2 + " >" + catFile) #concatenate grader file
    str1 = "get_ipython"
    bashRep( str1,"#"+str1, catFile) #remove matplotlib inline
    str1 = "pl.show"
    bashRep(str1,"#"+str1, catFile) #Dont show plots
    bexec("mv " + catFile + " " +currDir) #mv cat file to currDir
    #if running in top directory dont need to move datafile, just have in this location
    print( "Running Grader: " )
    #need to strip directory from name now
    cf2_path, cf2 = os.path.split(catFile[1:-1])
    print( bexec("python \""+cf2+"\" "+dfil+" 2>>errlog") )

    dArr = pd.read_csv(dfil, skipinitialspace=True)
    ckey = dArr.keys()[1] #Comments Key
    gkey = dArr.keys()[2] #Grade Key
    gread = str(dArr.get_value(0,gkey))
    cread = str(dArr.get_value(0,ckey))
    return (gread, cread)

# In[109]:

for i in range(snum): #Go over all students in gradebook

    print("\n==== Checking StudentID: "+arr2[0][i]+" ====")
    if( str(arr2[1][i]) != "None" ): #Check for folder corresponding to studentID

        #check the submission folder
        for root, dirs, files in         os.walk(os.path.join(currDir,arr2[1][i],subF), topdown=True):
            found = 0
            for name in [f for f in files if not f[0] == '.']: #ignore hidden directories
                fnam, fext = os.path.splitext(name)
                if( fext == ext1 ) :
                    found += 1
                    print("--ipynb found--: " + fnam)
                    if (found == 1) :
                        g1, c1 = grade(os.path.join(root, fnam), cfil)
                        print("Grade, Comment: ")
                        print(g1)
                        print(c1)
                        print("---------------")
                    else :
                        print("!!!DUPLICATES!!!, found "+str(found-1))
                        g2, c2 = grade(os.path.join(root, fnam), cfil)
                        print("Grade, Comment: ")
                        print(g2)
                        print(c2)
                        print("---------------")
                        if(g2>g1): #replace g1 & c1 with max
                            g1 = g2
                            c1 = c2

            #print('---')
            if found == 0 :
                com1 = "No ipynb found. "
                print(com1)
                setGrade(0.0, com1, arr, i, gkey, ckey)
            elif found == 1:
                setGrade(g1, c1, arr, i, gkey, ckey)
            else :
                com1 = str(found)+" different ipynb's found. Max Grade Comment: "
                setGrade(g1, com1+c1, arr, i, gkey, ckey)

    else : #No folder found with that studentID
        com1 = "No submission. "
        print(com1)
        setGrade(0.0, com1, arr, i, gkey, ckey)

# In[110]:

# In[ ]:

gnam, gext = os.path.splitext(graCSV)
gnam += "+graded"
gnam += gext
arr.to_csv(gnam)

# In[ ]:

# In[ ]:


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
CS224N 2019-20: Homework 4
sanity_check.py: sanity checks for assignment 4
Sahil Chopra <schopra8@stanford.edu>
Michael Hahn <>
Vera Lin <veralin@stanford.edu>

If you are a student, please don't run overwrite_output_for_sanity_check as it will overwrite the correct output!

Usage:
    sanity_check.py 1d
    sanity_check.py 1e
    sanity_check.py 1f
    sanity_check.py overwrite_output_for_sanity_check
"""
import sys

import numpy as np

from docopt import docopt
from utils import batch_iter
from utils import read_corpus
from vocab import Vocab, VocabEntry

from nmt_model import NMT

import torch
import torch.nn as nn
import torch.nn.utils

#----------
# CONSTANTS
#----------
BATCH_SIZE = 5
EMBED_SIZE = 3
HIDDEN_SIZE = 3
DROPOUT_RATE = 0.0

def reinitialize_layers(model):
    """ Reinitialize the Layer Weights for Sanity Checks.
    """
    def init_weights(m):
        if type(m) == nn.Linear:
            m.weight.data.fill_(0.3)
            if m.bias is not None:
                m.bias.data.fill_(0.1)
        elif type(m) == nn.Embedding:
            m.weight.data.fill_(0.15)
        elif type(m) == nn.Dropout:
            nn.Dropout(DROPOUT_RATE)
    with torch.no_grad():
        model.apply(init_weights)

def generate_outputs(model, source, target, vocab):
    """ Generate outputs.
    """
    print ("-"*80)
    print("Generating Comparison Outputs")
    reinitialize_layers(model)
    model.gen_sanity_check = True
    model.counter = 0

    # Compute sentence lengths
    source_lengths = [len(s) for s in source]

    # Convert list of lists into tensors
    source_padded = model.vocab.src.to_input_tensor(source, device=model.device)
    target_padded = model.vocab.tgt.to_input_tensor(target, device=model.device)

    # Run the model forward
    with torch.no_grad():
        enc_hiddens, dec_init_state = model.encode(source_padded, source_lengths)
        enc_masks = model.generate_sent_masks(enc_hiddens, source_lengths)
        combined_outputs = model.decode(enc_hiddens, enc_masks, dec_init_state, target_padded)

    # Save Tensors to disk
    torch.save(enc_hiddens, './sanity_check_en_es_data/enc_hiddens.pkl')
    torch.save(dec_init_state, './sanity_check_en_es_data/dec_init_state.pkl')
    torch.save(enc_masks, './sanity_check_en_es_data/enc_masks.pkl')
    torch.save(combined_outputs, './sanity_check_en_es_data/combined_outputs.pkl')
    torch.save(target_padded, './sanity_check_en_es_data/target_padded.pkl')

    # 1f
    # Inputs
    Ybar_t = torch.load('./sanity_check_en_es_data/Ybar_t.pkl')
    enc_hiddens_proj = torch.load('./sanity_check_en_es_data/enc_hiddens_proj.pkl')
    reinitialize_layers(model)
    # Run Tests
    with torch.no_grad():
        dec_state_target, o_t_target, e_t_target = model.step(Ybar_t, dec_init_state, enc_hiddens, enc_hiddens_proj,
                                                        enc_masks)
    torch.save(dec_state_target, './sanity_check_en_es_data/dec_state.pkl')
    torch.save(o_t_target, './sanity_check_en_es_data/o_t.pkl')
    torch.save(e_t_target, './sanity_check_en_es_data/e_t.pkl')

    model.gen_sanity_check = False

def question_1d_sanity_check(model, src_sents, tgt_sents, vocab):
    """ Sanity check for question 1d.
        Compares student output to that of model with dummy data.
    """
    print("Running Sanity Check for Question 1d: Encode")
    print ("-"*80)

    # Configure for Testing
    reinitialize_layers(model)
    source_lengths = [len(s) for s in src_sents]
    source_padded = model.vocab.src.to_input_tensor(src_sents, device=model.device)

    # Load Outputs
    enc_hiddens_target = torch.load('./sanity_check_en_es_data/enc_hiddens.pkl')
    dec_init_state_target = torch.load('./sanity_check_en_es_data/dec_init_state.pkl')

    # Test
    with torch.no_grad():
        enc_hiddens_pred, dec_init_state_pred = model.encode(source_padded, source_lengths)
    assert(np.allclose(enc_hiddens_target.numpy(), enc_hiddens_pred.numpy())), "enc_hiddens is incorrect: it should be:\n {:} but is:\n{}".format(enc_hiddens_target, enc_hiddens_pred)
    print("enc_hiddens Sanity Checks Passed!")
    assert(np.allclose(dec_init_state_target[0].numpy(), dec_init_state_pred[0].numpy())), "dec_init_state[0] is incorrect: it should be:\n {} but is:\n{}".format(dec_init_state_target[0], dec_init_state_pred[0])
    print("dec_init_state[0] Sanity Checks Passed!")
    assert(np.allclose(dec_init_state_target[1].numpy(), dec_init_state_pred[1].numpy())), "dec_init_state[1] is incorrect: it should be:\n {} but is:\n{}".format(dec_init_state_target[1], dec_init_state_pred[1])
    print("dec_init_state[1] Sanity Checks Passed!")
    print("-"*80)
    print("All Sanity Checks Passed for Question 1d: Encode!")
    print("-"*80)

def question_1e_sanity_check(model, src_sents, tgt_sents, vocab):
    """ Sanity check for question 1e.
        Compares student output to that of model with dummy data.
    """
    print("-"*80)
    print("Running Sanity Check for Question 1e: Decode")
    print("-"*80)

    # Load Inputs
    dec_init_state = torch.load('./sanity_check_en_es_data/dec_init_state.pkl')
    enc_hiddens = torch.load('./sanity_check_en_es_data/enc_hiddens.pkl')
    enc_masks = torch.load('./sanity_check_en_es_data/enc_masks.pkl')
    target_padded = torch.load('./sanity_check_en_es_data/target_padded.pkl')

    # Load Outputs
    combined_outputs_target = torch.load('./sanity_check_en_es_data/combined_outputs.pkl')
    print(combined_outputs_target.shape)

    # Configure for Testing
    reinitialize_layers(model)
    COUNTER = [0]
    def stepFunction(Ybar_t, dec_state, enc_hiddens, enc_hiddens_proj, enc_masks):
       dec_state = torch.load('./sanity_check_en_es_data/step_dec_state_{}.pkl'.format(COUNTER[0]))
       o_t = torch.load('./sanity_check_en_es_data/step_o_t_{}.pkl'.format(COUNTER[0]))
       COUNTER[0]+=1
       return dec_state, o_t, None
    model.step = stepFunction

    # Run Tests
    with torch.no_grad():
        combined_outputs_pred = model.decode(enc_hiddens, enc_masks, dec_init_state, target_padded)
    assert(np.allclose(combined_outputs_pred.numpy(), combined_outputs_target.numpy())), "combined_outputs is incorrect: it should be:\n {} but is:\n{}".format(combined_outputs_target, combined_outputs_pred)
    print("combined_outputs Sanity Checks Passed!")
    print("-"*80)
    print("All Sanity Checks Passed for Question 1e: Decode!")
    print("-"*80)

def question_1f_sanity_check(model, src_sents, tgt_sents, vocab):
    """ Sanity check for question 1f.
        Compares student output to that of model with dummy data.
    """
    print ("-"*80)
    print("Running Sanity Check for Question 1f: Step")
    print ("-"*80)
    reinitialize_layers(model)

    # Inputs
    Ybar_t = torch.load('./sanity_check_en_es_data/Ybar_t.pkl')
    dec_init_state = torch.load('./sanity_check_en_es_data/dec_init_state.pkl')
    enc_hiddens = torch.load('./sanity_check_en_es_data/enc_hiddens.pkl')
    enc_masks = torch.load('./sanity_check_en_es_data/enc_masks.pkl')
    enc_hiddens_proj = torch.load('./sanity_check_en_es_data/enc_hiddens_proj.pkl')

    # Output
    dec_state_target = torch.load('./sanity_check_en_es_data/dec_state.pkl')
    o_t_target = torch.load('./sanity_check_en_es_data/o_t.pkl')
    e_t_target = torch.load('./sanity_check_en_es_data/e_t.pkl')

    # Run Tests
    with torch.no_grad():
        dec_state_pred, o_t_pred, e_t_pred= model.step(Ybar_t, dec_init_state, enc_hiddens, enc_hiddens_proj, enc_masks)
    assert(np.allclose(dec_state_target[0].numpy(), dec_state_pred[0].numpy())), "decoder_state[0] is incorrect: it should be:\n {} but is:\n{}".format(dec_state_target[0], dec_state_pred[0])
    print("dec_state[0] Sanity Checks Passed!")
    assert(np.allclose(dec_state_target[1].numpy(), dec_state_pred[1].numpy())), "decoder_state[1] is incorrect: it should be:\n {} but is:\n{}".format(dec_state_target[1], dec_state_pred[1])
    print("dec_state[1] Sanity Checks Passed!")
    assert(np.allclose(o_t_target.numpy(), o_t_pred.numpy())), "combined_output is incorrect: it should be:\n {} but is:\n{}".format(o_t_target, o_t_pred)
    print("combined_output  Sanity Checks Passed!")
    assert(np.allclose(e_t_target.numpy(), e_t_pred.numpy())), "e_t is incorrect: it should be:\n {} but is:\n{}".format(e_t_target, e_t_pred)
    print("e_t Sanity Checks Passed!")
    print("-"*80)
    print("All Sanity Checks Passed for Question 1f: Step!")
    print("-"*80)

def main():
    """ Main func.
    """
    # args = docopt(__doc__)
    args = {'1d': False,
     '1e': False,
     '1f': True,
     'overwrite_output_for_sanity_check': False}

    # print(args)
    # Check Python & PyTorch Versions
    assert (sys.version_info >= (3, 5)), "Please update your installation of Python to version >= 3.5"
    assert(torch.__version__ >= "1.0.0"), "Please update your installation of PyTorch. You have {} and you should have version 1.0.0".format(torch.__version__)

    # Seed the Random Number Generators
    seed = 1234
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    np.random.seed(seed * 13 // 7)

    # Load training data & vocabulary
    train_data_src = read_corpus('./sanity_check_en_es_data/train_sanity_check.es', 'src')
    train_data_tgt = read_corpus('./sanity_check_en_es_data/train_sanity_check.en', 'tgt')
    train_data = list(zip(train_data_src, train_data_tgt))

    for src_sents, tgt_sents in batch_iter(train_data, batch_size=BATCH_SIZE, shuffle=True):
        src_sents = src_sents
        tgt_sents = tgt_sents
        break
    vocab = Vocab.load('./sanity_check_en_es_data/vocab_sanity_check.json')

    # Create NMT Model
    model = NMT(
        embed_size=EMBED_SIZE,
        hidden_size=HIDDEN_SIZE,
        dropout_rate=DROPOUT_RATE,
        vocab=vocab)

    if args['1d']:
        question_1d_sanity_check(model, src_sents, tgt_sents, vocab)
    elif args['1e']:
        question_1e_sanity_check(model, src_sents, tgt_sents, vocab)
    elif args['1f']:
        question_1f_sanity_check(model, src_sents, tgt_sents, vocab)
    elif args['overwrite_output_for_sanity_check']:
        generate_outputs(model, src_sents, tgt_sents, vocab)
    else:
        raise RuntimeError('invalid run mode')

if __name__ == '__main__':
    main()


import numpy as np

# The copy method makes a complete copy of the array and its data.
a = np.arange(10)
# a new array object with new data is created
b = a.copy()
print("is operator : ",b is a)
print("base attribute : ",b.base is a)
b.shape = 2,5
print("shape of a (not changed): ",a.shape)
b[0,4] = 10
# data also not changed
print("array a (not changed) : ",a)

# Sometimes copy should be called after slicing if the original array is not required anymore.
# For example, suppose a is a huge intermediate result and the final result b only contains a
# small fraction of a, a deep copy should be made when constructing b with slicing:

#  XeY means x * 10^Y
a = np.arange(int(1e3))
b = a[0:100].copy()
del a  # the memory of ``a`` can be released.
# If b = a[:100] is used instead, a is referenced by b and will persist in memory even if del a is executed.


# what is pandas?
# 1) Pandas is a powerful python data analysis toolkit.
# 2) Open source
# 3) A fast and efficient dataframe object for data manipulation.
# 4) Reading and writing data structures and different formates:
# csv, tsv, txt, XML, JSON, ZIP, etc.
# series: 1D array in pandas(only one column), dataframe: 2D array in pandas(multiple columns).
# numpy vs pandas: numpy array is used for implementation of pandas data objects

import pandas as pd
# print(pd.__version__)               # to check version of pandas module
# series1 = pd.Series([1, -2, "omkar", 2.5])         # basic pandas series creation
# series2 = pd.Series([1, -2], index=["a", "b"], dtype=float)     # set index and datatype while create series in pandas
# print(series2[0:2])                              # to retrieve particular data from dataframe use colon
# print(series1 + series2)                        # performing arithmetic operation on two pandas series

# df1 = pd.DataFrame([[1, 2, 3], [-2, "omkar", 2.5]])     # creating dataframe with only lists
# df2 = pd.DataFrame([{"a": 1, "b": 2}, {"a": 11, "b": 22, "c": 33}])# another way to create dataframes (NaN=miss value)

# print(df.columns)                           # to get all columns name from dataframe
# print(pd.read_csv("friends.csv", nrows=2))    # to retrieve certain rows from dataframe
# print(pd.read_csv("friends.csv", usecols=[1, 2]))    # to retrieve specific column from dataframe
# print(pd.read_csv("friends.csv", index_col=2))    # to set any column as index

# print(pd.read_csv("friends.csv", header=None, prefix="col"))  # to create header to dataframe as col0, col1, so on.

# print(pd.read_csv("friends.csv", dtype={"marks": "float64"}))      # change dtype of data in columns
# print(pd.read_csv("friends.csv", true_values=["yes"], false_values=["no"]))
# # give True value where "yes" value and False value where "no" values in dataset

# print(pd.read_csv("friends.csv", na_values={"1stcolumn": "not", "3rdcol": "null"}))
# # this is the way we can remove specific values from specific columns
# print(pd.read_csv("friends.csv", keep_default_na=False))  # keep null, none, na, etc. values as it is
# print(pd.read_csv("friends.csv", na_filter=False))
# # pandas check each values for converting in na_values so to stop this long process and boost the process
# # we can use na_filter as false if we don't have na values in dataset

# import numpy as np
df = pd.DataFrame({"omkar": [24, np.nan], "sutar": [np.NaN, 77]})
# print(df.isnull().sum())              # gives how many null values are present in each column
# print(df.isnull().sum().sum())        # gives total number of null values present in all dataframe
# # same with notnull().sum().sum()     # gives false where null values are present

# print(df.dropna(subset=["omkar"]))      # drop rows where null values present in omkar column
# df.dropna(inplace=True)                 # inplace=True make changes in original dataset

# df.loc([0, 3])                  # show 0th and 3rd row from table
# newdf = df.loc[3, 5]            # show values which at 3rd row in 5th column
# newdf2 = df.loc[0:2, 4:5]       # show values between row 0-2 and column 4-5
# n2 = df.loc[df["class"] < 11, ["percentage"]]     # show percentage where class number is less than 11

# df.join(newdf)                 # join second dataset on first dataset
# df.join(newdf, how="left")     # left is by default
# # left show all indexes from first dataframe and put nan values if index not in second dataframe
# # right show all indexes from second dataframe and put nan values if index not in first dataframe
# # inner show common indexes from both dataframe
# # outer show all indexes from both dataframe and put nan values if index not common in both dataframe
# df.join(newdf, lsuffix="_1")     # if both dataframe have same column names then it will put "_1" at end of left column
# df.join(newdf, rsuffix="_1")     # if both dataframe have same column names then it will put "_1" at end of right column

# df.append(newdf)          # append second dataframe to first dataframe
# df.append(newdf, ignore_index=True)   # this will create new index for second dataframe for continuous indexing

pd.melt(df)           # gives two column first with all column names and second with all values of tables
pd.melt(df, id_vars=["city"])    # id_vars set city as variable(first column) in melt table
pd.melt(df, id_vars=["city"], value_vars=["year"])
# this gives city as main column and gives values where values are year

Python/Pandas/pandas 3 - Indian AI Production.py