#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import ipysheet as ips
import panel as pn
from scipy import stats
pn.extension("katex")

# # Tutorial 4 #
#
# + **solutions for homework problems 1 – 4**
#
# + **tutorial problems on effective conductivity and flow nets**
#
# + **homework problems on effective conductivity and flow nets**
#
#
#
#
#
#
#
# ### Solutions for Homework Problems 1 – 2 ###
#
#
#

# In[2]:

#
r1_1 = pn.pane.Markdown("""

### Homework Problem 1 ###

The pressure head in an aquifer extending over 200 km2 is decreased by 1.60 m.
Determine the loss of groundwater in the aquifer for two scenarios:
A. The aquifer is unconfined (storage coefficient 0.13).
B. The aquifer is confined (storage coefficient 0.0005).

""",width = 800, style={'font-size': '13pt'})

r1_2= pn.pane.PNG("images/T03_H1.png", width=350)
#r1_2 = pn.pane.PNG("images/T03_H1.PNG")

### Tutorial Problem 7 – Solution ###

#

r1_3 = pn.pane.Markdown("""
### Solution - Homework Problem 1 ###

Relevant information can be found in Lecture L03, Slides- 28-30

""",width = 800, style={'font-size': '13pt'})

r1_3b = pn.pane.LaTeX(r"""

The relevant equations is:

$$
S = \Delta V_w/(A\cdot \Delta H)
$$

""",width = 800, style={'font-size': '13pt'})
pn.Column(r1_1, r1_2, r1_3, r1_3b)

# In[3]:

# Given
A = 200 # km^2, aquifer area
D_h = 1.6 # m, head decrease
S_u = 0.13 # (-), Storativity unconfined aquifer
S_c = 0.0005 # (-) Storage coefficient, confined aquifer

# Solution
DV_wu = A*S_u*D_h * 10**6 # m^3 change in water volume unconfined aquifer
DV_wc = A*S_c*D_h* 10**6  # m^3 change in water volume unconfined aquifer

# output

print("Change in water volume in unconfined aquifer is: {0:1.1e}".format(DV_wu),"m\u00b3 \n")
print("Change in water volume in confined aquifer is: {0:1.1e}".format(DV_wc),"m\u00b3")

# ## Homework Problem 2
#
# Conduct a sieve analysis for a dried soil sample (see data in the table below)
#
# 1. Draw the granulometric curve (cumulative mass distribution) and briefly characterise the sediment with regard to its major constituent(s).
# 2. What is the coefficient of uniformity?
#

# In[4]:

#
title = ["mesh   size  [mm] ", "residue in the sieve [g] ", "∑Retained %", "Commulative Passed %"]
Size = [6.3, 2, 0.63, 0.2, 0.063, "< 0.063 /cup"]
passed = [11, 62, 288, 189, 42, 10]
s2 = ips.sheet(rows=6, columns=4, row_headers=False, column_headers=title)
ips.column(0, Size, row_start=0)
ips.column(1, passed, row_start=0); s2

# In[5]:

# Solution of problem 2

t_sample = np.sum(passed) # g, add the residue column to get total mass
retained_per = passed/t_sample *100 # %, # retain percentage residue/total mass
retain_per_cumsum =np.cumsum(retained_per) # get the cummulative sum of the reatined
passing_cumper = 100 - retain_per_cumsum # substract 100-cummsum to get passing % - the last column

#Output
s3 = ips.sheet(rows=6, columns=4, row_headers=False, column_headers=title)
ips.column(0, Size, row_start=0)
ips.column(1, passed, row_start=0);
ips.column(2, retained_per, row_start=0);
ips.column(3, passing_cumper, row_start=0); s3

# In[6]:

# Plotting granulometric curve

plt.rcParams['axes.linewidth']=2
plt.rcParams['grid.linestyle']='--'
plt.rcParams['grid.linewidth']=1
x = np.append([20], Size[:5]) # adding for all left over.
y = np.append([100],passing_cumper[:5])
fig = plt.figure(figsize=(9,6));
plt.plot(x, y, 'x-', color='red', lw=2.5);
tics=x.tolist()
plt.xscale('log');lw=2.5
plt.grid(which='major', color='k', alpha=0.7)
plt.grid(which='minor', color='k', alpha=0.3)
plt.xticks(x, tics);
plt.yticks(np.arange(0,110,10));
#plt.title('grain size distribution (combined wet sieving and sedimentation analysis)');
plt.xlabel('grain size d [mm]');
plt.ylabel('Cummulative Passed fraction %');

plt.annotate('', xy=(0.20, 10),  xycoords='data', xytext=(0.045, 10), arrowprops=dict(arrowstyle='->', color="b", lw=2.5),ha='right', va='top',)
plt.annotate('', xy=(1.1, 60),  xycoords='data', xytext=(0.045, 60), arrowprops=dict(arrowstyle='->', color="b", lw=2.5),ha='right', va='top',)
plt.annotate(r'$d_{60}$', xy=(1, 60),  xycoords="data", xytext=(0.85, -3),color='red',size=12, arrowprops=dict(arrowstyle='<-', color="b", lw=2.5),ha='left', va='bottom',)
plt.annotate(r'$d_{10}$', xy=(0.20, 10),  xycoords='data', xytext=(0.235, 1.5),color='red',size=12, arrowprops=dict(arrowstyle='<-', color="b", lw=2.5),ha='right', va='top',)
plt.rcParams["font.weight"] = "bold"

plt.savefig("fig6.png")

mpl_pane = pn.pane.Matplotlib(fig, dpi=144)

# In[7]:

# From the figure
d_10 = 0.22 # mm,approx, diameter 10% passing, see the arrow bottom in x-axis
d_60 = 1.0 # mm, approx diameter 10% passing, see the arrow bottom in x-axis

c_u = d_60/d_10 # [], coefficient of uniformity

#Output
print("The coefficient of uniformity is: {0:1.1f}".format(c_u))
r2_1 = pn.pane.Markdown("""
**Major constituents: coarse sand/medium sand** """, width=600, style={'font-size': '13pt', 'color': 'blue'} )
pn.Row(r2_1)

# In[8]:

# Tutorial Problem 11- Effective Conductivity and flow nets
r5_1 = pn.pane.Markdown("""
#Tutorial Problems on Effective Conductivity and Flow Nets
""", width = 900)

r5_2 = pn.pane.Markdown("""
###Tutorial Problem 11: Effective Hydraulic Conductivity
A sandy layer with a thickness of 2.5 m is embedded between two gravel layers. B
oth gravel layers have a thickness of 1.5 m and a hydraulic conductivity of 3.7·10-3 m/s.
Steady-state groundwater flow is in parallel to the layering.
A hydraulic gradient of 0.001 and an overall discharge of 1 m³/d per unit width have been determined.

a. Determine the effective hydraulic conductivity.

b. What is the hydraulic conductivity of the sand layer?

c. Which effective hydraulic conductivity would be obtained if flow was assumed perpendicular to the layering?

d. Calculate effective hydraulic conductivity if the angle between the flow direction and the layering equals 45°.
""", style={'font-size': '13pt'})

pn.Column(r5_1, r5_2)

# In[9]:

# Solution of Problem 11
r5_3 = pn.pane.PNG("images/T03_TP11_a.png", width=400)
r5_4 = pn.pane.LaTeX(r"""
Known relationships are (see Lecture 05, Slides 8-13, 22):
$$
Q = WmK\frac{\Delta H}{L}
$$
$$
K = \frac{Q/W}{m\cdot \Delta H \cdot L}
$$
Weighted arithmetic mean to determine hydraulic conductivity for sand:

$$
K = \frac{1}{m}\sum_{i=1}^n m_i\cdot K_i
$$
where $i$ is different layers
""", width = 500, style={'font-size': '13pt'})
spacer2 = pn.Spacer(width=100)

pn.Row(r5_3, spacer2, r5_4)

# In[10]:

#Given Solution of 11 a, b

Q = 2 # m^3/d, discharge
W = 1 # m, per unit width
K_g = 3.7*1E-3# m/s, conductivity of gravel layer
m_g = 1.5 # m, thickness of gravel layer
m_s = 2.5 # m, thickness of sand layer
m = 2*m_g + m_s # m. total thickness of aquifer
Dh_L = 0.001 # (-), hydraulic gradient

#Solution of 11a
Keff_h = (Q/W)/(m*Dh_L) # m/d, conductivity
Keff_hs = Keff_h/(24*3600)# m/s, conductivity unit changed

#Solution of 11b
# K_eff = (2*m_g*K_g + m_s*K_g)/m

K_s = ((m*Keff_hs - 2*m_g*K_g))/m_s

print("Effective horizontal hydraulic conductivity (Keff_h) = {0:1.2f}".format(Keff_h), "m/d\n" )
print("Effective horizontal hydraulic conductivity (Keff_hs) = {0:1.3E}".format(Keff_hs), "m/s\n" )
print("Hydraulic conductivity of sand layer (K_s) = {0:1.1E}".format(K_s), "m/s" )

# In[11]:

#Given Solution of 11 c, d

r5_5 = pn.pane.PNG("images/T03_TP11_b.png", width=200)
r5_6 = pn.pane.PNG("images/T03_TP11_c.png", width=200)

r5_7 = pn.Column(r5_5, r5_6)

r5_8 = pn.pane.LaTeX(r"""
Vertical effective conductivity is given by weighted harmoninc mean
$$
K = \frac{m}{2\cdot \frac{m_g}{K_g} + \frac{m_s}{K_s} }
$$

For inclined aquider the effective conductivity is:

$$
K = \frac{1}{\frac{\cos^2\theta}{K_h} + \frac{\sin^2\theta}{K_v}}
$$

""", style={'font-size': '13pt'})

pn.Row(r5_7,spacer2, r5_8)

# In[12]:

# Solution of 11c

Keff_v = m/(2*(m_g/K_g)+ (m_s/K_s))

#Given
theta = 45 # theta
theta_r = 45*(np.pi)/180 # degree to radian conversion
K_h = Keff_hs # m/s, solution from 11a
K_v = Keff_v # m/s, solution from 11c

# solution from 11d
Keff_i = 1/((np.cos(theta_r)**2/K_h)+(np.sin(theta_r)**2/K_v))

print("Effective vertical hydraulic conductivity (Keff_v) = {0:1.2E}".format(Keff_v), "m/s\n" )
print("Effective inclined hydraulic conductivity (Keff_i) = {0:1.2E}".format(Keff_i), "m/s" )

# In[13]:

#
r6_1 = pn.pane.Markdown("""
### Tutorial Problem 12: Hydrologic Triangle
The figure below shows the position of four groundwater observation wells with measured hydraulic heads in m a.s.l.

**a.** Sketch head isolines for intervals of 1 m by applying the hydrologic triangle method.

**b.** Indicate the flow direction.

""",width = 400, style={'font-size': '13pt'})

r6_2 = pn.pane.PNG("images/T03_TP12_a.png", width=400)

pn.Row(r6_1,spacer2, r6_2)

# In[14]:

#
r6_3 = pn.pane.Markdown("""
### Solution of Tutotrial Problem 12

Step 1. Connects all the points
""", width=600)

r6_2.object = "images/T03_TP12_b.png"
r6_3

# In[15]:

#
r6_4 = pn.pane.Markdown("""
### Solution of Tutotrial Problem 12
Step 2. Divide the connected lines at equal head-level (here = 1 m)
""", width=600)
r6_2.object = "images/T03_TP12_c.png"

# In[16]:

#
r6_4 = pn.pane.Markdown("""
### Solution of Tutotrial Problem 12
Step 3. Join all the equal head lines
""", width=600)
r6_2.object = "images/T03_TP12_d.png"

# In[17]:

r6_4 = pn.pane.Markdown("""
### Solution of Tutotrial Problem 12
Step 4. Mark the flow direction from higher head towards lower head
""", width=600)
r6_2.object = "images/T03_TP12_e.png"

# In[18]:

#
r7_1 = pn.pane.Markdown("""
##Tutorial Problem 13: Flow Nets##

Sketch head isolines and streamlines for the two configurations a) and b) of a well doublette shown below. In both cases flow nets should be sketched without and with the uniform flow component.

""",width=800,  style={'font-size': '13pt'})

r7_2 = pn.pane.Markdown("""
 a) withdrawal at both wells:

""",width=400,  style={'font-size': '13pt'})

r7_3 = pn.pane.PNG("images/T03_TP13_a.png", width=200)

r7_4 = pn.Column(r7_2,r7_3)

r7_5 = pn.pane.Markdown("""
 b) Injection and withdrawl wells:

""",width=400,  style={'font-size': '13pt'})

r7_6 = pn.pane.PNG("images/T03_TP13_b.png", width=200)

r7_7 = pn.Column(r7_5,r7_6)
r7_8 = pn.Row(r7_4, r7_7)
pn.Column(r7_1, r7_8)

# In[19]:

r8_1= pn.pane.Markdown("""
#Homework Problems on  Effective Conductivity and Flow Nets

""", width = 800, style={'font-size': '13pt'})

r8_2= pn.pane.Markdown("""
#There is no obligation to solve homework problems!
""", width = 800, style={'font-size': '13pt', 'color':'red'})

pn.Column(r8_1,r8_2)

# In[20]:

#
r9_1= pn.pane.Markdown("""
###Homework Problem 5: Effective Hydraulic Conductivity
A gravel layer with a thickness of 2.5 m is embedded between two sand layers. Both sand layers have a thickness of
1.5 m and a hydraulic conductivity of 3.7·10-4 m/s. Steady-state groundwater flow is perpendicular to the layering.
An overall head difference of 5.5 cm and a discharge of 500 l/d per unit area have been determined

**a.** Determine the effective hydraulic conductivity.

**b.** What is the hydraulic conductivity of the gravel layer?

**c.** Which effective hydraulic conductivity would be obtained if flow was assumed to be in parallel with the layering?

**d.** Calculate effective hydraulic conductivity if the angle between the flow direction and the layering equals 30°.

""", width = 900, style={'font-size': '13pt'})
r9_1

# In[21]:

#
r10_1= pn.pane.Markdown("""
###Homework Problem 6: Hydrologic Triangle
The figure below shows the position of five groundwater observation wells with measured hydraulic heads in m a.s.l.

**a.** Sketch head isolines for intervals of 1 m by applying the hydrologic triangle method.

**b.** Indicate the flow direction.

""", width = 500, style={'font-size': '13pt'})
r10_2 = pn.pane.PNG("images/T03_TH6.png", width=400)

pn.Row(r10_1, r10_2)

# In[22]:

#
r11_1= pn.pane.Markdown("""
###Homework Problem 7: Flow Nets
Sketch head isolines and streamlines for the well doublette shown below.
In this case, injection and withdrawal of groundwater is superimposed to a uniform flow component.

 """, width = 900, style={'font-size': '13pt'})

r11_2 = pn.pane.PNG("images/T03_TH7.png", width=400)

r11_3= pn.pane.Markdown("""

 """, width = 900, style={'font-size': '13pt'})
pn.Column(r11_1, r11_2, r11_3)

# In[ ]:


# -*- coding: utf-8 -*-
"""chennai_house_price_prediction_final_ipython_notebook_for_github_project_gallery_insaid.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/15gfBJ9NTYi51RPMMZzKU6VgGnQqLlmdr

----
# **Table of Contents**
----

**1.** [**Problem Statement**](#section1)

**2.** [**Installing and Importing Packages**](#section2)

  - **2.1** [**Description of the Dataset**](#section301)

  - **2.2** [**Upgrading Libraries**](#section302)

  - **2.2** [**Importing Libraries**](#section302)

**3.** [**Loading Data**](#section3)

**4.** [**Data Acquistititon and Description**](#section4)

  - **4.1** [**Description of the Dataset**](#section401)

  - **4.2** [**Data Cleaning**](#section40201)

**5.** [**Data Pre-Processing**](#section6)

  - **5.1** [**Pre-Profiling Report**](#section401)

  - **5.2** [**Data Information**](#section40201)

**6.** [**Exploratory Data Analysis**](#section7)

  - **6.1** [**Univariate Analysis**](#section401)

  - **6.2** [**Bivariate Analysis**](#section40201)

**7.** [**Data Postprocessing**](#section8)
  - **7.1** [**Encoding Categorical Variables**](#section401)

  - **7.2** [**Feature Engineering**](#section40201)

  - **7.3** [**Separating Train and Test Data**](#section40201)

**8.** [**Modelling**](#section8)
  - **8.1** [**Defining Baseline Models**](#section401)

  - **8.2** [**Hyperparameter Tuning**](#section40201)

**9.** [**Test Set**](#section8)

**10.** [**Conclusion**](#section8)

---
# **1. Problem Statement:-**
---

- **ChennaiEstate** is a **real estate firm** based in **Chennai** that is involved in the property business for the past 5 years.

- Since, they are in the business for so long, they have enough data of all the real estate transactions in the city.

- They decided to venture into Analytics and have now started a division called **Chennai Estate Analytics** to give consumers as much information as possible about housings and the real estate market in Chennai.

-  A home is often the largest and most expensive purchase a person makes in his or her lifetime. Ensuring real-estate owners have a
trusted way to monitor the asset is incredibly important.

-  Hence, they have hired you as a consultant to help them give insights and develop a model to accurately predict real estate prices.

- Based on the **train** dataset, you will need to develop a model that accurately predicts the real estate price in **Chennai**.

### **Scenario**

- You are given a dataset consisting of **required details**

- Your task is to build a **regression model** using the dataset

- Because there was **no machine learning model for this problem** in the company, you don’t have a quantifiable win condition. You need to build the best possible model

---
# **2. Importing Libraries**
---
"""

import sys                                                       # Importing System
!{sys.executable} -m pip install -U pandas-profiling[notebook]   # Installing pandas profiling
!jupyter nbextension enable --py widgetsnbextension              # enabling python notebook extention

!pip install pandas-profiling -q --upgrade                          # Installing pandas profiling
!pip install catboost -q                                            # Intalling Catboost regressor
!pip install xgboost -q                                             # Intalling XGboost regressor

"""
### **2.2 Upgrading Libraries**

- **After upgrading** the libraries, you need to **restart the runtime** to make the libraries in sync.

- Make sure not to execute the cell above (2.1) and below (2.2) again after restarting the runtime.
"""

!pip install pandas-profiling -q --upgrade                          # upgrading pandas profiling

"""
### **2.3 Importing Libraries**
- After the **installation** and **upgrading of all the libraries** we will now import the necessary libraries.
"""

# Commented out IPython magic to ensure Python compatibility.
#-------------------------------------------------------------------------------------------------------------------------------

import matplotlib.pyplot as plt                                     # Importing pyplot for visualization
# %matplotlib inline

#----------------------------------------------------------------------------------------------
import seaborn as sns
#-------------------------------------------------------------------------------------------------------------------------------
import pandas as pd                                                 # Importing for panel data analysis
from pandas_profiling import ProfileReport                          # Importing Pandas Profiling (To generate Univariate Analysis)
pd.set_option('display.max_columns', None)                          # Unfolding hidden features if the cardinality is high
pd.set_option('display.max_colwidth', None)                         # Unfolding the max feature width for better clearity
pd.set_option('display.max_rows', None)                             # Unfolding hidden data points if the cardinality is high
pd.set_option('mode.chained_assignment', None)                      # Removing restriction over chained assignments operations
pd.set_option('display.float_format', lambda x: '%.5f' % x)         # To suppress scientific notation over exponential values
#-------------------------------------------------------------------------------------------------------------------------------
import numpy as np                                                  # Importing package numpys (For Numerical Python)
from datetime import datetime as dt                                 # For datetime funcationality
import warnings                                                     # Importing warning to disable runtime warnings
warnings.filterwarnings("ignore")                                   # Warnings will appear only once
#-------------------------------------------------------------------------------------------------------------------------------
from sklearn import metrics                                         # Calling the metrics for calculating performence metrics
from sklearn import preprocessing                                   # Calling preprocessing for preprocessing of data
from sklearn.preprocessing import StandardScaler                    # Calling standardscaler for standerdization
#-------------------------------------------------------------------------------------------------------------------------------
from sklearn.svm import SVR                                         # Calling Support Vector Regressor for modelling
from sklearn.decomposition import PCA                               # Calling PCA for dimentionality reduction
from sklearn.metrics import make_scorer                             # Calling make_scorer for calculating score
from sklearn.metrics import mean_squared_error                      # Calling sean_squared_error for calculating mean squared error
import zipfile                                                      # Importing zipfile
#-------------------------------------------------------------------------------------------------------------------------------
from scipy import stats                                             # Importing stats from scipy
from scipy.stats import norm                                        # Importing norm
#-------------------------------------------------------------------------------------------------------------------------------
from sklearn.preprocessing import LabelEncoder, MinMaxScaler        # Importing Encoders
from sklearn.linear_model import LinearRegression, Ridge, Lasso     # Importing Linear Regressors
from sklearn.neighbors import KNeighborsRegressor                   # Importing KNN
from sklearn.ensemble import RandomForestRegressor                  # Importing Random Forest Regressor
from sklearn.ensemble import BaggingRegressor                       # Importing Bagging Regressor
from sklearn.ensemble import GradientBoostingRegressor              # Importing GradientBoostingRegressor
from sklearn.tree import DecisionTreeRegressor                      # Importing DecissionTreeeRegressor
from catboost import CatBoostRegressor                              # Importing CatBoostRegressor
from xgboost import XGBRegressor                                    # Importing XGBoost Regressor
#-------------------------------------------------------------------------------------------------------------------------------
from sklearn.model_selection import train_test_split                # Calling train_test_split for splitinng the dataset
from sklearn.model_selection import RandomizedSearchCV              # Calling RandomizedSearchCV for tuning the model
from sklearn.model_selection import cross_val_score                 # Importing cross_val_score
from sklearn.model_selection import GridSearchCV                    # Importing GridsearchCV
#-------------------------------------------------------------------------------------------------------------------------------
from sklearn.metrics import mean_squared_error                      # Importing MSE
from sklearn.metrics import r2_score                                # Importing R Squared
from sklearn.metrics import mean_absolute_error                     # Importing MAE
#-------------------------------------------------------------------------------------------------------------------------------
import time                                                         # Importing time
import re                                                           # Importing RegEx
import plotly.express as ex                                         # Importing Plotly Express for Dynamic Plotting
import plotly.graph_objs as go                                      # Importing Plotly graphs for Dynamic Plotting
import plotly.offline as pyo                                        # Importing offline Express for Dynamic Plotting
from plotly.subplots import make_subplots                           # Importing Plotly Subplots to plot Dynamic subplots
import plotly.figure_factory as ff                                  # Calling the figure factory to create unique chart types

"""----

# **3. Loading Data**
----

- In this step we will be **Loading the dataset**
"""

data = pd.read_csv("/content/chennai_house_price_prediction.csv")

data.head()

data.shape

"""----

# **4. Data Acquistion and Description**
----

### **4.1 Description of the Dataset**
- We have **71092 samples**  and for each of sample **19 different** properties are recorded.
"""

data.describe()

"""**Observation:**

- The emean of the **SALES_PRICE** is found to be **10894909.63919** which is deviating from the std.

### **4.2 Data Information**

- In this section we will see the **information about the types of features**.
"""

data.info()

"""### **Observation:**

- There are total **13 numerical data-type and 8 object type data files** recorded.

---
# **5. Data Pre-Processing**
---

### **5.1 Pre Profiling Report**

- For **quick analysis** pandas profiling is very handy.

- Generates profile reports from a pandas DataFrame.

- For each column **statistics** are presented in an interactive HTML report.
"""

profile = ProfileReport(df=data)
profile.to_file(output_file='Pre Profiling Report.html')
print('Accomplished!')

"""
### **5.2 Data Cleaning**

- In this section, we will clean out our data based on the information retrieved from the previous observations.

- Hence, we will have to perform thr following subtasks
  - Checking for missing values and manipulating them
  - Checking the datatype
  - Spelling Correction
"""

# Checking for missing values and manipulating them
data.isnull().sum()

#Checking the datatype
data.dtypes

# Treating the missing values:-
data["QS_OVERALL"] = data["QS_OVERALL"].fillna(data["QS_OVERALL"].mean())
data["N_BEDROOM"] = data["N_BEDROOM"].fillna(data["N_BEDROOM"].mean())
data["N_BATHROOM"] = data["N_BATHROOM"].fillna(data["N_BATHROOM"].mean())

data.isnull().sum()

# Checking Value Counts:-
for i in data.columns:
    print("************************************")
    print("The Value Count for " + i + " is :-" )
    print(data[i].value_counts())
    print("************************************")

# Dropping PRT_ID:-
data = data.drop(["PRT_ID"],axis=1)

# Replacing "AREA" with the respective values:
data["AREA"].replace(to_replace = ["Chrompt","Chormpet","Chrmpet"], value ="Chrompet",inplace=True)

data["AREA"].replace(to_replace=["Karapakam","KKNagar","Velchery","Ana Nagar","Ann Nagar","Adyr","TNagar"],
                     value=["Karapakkam","KK Nagar","Velachery","Anna Nagar","Anna Nagar","Adyar","T Nagar"]
                    ,inplace=True)

data["AREA"].value_counts()

pd.get_dummies(data["SALE_COND"])
data["SALE_COND"].replace(to_replace=["Adj Land","Partiall","PartiaLl","Ab Normal"],
                     value=["AdjLand","Partial","Partial","AbNormal"]
                    ,inplace=True)

data["SALE_COND"].value_counts()

data["SALE_COND"].value_counts()

data.head()

data["PARK_FACIL"].value_counts()

data["PARK_FACIL"].replace(to_replace=["Noo"],value = ["No"], inplace = True)

data["PARK_FACIL"].value_counts()

data["BUILDTYPE"].value_counts()

data["BUILDTYPE"].replace(to_replace=["Comercial","Other"],value = ["Commercial","Others"], inplace = True)

data["BUILDTYPE"].value_counts()

data["UTILITY_AVAIL"].value_counts()

data["UTILITY_AVAIL"].replace(to_replace=["All Pub"],value = ["AllPub"], inplace = True)

data["UTILITY_AVAIL"].value_counts()

data["STREET"].value_counts()

data["STREET"].replace(to_replace = ["Pavd","NoAccess"], value=["Paved","No Access"], inplace = True)

data["STREET"].value_counts()

data["MZZONE"].value_counts()

datatype = data.dtypes

datatype

"""

---
# **6. Exploratory Data Analysis**
---

### **6.1 Univariate Analysis**

- In this section we will see what information can be derived from each individual feature of the dataset.

#### **Question:** What insights can be drawn from the categorical features ?
"""

# Commented out IPython magic to ensure Python compatibility.
import matplotlib.pyplot as plt
# %matplotlib inline
import seaborn as sns

for i in data.columns:
    if datatype[i] == 'object':
        plt.figure(figsize = (10,5))
        sns.barplot(x = data[i].value_counts().values,
                    y = data[i].value_counts().index,
                    orient = 'h')
        plt.ylabel(i)

"""**Observations:**

- The freq of **RL** is maximum for **MZZONE**

- **Paved** type streets has been reported maximum number of times.

- **UTILITY_AVAIL** has **AllPub** recorded the maximum number of times.

#### **Question:** What insights can be drawn from the numerical features ?
"""

for i in data.columns:
    if datatype[i] != 'object':
        plt.figure(figsize = (10,5))
        data[i].plot.hist(bins=50)
        plt.xlabel(i)
        plt.grid()

"""### **6.2 Univariate Analysis**

- In this section we will see what information can be derived from more than one features at a time.
"""

# data_bivariate= data.copy()

"""#### **Question:** What is the relation in between **BUILDTYPE** and **PARK_FACIL** w.r.t average **SALES_PRICE**?"""

import seaborn as sns
sns.set()
temp = data.groupby(['BUILDTYPE', 'PARK_FACIL']).SALES_PRICE.median()
temp.plot(kind = 'bar', stacked = True)

"""#### **Question:** What is the deviation in **PARK_FACIL** w.r.t **SALES_PRICE**?"""

temp = data.groupby(['PARK_FACIL']).SALES_PRICE.mean()
temp_index = temp.index
temp_values = temp.values
sns.barplot(temp_values,
            temp_index,
            orient = 'h')
plt.grid()

"""#### **Question:** How **SALE_PRICE** effects the **distance from main road**?"""

plt.scatter(data['DIST_MAINROAD'],data['SALES_PRICE'])

"""----

# **7. Data Postprocecssing**
----

### **7.1 Data encoding**

- We will use **encode** the necessary features
"""

# OneHot Encode the Catagorical Variables
a,b,c,d,e,f = pd.get_dummies(data["AREA"]),pd.get_dummies(data["PARK_FACIL"]),pd.get_dummies(data["BUILDTYPE"]),pd.get_dummies(data["UTILITY_AVAIL"]),pd.get_dummies(data["STREET"]),pd.get_dummies(data["MZZONE"])

y= pd.get_dummies(data["SALE_COND"])

a.head()

b.head()

c.head()

d.head()

e.head()

f.head()

"""
### **7.2 Feature genaration**

- In this step we will use **genarate new features**
"""

for i in range(len(a.columns)):
    data.insert(0+i,a.columns[i],a[a.columns[i]])
for i in range(len(b.columns)):
    data.insert(11+i,b.columns[i],b[b.columns[i]])
for i in range(len(c.columns)):
    data.insert(12+i,c.columns[i],c[c.columns[i]])
for i in range(len(d.columns)):
    data.insert(13+i,d.columns[i],d[d.columns[i]])
for i in range(len(e.columns)):
    data.insert(14+i,e.columns[i],e[e.columns[i]])
for i in range(len(f.columns)):
    data.insert(15+i,f.columns[i],f[f.columns[i]])

for i in range(len(y.columns)):
    data.insert(7+i,y.columns[i],y[y.columns[i]])

"""
### **7.2 Feature Dropping**

- In this step we will use **genarate new features**
"""

# Dropping unnecessary features
data.drop(['AREA','SALE_COND','PARK_FACIL', 'BUILDTYPE', 'UTILITY_AVAIL', 'STREET', 'MZZONE'], axis=1 , inplace= True)

data.head()

"""
### **7.4 Feature Extraction**

- We will seperate the dataframe into train and validation sets.

- We will **seperate** the train set into X (independent) and y (dependent) dataframes.

- Finally, we will apply **train-test split** on the scaled data.
"""

x= data.drop(["SALES_PRICE"],axis=1)
y= data["SALES_PRICE"]

"""

---
# **8. Model Building**
---

### **8.1 Defining Baseline Models**
- In this section we will define all of the **best possible basine models** and train using the **training data**
"""

# Defining the scores list
model_scores = []

# Defining a list of useful regressors
regressors = [RandomForestRegressor(random_state=42),
              KNeighborsRegressor(),
              GradientBoostingRegressor(random_state=42),
              CatBoostRegressor(random_state=42),
              XGBRegressor(random_state=42),
              BaggingRegressor(random_state=42)]

"""### **8.2 Splitting training and Testing Data**
- In this section we will make the **train** and the **test** data.
"""

X_train, X_test, y_train, y_test = train_test_split(x, y,
                                                    test_size = 0.25,
                                                    random_state = 42)

for reg in regressors:
  # Extracting model name
  model_name = type(reg).__name__

  # Fit the model on train data
  reg.fit(X_train, y_train)

  # Make predictions using train data
  y_train_pred = reg.predict(X_train)

  # Make predictions using test data
  y_pred = reg.predict(X_test)

  # Calculate train rmse of the model
  reg_train_rmse = np.sqrt(mean_squared_error(y_train, y_train_pred))

  # Calculate test rmse of the model
  reg_test_rmse = np.sqrt(mean_squared_error(y_test, y_pred))

  # Calculating train R2 Score
  reg_train_r2 = r2_score(y_train, y_train_pred)

  # Calculating test R2 Score
  reg_test_r2 = r2_score(y_test, y_pred)

  # Display the accuracy of the model
  print('Performance Metrics for', model_name, ':')
  #print('[RMSE-Score Train]:', reg_train_rmse)
  #print('[RMSE-Score Test]:', reg_test_rmse)
  print('[R2-Score Train]:', reg_train_r2)
  print('[R2-Score Test]:', reg_test_r2)

  model_scores.append((model_name,
                       reg_train_r2,
                       reg_test_r2))

  print('--------------------\n')

models = pd.DataFrame(data=model_scores, columns=['Model', '[R2-Score Train]', '[R2-Score Test]'])
models

# Plotting the RMSE scores for each model
fig = plt.figure(figsize=(15,7))
sns.barplot((models['[R2-Score Test]']), models['Model'], palette='rocket')
plt.grid(b=True)

"""### **8.2 Hyperpameter tuning**

- In this section we will perform **hyperparameter tuning** and check how our models perform post tuning.

- We will be using **Random Search** in order to find the best values.

- We will consider **CatBoost Regressor** and **XGB Regresosor** as they have given best results
"""

# Creating a parameter grid for CatBoost Regressor
param_grid_cat  = {'iterations': [100, 150, 200],
                   'learning_rate': [0.03, 0.1],
                   'depth': [2, 4, 6, 8],
                   'l2_leaf_reg': [0.2, 0.5, 1, 3]}

cbr = CatBoostRegressor(random_state=42)
gscv = RandomizedSearchCV(estimator = cbr,
                          param_distributions = param_grid_cat,
                          scoring ='r2',
                          cv = 5)
gscv.fit(X_train, y_train)

# Printing metrics
print("[Hyperparameters]:", gscv.best_params_)
print("[Train Score]:", gscv.best_score_)
print("[Validation Score]:", r2_score(y_test,
                                      gscv.predict(X_test)))

"""

---
# **9. Test Set**
---
- We will use the **CatBoost regressor** as gives the best **R2 Score** among the three tuned models.
"""

# Predicting with the best fit parameters
best_fit = gscv.best_estimator_
best_fit.fit(X_train, y_train)

# storing all best tvalues in to y_pred_tuned
y_pred_tuned = best_fit.predict(X_test)

from sklearn import metrics
print('R2 Score in Test data is : ',r2_score(y_test, y_pred_tuned))

"""

---
# **10. Making Pickle File**
---
- We can see that our model performs **really good** in **unseen data**.
- Hence, we can say that our model is **Ready to Deploy**
- For this we will have to import **pickle**
- After that we need to form a **.pkl file**
"""

import pickle
# open a file, where you ant to store the data
file = open('chennai_house.pkl', 'wb')

# dump information to that file
pickle.dump(best_fit, file)

"""

---
# **10. Conclusion**
---

- In this case study the given data was analysed and on top of that a **regression model** was built.

- The model chosen for this case study was a **Catboost regressor** as it was retruning the least overfitting and best r2 score on **unseen data**

- The **r2 score** genarated in unseen data was **0.99** which means that the modedl performs really good and is generalizing well on unseen data.

- After modelling we dumped oue model into a pickle file and we can conclude that now our model is **ready to deploy**
"""


#!/usr/bin/env python
# coding: utf-8

# # [Memanggil Library Pandas](https://academy.dqlab.id/main/livecode/178/346/1682)

# In[1]:

import pandas as pd
import numpy as np

# # [DataFrame & Series](https://academy.dqlab.id/main/livecode/178/346/1683)

# In[2]:

import pandas as pd
# Series
number_list = pd.Series([1, 2, 3, 4, 5, 6])
print("Series:")
print(number_list)
# DataFrame
matrix = [[1, 2, 3],
          ['a','b','c'],
          [3, 4, 5],
          ['d',4,6]]
matrix_list = pd.DataFrame(matrix)
print("DataFrame:")
print(matrix_list)

# # [Atribut DataFrame & Series - Part 1](https://academy.dqlab.id/main/livecode/178/346/1684)

# In[3]:

import pandas as pd
# Series
number_list = pd.Series([1,2,3,4,5,6])
# DataFrame
matrix_list = pd.DataFrame([[1,2,3],
				            ['a','b','c'],
				            [3,4,5],
				            ['d',4,6]])
# [1] attribute .info()
print("[1] attribute .info()")
print(matrix_list.info())
# [2] attribute .shape
print("\n[2] attribute .shape")
print("    Shape dari number_list:", number_list.shape)
print("    Shape dari matrix_list:", matrix_list.shape)
# [3] attribute .dtypes
print("\n[3] attribute .dtypes")
print("    Tipe data number_list:", number_list.dtypes)
print("    Tipe data matrix_list:", matrix_list.dtypes)
# [4] attribute .astype()
print("\n[4] attribute .astype()")
print("    Konversi number_list ke str:", number_list.astype("str"))
print("    Konversi matrix_list ke str:", matrix_list.astype("str"))

# # [Atribut DataFrame & Series - Part 2](https://academy.dqlab.id/main/livecode/178/346/1685)

# In[4]:

import pandas as pd
# Series
number_list = pd.Series([1,2,3,4,5,6])
# DataFrame
matrix_list = pd.DataFrame([[1,2,3],
				            ['a','b','c'],
				            [3,4,5],
				            ['d',4,6]])
# [5] attribute .copy()
print("[5] attribute .copy()")
num_list = number_list.copy()
print("    Copy number_list ke num_list:", num_list)
mtr_list = matrix_list.copy()
print("    Copy matrix_list ke mtr_list:", mtr_list)
# [6] attribute .to_list()
print("[6] attribute .to_list()")
print(number_list.to_list())
# [7] attribute .unique()
print("[7] attribute .unique()")
print(number_list.unique())

# # [Atribut DataFrame & Series - Part 3](https://academy.dqlab.id/main/livecode/178/346/1686)

# In[5]:

import pandas as pd
# Series
number_list = pd.Series([1,2,3,4,5,6])
# DataFrame
matrix_list = pd.DataFrame([[1,2,3],
				            ['a','b','c'],
				            [3,4,5],
				            ['d',4,6]])
# [8] attribute .index
print("[8] attribute .index")
print("    Index number_list:", number_list.index)
print("    Index matrix_list:", matrix_list.index)
# [9] attribute .columns
print("[9] attribute .columns")
print("    Column matrix_list:", matrix_list.columns)
# [10] attribute .loc
print("[10] attribute .loc")
print("    .loc[0:1] pada number_list:", number_list.loc[0:1])
print("    .loc[0:1] pada matrix_list:", matrix_list.loc[0:1])
# [11] attribute .iloc
print("[11] attribute .iloc")
print("    iloc[0:1] pada number_list:", number_list.iloc[0:1])
print("    iloc[0:1] pada matrix_list:", matrix_list.iloc[0:1])

# # [Creating Series & Dataframe from List](https://academy.dqlab.id/main/livecode/178/346/1688)

# In[6]:

import pandas as pd
# Creating series from list
ex_list = ['a',1,3,5,'c','d']
ex_series = pd.Series(ex_list)
print(ex_series)
# Creating dataframe from list of list
ex_list_of_list = [[1, 'a', 'b', 'c'],
                   [2.5, 'd', 'e', 'f'],
		           [5, 'g', 'h', 'i'],
		           [7.5, 'j', 10.5, 'l']]
index = ['dq', 'lab', 'kar', 'lan']
cols = ['float', 'char', 'obj', 'char']
ex_df = pd.DataFrame(ex_list_of_list, index=index, columns=cols)
print(ex_df)

# # [Creating Series & Dataframe from Dictionary](https://academy.dqlab.id/main/livecode/178/346/1689)

# In[7]:

import pandas as pd
# Creating series from dictionary
dict_series = {'1':'a',
			   '2':'b',
			   '3':'c'}
ex_series = pd.Series(dict_series)
print(ex_series)
# Creating dataframe from dictionary
df_series = {'1':['a','b','c'],
             '2':['b','c','d'],
             '4':[2,3,'z']}
ex_df = pd.DataFrame(df_series)
print(ex_df)

# # [Creating Series & Dataframe from Numpy Array](https://academy.dqlab.id/main/livecode/178/346/1690)

# In[9]:

# import pandas as pd
import numpy as np
# Creating series from numpy array (1D)
arr_series = np.array([1,2,3,4,5,6,6,7])
ex_series = pd.Series(arr_series)
print(ex_series)
# Creating dataframe from numpy array (2D)
arr_df = np.array([[1, 2, 3, 5],
                   [5, 6, 7, 8],
                   ['a','b','c',10]])
ex_df = pd.DataFrame(arr_df)
print(ex_df)

# # [Read Dataset - CSV dan TSV](https://academy.dqlab.id/main/livecode/178/347/1694)

# In[10]:

import pandas as pd
# File CSV
df_csv = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_csv.csv")
print(df_csv.head(3)) # Menampilkan 3 data teratas
# File TSV
df_tsv = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_tsv.tsv", sep='\t')
print(df_tsv.head(3)) # Menampilkan 3 data teratas

# # [Read Dataset - Excel](https://academy.dqlab.id/main/livecode/178/347/1695)

# In[11]:

import pandas as pd
# File xlsx dengan data di sheet "test"
df_excel = pd.read_excel("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_excel.xlsx", sheet_name="test")
print(df_excel.head(4)) # Menampilkan 4 data teratas

# # [Read Dataset - JSON](https://academy.dqlab.id/main/livecode/178/347/1698)

# In[13]:

import pandas as pd
# File JSON
url = "https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/covid2019-api-herokuapp-v2.json"
df_json = pd.read_json(url)
print(df_json.head(10)) # Menampilkan 10 data teratas

# # [Head & Tail](https://academy.dqlab.id/main/livecode/178/347/2143)

# In[14]:

import pandas as pd
# Baca file sample_csv.csv
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_csv.csv")
# Tampilkan 3 data teratas
print("Tiga data teratas:\n", df.head(3))
# Tampilkan 3 data terbawah
print("Tiga data terbawah:\n", df.tail(3))

# # [Indexing - Part 2](https://academy.dqlab.id/main/livecode/178/429/2133)

# In[15]:

import pandas as pd
# Baca file TSV sample_tsv.tsv
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_tsv.tsv", sep="\t")
# Index dari df
print("Index:", df.index)
# Column dari df
print("Columns:", df.columns)

# # [Indexing - Part 3](https://academy.dqlab.id/main/livecode/178/429/2134)

# In[16]:

import pandas as pd
# Baca file TSV sample_tsv.tsv
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_tsv.tsv", sep="\t")
# Set multi index df
df_x = df.set_index(['order_date', 'city', 'customer_id'])
# Print nama dan level dari multi index
for name, level in zip(df_x.index.names, df_x.index.levels):
    print(name,':',level)

# # [Indexing - Part 4](https://academy.dqlab.id/main/livecode/178/429/2135)

# In[17]:

import pandas as pd
# Baca file sample_tsv.tsv untuk 10 baris pertama saja
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_tsv.tsv", sep="\t", nrows=10)
# Cetak data frame awal
print("Dataframe awal:\n", df)
# Set index baru
df.index = ["Pesanan ke-" + str(i) for i in range(1, 11)]
# Cetak data frame dengan index baru
print("Dataframe dengan index baru:\n", df)

# # [Indexing - Part 5](https://academy.dqlab.id/main/livecode/178/429/2138)

# In[18]:

import pandas as pd
# Baca file sample_tsv.tsv dan set lah index_col sesuai instruksi
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_tsv.tsv", sep="\t", index_col=["order_date","order_id"])
# Cetak data frame untuk 8 data teratas
print("Dataframe:\n", df.head(8))

# # [Slicing - Part 1](https://academy.dqlab.id/main/livecode/178/429/2136)

# In[19]:

import pandas as pd
# Baca file sample_csv.csv
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_csv.csv")
# Slice langsung berdasarkan kolom
df_slice = df.loc[(df["customer_id"] == "18055") &
		          (df["product_id"].isin(["P0029","P0040","P0041","P0116","P0117"]))]
print("Slice langsung berdasarkan kolom:\n", df_slice)

# # [Slicing - Part 2](https://academy.dqlab.id/main/livecode/178/429/2139)

# In[20]:

import pandas as pd
# Baca file sample_csv.csv
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_csv.csv")
# Set index dari df sesuai instruksi
df = df.set_index(["order_date","order_id","product_id"])
# Slice sesuai intruksi
df_slice = df.loc[("2019-01-01",1612339,["P2154","P2159"]),:]
print("Slice df:\n", df_slice)

# # [Transforming - Part 1](https://academy.dqlab.id/main/livecode/178/429/2142)

# In[21]:

import pandas as pd
# Baca file sample_csv.csv
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_csv.csv")
# Tampilkan tipe data
print("Tipe data df:\n", df.dtypes)
# Ubah tipe data kolom order_date menjadi datetime
df["order_date"] = pd.to_datetime(df["order_date"])
# Tampilkan tipe data df setelah transformasi
print("\nTipe data df setelah transformasi:\n", df.dtypes)

# # [Transforming - Part 2](https://academy.dqlab.id/main/livecode/178/429/2144)

# In[22]:

import pandas as pd
# Baca file sample_csv.csv
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_csv.csv")
# Tampilkan tipe data
print("Tipe data df:\n", df.dtypes)
# Ubah tipe data kolom quantity menjadi tipe data numerik float
df["quantity"] = pd.to_numeric(df["quantity"], downcast="float")
# Ubah tipe data kolom city menjadi tipe data category
df["city"] = df["city"].astype("category")
# Tampilkan tipe data df setelah transformasi
print("\nTipe data df setelah transformasi:\n", df.dtypes)

# # [Transforming - Part 3](https://academy.dqlab.id/main/livecode/178/429/2145)

# In[23]:

import pandas as pd
# Baca file sample_csv.csv
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/sample_csv.csv")
# Cetak 5 baris teratas kolom brand
print("Kolom brand awal:\n", df["brand"].head())
# Gunakan method apply untuk merubah isi kolom menjadi lower case
df["brand"] = df["brand"].apply(lambda x: x.lower())
# Cetak 5 baris teratas kolom brand
print("Kolom brand setelah apply:\n", df["brand"].head())
# Gunakan method map untuk mengambil kode brand yaitu karakter terakhirnya
df["brand"] = df["brand"].map(lambda x: x[-1])
# Cetak 5 baris teratas kolom brand
print("Kolom brand setelah map:\n", df["brand"].head())

# # [Transforming - Part 4](https://academy.dqlab.id/main/livecode/178/429/2146)

# In[24]:

import numpy as np
import pandas as pd
# number generator, set angka seed menjadi suatu angka, bisa semua angka, supaya hasil random nya selalu sama ketika kita run
np.random.seed(1234)
# create dataframe 3 baris dan 4 kolom dengan angka random
df_tr = pd.DataFrame(np.random.rand(3,4))
# Cetak dataframe
print("Dataframe:\n", df_tr)
# Cara 1 dengan tanpa define function awalnya, langsung pake fungsi anonymous lambda x
df_tr1 = df_tr.applymap(lambda x: x**2 + 3*x + 2)
print("\nDataframe - cara 1:\n", df_tr1)
# Cara 2 dengan define function
def qudratic_fun(x):
	return x**2 + 3*x + 2
df_tr2 = df_tr.applymap(qudratic_fun)
print("\nDataframe - cara 2:\n", df_tr2)

# # [Inspeksi Missing Value](https://academy.dqlab.id/main/livecode/178/430/2148)

# In[25]:

import pandas as pd
# Baca file "public data covid19 jhu csse eu.csv"
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/CHAPTER+4+-+missing+value+-+public+data+covid19+.csv")
# Cetak info dari df
print(df.info())
# Cetak jumlah missing value di setiap kolom
mv = df.isna().sum()
print("\nJumlah missing value per kolom:\n", mv)

# # [Treatment untuk Missing Value - Part 2](https://academy.dqlab.id/main/livecode/178/430/2150)

# In[26]:

import pandas as pd
# Baca file "public data covid19 jhu csse eu.csv"
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/CHAPTER+4+-+missing+value+-+public+data+covid19+.csv")
# Cetak ukuran awal dataframe
print("Ukuran awal df: %d baris, %d kolom." % df.shape)
# Drop kolom yang seluruhnya missing value dan cetak ukurannya
df = df.dropna(axis=1, how="all")
print("Ukuran df setelah buang kolom dengan seluruh data missing: %d baris, %d kolom." % df.shape)
# Drop baris jika ada satu saja data yang missing dan cetak ukurannya
df = df.dropna(axis=0, how="any")
print("Ukuran df setelah dibuang baris yang memiliki sekurangnya 1 missing value: %d baris, %d kolom." % df.shape)

# # [Treatment untuk Missing Value - Part 3](https://academy.dqlab.id/main/livecode/178/430/2152)

# In[27]:

import pandas as pd
# Baca file "public data covid19 jhu csse eu.csv"
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/CHAPTER+4+-+missing+value+-+public+data+covid19+.csv")
# Cetak unique value pada kolom province_state
print("Unique value awal:\n", df["province_state"].unique())
# Ganti missing value dengan string "unknown_province_state"
df["province_state"] = df["province_state"].fillna("unknown_province_state")
# Cetak kembali unique value pada kolom province_state
print("Unique value setelah fillna:\n", df["province_state"].unique())

# # [Treatment untuk Missing Value - Part 4](https://academy.dqlab.id/main/livecode/178/430/2151)

# In[28]:

import pandas as pd
# Baca file "https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/CHAPTER+4+-+missing+value+-+public+data+covid19+.csv"
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/CHAPTER+4+-+missing+value+-+public+data+covid19+.csv")
# Cetak nilai mean dan median awal
print("Awal: mean = %f, median = %f." % (df["active"].mean(), df["active"].median()))
# Isi missing value kolom active dengan median
df_median = df["active"].fillna(df["active"].median())
# Cetak nilai mean dan median awal setelah diisi dengan median
print("Fillna median: mean = %f, median = %f." % (df_median.mean(), df_median.median()))
# Isi missing value kolom active dengan mean
df_mean = df["active"].fillna(df["active"].mean())
# Cetak nilai mean dan median awal setelah diisi dengan mean
print("Fillna mean: mean = %f, median = %f." % (df_mean.mean(), df_mean.median()))

# # [Treatment untuk Missing Value - Part 5](https://academy.dqlab.id/main/livecode/178/430/2155)

# In[29]:

import numpy as np
import pandas as pd
# Data
ts = pd.Series({
   "2020-01-01":9,
   "2020-01-02":np.nan,
   "2020-01-05":np.nan,
   "2020-01-07":24,
   "2020-01-10":np.nan,
   "2020-01-12":np.nan,
   "2020-01-15":33,
   "2020-01-17":np.nan,
   "2020-01-16":40,
   "2020-01-20":45,
   "2020-01-22":52,
   "2020-01-25":75,
   "2020-01-28":np.nan,
   "2020-01-30":np.nan
})
# Isi missing value menggunakan interpolasi linier
ts = ts.interpolate()
# Cetak time series setelah interpolasi linier
print("Setelah diisi missing valuenya:\n", ts)

# # [Project dari Andra](https://academy.dqlab.id/main/livecode/178/431/2156)

# In[30]:

import pandas as pd

# 1. Baca dataset
print("[1] BACA DATASET")
df = pd.read_csv("https://dqlab-dataset.s3-ap-southeast-1.amazonaws.com/retail_raw_test.csv", low_memory=False)
print(" Dataset:\n", df.head())
print(" Info:\n", df.info())

# 2. Ubah tipe data
print("\n[2] UBAH TIPE DATA")
df["customer_id"] = df["customer_id"].apply(lambda x: x.split("'")[1]).astype("int64")
df["quantity"] = df["quantity"].apply(lambda x: x.split("'")[1]).astype("int64")
df["item_price"] = df["item_price"].apply(lambda x: x.split("'")[1]).astype("int64")
print(" Tipe data:\n", df.dtypes)

# 3. Transform "product_value" supaya bentuknya seragam dengan format "PXXXX", assign ke kolom baru "product_id", dan drop kolom "product_value", jika terdapat nan gantilah dengan "unknown"
print("\n[3] TRANSFORM product_value MENJADI product_id")
# Buat fungsi
import math
def impute_product_value(val):
    if math.isnan(val):
        return "unknown"
    else:
        return 'P' + '{:0>4}'.format(str(val).split('.')[0])
# Buat kolom "product_id"
df["product_id"] = df["product_value"].apply(lambda x: impute_product_value(x))
# Hapus kolom "product_value"
df.drop(["product_value"], axis=1, inplace=True)
# Cetak 5 data teratas
print(df.head())

# 4. Tranform order_date menjadi value dengan format "YYYY-mm-dd"
print("\n[4] TRANSFORM order_date MENJADI FORMAT YYYY-mm-dd")
months_dict = {
   	"Jan":"01",
	"Feb":"02",
	"Mar":"03",
	"Apr":"04",
	"May":"05",
	"Jun":"06",
	"Jul":"07",
	"Aug":"08",
	"Sep":"09",
	"Oct":"10",
	"Nov":"11",
	"Dec":"12"
}
df["order_date"] = pd.to_datetime(df["order_date"].apply(lambda x: str(x)[-4:] + "-" + months_dict[str(x)[:3]] + "-" + str(x)[4:7]))
print(" Tipe data:\n", df.dtypes)

# 5. Mengatasi data yang hilang di beberapa kolom
print("\n[5] HANDLING MISSING VALUE")
# Kolom "city" dan "province" masih memiliki missing value, nilai yang hilang di kedua kolom ini diisi saja dengan "unknown"
df[["city","province"]] = df[["city","province"]].fillna("unknown")
# Kolom brand juga masih memiliki missing value, Ganti value NaN menjadi "no_brand"
df["brand"] = df["brand"].fillna("no_brand")
# Cek apakah masih terdapat missing value di seluruh kolom
print(" Info:\n", df.info())

# 6. Membuat kolom baru "city/province" dengan menggabungkan kolom "city" dan kolom "province" dan delete kolom asalnya
print("\n[6] MEMBUAT KOLOM BARU city/province")
df["city/province"] = df["city"] + "/" + df["province"]
# drop kolom "city" dan "province" karena telah digabungkan
df.drop(["city","province"], axis=1, inplace=True)
# Cetak 5 data teratas
print(df.head())

# 7. Membuat hierarchical index yang terdiri dari kolom "city/province", "order_date", "customer_id", "order_id", "product_id"
print("\n[7] MEMBUAT HIERACHICAL INDEX")
df = df.set_index(["city/province","order_date","customer_id","order_id","product_id"])
# urutkanlah berdasarkan index yang baru
df = df.sort_index()
# Cetak 5 data teratas
print(df.head())

# 8. Membuat kolom "total_price" yang formula nya perkalian antara kolom "quantity" dan kolom "item_price"
print("\n[8] MEMBUAT KOLOM total_price")
df["total_price"] = df["quantity"] * df["item_price"]
# Cetak 5 data teratas
print(df.head())

# 9. Slice dataset agar hanya terdapat data bulan Januari 2019
print("\n[9] SLICE DATASET UNTUK BULAN JANUARI 2019 SAJA")
idx = pd.IndexSlice
df_jan2019 = df.loc[idx[:, "2019-01-01":"2019-01-31"],:]
print("Dataset akhir:\n", df_jan2019)

# END OF PROJECT

Learn/Python/Fundamental/Data Manipulation with Pandas - Part 1/Data Manipulation with Pandas - Part 1.py


import numpy as np
from skimage.transform import resize
from skimage import measure
from skimage.measure import regionprops
import matplotlib.patches as patches
import matplotlib.pyplot as plt
from skimage.color import rgb2gray

from skimage.io import imread
from skimage.filters import threshold_otsu
import sys
import pytesseract
import cv2
from PIL import Image

from skimage import measure
from skimage.measure import regionprops
import matplotlib.pyplot as plt
import matplotlib.patches as patches

filenames = sys.argv[1:]

index = 1260

for filename in filenames:
    plate_like_objects = []
    print(filename)
    file_splits = filename.split('.')
    if len(file_splits) < 2:
        continue
    file_type = file_splits[1]
    if not file_type == 'jpg':
        continue

    import cv2
    cap = cv2.VideoCapture(filename)
    # cap = cv2.VideoCapture(0)
    count = 0
    while cap.isOpened():
        ret,frame = cap.read()
        if ret == True:
            # cv2.imshow('window-name',frame)
            cv2.imwrite("./output/frame%d.jpg" % count, frame)
            count = count + 1
            if cv2.waitKey(10) & 0xFF == ord('q'):
                break
        else:
            break
    cap.release()
    cv2.destroyAllWindows()

    # car image -> grayscale image -> binary image
    import imutils
    car_image = imread("./output/frame%d.jpg"%(count-1), as_gray=True)
    print(car_image.shape)

    gray_car_image = car_image * 255
    # fig, (ax1, ax2) = plt.subplots(1, 2)
    # ax1.imshow(gray_car_image, cmap="gray")
    threshold_value = threshold_otsu(gray_car_image)
    binary_car_image = gray_car_image > threshold_value

    # this gets all the connected regions and groups them together
    label_image = measure.label(binary_car_image)

    # print(label_image.shape[0]) #width of car img

    # getting the maximum width, height and minimum width and height that a license plate can be
    plate_dimensions = (0.04*label_image.shape[0], 0.2*label_image.shape[0], 0.2*label_image.shape[1], 0.6*label_image.shape[1])
    # plate_dimensions2 = (0.08*label_image.shape[0], 0.2*label_image.shape[0], 0.15*label_image.shape[1], 0.4*label_image.shape[1])
    plate_dimensions2 = (0.04*label_image.shape[0], 0.5*label_image.shape[0], 0.2*label_image.shape[1], 0.6*label_image.shape[1])
    min_height, max_height, min_width, max_width = plate_dimensions
    plate_objects_cordinates = []
    # plate_like_objects = []

    flag =0

    if(flag==0):
        min_height, max_height, min_width, max_width = plate_dimensions2
        plate_objects_cordinates = []
        # plate_like_objects = []

        fig, (ax1) = plt.subplots(1)
        # ax1.imshow(gray_car_image, cmap="gray")

        # regionprops creates a list of properties of all the labelled regions
        for region in regionprops(label_image):
            if region.area < 50:
                #if the region is so small then it's likely not a license plate
                continue
                # the bounding box coordinates
            min_row, min_col, max_row, max_col = region.bbox

            region_height = max_row - min_row
            region_width = max_col - min_col

            # ensuring that the region identified satisfies the condition of a typical license plate
            if region_height >= min_height and region_height <= max_height and region_width >= min_width and region_width <= max_width and region_width > region_height:

                plate_like_objects.append(binary_car_image[min_row:max_row,
                                            min_col:max_col])
                plate_objects_cordinates.append((min_row, min_col,
                                                    max_row, max_col))
                rectBorder = patches.Rectangle((min_col, min_row), max_col - min_col, max_row - min_row, edgecolor="red",
                                                linewidth=2, fill=False)
                ax1.add_patch(rectBorder)
                Cropped = gray_car_image[min_row:max_row, min_col:max_col]
                # plt.imshow(Cropped)
                # plt.show()
                text = pytesseract.image_to_string(Cropped, config='--psm 11')
                print("Detected Number is:",text)
                # break
                # let's draw a red rectangle over those regions
        # print(plate_like_objects[0])
        # plt.show()
        # plt.imshow(Cropped,cmap='gray')

        # Read the number plate
        # text = pytesseract.image_to_string(Cropped, config='--psm 11')
        # print("Detected Number is:",text)

        # plt.show()

    # print(DetectPlate.plate_like_objects)

    # The invert was done so as to convert the black pixel to white pixel and vice versa
    for lp in plate_like_objects:
        license_plate = np.invert(lp)

        labelled_plate = measure.label(license_plate)

        fig, ax1 = plt.subplots(1)
        license_plate = rgb2gray(license_plate)
        # ax1.imshow(license_plate, cmap="gray")

        character_dimensions = (0.5*license_plate.shape[0], 1.0*license_plate.shape[0], 0.00*license_plate.shape[1], 0.4*license_plate.shape[1])
        min_height, max_height, min_width, max_width = character_dimensions

        characters = []
        counter=0
        column_list = []

        rois = []

        for regions in regionprops(labelled_plate):

            y0, x0, y1, x1 = regions.bbox
            region_height = y1 - y0
            region_width = x1 - x0

            if region_height > min_height and region_height < max_height and region_width > min_width and region_width < max_width:
                roi = license_plate[y0:y1, x0:x1]

                # draw a red bordered rectangle over the character.
                rect_border = patches.Rectangle((x0, y0), x1 - x0, y1 - y0, edgecolor="red",
                                            linewidth=2, fill=False)
                ax1.add_patch(rect_border)
                # plt.imshow(roi)
                # plt.show()
                rois.append(roi)

                # resize the characters to 20X20 and then append each character into the characters list
                resized_char = resize(roi, (20, 20))
                for x in range(20):
                    for y in range(20):
                        if(resized_char[x][y]>=0.5):
                            print("X"),
                        else:
                            print(" "),
                    print("\n")
                face = raw_input("Specify Character: ")
                print(face)
                characters.append(resized_char)

                # this is just to keep track of the arrangement of the characters
                column_list.append(x0)
        # print(characters)
        # plt.show()
                if face != "-":
                    print("print")
                    index +=1
                    with open("trained/" + str(index)+".txt",  "w+") as file_handler:
                        for x in range(20):
                            for y in range(20):
                                file_handler.write(str(resized_char[x][y]) + " ")
                            file_handler.write("\n")
                    file_handler.close()

                    with open("trained/data.txt", "a") as num_file_handler:
                        num_file_handler.write(face + "\n")
                    num_file_handler.close()
print("Please change index to: "+str(index+1))