"""
Copyright 2018-2019 CS Systèmes d'Information

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

"""

import logging
import unittest

import numpy as np

from ikats.algo.resampling.resampling_computation import downsampling_ts, LOGGER
from ikats.core.resource.api import IkatsApi

def log_to_stdout(logger_to_use):
    """
    Allow to print some loggers to stdout
    :param logger_to_use: the LOGGER object to redirect to stdout
    """

    logger_to_use.setLevel(logging.DEBUG)
    formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(funcName)s:%(message)s')
    stream_handler = logging.StreamHandler()
    stream_handler.setLevel(logging.DEBUG)
    stream_handler.setFormatter(formatter)
    logger_to_use.addHandler(stream_handler)

# Prints the logger to display
log_to_stdout(LOGGER)

def gen_ts(ts_id):
    """
    Generate a TS in database used for test bench where id is defined

    :param ts_id: Identifier of the TS to generate (see content below for the structure)
    :type ts_id: int

    :return: the TSUID and funcId
    :rtype: dict
    """

    # Build TS identifier
    fid = "UNIT_TEST_Downsampling_%s" % ts_id

    if ts_id == 1:
        ts_content = [
            [1e12, 5.0],
            [1e12 + 1000, 6.0],
            [1e12 + 2000, 6.0],
            [1e12 + 3000, 8.0],
            [1e12 + 4000, -15.0],
            [1e12 + 5000, 2.0],
            [1e12 + 6000, 6.0],
            [1e12 + 7000, 3.0],
            [1e12 + 8000, 2.0],
            [1e12 + 9000, 42.0],
            [1e12 + 10000, 8.0],
            [1e12 + 11000, 8.0],
            [1e12 + 12000, 8.0],
            [1e12 + 13000, 8.0]
        ]
    elif ts_id == 2:
        ts_content = [
            [1e12, 5.0],
            [1e12 + 1000, 6.0],
            [1e12 + 2000, 6.0],
            [1e12 + 3600, 8.0],
            [1e12 + 4000, -15.0],
            [1e12 + 5000, 2.0],
            [1e12 + 6000, 6.0],
            [1e12 + 7200, 3.0],
            [1e12 + 8000, 2.0],
            [1e12 + 9000, 5.0],
            [1e12 + 13000, 10.0]
        ]
    elif ts_id == 3:
        ts_content = [
            [1e12, 5.0],
            [1e12 + 1000, 6.0],
            [1e12 + 2000, 6.0],
            [1e12 + 3000, 8.0],
            [1e12 + 100000, 5.0],
            [1e12 + 101000, 9.0],
            [1e12 + 102000, 5.0],
            [1e12 + 103000, 10.0]
        ]
    elif ts_id == 4:
        ts_content = [
            [1e12, 42.0]
        ]
    else:
        raise NotImplementedError

    # Remove former potential story having this name
    try:
        tsuid = IkatsApi.fid.tsuid(fid=fid)
        IkatsApi.ts.delete(tsuid=tsuid, no_exception=True)
    except ValueError:
        # No TS to delete
        pass

    # Create the timeseries
    result = IkatsApi.ts.create(fid=fid, data=np.array(ts_content))
    IkatsApi.md.create(tsuid=result['tsuid'], name="qual_ref_period", value=1000, force_update=True)
    IkatsApi.md.create(tsuid=result['tsuid'], name="qual_nb_points", value=len(ts_content), force_update=True)
    if not result['status']:
        raise SystemError("Error while creating TS %s" % ts_id)

    return {"tsuid": result['tsuid'], "funcId": fid}

class TestDownsampling(unittest.TestCase):
    """
    Test of Downsampling computation
    """

    def _check_results(self, ts_list, result, expected_data):
        """
        Check the results of the downsampling and compare it to the expected data

        :param ts_list: list of duet tsuid/funcId to match input to output
        :param result: raw result of the operator
        :param expected_data: expected data to be used as comparison reference

        :type ts_list: list of dict
        :type result: dict
        :type expected_data: dict
        """

        # Check number of results is the same
        self.assertEqual(len(ts_list), len(result))

        # Check data content
        for index, ts_item in enumerate(ts_list):
            original_tsuid = ts_item["tsuid"]
            obtained_tsuid = result[original_tsuid]["tsuid"]
            obtained_data = IkatsApi.ts.read([obtained_tsuid])[0]

            # Compare values
            try:
                self.assertTrue(np.allclose(
                    np.array(expected_data[original_tsuid], dtype=np.float64),
                    np.array(obtained_data, dtype=np.float64),
                    atol=1e-2))
            except Exception:
                print("ts_item:%s" % ts_item)
                print("Expected (%d points)" % len(expected_data[original_tsuid]))
                print(expected_data[original_tsuid])
                print("Obtained (%d points)" % len(obtained_data))
                print(obtained_data)
                raise

    @staticmethod
    def _cleanup_ts(obtained_result=None, ts_list=None):
        """
        Cleanup the time series used as inputs + resulting time series.

        :param obtained_result: raw results obtained by algorithm
        :type obtained_result: dict
        """
        if obtained_result is not None:
            for original_ts in obtained_result:
                IkatsApi.ts.delete(tsuid=obtained_result[original_ts]['tsuid'], no_exception=True)
                IkatsApi.ts.delete(tsuid=original_ts, no_exception=True)
        if ts_list is not None:
            for ts_item in ts_list:
                IkatsApi.ts.delete(tsuid=ts_item['tsuid'], no_exception=True)

    def test_nominal(self):
        """
        Compute the downsampling on a single time series without any constraint
        Check the time series is processed
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 6.0],
                [1e12 + 2000, 8.0],
                [1e12 + 4000, 2.0],
                [1e12 + 6000, 6.0],
                [1e12 + 8000, 42.0],
                [1e12 + 10000, 8.0],
                [1e12 + 12000, 8.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=2000, timestamp_position="BEG", aggregation_method="MAX",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)

        finally:
            # Cleanup
            self._cleanup_ts(result)

    def test_not_aligned(self):
        """
        Compute the downsampling on a single time series with a resampling period not a multiple of the original period
        Check the time series is processed
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 6.0],
                [1e12 + 1400, 6.0],
                [1e12 + 2800, 8.0],
                [1e12 + 4200, 2.0],
                [1e12 + 5600, 6.0],
                [1e12 + 7000, 3.0],
                [1e12 + 8400, 42.0],
                [1e12 + 9800, 8.0],
                [1e12 + 11200, 8.0],
                [1e12 + 12600, 8.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=1400, timestamp_position="BEG", aggregation_method="MAX",
                                     nb_points_by_chunk=4, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)

        finally:
            # Cleanup
            self._cleanup_ts(result)

    def test_aggregation_max(self):
        """
        Compute the downsampling on a single time series using MAX aggregation
        Check the time series is processed and result matches the aggregation method
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 6.0],
                [1e12 + 2000, 8.0],
                [1e12 + 4000, 2.0],
                [1e12 + 6000, 6.0],
                [1e12 + 8000, 42.0],
                [1e12 + 10000, 8.0],
                [1e12 + 12000, 8.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=2000, timestamp_position="BEG", aggregation_method="MAX",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)
        finally:

            # Cleanup
            self._cleanup_ts(result)

    def test_aggregation_min(self):
        """
        Compute the downsampling on a single time series using MIN aggregation
        Check the time series is processed and result matches the aggregation method
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 5.0],
                [1e12 + 2000, 6.0],
                [1e12 + 4000, -15.0],
                [1e12 + 6000, 3.0],
                [1e12 + 8000, 2.0],
                [1e12 + 10000, 8.0],
                [1e12 + 12000, 8.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=2000, timestamp_position="BEG", aggregation_method="MIN",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)
        finally:

            # Cleanup
            self._cleanup_ts(result)

    def test_aggregation_med(self):
        """
        Compute the downsampling on a single time series using MED aggregation (median)
        To compute the median, sort (ascending) the values in the desired period and take the middle point
        (or apply a linear interpolation in case of even values)
        Check the time series is processed and result matches the aggregation method
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 5.5],
                [1e12 + 10000, 8.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=10000, timestamp_position="BEG", aggregation_method="MED",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)
        finally:

            # Cleanup
            self._cleanup_ts(result)

    def test_aggregation_first(self):
        """
        Compute the downsampling on a single time series using FIRST aggregation
        Check the time series is processed and result matches the aggregation method
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 5.0],
                [1e12 + 10000, 8.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=10000, timestamp_position="BEG", aggregation_method="FIRST",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)
        finally:

            # Cleanup
            self._cleanup_ts(result)

    def test_aggregation_last(self):
        """
        Compute the downsampling on a single time series using LAST aggregation
        Check the time series is processed and result matches the aggregation method
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 42.0],
                [1e12 + 10000, 8.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=10000, timestamp_position="BEG", aggregation_method="LAST",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)
        finally:

            # Cleanup
            self._cleanup_ts(result)

    def test_alignment_mid(self):
        """
        Compute the downsampling on a single time series using MID alignment
        Check the time series is processed and result matches the alignment method
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12 + 3000, 8.0],
                [1e12 + 9000, 42.0],
                [1e12 + 15000, 8.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=6000, timestamp_position="MID", aggregation_method="MAX",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)
        finally:

            # Cleanup
            self._cleanup_ts(result)

    def test_alignment_end(self):
        """
        Compute the downsampling on a single time series using END alignment
        Check the time series is processed and result matches the alignment method
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12 + 6000, 8.0],
                [1e12 + 12000, 42.0],
                [1e12 + 18000, 8.0]
            ]
        }
        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=6000, timestamp_position="END", aggregation_method="MAX",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)

        finally:
            # Cleanup
            self._cleanup_ts(result)

    def test_multiple_ts(self):
        """
        Compute the downsampling on multiple time series without any constraint
        Check all time series are processed
        """

        # Prepare inputs
        ts_list = [gen_ts(1), gen_ts(2)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 5.5],
                [1e12 + 2000, 7.0],
                [1e12 + 4000, -6.5],
                [1e12 + 6000, 4.5],
                [1e12 + 8000, 22.0],
                [1e12 + 10000, 8.0],
                [1e12 + 12000, 8.0]
            ],
            ts_list[1]['tsuid']: [
                [1e12, 5.5],
                [1e12 + 2000, 7.0],
                [1e12 + 4000, -6.5],
                [1e12 + 6000, 4.5],
                [1e12 + 8000, 3.5],
                [1e12 + 12000, 10.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=2000, timestamp_position="BEG", aggregation_method="AVG",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)

        finally:
            # Cleanup
            self._cleanup_ts(result)

    def test_multiple_chunks(self):
        """
        Compute the downsampling on multiple time series having many points
        Check the downsampling works with multiple chunks
        """

        # Prepare inputs
        ts_list = [gen_ts(1)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 5.5],
                [1e12 + 2000, 7.0],
                [1e12 + 4000, -6.5],
                [1e12 + 6000, 4.5],
                [1e12 + 8000, 22.0],
                [1e12 + 10000, 8.0],
                [1e12 + 12000, 8.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=2000, timestamp_position="BEG", aggregation_method="AVG",
                                     nb_points_by_chunk=4, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)

        finally:
            # Cleanup
            self._cleanup_ts(result)

    def test_empty_chunks(self):
        """
        Compute the downsampling on multiple time series having many points not evenly distributed
        Check the downsampling works with empty chunks
        """

        # Prepare inputs
        ts_list = [gen_ts(3)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 5.5],
                [1e12 + 2000, 7.0],
                [1e12 + 100000, 7.0],
                [1e12 + 102000, 7.5]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=2000, timestamp_position="BEG", aggregation_method="AVG",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)

        finally:
            # Cleanup
            self._cleanup_ts(result)

    def test_single_point(self):
        """
        Compute the downsampling on time series having 1 point
        Check the new time series is the same as the original one
        """
        # Prepare inputs
        ts_list = [gen_ts(4)]

        # Prepare expected output
        expected_results = {
            ts_list[0]['tsuid']: [
                [1e12, 42.0]
            ]
        }

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=2000, timestamp_position="BEG", aggregation_method="MAX",
                                     nb_points_by_chunk=50000, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)

        finally:
            # Cleanup
            self._cleanup_ts(result)

    def test_robustness_no_ts(self):
        """
        No time series is provided
        Check that the result is empty
        """

        # Prepare inputs
        ts_list = []

        # Prepare expected output
        expected_results = {}

        result = None
        try:
            # Call algorithm
            result = downsampling_ts(ts_list=ts_list,
                                     resampling_period=2000, timestamp_position="BEG", aggregation_method="AVG",
                                     nb_points_by_chunk=4, generate_metadata=True)

            # Check the results
            self._check_results(ts_list, result, expected_results)

        finally:
            # Cleanup
            self._cleanup_ts(result)


"""
Checking Imported Data III - DataFrame Labels
"""

import numpy as np
import pandas as pd
wine_reviews = pd.read_csv('../winemag-data-130k.csv')

# Access the labels on the rows of data.

# Access the labels on the columns of data.

# Return the labels for the rows and columns in wine_reviews in one command.


# ---
# jupyter:
#   jupytext:
#     formats: ipynb,py:light
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.11.3
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # 2021 Oceanography Camp for Girls Saildrone Lesson
# Developed by Nancy Williams, Veronica Tamsitt, Nicola Guisewhite at University of South Florida College of Marine Science

# ## To Do List:
# * remove all fronts except SAF and SBDY for simplicity
# * make a function for plotting instead of copy/pasting the map each time (or not, if we want to keep it simple)
# * add more markdown in the form of instructions, pictures, pulling variables out into their own cell so girls know where they can make changes to the code
# * in figure titles and filenames, change the variables from using the first four characters (currently var[:4]) to instead cutting off at the first underscore
# * try plotting previous 8-day chl-a snapshot to see if it has better coverage for the eddy crossing
# * edit to make it easy to adjust time series x-axis limits
# * Check Veronica's carbon flux calculation is correct (Nancy)

# ## Data Sources:
# * (too big for Binder so had to remove it) Saildrone 1-minute physical and ADCP data available from: https://data.saildrone.com/data/sets/antarctica-circumnavigation-2019
# (login required, so cannot be accessed using an FTP. Will need to download ahead)
# * Saildrone hourly-ish CO2, pH, and physical data available from: https://www.ncei.noaa.gov/access/metadata/landing-page/bin/iso?id=gov.noaa.nodc:0221912
# * Satellite Chlorophyll: https://neo.sci.gsfc.nasa.gov/view.php?datasetId=MY1DMW_CHLORA&year=2019
# * SSH: https://cds.climate.copernicus.eu/cdsapp#!/dataset/satellite-sea-level-global?tab=overview
# (login required for chla and SSH, download ahead of time. Can also be downloaded using motuclient, login also required https://github.com/clstoulouse/motu-client-python)

# +
# Import the tools you need
import os
import numpy as np
import pandas as pd
import datetime as dt
import xarray as xr
import matplotlib.pyplot as plt
import matplotlib.path as mpath
import cartopy.crs as ccrs
import cartopy.feature
from datetime import datetime

# add something
# -

# Set the paths
output_dir = 'Output/'
data_dir = 'Data/'

# Go and download the hourly Saildrone CO2 data and put it in the `Data/` folder
os.chdir(data_dir) # Change the directory to the `Data/` folder
os.getcwd() # Check that you're now in the `Data/` folder
# Curl downloads the data files directly from the web and shows you the status while it works.
# `!` at the beginning of the line tells you that this command is a unix shell command (not python code)
# ! curl -o 32DB20190119_ASV_Saildrone1020_Antarctic_Jan2019_Aug2019.csv https://www.ncei.noaa.gov/data/oceans/ncei/ocads/data/0221912/32DB20190119_ASV_Saildrone1020_Antarctic_Jan2019_Aug2019.csv
os.chdir("..") # Use ".." to move back up one directory now that we've imported the MLD climatology data

# Import the hourly Saildrone CO2 data file
Saildrone_CO2 = pd.read_csv(
    (data_dir + '32DB20190119_ASV_Saildrone1020_Antarctic_Jan2019_Aug2019.csv'),
    header=4,
    na_values=-999,
)

# Create a datetime object
Saildrone_CO2['datetime'] = pd.to_datetime(Saildrone_CO2['Date'] + ' ' + Saildrone_CO2['Time'])
# Check that the Saildrone data was imported correctly
Saildrone_CO2

# Import the Southern Ocean fronts for mapping
stf = pd.read_csv(data_dir + 'fronts/stf.txt', header=None, sep='\s+',
                  na_values='%', names=['lon','lat'])
saf = pd.read_csv(data_dir + 'fronts/saf.txt', header=None, sep='\s+',
                  na_values='%', names=['lon','lat'])
pf = pd.read_csv(data_dir + 'fronts/pf.txt', header=None, sep='\s+',
                 na_values='%', names=['lon','lat'])
saccf = pd.read_csv(data_dir + 'fronts/saccf.txt', header=None, sep='\s+',
                    na_values='%', names=['lon','lat'])
sbdy = pd.read_csv(data_dir + 'fronts/sbdy.txt', header=None, sep='\s+',
                   na_values='%', names=['lon','lat'])

# +
# Plot the Saildrone track on a map

# Make the "bones" of the figure
plt.figure(figsize=(10, 10))
ax = plt.axes(projection=ccrs.SouthPolarStereo())
ax.set_extent([-180, 180, -90, -30],ccrs.PlateCarree())
ax.add_feature(cartopy.feature.LAND)
ax.add_feature(cartopy.feature.OCEAN, color='lightblue')
ax.gridlines()

# Compute a circle in axes coordinates, which we can use as a boundary
# for the map. We can pan/zoom as much as we like - the boundary will be
# permanently circular.
theta = np.linspace(0, 2 * np.pi, 100)
center, radius = [0.5, 0.5], 0.5
verts = np.vstack([np.sin(theta), np.cos(theta)]).T
circle = mpath.Path(verts * radius + center)

# Plot the ACC fronts in various colors
ax.set_boundary(circle, transform=ax.transAxes)
plt.plot(stf['lon'], stf['lat'], color='Red', transform=ccrs.PlateCarree(),
         label = 'Subtropical Front')
plt.plot(saf['lon'], saf['lat'], color='Orange', transform=ccrs.PlateCarree(),
         label = 'Subantarctic Front')
plt.plot(pf['lon'], pf['lat'], color='Yellow', transform=ccrs.PlateCarree(),
         label = 'Polar Front')
plt.plot(saccf['lon'], saccf['lat'], color='Green', transform=ccrs.PlateCarree(),
         label = 'Southern ACC Front')
plt.plot(sbdy['lon'], sbdy['lat'], color='Blue', transform=ccrs.PlateCarree(),
         label = 'Southern Boundary of ACC')

# Plot the Saildrone in black dots
plt.scatter(Saildrone_CO2.Longitude, Saildrone_CO2.Latitude,
           transform=ccrs.PlateCarree(), c='black', s=3, label='Saildrone', zorder=1000)

# Turn on the legend
plt.legend()

# Save the figure in the output folder
plt.title('2019 Saildrone Antarctic Circumnavigation Track')
plt.savefig(output_dir + 'SaildroneMap' + '.jpg') # Changing the suffix will change the format
plt.show()

# +
# Now plot some variable "var" on the map with colored dots
var = 'SST (C)'

# Make the "bones" of the figure
plt.figure(figsize=(10, 10))
ax = plt.axes(projection=ccrs.SouthPolarStereo())
ax.set_extent([-180, 180, -90, -30],ccrs.PlateCarree())
ax.add_feature(cartopy.feature.LAND)
ax.add_feature(cartopy.feature.OCEAN, color='lightblue')
ax.gridlines()

# Compute a circle in axes coordinates, which we can use as a boundary
# for the map. We can pan/zoom as much as we like - the boundary will be
# permanently circular.
theta = np.linspace(0, 2 * np.pi, 100)
center, radius = [0.5, 0.5], 0.5
verts = np.vstack([np.sin(theta), np.cos(theta)]).T
circle = mpath.Path(verts * radius + center)

# Plot the ACC fronts in various colors
ax.set_boundary(circle, transform=ax.transAxes)
plt.plot(stf['lon'], stf['lat'], color='Red', transform=ccrs.PlateCarree(),
         label = 'Subtropical Front')
plt.plot(saf['lon'], saf['lat'], color='Orange', transform=ccrs.PlateCarree(),
         label = 'Subantarctic Front')
plt.plot(pf['lon'], pf['lat'], color='Yellow', transform=ccrs.PlateCarree(),
         label = 'Polar Front')
plt.plot(saccf['lon'], saccf['lat'], color='Green', transform=ccrs.PlateCarree(),
         label = 'Southern ACC Front')
plt.plot(sbdy['lon'], sbdy['lat'], color='Blue', transform=ccrs.PlateCarree(),
         label = 'Southern Boundary of ACC')

# Plot the Saildrone in black dots
plt.scatter(Saildrone_CO2.Longitude, Saildrone_CO2.Latitude,
            c=Saildrone_CO2[var], cmap='bwr',
            transform=ccrs.PlateCarree(), s=5, zorder=1000)

# Turn on the legend
plt.legend()
cb1 = plt.colorbar()

# Save the figure in the output folder
plt.title('2019 Saildrone ' + var[:4])
plt.savefig(output_dir + var[:4] + 'SaildroneMap' + '.jpg') # Changing the suffix will change the format
plt.show()
# -

# Now let's make a map of some satellite data to show the Saildrone crossing an ocean eddy.
#
# First we need to load in a single daily satellite sea surface height data file from Feb 10th 2019, the day the Saildrone crossed a large eddy.

satellite_ssh = xr.open_dataset(data_dir + 'ssh_2019_02_10.nc')

# Now plot the Saildrone path on a map of sea surface height for a region surrounding the Saildrone on Feb 10th

#set plot parameters (contour levels, colormap etc)
levels_1 = np.arange(-1.2,0.8,0.1) #contour levels
cmap_1 = 'viridis' #contour map colormap
c1 = 'black' #Saildrone track color

# +
#finding position of Saildrone on Feb 10
time_index = np.where(Saildrone_CO2['Date']=='02/10/2019')
tlon = Saildrone_CO2.Longitude.values[time_index]
tlat = Saildrone_CO2.Latitude.values[time_index]

#make a contour plot of satellite ssh
xr.plot.contourf(satellite_ssh.adt[0,:,:],levels=levels_1,cmap=cmap_1,size=8,aspect=2)
xr.plot.contour(satellite_ssh.adt[0,:,:],levels=levels_1,colors='k',linewidths=0.75)
plt.xlim(tlon.min()+360-5,tlon.max()+360+5)
plt.ylim(tlat.min()-5,tlat.max()+5)

#add Saildrone track
plt.scatter(Saildrone_CO2.Longitude+360, Saildrone_CO2.Latitude, c=c1, s=3, label='Saildrone', zorder=1000)
plt.legend()

#give the plot a title and save figure in the output folder
plt.title('Saildrone path across an eddy on Feb 10th')
plt.savefig(output_dir + 'Sea_surface_height_Saildrone_Feb10' + '.jpg')

# -

# Ocean eddies can be identified by closed rings of constant absolute dynamic topography (this is the anomaly in sea surface height from average sea level in meters, which represents changes in pressure). You can see the Saildrone's path crossing near the center of an eddy.
#
# We can do the same thing with satellite chlorophyll-a data. The chlorophyll-a data gives an approximate estimate of the relative phytoplankton biomass (in units of mg/m3) at the sea surface in different locations.

#load satellite chl-a data file
satellite_chla = xr.open_dataset(data_dir + 'A20190412019048.L3m_8D_CHL_chlor_a_4km.nc')

# Here you can edit parameters (colors, range etc) for the map

#set plot parameters (contour levels, colormap etc)
levels_1 = np.arange(0,1.0,0.01) #contour levels
cmap_1 = 'YlGnBu' #contour map colormap
c1 = 'black' #Saildrone track color

# +
#make a contour plot of chl-a data
satellite_chla.chlo_a.values[satellite_chla.chlo_a>1000] = np.nan
xr.plot.contourf(satellite_chla.chlo_a, levels = levels_1, cmap=cmap_1,size=8,aspect=2)
plt.xlim(tlon.min()-5,tlon.max()+5)
plt.ylim(tlat.min()-5,tlat.max()+5)

#add Saildrone track
plt.scatter(Saildrone_CO2.Longitude, Saildrone_CO2.Latitude, c=c1, s=3, label='Saildrone', zorder=1000)
plt.legend()

#save figure
plt.title('Saildrone path and chlorophyll-a concentration')
plt.savefig(output_dir + 'Sea_surface_chlorophylla_Saildrone_Feb10' + '.jpg')
# -

# Now we can add the Saildrone data observations on the map to start to see if there is a relationship between the satellite observations and what the Saildrone measured directly. Note that the Saildrone took a few days to cross this region, while the satellite data shown here is a snapshot for a single day, so it can be tricky to compare the two types of data because the Saildrone is moving in space AND time.

#choose which variable to plot
var = 'SST (C)'
#set minimum and maximum colorbar limits
v_min = 6
v_max = 12
#choose colormap for map
cmap_1 = 'viridis'
#choose colormap for Saildrone variable
cmap_2 = 'RdBu_r'

# +
#make a contour plot of satellite ssh
xr.plot.contourf(satellite_ssh.adt[0,:,:],levels=np.arange(-1.2,0.8,0.1),cmap=cmap_1,size=8,aspect=2)
xr.plot.contour(satellite_ssh.adt[0,:,:],levels=np.arange(-1.2,0.8,0.1),colors='black',linewidths=0.75)
plt.xlim(tlon.min()+360-5,tlon.max()+360+5)
plt.ylim(tlat.min()-5,tlat.max()+5)

#add Saildrone data scattered on top
plt.scatter(Saildrone_CO2.Longitude+360, Saildrone_CO2.Latitude, c=Saildrone_CO2[var], s=15, cmap = cmap_2,
            vmin=v_min,vmax=v_max,label='Saildrone', zorder=1000)
plt.legend()
plt.colorbar()

#add title and save figure
plt.title('Saildrone ' + var[:4] + ' across an eddy on Feb 10th')
plt.savefig(output_dir + 'Sea_surface_height_Saildrone_' + var[:4] + '_Feb10' + '.jpg')
# -

# Now that we've plotted some Saildrone and satellite data on maps to see how different ocean variables are related, there are other ways we can look at the relationship between variables.
#
# This includes scatter plots, which is a useful way to compare data from two variables collected at the same time and location to look for a relationship. In our case, we can compare two different variables collected simulatneously by the Saildrone.

# +
#choose two variables from the Saildrone to compare
var1 = 'SST (C)'
var2 = 'pCO2 SW (sat) uatm'

#choose lower and upper limits for the two variables for plotting
var1_min = -2
var1_max = 18
#var2_min = 98
#var2_max = 102

# +
#create scatter plot
plt.figure(figsize=(12,8))
plt.scatter(Saildrone_CO2[var1], Saildrone_CO2[var2], s=10)
plt.xlim(var1_min,var1_max)
#plt.ylim(var2_min,var2_max)
plt.xlabel(var1)
plt.ylabel(var2)
plt.grid()

#add title and save figure
plt.title('Saildrone '+ var1[:4] + ' vs ' + var2[:4])
plt.savefig(output_dir + 'Saildrone_' + var1[:4] + '_vs_' + var2[:4] + '.jpg')
# -

# If we want to look at the relationship between more than two variables, one way we can look at this is by using a third variable to change the color of the scatter plot points.
#
# In this example, the scatter plot shows the same variable 1 and 2 from the Saildrone on the x and y axes as above, but we can choose a third Saildrone variable as the color of the scatter plot points.

# +
#choose a third variable
var3 = 'Latitude'

#set lower and upper limits of variable 3 for plotting
var3_min = -65
var3_max = -40

# +
#create scatter plot
plt.figure(figsize=(12,8))
plt.scatter(Saildrone_CO2[var1], Saildrone_CO2[var2], c=Saildrone_CO2[var3],
            s=10, vmin = var3_min, vmax = var3_max)
plt.xlim(var1_min,var1_max)
#plt.ylim(var2_min,var2_max)
plt.xlabel(var1)
plt.ylabel(var2)
plt.grid()
cbar = plt.colorbar()
cbar.set_label(var3)

#add title and save figure
plt.title('Saildrone '+ var1[:4] + ' vs ' + var2[:4])
plt.savefig(output_dir + 'Saildrone_' + var1[:4] + '_vs_' + var2[:4] + '_vs_' + var3[:4] + '.jpg')
# -

# Plot time series of wind speed and pressure

# +
#input plot parameters

#variables to plot
var1 = 'WSPD (m/s)'
var2 = 'Licor Atm Pressure (hPa)'

#set x axis limits

# +
#plot time series
plt.figure(figsize=(12,5))
ax1 = plt.subplot(211)
ax1.plot(Saildrone_CO2.datetime,Saildrone_CO2[var1])
plt.xlim(Saildrone_CO2.datetime.values[0],Saildrone_CO2.datetime.values[-1])

ax2 = plt.subplot(212)
ax2.plot(Saildrone_CO2.datetime,Saildrone_CO2[var2])
plt.xlim(Saildrone_CO2.datetime.values[0],Saildrone_CO2.datetime.values[-1])
plt.show()
# -
# Next, we can calculate the flux of carbon between the ocean and the atmosphere based on the difference in pCO2 between the atmosphere and the ocean.

# +
#constants for CO2 flux calculation
#ocean/atmosphere variables needed as inputs
T = Saildrone_CO2['SST (C)'] #sea surface temperature
S  = Saildrone_CO2['Salinity'] #sea surface salinity
u = Saildrone_CO2['WSPD (m/s)'] #surface wind speed
dpCO2 = Saildrone_CO2['dpCO2'] #difference between ocean and atmosphere pCO2

#1. Calculate the transfer velocity (Wanninkhof et al. 2014)
#Schmidt number as a function of temperature
Sc = 2116.8-136.25*T  + 4.7353*np.power(T,2) - 0.092307*np.power(T,3) + 0.000755*np.power(T,4)
K = 0.251*(u*u)*np.power((Sc/660),-0.5)
K = K

#2. calculate solubility constant as a function of temperature and salinity
T_K = T + 273.15
K0 = -58.0931 + ( 90.5069*(100.0 /T_K) ) \
    + (22.2940 * (np.log(T_K/100.0))) + (S * (0.027766 +  ( (-0.025888)*(T_K/100.0)) \
    + (0.0050578*( (T_K/100.0)*(T_K/100.0) ) ) ) )
a = np.exp(K0)

#CO2 flux equation
Saildrone_CO2['FCO2'] = 0.24 * K * a * dpCO2  #FCO2 = K*a(dpCO2)
# -

# Let's plot the time series of carbon fluxes together with the time series of wind speed to see how they are related. Sign of FCO2?

# +
#variables to plot
var1 = 'WSPD (m/s)'
var2 = 'FCO2'

#colors
c1 = 'darkblue'
c2 = 'darkorange'

# +
#plot time series
fig, ax1 = plt.subplots(figsize=(12,5))

#y axis 1
ax1.plot(Saildrone_CO2['datetime'],Saildrone_CO2[var1],color=c1)
ax1.set_xlabel('date')
ax1.set_ylabel(var1, color=c1)
ax1.tick_params(axis='y', labelcolor=c1)

#y axis 2
ax2 = ax1.twinx()
ax2.plot(Saildrone_CO2['datetime'],-Saildrone_CO2[var2],color=c2)
ax2.set_xlabel('date')
ax2.set_ylabel(var2, color=c2)
ax2.tick_params(axis='y', labelcolor=c2)

ax2.plot([Saildrone_CO2['datetime'].values[0], Saildrone_CO2['datetime'].values[-1]],[0,0],
         color='black', linewidth=0.5)
plt.xlim(Saildrone_CO2['datetime'][0],Saildrone_CO2['datetime'][1800])
fig.tight_layout()
plt.show()
# -


__copyright__ = "Copyright (C) 2019 Zachary J Weiner"

__license__ = """
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
"""

import pystella as ps
from pymbolic import parse, var
from pystella.field import shift_fields
import pytest

def test_field(proc_shape):
    if proc_shape != (1, 1, 1):
        pytest.skip("test field only on one rank")

    y = ps.Field("y", offset="h")
    result = ps.index_fields(y)
    assert result == parse("y[i + h, j + h, k + h]"), result

    y = ps.Field("y", offset="h", indices=("a", "b", "c"))
    result = ps.index_fields(y)
    assert result == parse("y[a + h, b + h, c + h]"), result

    y = ps.Field("y", ignore_prepends=True)
    result = ps.index_fields(y, prepend_with=(0, 1))
    assert result == parse("y[i, j, k]"), result

    y = ps.Field("y[4, 5]", ignore_prepends=True)
    result = ps.index_fields(y, prepend_with=(0, 1))
    assert result == parse("y[4, 5, i, j, k]"), result

    y = ps.Field("y", ignore_prepends=True)
    result = ps.index_fields(y[2, 3], prepend_with=(0, 1))
    assert result == parse("y[2, 3, i, j, k]"), result

    y = ps.Field("y[4, 5]", ignore_prepends=True)
    result = ps.index_fields(y[2, 3], prepend_with=(0, 1))
    assert result == parse("y[2, 3, 4, 5, i, j, k]"), result

    y = ps.Field("y", ignore_prepends=False)
    result = ps.index_fields(y, prepend_with=(0, 1))
    assert result == parse("y[0, 1, i, j, k]"), result

    y = ps.Field("y[4, 5]", ignore_prepends=False)
    result = ps.index_fields(y, prepend_with=(0, 1))
    assert result == parse("y[0, 1, 4, 5, i, j, k]"), result

    y = ps.Field("y", ignore_prepends=False)
    result = ps.index_fields(y[2, 3], prepend_with=(0, 1))
    assert result == parse("y[0, 1, 2, 3, i, j, k]"), result

    y = ps.Field("y[4, 5]", ignore_prepends=False)
    result = ps.index_fields(y[2, 3], prepend_with=(0, 1))
    assert result == parse("y[0, 1, 2, 3, 4, 5, i, j, k]"), result

    y = ps.Field("y", offset=("hx", "hy", "hz"))
    result = ps.index_fields(shift_fields(y, (1, 2, 3)))
    assert result == parse("y[i + hx + 1, j + hy + 2, k + hz + 3]"), result

    y = ps.Field("y", offset=("hx", var("hy"), "hz"))
    result = ps.index_fields(shift_fields(y, (1, 2, var("a"))))
    assert result == parse("y[i + hx + 1, j + hy + 2, k + hz + a]"), result

def test_dynamic_field(proc_shape):
    if proc_shape != (1, 1, 1):
        pytest.skip("test field only on one rank")

    y = ps.DynamicField("y", offset="h")

    result = ps.index_fields(y)
    assert result == parse("y[i + h, j + h, k + h]"), result

    result = ps.index_fields(y.lap)
    assert result == parse("lap_y[i, j, k]"), result

    result = ps.index_fields(y.dot)
    assert result == parse("dydt[i + h, j + h, k + h]"), result

    result = ps.index_fields(y.pd[var("x")])
    assert result == parse("dydx[x, i, j, k]"), result

    result = ps.index_fields(y.d(1, 0))
    assert result == parse("dydt[1, i + h, j + h, k + h]"), result

    result = ps.index_fields(y.d(1, 1))
    assert result == parse("dydx[1, 0, i, j, k]"), result

def test_field_diff(proc_shape):
    if proc_shape != (1, 1, 1):
        pytest.skip("test field only on one rank")

    from pystella import diff

    y = ps.Field("y")
    assert diff(y, y) == 1
    assert diff(y[0], y[0]) == 1
    assert diff(y[0], y[1]) == 0

    y = ps.DynamicField("y")
    assert diff(y, y) == 1
    assert diff(y[0], y[0]) == 1
    assert diff(y[0], y[1]) == 0

    import pymbolic.primitives as pp
    assert diff(y**3, y, "t") == pp.Product((3, 2, y, y.d(0)))
    assert diff(y**3, "t", y) == pp.Product((3, y.d(0), 2, y))

    for i, x in enumerate(["t", "x", "y", "z"]):
        assert diff(y, x) == y.d(i)
        assert diff(y[1, 3], x) == y.d(1, 3, i)
        assert diff(y[1]**2, x) == 2 * y[1] * y.d(1, i)

def test_get_field_args(proc_shape):
    if proc_shape != (1, 1, 1):
        pytest.skip("test field only on one rank")

    from pystella import Field, DynamicField, get_field_args

    x = Field("x", offset=(1, 2, 3))
    y = Field("y", offset="h")
    z = DynamicField("z", shape=(2, "a"))

    import loopy as lp
    true_args = [
        lp.GlobalArg("x", shape="(Nx+2, Ny+4, Nz+6)", offset=lp.auto),
        lp.GlobalArg("y", shape="(Nx+2*h, Ny+2*h, Nz+2*h)", offset=lp.auto),
        lp.GlobalArg("z", shape="(2, a, Nx, Ny, Nz)", offset=lp.auto),
        lp.GlobalArg("dzdx", shape="(2, a, 3, Nx, Ny, Nz)", offset=lp.auto),
    ]

    def lists_equal(a, b):
        equal = True
        for x in a:
            equal *= x in b
        for x in b:
            equal *= x in a
        return equal

    expressions = {x: y, y: x * z + z.pd[0]}
    args = get_field_args(expressions)
    assert lists_equal(args, true_args)

    expressions = x * y + z + z.pd[2]
    args = get_field_args(expressions)
    assert lists_equal(args, true_args)

    expressions = [x, y, y * z**2, 3 + z.pd[0] + z.pd[1]]
    args = get_field_args(expressions)
    assert lists_equal(args, true_args)

    expressions = [shift_fields(x, (1, 2, 3)), y + z.pd[0], y * z**2]
    args = get_field_args(expressions)
    assert lists_equal(args, true_args)

def test_collect_field_indices(proc_shape):
    if proc_shape != (1, 1, 1):
        pytest.skip("test field only on one rank")

    from pystella import Field, DynamicField
    from pystella.field import collect_field_indices

    x = Field("x", offset=(1, 2, 3))
    y = Field("y", indices=("i", "x"), offset="h")
    z = DynamicField("z", shape=(2, "a"))

    expressions = {x: y, y: x * z + z.pd[0]}
    indices = collect_field_indices(expressions)
    assert indices == {"i", "j", "k", "x"}

    expressions = [x, z]
    indices = collect_field_indices(expressions)
    assert indices == {"i", "j", "k"}

    expressions = [shift_fields(x, (1, 2, 3)), y + z.pd[0], y * z**2]
    indices = collect_field_indices(expressions)
    assert indices == {"i", "j", "k", "x"}

def test_sympy_interop(proc_shape):
    if proc_shape != (1, 1, 1):
        pytest.skip("test field only on one rank")

    from pystella.field.sympy import pymbolic_to_sympy, sympy_to_pymbolic
    import sympy as sym

    f = ps.Field("f", offset="h")
    g = ps.Field("g", offset="h")

    expr = f[0]**2 * g + 2 * g[1] * f
    sympy_expr = pymbolic_to_sympy(expr)
    new_expr = sympy_to_pymbolic(sympy_expr)
    sympy_expr_2 = pymbolic_to_sympy(new_expr)
    assert sym.simplify(sympy_expr - sympy_expr_2) == 0, \
        "sympy <-> pymbolic conversion not invertible"

    expr = f + shift_fields(f, (1, 2, 3))
    sympy_expr = pymbolic_to_sympy(expr)
    new_expr = sympy_to_pymbolic(sympy_expr)
    sympy_expr_2 = pymbolic_to_sympy(new_expr)
    assert sym.simplify(sympy_expr - sympy_expr_2) == 0, \
        "sympy <-> pymbolic conversion not invertible with shifted indices"

    # from pymbolic.functions import fabs, exp, exmp1
    fabs = parse("math.fabs")
    exp = parse("math.exp")
    expm1 = parse("math.expm1")
    x = sym.Symbol("x")

    expr = sym.Abs(x)
    assert sympy_to_pymbolic(expr) == fabs(var("x"))

    expr = sym.exp(x)
    assert sympy_to_pymbolic(expr) == exp(var("x"))

    expr = sym.Function("expm1")(x)  # pylint: disable=E1102
    assert sympy_to_pymbolic(expr) == expm1(var("x"))

    expr = sym.Function("aaa")(x)  # pylint: disable=E1102
    from pymbolic.primitives import Call, Variable
    assert sympy_to_pymbolic(expr) == Call(Variable("aaa"), (Variable("x"),))

if __name__ == "__main__":
    test_field((1, 1, 1))
    test_dynamic_field((1, 1, 1))
    test_field_diff((1, 1, 1))
    test_get_field_args((1, 1, 1))
    test_collect_field_indices((1, 1, 1))
    test_sympy_interop((1, 1, 1))


#!/usr/bin/env python
# coding: utf-8

#
Problem Statement

#
# **About Company**
# Dream Housing Finance company deals in all home loans. They have presence across all urban, semi urban and rural areas. Customer first apply for home loan after that company validates the customer eligibility for loan.
#
# **Problem**
# Company wants to automate the loan eligibility process (real time) based on customer detail provided while filling online application form. These details are Gender, Marital Status, Education, Number of Dependents, Income, Loan Amount, Credit History and others. To automate this process, they have given a problem to identify the customers segments, those are eligible for loan amount so that they can specifically target these customers. Here they have provided a partial data set.
# ________________________________________
# #### Data
#
# |Variable|Description|
# |--------|-----------|
# |Loan_ID|Unique Loan ID|
# |Gender|Male/ Female|
# |Married|Applicant married (Y/N)|
# |Dependents|Number of dependents|
# |Education|Applicant Education (Graduate/ Under Graduate)|
# |Self_Employed|Self employed (Y/N)|
# |ApplicantIncome|Applicant income|
# |CoapplicantIncome|Coapplicant income|
# |LoanAmount|Loan amount in thousands|
# |Loan_Amount_Term|Term of loan in months|
# |Credit_History|credit history meets guidelines|
# |Property_Area|Urban/ Semi Urban/ Rural|
# |Loan_Status|Loan approved (Y/N)|
#
#

#
# Note:
# 1.	Evaluation Metric is accuracy i.e. percentage of loan approval you correctly predict[Accuracy Metric,Precision Metric,Classification Report ,Confustion Matrix].
# 2.	You are expected to upload the solution in the format of "sample_submission.csv"
#
# #### Loading Packages

# In[109]:

import pandas as pd
import numpy as np                     # For mathematical calculations
import seaborn as sns                  # For data visualization
import matplotlib.pyplot as plt        # For plotting graphs
get_ipython().run_line_magic('matplotlib', 'inline')
import pandas_profiling                # Report Generation
import warnings                        # To ignore any warnings
warnings.filterwarnings("ignore")

# #### Data
#
# For this practice problem, we have been given three CSV files: train, test and sample submission.
#
# * Train file will be used for training the model, i.e. our model will learn from this file. It contains all the independent variables and the target variable.
# * Test file contains all the independent variables, but not the target variable. We will apply the model to predict the target variable for the test data.
# * Sample submission file contains the format in which we have to submit our predictions.

# In[110]:

train=pd.read_csv("Data/train_ctrUa4K.csv")
test=pd.read_csv("Data/test_lAUu6dG.csv")

# The file name should be replaced with the name of the train and test file that you have downloaded from the github
#
# Let’s make a copy of train and test data so that even if we have to make any changes in these datasets we would not lose the original datasets.

# In[111]:

train_original=train.copy()
test_original=test.copy()

# In[112]:

train_original.shape

# In[113]:

test_original.shape

# In this section, we will look at the structure of the train and test datasets. Firstly, we will check the features present in our data and then we will look at their data types.

# In[114]:

train.columns

# We have 12 independent variables and 1 target variable, i.e. Loan_Status in the train dataset. Let’s also have a look at the columns of test dataset.

# In[115]:

test.columns

# We have similar features in the test dataset as the train dataset except the Loan_Status. We will predict the Loan_Status using the model built using the train data.
#
# Given below is the description for each variable.

# |Variable|Description|
# |--------|-----------|
# |Loan_ID|Unique Loan ID|
# |Gender|Male/ Female|
# |Married|Applicant married (Y/N)|
# |Dependents|Number of dependents|
# |Education|Applicant Education (Graduate/Under Graduate)|
# |Self_Employed|Self employed (Y/N)|
# |ApplicantIncome|Applicant income|
# |CoapplicantIncome|Coapplicant income|
# |LoanAmount|Loan amount in thousands|
# |Loan_Amount_Term|Term of loan in months|
# |Credit_History|Credit history meets guidelines|
# |Property_Area|Urban/ Semi Urban/ Rural|
# |Loan_Status|Loan approved (Y/N)|

# In[116]:

pd.DataFrame({"Train_DataTypes":train.dtypes,
              "Test_DataTypes":test.dtypes})

# We can see there are three format of data types:
#
# * `object:` Object format means variables are categorical. Categorical variables in our dataset are: Loan_ID, Gender, Married, Dependents, Education, Self_Employed, Property_Area, Loan_Status
#
# * `int64:` It represents the integer variables. ApplicantIncome is of this format.
#
# * `float64:` It represents the variable which have some decimal values involved. They are also numerical variables. Numerical variables in our dataset are: CoapplicantIncome, LoanAmount, Loan_Amount_Term, and Credit_History
# Let’s look at the shape of the dataset.

# In[117]:

train.shape

# In[118]:

test.shape

# We have 614 rows and 13 columns in the train dataset and 367 rows and 12 columns in test dataset.
#
#
# In this section, we will do univariate analysis. It is the simplest form of analyzing data where we examine each variable individually. For categorical features we can use frequency table or bar plots which will calculate the number of each category in a particular variable. For numerical features, probability density plots can be used to look at the distribution of the variable.
#
# #### Target Variable
#
# We will first look at the target variable, i.e., Loan_Status. As it is a categorical variable, let us look at its frequency table, percentage distribution and bar plot.
#
# Frequency table of a variable will give us the count of each category in that variable.

# In[119]:

train['Loan_Status'].value_counts()

# In[120]:

# Normalize can be set to True to print proportions instead of number
train['Loan_Status'].value_counts(normalize=True)

# In[121]:

train['Loan_Status'].value_counts().plot.bar()

# The loan of 422(around 69%) people out of 614 was approved.
#
# Now lets visualize each variable separately. Different types of variables are Categorical, ordinal and numerical.
#
# * Categorical features: These features have categories (Gender, Married, Self_Employed, Credit_History, Loan_Status)
# Ordinal features: Variables in categorical features having some order involved (Dependents, Education, Property_Area)
# * Numerical features: These features have numerical values (ApplicantIncome, CoapplicantIncome, LoanAmount, Loan_Amount_Term)
# Let’s visualize the categorical and ordinal features first.

# In[122]:

plt.figure(1)
plt.subplot(221)
train['Gender'].value_counts(normalize=True).plot.bar(figsize=(20,10), title= 'Gender')
plt.subplot(222)
train['Married'].value_counts(normalize=True).plot.bar(title= 'Married')
plt.subplot(223)
train['Self_Employed'].value_counts(normalize=True).plot.bar(title= 'Self_Employed')
plt.subplot(224)
train['Credit_History'].value_counts(normalize=True).plot.bar(title= 'Credit_History')
plt.show()

# It can be inferred from the above bar plots that:
#
# * 80% applicants in the dataset are male.
# * Around 65% of the applicants in the dataset are married.
# * Around 15% applicants in the dataset are self employed.
# * Around 85% applicants have repaid their debts.
#
# Now let’s visualize the ordinal variables.

# #### Independent Variable (Ordinal)

# In[123]:

plt.figure(1)
plt.subplot(131)
train['Dependents'].value_counts(normalize=True).plot.bar(figsize=(24,6), title= 'Dependents')
plt.subplot(132)
train['Education'].value_counts(normalize=True).plot.bar(title= 'Education')
plt.subplot(133)
train['Property_Area'].value_counts(normalize=True).plot.bar(title= 'Property_Area')
plt.show()

# Following inferences can be made from the above bar plots:
#
# * Most of the applicants don’t have any dependents.
# * Around 80% of the applicants are Graduate.
# * Most of the applicants are from Semiurban area.
#
# #### Independent Variable (Numerical)
# Till now we have seen the categorical and ordinal variables and now lets visualize the numerical variables. Lets look at the distribution of Applicant income first.

# In[124]:

plt.figure(1)
plt.subplot(121)
sns.distplot(train['ApplicantIncome']);
plt.subplot(122)
train['ApplicantIncome'].plot.box(figsize=(16,5))
plt.show()

# It can be inferred that most of the data in the distribution of applicant income is towards left which means it is not normally distributed. We will try to make it normal in later sections as algorithms works better if the data is normally distributed.
#
# The boxplot confirms the presence of a lot of outliers/extreme values. This can be attributed to the income disparity in the society. Part of this can be driven by the fact that we are looking at people with different education levels. Let us segregate them by Education:

# In[125]:

train.boxplot(column='ApplicantIncome', by = 'Education')
plt.suptitle("")

# We can see that there are a higher number of graduates with very high incomes, which are appearing to be the outliers.
#
# Let’s look at the Coapplicant income distribution.

# In[126]:

plt.figure(1)
plt.subplot(121)
sns.distplot(train['CoapplicantIncome']);
plt.subplot(122)
train['CoapplicantIncome'].plot.box(figsize=(16,5))
plt.show()

# We see a similar distribution as that of the applicant income. Majority of coapplicant’s income ranges from 0 to 5000. We also see a lot of outliers in the coapplicant income and it is not normally distributed.
#
# Let’s look at the distribution of LoanAmount variable.

# In[127]:

plt.figure(1)
plt.subplot(121)
df=train.dropna(inplace=True)
sns.distplot(train['LoanAmount']);
plt.subplot(122)
train['LoanAmount'].plot.box(figsize=(16,5))
plt.show()

# We see a lot of outliers in this variable and the distribution is fairly normal. We will treat the outliers in later sections.
#
# Now we would like to know how well each feature correlate with Loan Status. So, in the next section we will look at bivariate analysis.
#
#
# Lets recall some of the hypotheses that we generated earlier:
#
# * Applicants with high income should have more chances of loan approval.
# * Applicants who have repaid their previous debts should have higher chances of loan approval.
# * Loan approval should also depend on the loan amount. If the loan amount is less, chances of loan approval should be high.
# * Lesser the amount to be paid monthly to repay the loan, higher the chances of loan approval.
#
# Lets try to test the above mentioned hypotheses using bivariate analysis
#
# After looking at every variable individually in univariate analysis, we will now explore them again with respect to the target variable.
#
# #### Categorical Independent Variable vs Target Variable
#
# First of all we will find the relation between target variable and categorical independent variables. Let us look at the stacked bar plot now which will give us the proportion of approved and unapproved loans.

# In[128]:

Gender=pd.crosstab(train['Gender'],train['Loan_Status'])
Gender.div(Gender.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True, figsize=(16,9))

# It can be inferred that the proportion of male and female applicants is more or less same for both approved and unapproved loans.
#
# Now let us visualize the remaining categorical variables vs target variable.

# In[129]:

Married=pd.crosstab(train['Married'],train['Loan_Status'])
Dependents=pd.crosstab(train['Dependents'],train['Loan_Status'])
Education=pd.crosstab(train['Education'],train['Loan_Status'])
Self_Employed=pd.crosstab(train['Self_Employed'],train['Loan_Status'])

# In[130]:

Married.div(Married.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True, figsize=(16,9))
plt.show()

# In[131]:

Dependents.div(Dependents.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True,figsize=(16,9))
plt.show()

# In[132]:

Education.div(Education.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True, figsize=(16,9))
plt.show()

# In[133]:

Self_Employed.div(Self_Employed.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True, figsize=(16,9))
plt.show()

# * Proportion of married applicants is higher for the approved loans.
# * Distribution of applicants with 1 or 3+ dependents is similar across both the categories of Loan_Status.
# * There is nothing significant we can infer from Self_Employed vs Loan_Status plot.
#
# Now we will look at the relationship between remaining categorical independent variables and Loan_Status.

# In[134]:

Credit_History=pd.crosstab(train['Credit_History'],train['Loan_Status'])
Property_Area=pd.crosstab(train['Property_Area'],train['Loan_Status'])

# In[135]:

Credit_History.div(Credit_History.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True, figsize=(16,9))
plt.show()

# In[136]:

Property_Area.div(Property_Area.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True,figsize=(16,9))
plt.show()

# * It seems people with credit history as 1 are more likely to get their loans approved.
# * Proportion of loans getting approved in semiurban area is higher as compared to that in rural or urban areas.
#
# Now let’s visualize numerical independent variables with respect to target variable.
#
# ### Numerical Independent Variable vs Target Variable
# We will try to find the mean income of people for which the loan has been approved vs the mean income of people for which the loan has not been approved.

# In[137]:

train.groupby('Loan_Status')['ApplicantIncome'].mean().plot.bar(figsize=(16,9))

# Here the y-axis represents the mean applicant income. We don’t see any change in the mean income. So, let’s make bins for the applicant income variable based on the values in it and analyze the corresponding loan status for each bin.

# In[138]:

bins=[0,2500,4000,6000,81000]
group=['Low','Average','High', 'Very high']
train['Income_bin']=pd.cut(train['ApplicantIncome'],bins,labels=group)
Income_bin=pd.crosstab(train['Income_bin'],train['Loan_Status'])

# In[139]:

Income_bin.div(Income_bin.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True,figsize=(16,9))
plt.xlabel('ApplicantIncome')
P = plt.ylabel('Percentage')

# It can be inferred that Applicant income does not affect the chances of loan approval which contradicts our hypothesis in which we assumed that if the applicant income is high the chances of loan approval will also be high.
#
# We will analyze the coapplicant income and loan amount variable in similar manner.

# In[140]:

bins=[0,1000,3000,42000]
group=['Low','Average','High']
train['Coapplicant_Income_bin']=pd.cut(train['CoapplicantIncome'],bins,labels=group)
Coapplicant_Income_bin=pd.crosstab(train['Coapplicant_Income_bin'],train['Loan_Status'])

# In[141]:

Coapplicant_Income_bin.div(Coapplicant_Income_bin.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True,figsize=(16,9))
plt.xlabel('CoapplicantIncome')
plt.ylabel('Percentage')

# It shows that if coapplicant’s income is less the chances of loan approval are high. But this does not look right. The possible reason behind this may be that most of the applicants don’t have any coapplicant so the coapplicant income for such applicants is 0 and hence the loan approval is not dependent on it. So we can make a new variable in which we will combine the applicant’s and coapplicant’s income to visualize the combined effect of income on loan approval.
#
# Let us combine the Applicant Income and Coapplicant Income and see the combined effect of Total Income on the Loan_Status.

# In[142]:

train['Total_Income']=train['ApplicantIncome']+train['CoapplicantIncome']
bins=[0,2500,4000,6000,81000]
group=['Low','Average','High', 'Very high']
train['Total_Income_bin']=pd.cut(train['Total_Income'],bins,labels=group)
Total_Income_bin=pd.crosstab(train['Total_Income_bin'],train['Loan_Status'])

# In[143]:

Total_Income_bin.div(Total_Income_bin.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True,figsize=(20,9))
plt.xlabel('Total_Income')
plt.ylabel('Percentage')

# We can see that Proportion of loans getting approved for applicants having low Total_Income is very less as compared to that of applicants with Average, High and Very High Income.
#
# Let’s visualize the Loan amount variable.

# In[144]:

bins=[0,100,200,700]
group=['Low','Average','High']
train['LoanAmount_bin']=pd.cut(train['LoanAmount'],bins,labels=group)
LoanAmount_bin=pd.crosstab(train['LoanAmount_bin'],train['Loan_Status'])

# In[145]:

LoanAmount_bin.div(LoanAmount_bin.sum(1).astype(float), axis=0).plot(kind="bar", stacked=True)
plt.xlabel('LoanAmount')
plt.ylabel('Percentage')

# It can be seen that the proportion of approved loans is higher for Low and Average Loan Amount as compared to that of High Loan Amount which supports our hypothesis in which we considered that the chances of loan approval will be high when the loan amount is less.
#
# Let’s drop the bins which we created for the exploration part. We will change the 3+ in dependents variable to 3 to make it a numerical variable.We will also convert the target variable’s categories into 0 and 1 so that we can find its correlation with numerical variables. One more reason to do so is few models like logistic regression takes only numeric values as input. We will replace N with 0 and Y with 1.

# In[146]:

train=train.drop(['Income_bin', 'Coapplicant_Income_bin', 'LoanAmount_bin', 'Total_Income_bin', 'Total_Income'], axis=1)
train['Dependents'].replace('3+', 3,inplace=True)
test['Dependents'].replace('3+', 3,inplace=True)
train['Loan_Status'].replace('N', 0,inplace=True)
train['Loan_Status'].replace('Y', 1,inplace=True)

# Now lets look at the correlation between all the numerical variables. We will use the heat map to visualize the correlation. Heatmaps visualize data through variations in coloring. The variables with darker color means their correlation is more.

# In[149]:

matrix = train.corr()
f, ax = plt.subplots(figsize=(16, 9))
sns.heatmap(matrix, vmax=.8, square=True, cmap="BuPu");

# We see that the most correlated variables are (ApplicantIncome - LoanAmount) and (Credit_History - Loan_Status). LoanAmount is also correlated with CoapplicantIncome.

# After exploring all the variables in our data, we can now impute the missing values and treat the outliers because missing data and outliers can have adverse effect on the model performance.
#
# ### Missing value imputation
# Let’s list out feature-wise count of missing values.

# In[151]:

train.isnull().sum()

# There are missing values in Gender, Married, Dependents, Self_Employed, LoanAmount, Loan_Amount_Term and Credit_History features.
#
# We will treat the missing values in all the features one by one.
#
# We can consider these methods to fill the missing values:
#
# * For numerical variables: imputation using mean or median
# * For categorical variables: imputation using mode
# There are very less missing values in Gender, Married, Dependents, Credit_History and Self_Employed features so we can fill them using the mode of the features.

# In[154]:

train['Gender'].fillna(train['Gender'].mode()[0], inplace=True)
train['Married'].fillna(train['Married'].mode()[0], inplace=True)
train['Dependents'].fillna(train['Dependents'].mode()[0], inplace=True)
train['Self_Employed'].fillna(train['Self_Employed'].mode()[0], inplace=True)
train['Credit_History'].fillna(train['Credit_History'].mode()[0], inplace=True)

# Now let’s try to find a way to fill the missing values in Loan_Amount_Term. We will look at the value count of the Loan amount term variable.

# In[156]:

train['Loan_Amount_Term'].value_counts()

# It can be seen that in loan amount term variable, the value of 360 is repeating the most. So we will replace the missing values in this variable using the mode of this variable.

# In[158]:

train['Loan_Amount_Term'].fillna(train['Loan_Amount_Term'].mode()[0], inplace=True)

# Now we will see the LoanAmount variable. As it is a numerical variable, we can use mean or median to impute the missing values. We will use median to fill the null values as earlier we saw that loan amount have outliers so the mean will not be the proper approach as it is highly affected by the presence of outliers.

# In[160]:

train['LoanAmount'].fillna(train['LoanAmount'].median(), inplace=True)

# Now lets check whether all the missing values are filled in the dataset.

# In[162]:

train.isnull().sum()

# As we can see that all the missing values have been filled in the test dataset. Let’s fill all the missing values in the test dataset too with the same approach.

# In[164]:

test['Gender'].fillna(train['Gender'].mode()[0], inplace=True)
test['Dependents'].fillna(train['Dependents'].mode()[0], inplace=True)
test['Self_Employed'].fillna(train['Self_Employed'].mode()[0], inplace=True)
test['Credit_History'].fillna(train['Credit_History'].mode()[0], inplace=True)
test['Loan_Amount_Term'].fillna(train['Loan_Amount_Term'].mode()[0], inplace=True)
test['LoanAmount'].fillna(train['LoanAmount'].median(), inplace=True)

# ### Outlier Treatment
#
# As we saw earlier in univariate analysis, LoanAmount contains outliers so we have to treat them as the presence of outliers affects the distribution of the data. Let's examine what can happen to a data set with outliers. For the sample data set:
#
#
# 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4
#
# We find the following: mean, median, mode, and standard deviation
#
# Mean = 2.58
#
# Median = 2.5
#
# Mode = 2
#
# Standard Deviation = 1.08
#
# If we add an outlier to the data set:
#
# 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 400
#
# The new values of our statistics are:
#
# Mean = 35.38
#
# Median = 2.5
#
# Mode = 2
#
# Standard Deviation = 114.74
#
# It can be seen that having outliers often has a significant effect on the mean and standard deviation and hence affecting the distribution. We must take steps to remove outliers from our data sets.
#
# Due to these outliers bulk of the data in the loan amount is at the left and the right tail is longer. This is called right skewness. One way to remove the skewness is by doing the log transformation. As we take the log transformation, it does not affect the smaller values much, but reduces the larger values. So, we get a distribution similar to normal distribution.
#
# Let’s visualize the effect of log transformation. We will do the similar changes to the test file simultaneously.

# In[167]:

train['LoanAmount_log'] = np.log(train['LoanAmount'])
train['LoanAmount_log'].hist(bins=20)
test['LoanAmount_log'] = np.log(test['LoanAmount'])

# Now the distribution looks much closer to normal and effect of extreme values has been significantly subsided. Let’s build a logistic regression model and make predictions for the test dataset.
#
# The process of model building is not complete without evaluation of model’s performance. Suppose we have the predictions from the model, how can we decide whether the predictions are accurate? We can plot the results and compare them with the actual values, i.e. calculate the distance between the predictions and actual values. Lesser this distance more accurate will be the predictions. Since this is a classification problem, we can evaluate our models using any one of the following evaluation metrics:
#
# Accuracy: Let us understand it using the confusion matrix which is a tabular representation of Actual vs Predicted values. This is how a confusion matrix looks like:
#
#
#
# * True Positive - Targets which are actually true(Y) and we have predicted them true(Y)
# * True Negative - Targets which are actually false(N) and we have predicted them false(N)
# * False Positive - Targets which are actually false(N) but we have predicted them true(T)
# * False Negative - Targets which are actually true(T) but we have predicted them false(N)
#
# Using these values, we can calculate the accuracy of the model. The accuracy is given by:

#

# **Precision:** It is a measure of correctness achieved in true prediction i.e. of observations labeled as true, how many are actually labeled true.
# Precision = TP / (TP + FP)
#
# **Recall(Sensitivity)** - It is a measure of actual observations which are predicted correctly i.e. how many observations of true class are labeled correctly. It is also known as ‘Sensitivity’.
# Recall = TP / (TP + FN)
#
# **Specificity** - It is a measure of how many observations of false class are labeled correctly.
# Specificity = TN / (TN + FP)
#
# Specificity and Sensitivity plays a crucial role in deriving ROC curve.
#
# **ROC curve**
# Receiver Operating Characteristic(ROC) summarizes the model’s performance by evaluating the trade offs between true positive rate (sensitivity) and false positive rate(1- specificity).
# The area under curve (AUC), referred to as index of accuracy(A) or concordance index, is a perfect performance metric for ROC curve. Higher the area under curve, better the prediction power of the model.
# This is how a ROC curve looks like:
#
#
#
# *  The area of this curve measures the ability of the model to correctly classify true positives and true negatives. We want our model to predict the true classes as true and false classes as false.
# * So it can be said that we want the true positive rate to be 1. But we are not concerned with the true positive rate only but the false positive rate too. For example in our problem, we are not only concerned about predicting the Y classes as Y but we also want N classes to be predicted as N.
# * We want to increase the area of the curve which will be maximum for class 2,3,4 and 5 in the above example.
# * For class 1 when the false positive rate is 0.2, the true positive rate is around 0.6. But for class 2 the true positive rate is 1 at the same false positive rate. So, the AUC for class 2 will be much more as compared to the AUC for class 1. So, the model for class 2 will be better.
# * The class 2,3,4 and 5 model will predict more accurately as compared to the class 0 and 1 model as the AUC is more for those classes.
# At the competition’s page, it has been mentioned that our submission data would be evaluated based on the accuracy. Hence, we will use accuracy as our evaluation metric.
#
#
#
Model Evaluation Metrics for Machine Learning

# 1. Confusion Matrix
# 2. F1 Score
# 3. Gain and Lift Charts
# 4. Kolmogorov Smirnov Chart
# 5. AUC – ROC
# 6. Log Loss
# 7. Gini Coefficient
# 8. Concordant – Discordant Ratio
# 9. Root Mean Squared Error

# #### Model Building

# Let us make our first model to predict the target variable. We will start with Logistic Regression which is used for predicting binary outcome.
#
# * Logistic Regression is a classification algorithm. It is used to predict a binary outcome (1 / 0, Yes / No, True / False) given a set of independent variables.
# * Logistic regression is an estimation of Logit function. Logit function is simply a log of odds in favor of the event.
# * This function creates a s-shaped curve with the probability estimate, which is very similar to the required step wise function
#
# Lets drop the Loan_ID variable as it do not have any effect on the loan status. We will do the same changes to the test dataset which we did for the training dataset.
#

# In[ ]:

train=train.drop('Loan_ID',axis=1)
test=test.drop('Loan_ID',axis=1)

# We will use scikit-learn (sklearn) for making different models which is an open source library for Python. It is one of the most efficient tool which contains many inbuilt functions that can be used for modeling in Python.

# Sklearn requires the target variable in a separate dataset. So, we will drop our target variable from the train dataset and save it in another dataset.

# In[172]:

X = train.drop('Loan_Status',1)
y = train.Loan_Status

# Now we will make dummy variables for the categorical variables. Dummy variable turns categorical variables into a series of 0 and 1, making them lot easier to quantify and compare. Let us understand the process of dummies first:
#
# * Consider the “Gender” variable. It has two classes, Male and Female.
# * As logistic regression takes only the numerical values as input, we have to change male and female into numerical value.
# * Once we apply dummies to this variable, it will convert the “Gender” variable into two variables(Gender_Male and Gender_Female), one for each class, i.e. Male and Female.
# * Gender_Male will have a value of 0 if the gender is Female and a value of 1 if the gender is Male.

# In[174]:

X=pd.get_dummies(X)
train=pd.get_dummies(train)
test=pd.get_dummies(test)

# Now we will train the model on training dataset and make predictions for the test dataset. But can we validate these predictions? One way of doing this is we can divide our train dataset into two parts: train and validation. We can train the model on this train part and using that make predictions for the validation part. In this way we can validate our predictions as we have the true predictions for the validation part (which we do not have for the test dataset).
#
# We will use the train_test_split function from sklearn to divide our train dataset. So, first let us import train_test_split.

# In[176]:

from sklearn.model_selection import train_test_split
x_train, x_cv, y_train, y_cv = train_test_split(X,y, test_size =0.3)

# The dataset has been divided into training and validation part. Let us import LogisticRegression and accuracy_score from sklearn and fit the logistic regression model.

# In[178]:

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score
model = LogisticRegression()
model.fit(x_train, y_train)
LogisticRegression(C=1.0, class_weight=None, dual=False,
                   fit_intercept=True,
                   intercept_scaling=1,
                   max_iter=100,
                   multi_class='ovr',
                   n_jobs=1,
                   penalty='l2',
                   random_state=1,
                   solver='liblinear',
                   tol=0.0001,
                   verbose=0,
                   warm_start=False)

# Here the C parameter represents inverse of regularization strength. Regularization is applying a penalty to increasing the magnitude of parameter values in order to reduce overfitting. Smaller values of C specify stronger regularization

# Let’s predict the Loan_Status for validation set and calculate its accuracy.

# In[179]:

pred_cv = model.predict(x_cv)

# In[181]:

accuracy_score(y_cv,pred_cv)

# So our predictions are almost 80% accurate, i.e. we have identified 80% of the loan status correctly.
#

# In[184]:

# Let’s make predictions for the test dataset.

pred_test = model.predict(test)

#
# Lets import the submission file which we have to submit on the solution checker.

# In[187]:

submission=pd.read_csv("Data/sample_submission_49d68Cx.csv")

# We only need the Loan_ID and the corresponding Loan_Status for the final submission. we will fill these columns with the Loan_ID of test dataset and the predictions that we made, i.e., pred_test respectively.

# In[188]:

submission['Loan_Status']=pred_test
submission['Loan_ID']=test_original['Loan_ID']

# Remember we need predictions in Y and N. So let’s convert 1 and 0 to Y and N.

# In[190]:

submission['Loan_Status'].replace(0, 'N',inplace=True)
submission['Loan_Status'].replace(1, 'Y',inplace=True)

# In[191]:

df = pd.DataFrame(submission, columns=['Loan_ID','Loan_Status']).to_csv('Data/logistic.csv')
df

# In[192]:

df


# -*- coding: utf-8 -*-
"""tflite_c03_exercise_convert_model_to_tflite.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1EuauRPZHxAsojcDIz-vsHO9kalFj6WkV

##### Copyright 2018 Shivansh Gour. Based on a work by The TensorFlow Authors.
"""

#@title Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""# Train Your Own Model and Convert It to TFLite

This notebook uses the [Fashion MNIST](https://github.com/zalandoresearch/fashion-mnist) dataset which contains 70,000 grayscale images in 10 categories. The images show individual articles of clothing at low resolution (28 by 28 pixels), as seen here:

    Figure 1. Fashion-MNIST samples (by Zalando, MIT License).

Fashion MNIST is intended as a drop-in replacement for the classic [MNIST](http://yann.lecun.com/exdb/mnist/) dataset—often used as the "Hello, World" of machine learning programs for computer vision. The MNIST dataset contains images of handwritten digits (0, 1, 2, etc.) in a format identical to that of the articles of clothing we'll use here.

This uses Fashion MNIST for variety, and because it's a slightly more challenging problem than regular MNIST. Both datasets are relatively small and are used to verify that an algorithm works as expected. They're good starting points to test and debug code.

We will use 60,000 images to train the network and 10,000 images to evaluate how accurately the network learned to classify images. You can access the Fashion MNIST directly from TensorFlow. Import and load the Fashion MNIST data directly from TensorFlow:

# Setup
"""

# Commented out IPython magic to ensure Python compatibility.
try:
#   %tensorflow_version 2.x
except:
  pass

from __future__ import absolute_import, division, print_function, unicode_literals

# TensorFlow and tf.keras
import tensorflow as tf
from tensorflow import keras

import tensorflow_datasets as tfds
tfds.disable_progress_bar()

# Helper libraries
import numpy as np
import matplotlib.pyplot as plt
import pathlib

print(tf.__version__)

"""# Download Fashion MNIST Dataset"""

splits = tfds.Split.ALL.subsplit(weighted=(80, 10, 10))

splits, info = tfds.load('fashion_mnist', with_info=True, as_supervised=True, split=splits)

(train_examples, validation_examples, test_examples) = splits

num_examples = info.splits['train'].num_examples
num_classes = info.features['label'].num_classes

class_names = ['T-shirt_top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']

with open('labels.txt', 'w') as f:
  f.write('\n'.join(class_names))

IMG_SIZE = 28,28

"""# Preprocessing data

## Preprocess
"""

# Write a function to normalize and resize the images

def format_example(image, label):
  # Cast image to float32
  image = tf.image.resize(image, IMG_SIZE) / 225.0
  return image, label

BATCH_SIZE = 32

"""## Create a Dataset from images and labels"""

# Prepare the examples by preprocessing the them and then batching them (and optionally prefetching them)

# If you wish you can shuffle train set here
train_batches = train_examples.shuffle(num_examples // 4).map(format_example).batch(BATCH_SIZE).prefetch(1)
validation_batches = validation_examples.map(format_example).batch(BATCH_SIZE).prefetch(1)
test_batches = test_examples.map(format_example).batch(1)

"""# Building the model

Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
conv2d (Conv2D)              (None, 26, 26, 16)        160
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 13, 13, 16)        0
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 11, 11, 32)        4640
_________________________________________________________________
flatten (Flatten)            (None, 3872)              0
_________________________________________________________________
dense (Dense)                (None, 64)                247872
_________________________________________________________________
dense_1 (Dense)              (None, 10)                650
=================================================================
Total params: 253,322
Trainable params: 253,322
Non-trainable params: 0
"""

# Build the model shown in the previous cell

model = tf.keras.Sequential([
  # Set the input shape to (28, 28, 1), kernel size=3, filters=16 and use ReLU activation,
  tf.keras.layers.Conv2D(kernel_size=3, filters=16, activation='relu', input_shape=(28,28,1)),
  # model.add(Conv2D (kernel_size = (20,30), filters = 400, activation='relu'))
  tf.keras.layers.MaxPooling2D(),
  # Set the number of filters to 32, kernel size to 3 and use ReLU activation
  tf.keras.layers.Conv2D(kernel_size=3, filters=32, activation='relu'),
  # Flatten the output layer to 1 dimension
  tf.keras.layers.Flatten(),
  # Add a fully connected layer with 64 hidden units and ReLU activation
  tf.keras.layers.Dense(units=64, activation='relu'),
  # Attach a final softmax classification head
  tf.keras.layers.Dense(units=64, activation='softmax')
  # model.add(Dense(64,activation='softmax'))
])

# Set the loss and accuracy metrics
model.compile(
     optimizer='adam',
     loss='sparse_categorical_crossentropy',
     metrics=['accuracy']
)

"""## Train"""

model.fit(train_batches,
          epochs=10,
          validation_data=validation_batches)

"""# Exporting to TFLite"""

export_dir = 'saved_model/1'

# Use the tf.saved_model API to export the SavedModel
tf.saved_model.save(model, export_dir)
# Your Code Here
# saved_model_cli show --dir $1 --tag_set serve --signature_def serving_default
# loaded = tf.saved_model.load(export_dir)
# print(list(loaded.signatures.keys()))
# infer = loaded.signatures["serving_default"]
# print(infer.structured_input_signature)
# print(infer.structured_outputs)

#@title Select mode of optimization
mode = "Speed" #@param ["Default", "Storage", "Speed"]

if mode == 'Storage':
  optimization = tf.lite.Optimize.OPTIMIZE_FOR_SIZE
elif mode == 'Speed':
  optimization = tf.lite.Optimize.OPTIMIZE_FOR_LATENCY
else:
  optimization = tf.lite.Optimize.DEFAULT

optimization

# Use the TFLiteConverter SavedModel API to initialize the converter
converter = tf.lite.TFLiteConverter.from_saved_model(export_dir)
# converter = tf.lite.TFLiteConverter.from_saved_model(CATS_VS_DOGS_SAVED_MODEL)
# Set the optimzations
converter.optimizations = [tf.lite.Optimize.DEFAULT]
# converter.optimizations = [tf.lite.Optimize.DEFAULT]
# Invoke the converter to finally generate the TFLite model
tflite_model = converter.convert()
# tflite_model = converter.convert()

tflite_model_file = 'model.tflite'

with open(tflite_model_file, "wb") as f:
  f.write(tflite_model)

"""# Test if your model is working"""

# Load TFLite model and allocate tensors.
interpreter = tf.lite.Interpreter(model_content=tflite_model)
interpreter.allocate_tensors()

input_index = interpreter.get_input_details()[0]["index"]
output_index = interpreter.get_output_details()[0]["index"]

# Gather results for the randomly sampled test images
predictions = []
test_labels = []
test_images = []

for img, label in test_batches.take(50):
  interpreter.set_tensor(input_index, img)
  interpreter.invoke()
  predictions.append(interpreter.get_tensor(output_index))
  test_labels.append(label[0])
  test_images.append(np.array(img))

#@title Utility functions for plotting
# Utilities for plotting

def plot_image(i, predictions_array, true_label, img):
  predictions_array, true_label, img = predictions_array[i], true_label[i], img[i]
  plt.grid(False)
  plt.xticks([])
  plt.yticks([])

  img = np.squeeze(img)

  plt.imshow(img, cmap=plt.cm.binary)

  predicted_label = np.argmax(predictions_array)
  if predicted_label == true_label.numpy():
    color = 'green'
  else:
    color = 'red'

  plt.xlabel("{} {:2.0f}% ({})".format(class_names[predicted_label],
                                100*np.max(predictions_array),
                                class_names[true_label]),
                                color=color)

def plot_value_array(i, predictions_array, true_label):
  predictions_array, true_label = predictions_array[i], true_label[i]
  plt.grid(False)
  plt.xticks(list(range(10)), class_names, rotation='vertical')
  plt.yticks([])
  thisplot = plt.bar(range(10), predictions_array[0], color="#777777")
  plt.ylim([0, 1])
  predicted_label = np.argmax(predictions_array[0])

  thisplot[predicted_label].set_color('red')
  thisplot[true_label].set_color('green')

#@title Visualize the outputs { run: "auto" }
index = 1 #@param {type:"slider", min:1, max:10, step:1}
plt.figure(figsize=(6,3))
plt.subplot(1,2,1)
plot_image(index, predictions, test_labels, test_images)
plt.show()
#plot_value_array(index, predictions, test_labels)
#plt.show()

"""# Download TFLite model and assets

**NOTE: You might have to run to the cell below twice**
"""

try:
  from google.colab import files
  files.download(tflite_model_file)
  files.download('labels.txt')
except:
  pass

"""# Deploying TFLite model

Now once you've the trained TFLite model downloaded, you can ahead and deploy this on an Android/iOS application by placing the model assets in the appropriate location.

# Prepare the test images for download (Optional)
"""

!mkdir -p test_images

from PIL import Image

for index, (image, label) in enumerate(test_batches.take(50)):
  image = tf.cast(image * 255.0, tf.uint8)
  image = tf.squeeze(image).numpy()
  pil_image = Image.fromarray(image)
  pil_image.save('test_images/{}_{}.jpg'.format(class_names[label[0]].lower(), index))

!ls test_images

!zip -qq fmnist_test_images.zip -r test_images/

try:
  files.download('fmnist_test_images.zip')
except:
  pass


# -*- coding: utf-8 -*-
"""
HunterLab Dataset
=================

Defines the *HunterLab* illuminants dataset for the
*CIE 1931 2 Degree Standard Observer* and
*CIE 1964 10 Degree Standard Observer*.

The currently implemented data has been extracted from :cite:`HunterLab2008b`,
however you may want to use different data according to the tables given in
:cite:`HunterLab2008c`.

See Also
--------
`Illuminants Jupyter Notebook
<http://nbviewer.jupyter.org/github/colour-science/colour-notebooks/\
blob/master/notebooks/colorimetry/illuminants.ipynb>`_

References
----------
-   :cite:`HunterLab2008b` : HunterLab. (2008). Hunter L,a,b Color Scale.
    Retrieved from http://www.hunterlab.se/wp-content/uploads/2012/11/\
Hunter-L-a-b.pdf
-   :cite:`HunterLab2008c` : HunterLab. (2008). Illuminant Factors in Universal
    Software and EasyMatch Coatings. Retrieved from
    https://support.hunterlab.com/hc/en-us/article_attachments/201437785/\
an02_02.pdf
"""

from __future__ import division, unicode_literals

import numpy as np
from collections import namedtuple

from colour.utilities import CaseInsensitiveMapping

__author__ = 'Colour Developers'
__copyright__ = 'Copyright (C) 2013-2019 - Colour Developers'
__license__ = 'New BSD License - https://opensource.org/licenses/BSD-3-Clause'
__maintainer__ = 'Colour Developers'
__email__ = 'colour-science@googlegroups.com'
__status__ = 'Production'

__all__ = [
    'HunterLab_Illuminant_Specification',
    'HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER_DATA',
    'HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER',
    'HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER_DATA',
    'HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER',
    'HUNTERLAB_ILLUMINANTS'
]

HunterLab_Illuminant_Specification = namedtuple(
    'HunterLab_Illuminant_Specification', ('name', 'XYZ_n', 'K_ab'))

# yapf: disable
HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER_DATA = (
    ('A', np.array([109.83, 100.00, 35.55]), np.array([185.20, 38.40])),
    ('C', np.array([98.04, 100.00, 118.11]), np.array([175.00, 70.00])),
    ('D50', np.array([96.38, 100.00, 82.45]), np.array([173.51, 58.48])),
    ('D60', np.array([95.23, 100.00, 100.86]), np.array([172.47, 64.72])),
    ('D65', np.array([95.02, 100.00, 108.82]), np.array([172.30, 67.20])),
    ('D75', np.array([94.96, 100.00, 122.53]), np.array([172.22, 71.30])),
    ('FL2', np.array([98.09, 100.00, 67.53]), np.array([175.00, 52.90])),
    ('TL 4', np.array([101.40, 100.00, 65.90]), np.array([178.00, 52.30])),
    ('UL 3000', np.array([107.99, 100.00, 33.91]), np.array([183.70, 37.50])))
# yapf: enable

HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER = (
    CaseInsensitiveMapping({
        x[0]: HunterLab_Illuminant_Specification(*x)
        for x in HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER_DATA
    }))
"""
*Hunter L,a,b* illuminant dataset for *CIE 1931 2 Degree Standard Observer*.

References
----------
:cite:`HunterLab2008b`, :cite:`HunterLab2008c`

HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER :
    CaseInsensitiveMapping
"""

# yapf: disable
HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER_DATA = (
    ('A', np.array([111.16, 100.00, 35.19]), np.array([186.30, 38.20])),
    ('C', np.array([97.30, 100.00, 116.14]), np.array([174.30, 69.40])),
    ('D50', np.array([96.72, 100.00, 81.45]), np.array([173.82, 58.13])),
    ('D60', np.array([95.21, 100.00, 99.60]), np.array([172.45, 64.28])),
    ('D65', np.array([94.83, 100.00, 107.38]), np.array([172.10, 66.70])),
    ('D75', np.array([94.45, 100.00, 120.70]), np.array([171.76, 70.76])),
    ('FL2', np.array([102.13, 100.00, 69.37]), np.array([178.60, 53.60])),
    ('TL 4', np.array([103.82, 100.00, 66.90]), np.array([180.10, 52.70])),
    ('UL 3000', np.array([111.12, 100.00, 35.21]), np.array([186.30, 38.20])))
# yapf: enable

HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER = (
    CaseInsensitiveMapping({
        x[0]: HunterLab_Illuminant_Specification(*x)
        for x in
        HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER_DATA
    }))
"""
*Hunter L,a,b* illuminant dataset for *CIE 1964 10 Degree Standard Observer*.

References
----------
:cite:`HunterLab2008b`, :cite:`HunterLab2008c`

HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER :
    CaseInsensitiveMapping
"""

HUNTERLAB_ILLUMINANTS = CaseInsensitiveMapping({
    'CIE 1931 2 Degree Standard Observer':
        HUNTERLAB_ILLUMINANTS_CIE_1931_2_DEGREE_STANDARD_OBSERVER,
    'CIE 1964 10 Degree Standard Observer':
        HUNTERLAB_ILLUMINANTS_CIE_1964_10_DEGREE_STANDARD_OBSERVER
})
HUNTERLAB_ILLUMINANTS.__doc__ = """
Aggregated *Hunter L,a,b* illuminant dataset.

References
----------
:cite:`HunterLab2008b`, :cite:`HunterLab2008c`

HUNTERLAB_ILLUMINANTS : CaseInsensitiveMapping
    **{'CIE 1931 2 Degree Standard Observer',
    'CIE 1964 10 Degree Standard Observer'}**

Aliases:

-   'cie_2_1931': 'CIE 1931 2 Degree Standard Observer'
-   'cie_10_1964': 'CIE 1964 10 Degree Standard Observer'
"""
HUNTERLAB_ILLUMINANTS['cie_2_1931'] = (
    HUNTERLAB_ILLUMINANTS['CIE 1931 2 Degree Standard Observer'])
HUNTERLAB_ILLUMINANTS['cie_10_1964'] = (
    HUNTERLAB_ILLUMINANTS['CIE 1964 10 Degree Standard Observer'])


#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import print_function

import unittest
import numpy as np
import sys

sys.path.append("..")

import paddle

from op_test import OpTest
from op_test_xpu import XPUOpTest
from xpu.get_test_cover_info import create_test_class, get_xpu_op_support_types, XPUOpTestWrapper

paddle.enable_static()

class XPUTestSignOP(XPUOpTestWrapper):

    def __init__(self):
        self.op_name = 'sign'
        self.use_dynamic_create_class = False

    class TestSignOPBase(XPUOpTest):

        def setUp(self):
            self.place = paddle.XPUPlace(0)
            self.init_dtype()
            self.set_case()

        def set_case(self):
            self.op_type = 'sign'
            self.dtype = self.in_type
            self.init_config()
            self.x = np.random.uniform(-10, 10,
                                       self.input_shape).astype(self.dtype)
            self.inputs = {'X': self.x}
            self.outputs = {'Out': np.sign(self.x)}
            self.attrs = {'use_xpu': True}

        def init_dtype(self):
            self.dtype = np.float32

        def test_check_output(self):
            self.check_output_with_place(self.place)

        def test_check_grad(self):
            self.check_grad_with_place(self.place, ['X'], 'Out')

        def init_config(self):
            self.input_shape = [864]

    class XPUTestSign1(TestSignOPBase):

        def init_config(self):
            self.input_shape = [2, 768]

    class XPUTestSign2(TestSignOPBase):

        def init_config(self):
            self.input_shape = [3, 8, 4096]

    class XPUTestSign3(TestSignOPBase):

        def init_config(self):
            self.input_shape = [1024]

    class XPUTestSign4(TestSignOPBase):

        def init_config(self):
            self.input_shape = [2, 2, 255]

support_types = get_xpu_op_support_types('sign')
for stype in support_types:
    create_test_class(globals(), XPUTestSignOP, stype)

if __name__ == "__main__":
    unittest.main()