import warnings
import numpy as np
import datetime

from functools import lru_cache

from scipy.interpolate import interp1d
from scipy.optimize import OptimizeWarning

import astropy.units as u
from astropy.time import Time
from astropy.table import Table, MaskedColumn
from astropy.utils.console import ProgressBar
from astropy.coordinates import Latitude, Longitude, EarthLocation
from astropy.coordinates import AltAz
from astropy.coordinates.name_resolve import NameResolveError
from astropy.io.fits import ImageHDU

import h5py
from autologging import logged

from . import kids_calib
from . import kids_plots
from .kids_rawdata import KidsRawData
from .kiss_object import get_coords
from .read_kidsdata import _to_hdf5, _from_hdf5
from .utils import _import_from, pprint_list, interp_nan
from .telescope_positions import KissPositions

try:
    from .kiss_pointing_model import KISSPmodel

except ModuleNotFoundError:
    warnings.warn("kiss_pointing_model not installed", Warning)

    class KISSPmodel(object):
        def __init__(self, *args, **kwargs):
            warnings.warn("No pointing correction", Warning)
            pass

        def telescope2sky(self, *args):
            return args

# pylint: disable=no-member
@logged
class KissRawData(KidsRawData):
    """Class dealing with KISS raw data.

    Derive from KidsRawData and add specific attributes and methods

    Attributes
    ----------
    f_mod : float, cached
        the modulation frequency to be used for calibration
    mod_mask : array_like, cached
        the modulation mask to be used for calibration
    __calib : dict
        The calibrated data

    Methods
    -------
    calib_raw(calib_func="kidsdata.kids__calib.get_calfact", clean_raw=False, **kwargs)
        Calibrate the data
    get_object_altaz(npoints=None), cached
        Compute the object alt az positions for the observations
    plot_kidpar(*args, **kwargs)
        Deprecated -- Plot the current kidpar
    plot_calib(*args, **kwargs)
        Deprecated -- Plot the calibration
    plot_pointing(*args, **kwargs)
        Deprecated -- Plot the pointing
    """

    __calib = None
    continuum = None
    interferograms = None

    def __init__(self, *args, pointing_model="KISSMateoNov2020", **kwargs):
        """
        Parameters
        ----------
        pointing_model : str
            the key used for the KISS pointing model
        """
        super().__init__(*args, **kwargs)

        self.meta["pointing_model"] = pointing_model
        self.__calib = dict()

    def _write_data(self, filename=None, mode="a", file_kwargs=None, **kwargs):
        """write internal data to hdf5 file

        Parameters
        ----------
        filename : [str]
            output filename, default None, use the cache file name
        dataSd : bool, optional
            flag to output the fully sampled data, by default False
        mode : str
            the open mode for the h5py.File object, by default 'a'
        file_kwargs, dict, optionnal
            additionnal keyword for h5py.File object, by default None
        **kwargs
            additionnal keyword argument for the h5py.Dataset, see Note

        Notes
        -----
        Usual kwargs could be :

        kwargs={'chunks': True, 'compression': "gzip", 'compression_opts':9, 'shuffle':True}
        """
        super()._write_data(filename, mode=mode, file_kwargs=file_kwargs, **kwargs)

        if filename is None:
            filename = self._cache_filename

        if file_kwargs is None:
            file_kwargs = {}

        # writing extra data
        with h5py.File(filename, mode="a", **file_kwargs) as f:

            if self.pointing_model:
                self.__log.debug("Saving pointing model")
                _to_hdf5(f, "pointing_model", self.meta["pointing_model"], **kwargs)
            if self.__calib:
                self.__log.debug("Saving calibrated data")
                _to_hdf5(f, "calib", self.__calib, **kwargs)

    @property
    @lru_cache(maxsize=1)
    def mod_mask(self):
        """Retrieve one modulation mask for all kids."""
        # Check the *_masq values
        self.__log.debug("Checking the *_masq arrays")
        # Retrieve the kid boxes
        masq_names = np.unique(["{}_masq".format(item[1]) for item in self.list_detector])
        self.__check_attributes(masq_names, read_missing=False)
        # Check that they are all the same
        warnings.warn("Temporary fix to int8")
        masqs = [getattr(self, masq).astype(np.int8) for masq in masq_names]

        if np.any(np.std(masqs, axis=0) != 0):
            self.__log.error("*_masq is varying -- Please check : {}".format(pprint_list(masq_names, "_masq")))

        # AB private comm) main_flag should be the bitwise_or of all boxes
        # Well not exactly....
        # cast into 8 bit, is more than enough, only 3 bits used anyway...
        masq = np.bitwise_or.reduce(masqs, axis=0).astype(np.int8)

        # AB (#CONCERTO_DAQ January 11 13:02)
        # _flag_balayage_en_cours & _flag_blanking_synthe
        # Ainsi on aura la modulation en bit0 et 1 et le flag blanking en bit
        # AB (#CONCERTO_DAQ February 11 11:07)
        # bit 1 & 2 code the modulation as a signed integer -1 0 1 : 11 00 01 ie 3 0 1
        # bit 3 is a blanking bit, which does not exist for KISS, but should not be taken into account for CONCERTO

        # Thus as a temporary fix, let's clear the 3rd bit, actually a bad idea...
        # self.__log.warning("Temporary fix : clearing the 3rd bit of masq")
        # masq = masq & ~(1 << 2)

        return masq

    @property
    @lru_cache(maxsize=1)
    def u_obstime(self):
        """Compute undersampled obstime.

        Properly compute the undersampled timestamp at the middle of the observed bloc
        """
        mask = self.mod_mask == 0
        obstime = self.obstime

        _u_mjd = [np.mean(_mjd[_mask]) for _mjd, _mask in zip(obstime.mjd, mask)]

        if np.any(np.isnan(_u_mjd)):
            _u_mjd = interp_nan(_u_mjd)

        return Time(_u_mjd, format="mjd", scale=obstime.scale)

    @lru_cache(maxsize=2)
    def _get_positions(self, coord="pdiff", undersampled=True):
        """Retrieve interpolated telescope position, without shift.

        Parameters
        ----------
        coord : str
            the type of position to retrieve
        undersampled : bool
            retrieve undersampled positions, default True

        Returns
        ------
        lon, lat, mask : array_like
            the corresponding longitude, latitude and mask
        """
        if undersampled:
            mjd = self.u_obstime
        else:
            mjd = self.obstime.flatten()

        return self.telescope_positions.get_interpolated_positions(mjd, key=coord)

    @property
    @lru_cache(maxsize=1)
    def fmod(self):
        # Check on frequency modulation values, in principle one should use the one corresponding on the array/crate, but depending on the files this could lead to wrong result
        self.__log.debug("Checking the *-modulFreq values")
        fmod_names = sorted([key for key in self.param_c.keys() if "modulFreq" in key])

        # Exclude null values (CONCERTO crate 1 has 0, has it does not read kids)
        fmods = [self.param_c[key] for key in fmod_names if self.param_c[key] != 0]
        if np.std(fmods) != 0:
            self.__log.warning("modulFreq are varying over crates  {}".format(dict(zip(fmod_names, fmods))))
        # TODO: make an _or_ of all or use main_flag
        return fmods[0]

    def calib_raw(self, calib_func="kidsdata.kids_calib.get_calfact", clean_raw=False, **kwargs):
        """Calibrate the KIDS timeline."""

        if getattr(self, "__calib", None) is None:
            self.__log.debug("calibration using {}".format(calib_func))
            self.__check_attributes(["I", "Q"], read_missing=False)

            fmod = self.fmod
            mod_mask = self.mod_mask

            # Check about the 3rd bit and the fix_masq keyword
            if np.any(mod_mask & (1 << 2)) and kwargs.get("fix_masq") is True:
                self.__log.error("fix_masq should not be used when 3rd bit is set")

            self.__log.info("Calibrating with fmod={} and {}".format(fmod, kwargs))
            calib_func = _import_from(calib_func)
            self.__calib = calib_func(self.I, self.Q, mod_mask, fmod=fmod, **kwargs)

        else:
            self.__log.error("calibrated data already present")

        # Expand keys :
        # Does not double memory, but it will not be possible to
        # partially free memory : All attribute read at the same time
        # must be deleted together
        for ckey in self.__calib.keys():
            self.__dict__[ckey] = self.__calib[ckey]

        if clean_raw:
            self._clean_data("_KidsRawData__dataSd")

    # Check if we can merge that with the asserions in other functions
    # Beware that some are read so are computed...
    def __check_attributes(self, attr_list, **kwargs):
        """Check if the data has been read an attribute and read in it if not."""
        dependancies = [
            # I & Q will need A_masq
            (["I", "Q"], ["I", "Q", "A_masq"]),
            # Calibration data depends on the I, Q & A_masq raw data
            (["calfact", "Icc", "Qcc", "P0", "R0", "interferograms", "continuum"], ["I", "Q", "A_masq"]),
            # For any requested telescope position, read them all
            (["F_tl_Az", "F_tl_El", "F_sky_Az", "F_sky_El"], ["F_tl_Az", "F_tl_El"]),
        ]

        _dependancies = self._KidsRawData__check_attributes(attr_list, dependancies=dependancies, **kwargs)

        if _dependancies is not None:
            self.calib_raw()

    # Move most of that to __repr__ or __str__
    def info(self):
        super().info()
        print("No. of interfergrams:\t", self.nint)
        print("No. of points per interfergram:\t", self.nptint)
        print(
            "Typical size of undersampled data (MiB + mask):\t{:3.1f} (+{:3.1f})".format(
                self.nint * self.ndet * 32 / 8 / 1024 ** 2, self.nint * self.ndet * 8 / 8 / 1024 ** 2
            )
        )

    def plot_kidpar(self, *args, **kwargs):
        fig_geometry = kids_plots.show_kidpar(self)
        fig_fwhm = kids_plots.show_kidpar_fwhm(self)
        return fig_geometry, fig_fwhm

    def plot_calib(self, *args, **kwargs):
        self.__check_attributes(["Icc", "Qcc", "calfact", "interferograms"])
        return kids_plots.calibPlot(self, *args, **kwargs)

    def plot_pointing(self, *args, coord="tl", **kwargs):
        """Plot azimuth and elevation to check pointing."""
        # TODO: Generalize that function
        warnings.warn("Deprecated function needs update.", DeprecationWarning)
        self.__check_attributes(["F_{}_az".format(coord), " F_{}_el".format(coord)])
        return kids_plots.checkPointing(self, *args, **kwargs)

    def read_data(self, *args, cache=False, array=np.array, **kwargs):
        """Read raw data.

        Also read the calibrated data in the cache file if present.

        Notes
        -----

        The different Kiss telescope positions are also handled
        """
        super().read_data(*args, cache=cache, array=array, **kwargs)

        if cache and self._cache is not None:
            self.__log.info("Reading cached data :")
            datas = []
            for data in ["calib"]:
                datas.append(_from_hdf5(self._cache, data, array=array) if data in self._cache else {})

            (calib,) = datas

            self.__log.debug("Updating dictionnaries with cached data")
            self.__calib.update(calib)

            keys = [key for data in datas for key in data]
            self.__log.debug("Read cached data : {}".format(keys))

            # Expand keys :
            # Does not double memory, but it will not be possible to
            # partially free memory : All attribute read at the same time
            # must be deleted together

            for _dict in [self.__calib]:
                for ckey in _dict:
                    self.__dict__[ckey] = _dict[ckey]

            # TODO: list_detectors and nsamples

        # TODO: Check on NIKA data, if this can be moved here
        # self.nptint = self.header.nb_pt_bloc  # Number of points for one interferogram
        # self.nint = self.nsamples // self.nptint  # Number of interferograms

        if hasattr(self, "indice"):
            indice = self.indice
            assert self.nptint == np.int(indice.max() - indice.min() + 1), "Problem with 'indice' or header"

        # Support for old parameters
        if "F_azimuth" in self.__dict__ and "F_elevation" in self.__dict__:
            warnings.warn("F_azimuth and F_elevation are deprecated", DeprecationWarning)

            # Pointing have changed... from Interpolated in Sc to real sampling in Uc
            if self.F_azimuth.shape == (self.nint * self.nptint,):
                warnings.warn("Interpolated positions", PendingDeprecationWarning)
                self.F_tl_Az = np.median(self.F_azimuth.reshape((self.nint, self.nptint)), axis=1)
                self.F_tl_El = np.median(self.F_elevation.reshape((self.nint, self.nptint)), axis=1)
            elif self.F_azimuth.shape == (self.nint,):
                self.F_tl_Az = self.F_azimuth
                self.F_tl_El = self.F_elevation

            del (self.F_azimuth, self.F_elevation)

        # Replace BasicPositions with KissPositions
        pos_keys = self._KidsRawData__pos_keys()

        # Clean unused/recomputed positions
        # sky : pointing corrected values at the telescope
        # tel : oversampling of the position
        # diff : differential coordinates computed at the telescope
        for key in ["sky", "tel", "diff"]:
            if key in pos_keys:
                del pos_keys[key]

        if "tl" in pos_keys:
            self.__log.debug("Initializing KissPositions")

            lon, lat = pos_keys["tl"]

            pos = np.array([getattr(self, lon).flatten(), getattr(self, lat).flatten()])
            mjd = getattr(self, "obstime").flatten()

            if pos.shape[1] == self.nint * self.nptint:
                # fully sampled position, should not arrive
                self.__log.error('Fully sampled position with "tl" should not occur')
                mjd = self.obstime.flatten()
            elif pos.shape[1] == self.nint:
                # Undersampled position, assuming center of block
                mjd = Time(mjd.mjd.reshape(self.nint, self.nptint).mean(1), scale=mjd.scale, format="mjd")
            else:
                raise ValueError("Do not known how to handle position {}".format(key))

            args = (mjd, pos)
            kwargs = {"pointing_model": self.meta["pointing_model"], "source": self.meta["OBJECT"]}
            # # Do not copy data (reference)
            # pos_keys = {
            #     "tl": KissPositions(*args, **kwargs, position_key="tl"),
            #     "sky": KissPositions(*args, **kwargs, position_key="sky"),
            #     "pdiff": KissPositions(*args, **kwargs, position_key="pdiff"),
            # }
            self.telescope_positions = KissPositions(*args, **kwargs, position_key="pdiff")

import numpy,from scipy,import astropy,from astropy


# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import numpy as np
import pytest

from mindspore import log
import mindspore.dataset as ds
import mindspore.dataset.text as text
import mindspore.dataset.text.transforms as T

DATASET_ROOT_PATH = "../data/dataset/testVectors/"

def _count_unequal_element(data_expected, data_me, rtol, atol):
    assert data_expected.shape == data_me.shape
    total_count = len(data_expected.flatten())
    error = np.abs(data_expected - data_me)
    greater = np.greater(error, atol + np.abs(data_expected)*rtol)
    loss_count = np.count_nonzero(greater)
    assert (loss_count/total_count) < rtol,\
        "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".\
        format(data_expected[greater], data_me[greater], error[greater])

def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
    if np.any(np.isnan(data_expected)):
        assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan)
    elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan):
        _count_unequal_element(data_expected, data_me, rtol, atol)
    else:
        assert True

def test_char_n_gram_all_to_vectors_params_eager():
    """
    Feature: CharNGram
    Description: test with all parameters which include `unk_init`
        and `lower_case_backup` in function ToVectors in eager mode
    Expectation: output is equal to the expected value
    """
    char_n_gram = text.CharNGram.from_file(DATASET_ROOT_PATH + "char_n_gram_20.txt", max_vectors=18)
    unk_init = (-np.ones(5)).tolist()
    to_vectors = T.ToVectors(char_n_gram, unk_init=unk_init, lower_case_backup=True)
    result1 = to_vectors("THE")
    result2 = to_vectors(".")
    result3 = to_vectors("To")
    res = [[-1.34121733e+00, 4.42693333e-02, -4.86969667e-01, 6.62939000e-01, -3.67669000e-01],
           [-1.00000000e+00, -1.00000000e+00, -1.00000000e+00, -1.00000000e+00, -1.00000000e+00],
           [-9.68530000e-01, -7.89463000e-01, 5.15762000e-01, 2.02107000e+00, -1.64635000e+00]]
    res_array = np.array(res, dtype=np.float32)

    allclose_nparray(res_array[0], result1, 0.0001, 0.0001)
    allclose_nparray(res_array[1], result2, 0.0001, 0.0001)
    allclose_nparray(res_array[2], result3, 0.0001, 0.0001)

def test_char_n_gram_build_from_file():
    """
    Feature: CharNGram
    Description: test with only default parameter
    Expectation: output is equal to the expected value
    """
    char_n_gram = text.CharNGram.from_file(DATASET_ROOT_PATH + "char_n_gram_20.txt")
    to_vectors = text.ToVectors(char_n_gram)
    data = ds.TextFileDataset(DATASET_ROOT_PATH + "words.txt", shuffle=False)
    data = data.map(operations=to_vectors, input_columns=["text"])
    ind = 0
    res = [[0., 0., 0., 0., 0.],
           [0., 0., 0., 0., 0.],
           [0.117336, 0.362446, -0.983326, 0.939264, -0.05648],
           [0.657201, 2.11761, -1.59276, 0.432072, 1.21395],
           [0., 0., 0., 0., 0.],
           [-2.26956, 0.288491, -0.740001, 0.661703, 0.147355],
           [0., 0., 0., 0., 0.]]
    for d in data.create_dict_iterator(num_epochs=1, output_numpy=True):
        res_array = np.array(res[ind], dtype=np.float32)
        allclose_nparray(res_array, d["text"], 0.0001, 0.0001)
        ind += 1

def test_char_n_gram_all_build_from_file_params():
    """
    Feature: CharNGram
    Description: test with all parameters which include `path` and `max_vector` in function BuildFromFile
    Expectation: output is equal to the expected value
    """
    char_n_gram = text.CharNGram.from_file(DATASET_ROOT_PATH + "char_n_gram_20.txt", max_vectors=100)
    to_vectors = text.ToVectors(char_n_gram)
    data = ds.TextFileDataset(DATASET_ROOT_PATH + "words.txt", shuffle=False)
    data = data.map(operations=to_vectors, input_columns=["text"])
    ind = 0
    res = [[0., 0., 0., 0., 0.],
           [0., 0., 0., 0., 0.],
           [0.117336, 0.362446, -0.983326, 0.939264, -0.05648],
           [0.657201, 2.11761, -1.59276, 0.432072, 1.21395],
           [0., 0., 0., 0., 0.],
           [-2.26956, 0.288491, -0.740001, 0.661703, 0.147355],
           [0., 0., 0., 0., 0.]]
    for d in data.create_dict_iterator(num_epochs=1, output_numpy=True):
        res_array = np.array(res[ind], dtype=np.float32)
        allclose_nparray(res_array, d["text"], 0.0001, 0.0001)
        ind += 1

def test_char_n_gram_all_build_from_file_params_eager():
    """
    Feature: CharNGram
    Description: test with all parameters which include `path` and `max_vector` in function BuildFromFile in eager mode
    Expectation: output is equal to the expected value
    """
    char_n_gram = text.CharNGram.from_file(DATASET_ROOT_PATH + "char_n_gram_20.txt", max_vectors=18)
    to_vectors = T.ToVectors(char_n_gram)
    result1 = to_vectors("the")
    result2 = to_vectors(".")
    result3 = to_vectors("to")
    res = [[-1.34121733e+00, 4.42693333e-02, -4.86969667e-01, 6.62939000e-01, -3.67669000e-01],
           [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00],
           [-9.68530000e-01, -7.89463000e-01, 5.15762000e-01, 2.02107000e+00, -1.64635000e+00]]
    res_array = np.array(res, dtype=np.float32)

    allclose_nparray(res_array[0], result1, 0.0001, 0.0001)
    allclose_nparray(res_array[1], result2, 0.0001, 0.0001)
    allclose_nparray(res_array[2], result3, 0.0001, 0.0001)

def test_char_n_gram_build_from_file_eager():
    """
    Feature: CharNGram
    Description: test with only default parameter in eager mode
    Expectation: output is equal to the expected value
    """
    char_n_gram = text.CharNGram.from_file(DATASET_ROOT_PATH + "char_n_gram_20.txt")
    to_vectors = T.ToVectors(char_n_gram)
    result1 = to_vectors("the")
    result2 = to_vectors(".")
    result3 = to_vectors("to")
    res = [[-8.40079000e-01, -2.70002500e-02, -8.33472250e-01, 5.88367000e-01, -2.10011750e-01],
           [0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00, 0.00000000e+00],
           [-9.68530000e-01, -7.89463000e-01, 5.15762000e-01, 2.02107000e+00, -1.64635000e+00]]
    res_array = np.array(res, dtype=np.float32)

    allclose_nparray(res_array[0], result1, 0.0001, 0.0001)
    allclose_nparray(res_array[1], result2, 0.0001, 0.0001)
    allclose_nparray(res_array[2], result3, 0.0001, 0.0001)

def test_char_n_gram_invalid_input():
    """
    Feature: CharNGram
    Description: test the validate function with invalid parameters.
    Expectation: Verification of correct error message for invalid input.
    """
    def test_invalid_input(test_name, file_path, error, error_msg, max_vectors=None,
                           unk_init=None, lower_case_backup=False, token="ok"):
        log.info("Test CharNGram with wrong input: {0}".format(test_name))
        with pytest.raises(error) as error_info:
            char_n_gram = text.CharNGram.from_file(file_path, max_vectors=max_vectors)
            to_vectors = T.ToVectors(char_n_gram, unk_init=unk_init, lower_case_backup=lower_case_backup)
            to_vectors(token)
        assert error_msg in str(error_info.value)

    test_invalid_input("Not all vectors have the same number of dimensions",
                       DATASET_ROOT_PATH + "char_n_gram_20_dim_different.txt", error=RuntimeError,
                       error_msg="all vectors must have the same number of dimensions, " +
                       "but got dim 4 while expecting 5")
    test_invalid_input("the file is empty.", DATASET_ROOT_PATH + "vectors_empty.txt",
                       error=RuntimeError, error_msg="invalid file, file is empty.")
    test_invalid_input("the count of `unknown_init`'s element is different with word vector.",
                       DATASET_ROOT_PATH + "char_n_gram_20.txt",
                       error=RuntimeError, error_msg="unk_init must be the same length as vectors, " +
                       "but got unk_init: 6 and vectors: 5", unk_init=np.ones(6).tolist())
    test_invalid_input("The file not exist", DATASET_ROOT_PATH + "not_exist.txt", RuntimeError,
                       error_msg="get real path failed")
    test_invalid_input("max_vectors parameter must be greater than 0",
                       DATASET_ROOT_PATH + "char_n_gram_20.txt", error=ValueError,
                       error_msg="Input max_vectors is not within the required interval", max_vectors=-1)
    test_invalid_input("invalid max_vectors parameter type as a float",
                       DATASET_ROOT_PATH + "char_n_gram_20.txt", error=TypeError,
                       error_msg="Argument max_vectors with value 1.0 is not of type [<class 'int'>],"
                       " but got <class 'float'>.", max_vectors=1.0)
    test_invalid_input("invalid max_vectors parameter type as a string",
                       DATASET_ROOT_PATH + "char_n_gram_20.txt", error=TypeError,
                       error_msg="Argument max_vectors with value 1 is not of type [<class 'int'>],"
                       " but got <class 'str'>.", max_vectors="1")
    test_invalid_input("invalid token parameter type as a float",
                       DATASET_ROOT_PATH + "char_n_gram_20.txt", error=RuntimeError,
                       error_msg="input tensor type should be string.", token=1.0)
    test_invalid_input("invalid lower_case_backup parameter type as a string", DATASET_ROOT_PATH + "char_n_gram_20.txt",
                       error=TypeError, error_msg="Argument lower_case_backup with " +
                       "value True is not of type [<class 'bool'>],"
                       " but got <class 'str'>.", lower_case_backup="True")
    test_invalid_input("invalid lower_case_backup parameter type as a string", DATASET_ROOT_PATH + "char_n_gram_20.txt",
                       error=TypeError, error_msg="Argument lower_case_backup with " +
                       "value True is not of type [<class 'bool'>],"
                       " but got <class 'str'>.", lower_case_backup="True")

if __name__ == '__main__':
    test_char_n_gram_all_to_vectors_params_eager()
    test_char_n_gram_build_from_file()
    test_char_n_gram_all_build_from_file_params()
    test_char_n_gram_all_build_from_file_params_eager()
    test_char_n_gram_build_from_file_eager()
    test_char_n_gram_invalid_input()


#!/usr/bin/env python
# coding: utf-8

# # Analysis Predictive: Prediction positive or negative stroke

# ### Oleh : [Ahmad Habib Husaini](https://www.linkedin.com/in/ahmad-habib-husaini-1705711b0/)
#
# #### Pendahuluan
# Pada proyek ini, topik yang dibahas adalah mengenai kesehatan yang di buat untuk memprediksi pasien apakah menderita diabeste atau tidak. Proyek ini dibuat untuk proyek Submission 1 - Machine Learning Terapan Dicoding.

# # 1. Import important package

# In[1]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import opendatasets
import wget
import zipfile
from tqdm import tqdm
import os

# # 2. Data loading

# ### 2.1 Data Acquisition

# In[2]:

if os.path.exists('stroke-prediction-dataset/healthcare-dataset-stroke-data.csv'):
    print("file sudah ada")
else:
    opendatasets.download_kaggle_dataset(dataset_url='https://www.kaggle.com/fedesoriano/stroke-prediction-dataset', data_dir='')

# Pada tahapan *data loading* berisikan akuisisi data dengan mengunduh dataset pada [link](https://www.kaggle.com/fedesoriano/stroke-prediction-datase)dengan menggunakan library `opendatasets` dengan sintaks seperti diatas. Ketika merunning code tersebut akan diminta mengisikan username dan key.
# 1. username silahkan disisi dengan username *account* kaggle
# 2. key didapatkan dengan :
#     1. Buka website [kaggle](https://www.kaggle.com/),
#     2. login dengan akun masing-masing.
#     3. Pilih your profile pada kanan atas
#     4. Pilih *account*
#     5. Scroll sedikit kebawah maka akan ada pilihan `Create new API token`
#

#     6. Setelah menekan pilihan tersebut akan terunduh file kaggle.json yang berisikan username dan key

# ### 2.2 Memuat data dalam format Dataframe

# In[3]:

df = pd.read_csv("stroke-prediction-dataset/healthcare-dataset-stroke-data.csv")
df.head()

# In[4]:

df.stroke.unique()

# # 3. Data Understanding

# # informasi data set :
# Attribute  | Keterangan
# :------------- | :-------------
# Sumber  | https://www.kaggle.com/fedesoriano/stroke-prediction-dataset
# ID | Nomor identitas pasien
# Gender | Jenis kelamin (male, female, other(**dianggap invalid data**))
# Hypertension | Nol jika pasien tidak menderita darah tinggi, satu sebaliknya
# Heart diseease | Nol jika pasien tidak menderita penyakit jantung, satu sebaliknya
# Ever married | Status pernah menikah (Yes or No)
# Work type | Jenis pekerjaan terdiri dari anak-anak, Pekerjaan pemerintah, Tidak pernah bekerja, Swasta atau Wiraswasta
# Residence type | Tipe tempat tinggal Rural (pedasaan) atau Urban (perkotaan)
# Avg glucose type | rata-rata kadar gula dalam darah
# BMI | body mass index
# Smoking status | formerly smoked (Sebelumnya merokok), never smoked (tidak pernah merokok), smokes (merokok) atau "Unknown" (dianggap invalid data)
# Stroke | satu jika pasien positif stroke nol sebaliknya

# ### 3.1 Melihat banyak data

# In[5]:

print("Jumlah baris          :", df.shape[0])
print("Jumlah kolom          :", df.shape[1])
print("Jumlah missing values :", df.isnull().sum().sum())

# `Terdapat 5110 baris, 12 kolom dan 201 missing values.`

# `Kolom apa saja yang terdapat missing values ?`

# ### 3.2 Cek missing values

# In[6]:

pd.DataFrame({
    'missing value':df.isnull().sum()
})

# `kolom tersebut akan diisikan dengan rata-rata, namum prosesnya akan dilakukan belakangan dengan bantuan pipeline sklearn`

# ### 3.3 cek tipe data tiap kolom

# In[7]:

df.info()

# `Jika dilihat terdapat 7 kolom numerik dan 5 kolom categorical`,`namun jika diperhatikan kolom hypertension dan heart_disease merupakan target yang merupakan data cagetorical, oleh karenanya kita perlu mengubah tipe data dari kolom tersebut`

# ### 3.4 Mengubah tipe data

# In[8]:

df.hypertension = df.hypertension.astype(object)
df.heart_disease = df.hypertension.astype(object)

# In[9]:

df.info()

# ### 3.5 Summary statistical descriptive

# `Numerical`
# 1. Count  adalah jumlah sampel pada data.
# 2. Mean adalah nilai rata-rata.
# 3. Std adalah standar deviasi.
# 4. Min yaitu nilai minimum setiap kolom.
# 5. 25% adalah kuartil pertama. Kuartil adalah nilai yang menandai batas interval dalam empat bagian sebaran yang sama.
# 6. 50% adalah kuartil kedua, atau biasa juga disebut median (nilai tengah).
# 7. 75% adalah kuartil ketiga.
# 8. Max adalah nilai maksimum.
#
# `categorical:`
# 1. unique adalah banyaknya kategori dari setiap kolom categorical
# 2. top adalah kategori paling banyak dari setiap kolom
# 3. freq adalah banyaknya frequensi dari top

# In[10]:

df.describe()

# `pada kolom age nilai min adalah 0.08, pertanyaanya apa maksud dari 0.08 ? `, `ini menunjukan terdapat indikasi invalid data`

# In[11]:

df.describe(include=object)

# `Pada categorical columns tidak terlihat ada yang aneh`

# ### 3.6 Visualisasi data

# ### 3.6.1 Memisahkan fitur kategorik, numerik dan target

# In[12]:

target = ['stroke']
num_feature = ['age', 'avg_glucose_level','bmi']
cat_feature = [i for i in df.columns if i not in (target + num_feature + ['id'])]

# ### 3.6.2 Visualisasi fitur kategorik

# In[13]:

fig, ax = plt.subplots(len(cat_feature),1, figsize=(30,100))
idx=0
for ft in cat_feature:
    sns.countplot(data=df, x=ft, ax=ax[idx])
    for p in ax[idx].patches:
        ax[idx].annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
    ax[idx].set_xticklabels(ax[idx].get_xticklabels(), fontsize=13)
    ax[idx].set_title("frequency of "+str(ax[idx].get_xlabel()), fontsize=20)
    ax[idx].set_xlabel("", fontsize=15)
    ax[idx].set_yticklabels(ax[idx].get_yticklabels(), fontsize=13)
    idx+=1
plt.show()

# In[14]:

fig, ax = plt.subplots(len(cat_feature),1, figsize=(30,100))
idx=0
for ft in cat_feature:
    sns.countplot(data=df, x=ft, ax=ax[idx], hue='stroke')
    for p in ax[idx].patches:
        ax[idx].annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
    ax[idx].set_xticklabels(ax[idx].get_xticklabels(), fontsize=13)
    ax[idx].set_title("frequency of "+str(ax[idx].get_xlabel()+" against the target"), fontsize=20)
#     ax[idx].set_xlabel(ax[idx].get_xlabel(), fontsize=15)
    ax[idx].set_xlabel("", fontsize=15)
    ax[idx].set_yticklabels(ax[idx].get_yticklabels(), fontsize=13)
    idx+=1
plt.show()

# ### 3.6.3 Visualisasi fitur numerik

# In[15]:

fig, ax = plt.subplots(len(num_feature),1, figsize=(25,30))
idx = 0
for feature in num_feature:
    sns.boxplot(data=df, y=feature, ax=ax[idx])
    ax[idx].set_title("boxplot of "+str(feature), fontsize=20)
    idx+=1

# In[16]:

fig, ax = plt.subplots(len(num_feature),1, figsize=(25,40))
idx=0
for ft in num_feature:
    sns.distplot(df[ft], bins=10, ax=ax[idx])
    ax[idx].set_xticklabels(ax[idx].get_xticks(), fontsize=15)
    ax[idx].set_title("Distibution of "+str(ax[idx].get_xlabel()), fontsize=20)
    ax[idx].set_xlabel("", fontsize=15)
    idx+=1
plt.show()

# ### 3.6.4 Visualisasi Variabel target

# In[17]:

plt.figure(figsize=(15,7))
ax = sns.countplot(data=df, x='stroke')
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_xlabel("", fontsize=15)
ax.set_title("frequent of target")
plt.show()

# ### 3.6.5 visualisasi missing value

# In[18]:

df_pos_null = df[df['stroke']==1].isnull().sum()[:-1].reset_index()
df_pos_null.columns = ["kolom","missing_value"]
df_pos_null

# In[19]:

plt.figure(figsize=(20,5))
sns.barplot(data=df_pos_null, x='kolom', y='missing_value')
plt.title("Missing values in positive stroke")
plt.show()

# In[20]:

df_neg_null = df[df['stroke']==0].isnull().sum()[:-1].reset_index()
df_neg_null.columns = ["kolom","missing_value"]
df_neg_null

# In[21]:

plt.figure(figsize=(20,5))
sns.barplot(data=df_neg_null, x='kolom', y='missing_value')
plt.title("Missing values in negative stroke")
plt.show()

# `Kesimpulan yang dapat diambil :`
# 1. Data target tidak seimbang antara pasien positif dan negatif
# 2. Data dengan value gender other dianggap **invalid data**
# 3. Data dengan value smoking_status unknown akan dihapus
# 4. Pada kolom numerik terdapat cukup banyak outlier
# 5. Kolom numerik masih condong atau skew

# # 4. Data Preparation

# ### 4.1  Handling invalid data

# `Kolom ID hanyalah nomer unik dari masing-masing pasien`,`kolo tersebut sangat kecil bahkan tidak berpengaruh sama sekali pada target`

# `Pada kolom gender dan smoking status terdapat data invalid`

# In[22]:

df_backup = df.copy(deep=True)

# In[23]:

df[(df.gender == 'Other')]

# In[24]:

df.groupby(by=[cat_feature[0],'stroke'])['id'].count().to_frame()

# In[25]:

df = df_backup.copy(deep=True)

# In[26]:

df = df[(df.gender != 'Other')]
df.shape

# In[27]:

df.groupby(by=[cat_feature[0],'stroke'])['id'].count().to_frame()

# In[28]:

df.groupby(by=[cat_feature[6],'stroke'])['id'].count().to_frame()

# `Jika diihat ternyata pada kolom smoking status = unknown value stroke = 1 ada 47, jika kita hapus tidak terlalu banyak sample stroke = 1 yang hilang`

# In[29]:

df = df[(df.smoking_status !='Unknown')]
df.shape

# In[30]:

df.groupby(by=[cat_feature[6],'stroke'])['id'].count().to_frame()

# ### 4.2 Handling Outlier

# In[31]:

fig, ax = plt.subplots(len(num_feature),1, figsize=(25,30))
idx = 0
for feature in num_feature:
    sns.boxplot(data=df, y=feature, ax=ax[idx])
    ax[idx].set_title(feature)
    idx+=1

# `Jika dilihat dari kolom numeric diatas terdapat cukup banyak outlier`

# `Hal pertama yang perlu dilakukan adalah membuat batas bawah dan batas atas.`

# `Untuk membuat batas bawah, kurangi Q1 dengan 1,5 * IQR.`

# `Kemudian, untuk membuat batas atas, tambahkan 1.5 * IQR dengan Q3.`

# In[32]:

Q1 = df[num_feature].quantile(0.25)
Q3 = df[num_feature].quantile(0.75)
IQR = Q3 - Q1
IQR.to_frame()

# `Syarat bukan outlier, (data < (Q1-1.5*IQR)) atau (data > (Q3+1.5*IQR))`

# In[33]:

batas_bawah = Q1 - 1.5*IQR
batas_atas = Q3 + 1.5*IQR

# In[34]:

df = df[~((df < batas_bawah) | (df > batas_atas)).any(axis=1)]
df.shape

# In[35]:

fig, ax = plt.subplots(len(num_feature),1, figsize=(25,30))
idx = 0
for feature in num_feature:
    sns.boxplot(data=df, y=feature, ax=ax[idx])
    ax[idx].set_title(feature)
    idx+=1

# In[36]:

df.stroke.unique()

# In[37]:

fig, ax = plt.subplots(len(num_feature),1, figsize=(25,40))
idx=0
for ft in num_feature:
    sns.distplot(df[ft], bins=10, ax=ax[idx])
    ax[idx].set_xticklabels(ax[idx].get_xticks(), fontsize=15)
    ax[idx].set_title("distibution of "+str(ax[idx].get_xlabel()), fontsize=20)
    ax[idx].set_xlabel("", fontsize=15)
    idx+=1
plt.show()

# `Jika dilihat dari distribusi, kolom sudah hampir bersidtribusi normal dan tidak terlalu condong atau skew`,`Tetapi dalam pipeline nanti tetap akan di transform dengan yeo-johnson`

# ### 4.3 Split Data

# In[38]:

from sklearn.model_selection import train_test_split

# In[39]:

df[cat_feature] = df[cat_feature].astype(object)

# In[40]:

X = df.drop(columns="stroke")
y = df.stroke

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)
X_train.shape, X_test.shape, y_train.shape, y_test.shape

# ### 4.4 Build Pipeline

# In[41]:

import sklearn
from sklearn.preprocessing import OneHotEncoder,OrdinalEncoder ,MinMaxScaler, StandardScaler, RobustScaler, PowerTransformer
from sklearn.impute import SimpleImputer
# from sklearn.pipeline import Pipeline
from imblearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer

# ### 4.4.1 Numerical Pipeline

# In[42]:

sklearn.set_config(display='diagram')
num_pipline = Pipeline([
    ('inputer', SimpleImputer(strategy='median')), # Handling missing values
    ('transformer', PowerTransformer('yeo-johnson')), # Tranformasi agar berdistribusi normal
    ('scaling', RobustScaler()) # Penyekalaan data
])

# ### 4.4.2 Categorical Pipeline

# In[43]:

cat_pipeline = Pipeline([
    ('encoding', OrdinalEncoder())
])

# ### 4.4.3 Create Preprocessor

# In[44]:

preprocessor = ColumnTransformer([
    ('numeric', num_pipline, num_feature),
    ('categorical', cat_pipeline, cat_feature)
])

# In[45]:

preprocessor

# `Tahapan diatas untuk menentukan kolom mana yang akan diterapkan numerical pipeline atau categorical pipeline`

# In[47]:

from jcopml.tuning.space import Real, Integer # Library buatan mas WiraDKP https://www.linkedin.com/in/wiradkputra/
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import RandomizedSearchCV

# # 5. Modeling

# ### 5.1 Model without resampling

# ### 5.1.1 Final pipeline

# In[48]:

pipeline = Pipeline([
    ('prep', preprocessor),
    ('algo', RandomForestClassifier(n_jobs=-1, random_state=42))
])

# In[49]:

pipeline

# In[50]:

params_rf = {
    'algo__n_estimators': Integer(low=100, high=200),
    'algo__max_depth':Integer(low=20, high=80),
    'algo__max_features': Real(low=0.1, high=1, prior='uniform'),
    'algo__min_samples_leaf':Integer(low=1, high=20)
}

# In[51]:

model_rf = RandomizedSearchCV(pipeline, params_rf, cv=3, n_iter=50,scoring='f1', n_jobs=-1, verbose=1, random_state=42)
model_rf.fit(X_train, y_train)

print(model_rf.best_params_)
print(model_rf.score(X_train, y_train), model_rf.best_score_, model_rf.score(X_test, y_test))

# ### 5.1.2 Cek confusion matrix model

# In[52]:

from sklearn.metrics import classification_report, confusion_matrix

# In[53]:

cf_matrix = confusion_matrix(y_test, model_rf.predict(X_test))
cf_matrix

# In[54]:

import seaborn as sns
ax = sns.heatmap(cf_matrix, annot=True,cmap='Blues', fmt='g')
ax.set_xticklabels(["Negatif", "Positif"])
ax.set_yticklabels(["Negatif", "Positif"])
plt.show()

# In[55]:

print(classification_report(y_test, model_rf.predict(X_test)))

# In[56]:

y_test.value_counts()

# `Jika hanya berpacu pada accuracy model memiliki accuracy yang sangat baik, tetapi jika dilihat ternyata model salah semua dalam memprediksi pasien positif`,`oleh karenanya coba lakukan pembobotan dan gunakan scoring f1-score`

# ### 5.2 Model dengan pembobotan

# In[57]:

[{0: x, 1: 1-x} for x in [0.05, 0.1, 0.25]]

# In[58]:

params_rf = {
    'algo__n_estimators': Integer(low=100, high=200),
    'algo__max_depth':Integer(low=20, high=80),
    'algo__max_features': Real(low=0.1, high=1, prior='uniform'),
    'algo__min_samples_leaf':Integer(low=1, high=20),
    'algo__class_weight':[{0: x, 1: 1-x} for x in [0.05, 0.1, 0.25]]
}

# In[59]:

pipeline = Pipeline([
    ('prep', preprocessor),
    ('algo', RandomForestClassifier(n_jobs=-1, random_state=42))
])
pipeline

# In[60]:

model_rf_2 = RandomizedSearchCV(pipeline, params_rf, cv=3, n_iter=50, scoring='f1', n_jobs=-1, verbose=1, random_state=42)
model_rf_2.fit(X_train, y_train)

print(model_rf_2.best_params_)
print(model_rf_2.score(X_train, y_train), model_rf_2.best_score_, model_rf_2.score(X_test, y_test))

# In[61]:

cf_matrix = confusion_matrix(y_test, model_rf_2.predict(X_test))
cf_matrix

# In[62]:

ax = sns.heatmap(cf_matrix, annot=True,cmap='Blues', fmt='g')
ax.set_xticklabels(["Negatif", "Positif"])
ax.set_yticklabels(["Negatif", "Positif"])
plt.show()

# In[63]:

print(classification_report(y_test, model_rf_2.predict(X_test)))

# `Dengan melakukan pembobotan prediksi pada data text menjadi lebih baik walau masih banyak salah prediksi`

# ### 5.3 Resampling data dengan SMOTE

# `Teknik resampling merupakan pembuatan data dummy dengan algoritma tertentu, pada proyek kali ini akan digunakan algoritma SMOTE yang berdasarkan pada algoritma KNN`

# In[64]:

import imblearn, sklearn
from imblearn.over_sampling import SMOTE
print(imblearn.__version__, sklearn.__version__)

# `Resampling dilakukan data train saja, mengapa ? jika data test ikut diresampling dan misalkan accuracy model bagus apa maknanya ? bisa saja data test yang benar diprediksi adalah data buatan hasil resampling, oleh karenanya untuk menilai model baik atau tidak uji dengan data yang asli`

# ### 5.3.1 New Pipeline

# In[65]:

res_pipeline = Pipeline([
    ('prep', preprocessor),
    ('smote', SMOTE(random_state=42, sampling_strategy='minority')),
    ('algo', RandomForestClassifier(n_jobs=-1, random_state=42))
])
res_pipeline

# ### 5.3.2 Model tanpa tunning parameter Smote

# In[66]:

model_rf_3_1 = RandomizedSearchCV(res_pipeline, params_rf, cv=3, n_iter=50, scoring='f1', n_jobs=-1, verbose=1, random_state=42)
model_rf_3_1.fit(X_train, y_train)

print(model_rf_3_1.best_params_)
print(model_rf_3_1.score(X_train, y_train), model_rf_3_1.best_score_, model_rf_3_1.score(X_test, y_test))

# In[67]:

cf_matrix = confusion_matrix(y_test, model_rf_3_1.predict(X_test))
cf_matrix

# In[68]:

ax = sns.heatmap(cf_matrix, annot=True,cmap='Blues', fmt='g')
ax.set_xticklabels(["Negatif", "Positif"])
ax.set_yticklabels(["Negatif", "Positif"])
plt.show()

# In[69]:

print(classification_report(y_test, model_rf_3_1.predict(X_test)))

# `Walaupun accuracy berkurang, tetapi model dapat memprediksi pasien positif lebih baik dari sebelumnya`

# ### 5.3.3 Model dengan melakukan tunning parameter smote

# In[70]:

params_rf = {
    'smote__k_neighbors': Integer(low=1, high=50),
    'algo__n_estimators': Integer(low=100, high=200),
    'algo__max_depth':Integer(low=20, high=80),
    'algo__max_features': Real(low=0.1, high=1, prior='uniform'),
    'algo__min_samples_leaf':Integer(low=1, high=20),
    'algo__class_weight':[{0: x, 1: 1-x} for x in [0.05, 0.1, 0.25]]
}

# In[71]:

model_rf_3_2 = RandomizedSearchCV(res_pipeline, params_rf, cv=5, n_iter=50, scoring='f1', n_jobs=-1, verbose=1, random_state=42)
model_rf_3_2.fit(X_train, y_train)

print(model_rf_3_2.best_params_)
print(model_rf_3_2.score(X_train, y_train), model_rf_3_2.best_score_, model_rf_3_2.score(X_test, y_test))

# In[72]:

cf_matrix = confusion_matrix(y_test, model_rf_3_2.predict(X_test))
cf_matrix

# In[73]:

ax = sns.heatmap(cf_matrix, annot=True,cmap='Blues', fmt='g')
ax.set_xticklabels(["Negatif", "Positif"])
ax.set_yticklabels(["Negatif", "Positif"])
plt.show()

# In[74]:

print(classification_report(y_test, model_rf_3_2.predict(X_test)))

# `Jika diperhatikan dari rumus dibawah ini :`

# $$ Accuracy = \frac{TP+TN}{TP+TN+FP+FN}$$

# $$ Precision = \frac{TP}{TP+FP} $$

# $$ Recall = \frac{TP}{TP+FN}$$

# $$ F1 = \frac{2*Precision*Recall}{Precision+Recall} = \frac{2*TP}{2*TP+FP+FN} $$

# `Untuk mengecilkan false negatif (salah dalam memprediksi pasien positif, metrics yang cocok adalah recall`

# In[75]:

model_rf_4 = RandomizedSearchCV(res_pipeline, params_rf, cv=5, n_iter=50, scoring='recall', n_jobs=-1, verbose=1, random_state=42)
model_rf_4.fit(X_train, y_train)

print(model_rf_4.best_params_)
print(model_rf_4.score(X_train, y_train), model_rf_4.best_score_, model_rf_4.score(X_test, y_test))

# In[76]:

cf_matrix = confusion_matrix(y_test, model_rf_4.predict(X_test))
cf_matrix

# In[77]:

ax = sns.heatmap(cf_matrix, annot=True,cmap='Blues', fmt='g')
ax.set_xticklabels(["Negatif", "Positif"])
ax.set_yticklabels(["Negatif", "Positif"])
plt.show()

# In[78]:

from sklearn.metrics import f1_score, recall_score

# # Evaluasi

# In[79]:

df_model = pd.DataFrame(columns=['model_1', 'model_2', 'model_3_1', 'model_3_2', 'model_4'], index=['f1_score','recall'])
df_model

# In[80]:

f1_score(y_test, model_rf.predict(X_test))

# In[81]:

models = {
    'model_1':model_rf,
    'model_2':model_rf_2,
    'model_3_1':model_rf_3_1,
    'model_3_2':model_rf_3_2,
    'model_4':model_rf_4
}
metrics = {
    'f1_score':f1_score,
    'recall':recall_score
}
for metric in metrics.keys():
    for model in models.keys():
        df_model.loc[metric, model] = metrics[metric](y_test, models[model].predict(X_test))

# In[82]:

df_model

# `Tetapi hal tersebut menyebabkan banyak prediksi yang salah dari kelas negatif.` `Jika model salah dalam memprediksi pasien negatif (aslinya negatif diprediksi positif) tentu jauh lebih baik ketimbang aslinya positif tetapi diprediksi negatif. Namun kembali lagi pada keputusan klien atau pihak berkepentingan`

# `Dan jika ingin memprebaiki kualitas model maka perbanyak sample dan `**`jangan imbalance`**


#!/usr/bin/env python

import numpy
import silx.io
from silx.gui import qt
from silx.gui import hdf5
import silx.gui.data.DataViewerFrame

def main_ex1():

    #
    # EXERCISE: Open the file 'data/ID16B_diatomee.h5'
    #

    import silx.io
    h5 = silx.io.open("data/ID16B_diatomee.h5")

    #
    # EXERCISE: Display the file into the HDF5 tree
    #

    from silx.gui import hdf5
    tree = hdf5.Hdf5TreeView()
    model = tree.findHdf5TreeModel()
    model.insertH5pyObject(h5)
    tree.setVisible(True)

    #
    # EXERCISE: Access to one frame of the image
    #

    print(h5["/data/0000"])

    #
    # EXERCISE: Display it with the data viwer
    #

    import silx.gui.data.DataViewerFrame
    viewer = silx.gui.data.DataViewerFrame.DataViewerFrame()
    viewer.setData(h5["/data/0000"])
    viewer.setVisible(True)

    return tree, viewer

###############################################################

def main_ex2():

    def correctedImage(data, background, flatfield):
        data = numpy.array(data, dtype=numpy.float32)
        flatfield = numpy.array(flatfield, dtype=numpy.float32)
        return (data - background) / (flatfield - background)

    #
    # EXERCISE: Reach one data frame, a background and a flatfield from 'data/ID16B_diatomee.h5'
    #

    import silx.io
    h5 = silx.io.open("data/ID16B_diatomee.h5")
    data = h5["/data/0000"][...]
    background = h5["/background/0000"][...]
    flatfield = h5["/flatfield/0000"][...]

    #
    # EXERCISE: Compute the corrected image
    #

    corrected = correctedImage(data, background, flatfield)

    #
    # EXERCISE: Display it with the data viewer
    #

    import silx.gui.data.DataViewerFrame
    viewer = silx.gui.data.DataViewerFrame.DataViewerFrame()
    viewer.setData(corrected)
    viewer.setVisible(True)
    return viewer

###############################################################

class ViewerEx3(qt.QMainWindow):

    def __init__(self, parent=None):
        qt.QMainWindow.__init__(self, parent)
        widget = self.createCentralWidget()
        self.setCentralWidget(widget)

    def createCentralWidget(self):
        splitter = qt.QSplitter(self)

        # the tree
        self.tree = silx.gui.hdf5.Hdf5TreeView(self)
        # the data viewer
        self.viewer = silx.gui.data.DataViewerFrame.DataViewerFrame(self)

        splitter.addWidget(self.tree)
        splitter.addWidget(self.viewer)
        splitter.setStretchFactor(1, 1)

        #
        # EXERCISE: Connect the callback onTreeActivated (bellow)
        #           to a mouse event from the tree
        #

        self.tree.activated.connect(self.onTreeActivated)

        return splitter

    def onTreeActivated(self):

        #
        # EXERCISE: Reach selected objects from the tree
        #

        selectedObjects = list(self.tree.selectedH5Nodes())

        #
        # EXERCISE: Provide it to the data viewer
        #

        if len(selectedObjects) == 0:
            self.viewer.setData("Nothing selected")
        elif len(selectedObjects) > 1:
            self.viewer.setData("Too much things selected")
        obj = selectedObjects[0]
        self.viewer.setData(obj)

    def appendFile(self, filename):
        model = self.tree.findHdf5TreeModel()
        model.insertFile(filename)
        print("Load %s" % filename)

def main_ex3():
    from silx.gui import qt
    viewer = ViewerEx3()
    viewer.appendFile('data/ID16B_diatomee.h5')
    viewer.setVisible(True)
    return viewer

###############################################################

class ViewerEx4(ViewerEx3):

    def onTreeActivated(self):
        selectedObjects = list(self.tree.selectedH5Nodes())
        if len(selectedObjects) == 0:
            self.viewer.setData("Nothing selected")

        elif len(selectedObjects) > 1:
            self.viewer.setData("Too much things selected")

        else:
            obj = selectedObjects[0]
            node = obj.h5py_object

            if "/data/" in node.name:
                # That's a data from the /data group
                data = self.computeCorrectedImage(node)
                self.viewer.setData(data)
            else:
                # Other data is displayed in a normal way
                self.viewer.setData(obj)

    def computeCorrectedImage(self, h5data):
        """
        :param h5data: H5py dataset selected from the group /data/
        """
        background = self.getBackground(h5data)
        flatfield = self.getFlatField(h5data)

        raw = numpy.array(h5data, dtype=numpy.float32)
        flatfield = numpy.array(flatfield, dtype=numpy.float32)
        background = background[...]
        return (raw - background) / (flatfield - background)

    def getBackground(self, h5data):
        """
        :param h5data: H5py dataset selected from the group /data/
        """

        #
        # EXERCISE: Return the background image from the dataset
        #

        return h5data.file["/background/0000"][...]

    def getFlatField(self, h5data):
        """
        :param h5data: H5py dataset selected from the group /data/
        """

        #
        # EXERCISE: Return the flatfield image from the dataset
        #
        #           1) you can return a flatfield by default
        #           2) you can return the closest flat field according to the index of the data
        #           3) you can return an interpolation of the 2 flatfields according to the index of the data

        # return self.getFlatField1(h5data)
        # return self.getFlatField2(h5data)
        return self.getFlatField3(h5data)

    def getFlatField1(self, h5data):
        return h5data.file["/flatfield/0000"][...]

    def getFlatField2(self, h5data):
        index = int(h5data.name.split("/")[-1])
        if index < 250:
            return h5data.file["/flatfield/0000"][...]
        else:
            return h5data.file["/flatfield/0500"][...]

    def getFlatField3(self, h5data):
        index = int(h5data.name.split("/")[-1])
        flatfield0 = h5data.file["/flatfield/0000"][...]
        flatfield500 = h5data.file["/flatfield/0500"][...]
        coef = index / 500.0
        return flatfield0 * (1.0 - coef) + flatfield500 * coef

def main_ex4():
    viewer = ViewerEx4()
    viewer.appendFile('data/ID16B_diatomee.h5')
    viewer.setVisible(True)
    return viewer

###############################################################

if __name__ == "__main__":
    def raise_param_error():
        raise Exception('One an only one argument is expected (ex1, ex2, ex3, or ex4). ')

    import sys
    if len(sys.argv) != 2:
        raise_param_error()

    from silx.gui import qt
    app = qt.QApplication([])

    arg = sys.argv[1]
    if arg == 'ex1':
        mem = main_ex1()
    elif arg == 'ex2':
        mem = main_ex2()
    elif arg == 'ex3':
        mem = main_ex3()
    elif arg == 'ex4':
        mem = main_ex4()
    else:
        raise_param_error()

    app.exec_()


#!/usr/bin/env python
# coding: utf-8

# # Recomendation system : Book Recommendation

# ### Oleh : [Ahmad Habib Husaini](https://www.linkedin.com/in/ahmad-habib-husaini-1705711b0/)
#
# #### Pendahuluan
# Pada proyek ini, topik yang dibahas adalah mengenai pembuatan sistem rekomendasi dengan data book . Proyek ini dibuat untuk proyek Submission 2 - Machine Learning Terapan Dicoding.

# # 1. Import important package

# In[1]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import opendatasets
import wget
import zipfile
from tqdm import tqdm
import os

# # 2. Data loading

# ## 2.1 Data Acquisition

# In[2]:

if os.path.exists('book-recommendation-dataset'):
    print("file sudah ada")
else:
    opendatasets.download_kaggle_dataset(dataset_url='https://www.kaggle.com/arashnic/book-recommendation-dataset', data_dir='')

# Pada tahapan *data loading* berisikan akuisisi data dengan mengunduh dataset pada [link](https://www.kaggle.com/arashnic/book-recommendation-dataset) dengan menggunakan library `opendatasets` dengan sintaks seperti diatas. Ketika merunning code tersebut akan diminta mengisikan username dan key.
# 1. username silahkan disisi dengan username *account* kaggle
# 2. key didapatkan dengan :
#     1. Buka website [kaggle](https://www.kaggle.com/),
#     2. login dengan akun masing-masing.
#     3. Pilih your profile pada kanan atas
#     4. Pilih *account*
#     5. Scroll sedikit kebawah maka akan ada pilihan `Create new API token`
#

#     6. Setelah menekan pilihan tersebut akan terunduh file kaggle.json yang berisikan username dan key

# ## 2.2 Memuat data dalam format Dataframe

# ### 2.2.1 Book Dataframe

# In[3]:

df_book = pd.read_csv('book-recommendation-dataset/Books.csv')
df_book.head(3)

# ### 2.2.2 Rating Dataframe

# In[4]:

df_rating = pd.read_csv('book-recommendation-dataset/Ratings.csv')
df_rating.head(3)

# ### 2.2.4 User Dataframe

# In[5]:

df_user = pd.read_csv('book-recommendation-dataset/Users.csv')
df_user.tail(3)

# # 3. Data Understanding

# ## 3.1 Informasi dataset

# ![image/kaggle_dataset.png](attachment:image.png)

# Pada berkas yang dapat diunduh pada link [berikut](https://www.kaggle.com/arashnic/book-recommendation-dataset?select=Users.csv) berisikan dataset berjumlah 3 buah file yakni Book.csv, Rating.csv dan User.csv
#

# ### 3.1.1 Book
#
# Berisikan data sebanyak 271360 baris dan 8 kolom yang terdiri dari:

#
# Kolom  | Keterangan
# :------------- | :-------------
# Sumber  | https://www.kaggle.com/arashnic/book-recommendation-dataset?select=Books.csv
# ISBN | International Standard Book Number merupakan kode unik masing masing buku
# Book-title | Merupakan judul dari tiap buku
# Book-Author | Merupakan penulis atau pengarang tiap buku
# Year-Of_Publication |Merupakan tahun terbit tiap buku
# Publisher | Merupakan lebaga penerbit tiap buku
# Image-URL-S | Link foto dari tiap buku berukuran kecil
# Image-URL-M | Link foto dari tiap buku berukuran sedang
# Image-URL-L | Link foto dari tiap buku berukuran besar

# ### 3.1.2 Rating
# Berisikan data sebanyak 1149780 baris dan 3 kolom yang terdiri dari:
#
# Kolom  | Keterangan
# :------------- | :-------------
# Sumber  | https://www.kaggle.com/arashnic/book-recommendation-dataset?select=Ratings.csv
# User-ID | Merupakan kode unik tiap user
# ISBN | International Standard Book Number merupakan kode unik masing masing buku
# Book-Rating |Merupakan rating dari tiap buku

# ### 3.1.3 User
#
# Berisikan data sebanyak 278858 baris dan 4 kolom yang terdiri dari:

#
# Kolom  | Keterangan
# :------------- | :-------------
# Sumber  | https://www.kaggle.com/arashnic/book-recommendation-dataset?select=Users.csv
# User-ID | Merupakan kode unik tiap user
# Age | Merupakan umur tiap user
# Location |Merupakan lokasi tiap user

# ## 3.2 Cek Missing Values, tipe data & summary statistical descriptive

# ### 3.2.1 Book

# In[6]:

pd.DataFrame(df_book.isnull().sum(), columns=['missing_values'])

# In[7]:

df_book.info()

# In[8]:

df_book.loc[:,:'Publisher'].describe(include=object)

# ### 3.2.2 Rating

# In[9]:

pd.DataFrame(df_rating.isnull().sum(), columns=['missing_values'])

# In[10]:

df_rating.info()

# In[11]:

df_rating[['Book-Rating']].describe()

# In[12]:

df_rating['Book-Rating'] = df_rating['Book-Rating'].astype(object)

# In[13]:

df_rating.describe(include=[object])

# In[14]:

df_rating.info()

# ### 3.2.3 User

# In[15]:

df_user.head()

# In[16]:

df_user.Location.unique()

# In[17]:

df_user.Location.str.split(',',expand=True).loc[:,:2][0].unique()

# In[18]:

len(df_user.Location.str.split(',',expand=True).loc[:,:2][0].unique())

# In[19]:

df_user.Location.str.split(',',expand=True).loc[:,:2][1].unique()

# In[20]:

len(df_user.Location.str.split(',',expand=True).loc[:,:2][1].unique())

# In[21]:

df_user['city'] = df_user.Location.str.split(',',expand=True)[0]
# df_user['nation_state'] = df_user.Location.str.split(',',expand=True)[1]
df_user['country'] = df_user.Location.str.split(',',expand=True)[2]
df_user.head()

# In[22]:

pd.DataFrame(df_user.isnull().sum(), columns=['missing_values'])

# In[23]:

df_user.info()

# In[24]:

pd.set_option('display.max_rows',105)

# In[25]:

df_user.groupby('country')[['country']].count().head(10)

# In[26]:

df_user.drop(columns=['country'], inplace=True)

# In[27]:

df_user[['Age']].describe()

# In[28]:

df_user.describe(include=object)

# `Kesimpulan dari Tahapan ini :`
# 1. Book
#     - Buku dengan judul terbanyak adalah Selected Poems dengan jumlah 27 buah
#     - Penulis terbanyak adalah Agatha Christie dengan total 632 karya
#     - Buku terbitan tahun 2002 menjadi buku terbanyak
#     - Lembaga penerbit yang paling banayk menerbitkan buku adalah Harlequin
# 2. Rating
#     - masih banyak buku yang belum dirating, yakni sebanyak 716109
# 3. User
#     - Banyak data umur yang kosong kemungkinan tidak akan digunakan
#     - london menjadi kota user terbanyak sebanyak 139187 user

# ## 3.3 Visualisasi Data

# ### 3.3.1 Book

# In[29]:

count_book = {key: len(df_book[key].unique()) for key in df_book.columns[1:5]}

# In[30]:

plt.figure(figsize=(15,9))
ax = sns.barplot(x=list(count_book.keys()), y=list(count_book.values()))
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Banyak Data Unik Tiap Kolom Dataset Book")
plt.show()

# ### 3.3.2 Rating

# In[31]:

count_rating = {key: len(df_rating[key].unique()) for key in df_rating.columns}

# In[32]:

plt.figure(figsize=(15,9))
ax = sns.barplot(x=list(count_rating.keys()), y=list(count_rating.values()))
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Banyak Data Unik Tiap Kolom Dataset Rating")
plt.show()

# In[33]:

plt.figure(figsize=(15,9))
ax = sns.countplot(data=df_rating, x='Book-Rating')
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Banyak Data Unik Kolom Rating")
plt.show()

# ### 3.3.3 Users

# In[34]:

df_user.head()

# In[35]:

count_users = {key: len(df_user[key].unique()) for key in df_user.columns}

# In[36]:

plt.figure(figsize=(12,9))
ax = sns.barplot(x=list(count_users.keys()), y=list(count_users.values()))
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Banyak Data Unik Tiap Kolom Dataset User")
plt.show()

# In[37]:

city_top10 = df_user.city.value_counts().head(10).reset_index()
city_top10.columns = ['city', 'count']
city_top10

# In[38]:

plt.figure(figsize=(12,9))
ax = sns.barplot(x=city_top10.city, y=city_top10['count'])
for p in ax.patches:
        ax.annotate(format(p.get_height(), '.1f'), (p.get_x() + p.get_width() / 2., p.get_height()), ha = 'center', va = 'center', xytext = (0, 10), textcoords = 'offset points')
ax.set_title("Top 10 Kota Terbanyak")
plt.show()

# # 4. Data Preparation & Data Preprocessing

# ## 4.1 Memilih fitur untuk sistem rekomendasi

# In[39]:

df_fix = df_book.iloc[:,0:4]
df_fix.head()

# In[40]:

pd.DataFrame({
    'missing_values':df_fix.isnull().sum(),
    'presentase':df_fix.isnull().sum()/len(df_fix)
})

# ## 4.2 Hapus Missing Value dan data duplicate pada dataset fix

# In[41]:

df_fix.dropna(inplace=True)
df_fix.drop_duplicates(inplace=True)

# In[42]:

df_fix.shape

# ## 4.3 Create Metadata

# In[43]:

df_fix['Year-Of-Publication'] = df_fix['Year-Of-Publication'].astype('str')

# In[44]:

df_fix['metadata'] = ""
for col in ['Book-Title', 'Book-Author', 'Year-Of-Publication']:
    df_fix['metadata'] += df_fix[col]
    df_fix['metadata'] += " "

# In[45]:

df_fix.tail()

# ## 4.4 Encoding dengan TF-IDF

# In[46]:

from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer

# In[47]:

bow = CountVectorizer()
bank = bow.fit_transform(df_fix.metadata)

# In[48]:

tfidf = TfidfVectorizer()
bank2 = tfidf.fit_transform(df_fix.metadata)

# # 5. Modeling

# ## 5.1 Encoding metadata index

# In[49]:

index = 0
content = df_fix.loc[index, 'metadata']
content

# In[50]:

code1 = bow.transform([content])
code1

# In[51]:

code2 = tfidf.transform([content])

# ## 5.2 Document Search

# In[52]:

from sklearn.metrics.pairwise import cosine_distances

# In[53]:

distance1 = cosine_distances(code1, bank)
distance1

# In[54]:

rec_idx = distance1.argsort()[0, 1:]
rec_idx

# In[55]:

df_fix.loc[rec_idx[:5], :]

# In[56]:

distance2 = cosine_distances(code2, bank)
distance2

# In[57]:

rec_idx2 = distance2.argsort()[0, 1:]
rec_idx2

# In[58]:

df_fix.loc[rec_idx2[:10], :]

# ## 5.3 Bungkus code agar rapih

# In[59]:

class RecommendationSystems():
    def __init__(self,data, metadata_col):
        self.df = data
        self.metadata = metadata_col
        self.encoder = None
        self.bank = None

    def fit(self, encoder='tfidf'):
        if encoder not in ['bow', 'tfidf']:
            print("hanya support BoW dan TF-IDF")
        else:
            self.encoder = TfidfVectorizer()
            if encoder=='bow':
                self.encoder = CountVectorizer()
            self.bank = self.encoder.fit_transform(self.df[self.metadata])
    def recommender(self, index, topn=10):
        content = self.df.loc[index, self.metadata]
        code = self.encoder.transform([content])
        distance = cosine_distances(code, self.bank)
        rec_idx = distance.argsort()[0, 1:]
        return self.df.loc[rec_idx[:topn], :]

# In[60]:

recsys = RecommendationSystems(df_fix, 'metadata')
recsys.fit()

# In[61]:

rec = recsys.recommender(100)
rec

# In[62]:

df_fix.loc[100,:]

# In[63]:

recsys2 = RecommendationSystems(df_fix, 'metadata')
recsys2.fit('bow')

# In[64]:

rec2 = recsys2.recommender(100)
rec2

# ## 5.4 Scoring

# ## 5.4.1 Try

# In[65]:

from sklearn.metrics.pairwise import cosine_similarity
import numpy as np

# In[66]:

book = pd.DataFrame(dict(df_fix.loc[100,:]), index=[0])
book

# In[67]:

rec2

# In[68]:

np.mean(cosine_similarity(bow.transform(book.metadata.values), bow.transform(rec.metadata)))

# In[69]:

np.mean(cosine_similarity(tfidf.transform(book.metadata.values), tfidf.transform(rec.metadata)))

# ## 5.4.2 Bungkus kembali kedalam class

# In[70]:

class RecommendationSystems():
    def __init__(self,data, metadata_col):
        self.df = data
        self.metadata = metadata_col
        self.encoder = None
        self.bank = None
        self.content = None
        self.index = None
        self.topn = None

    def fit(self, encoder='tfidf'):
        if encoder not in ['bow', 'tfidf']:
            print("hanya support BoW dan TF-IDF")
        else:
            self.encoder = TfidfVectorizer()
            if encoder=='bow':
                self.encoder = CountVectorizer()
            self.bank = self.encoder.fit_transform(self.df[self.metadata])
    def recommender(self, index, topn=10, include_book_search=False):
        self.index = index
        self.topn = topn
        self.content = self.df.loc[index, self.metadata]
        code = self.encoder.transform([self.content])
        distance = cosine_distances(code, self.bank)
        rec_idx = distance.argsort()[0, :]
        if include_book_search:
            topn+=1
            return self.df.loc[rec_idx[:topn], :]
        return self.df.loc[rec_idx[1:topn], :]

    def score(self):
        rec = self.recommender(index=self.index, topn=self.topn, include_book_search=True)
        bank = self.encoder.transform(rec.metadata)
        code = self.encoder.transform([self.content])
        return np.mean(cosine_similarity(code, bank)[:,1:])

# In[71]:

recsys3 = RecommendationSystems(df_fix, 'metadata')
recsys3.fit()
recsys3.recommender(index=100, include_book_search=True)

# In[72]:

recsys3.score()

# In[73]:

import time

# In[74]:

start = time.time()
recsys4 = RecommendationSystems(df_fix, 'metadata')
recsys4.fit('bow')
recsys4.recommender(index=100)
stop = time.time()
print(f"Training time: {stop - start} s")
print("Score : ",recsys4.score())

# ## 5.4.3 Scoring Model dengan *Bag Of Word* top 5 rekomendasi

# In[75]:

recsys = RecommendationSystems(data=df_fix, metadata_col='metadata')
recsys.fit(encoder='bow')

# In[76]:

recsys.recommender(index=100, topn=5, include_book_search=True)

# In[77]:

recsys.score()

# In[78]:

recsys.recommender(index=1000, topn=5, include_book_search=True)

# In[79]:

recsys.score()

# ## 5.4.4 Scoring Model Tf-idf

# In[80]:

recsys = RecommendationSystems(data=df_fix, metadata_col='metadata')
recsys.fit(encoder='tfidf')

# In[81]:

recsys.recommender(index=100, topn=5, include_book_search=True)

# In[82]:

recsys.score()

# In[83]:

recsys.recommender(index=1000, topn=5, include_book_search=True)

# In[84]:

recsys.score()

# # 6 Evaluasi

# ## 6.1 Generate data yang akan dijadikan sampel sebanyak 500 data

# In[85]:

np.random.randint(0,10, 5)

# In[86]:

idx_sample_list = np.random.randint(0,len(df_fix), 500)
idx_sample_list

# In[87]:

len(idx_sample_list)

# In[88]:

idx_sample_list2 = [i for i in np.random.randint(0,len(df_fix), 500) if i not in idx_sample_list]
len(idx_sample_list2)

# In[89]:

from tqdm import tqdm

# ## 6.2 Define Loop Function

# In[90]:

def loop_rec(data, metadata_col='metadata', encoder='tfidf', sample_list=[0,2,10],topn=10, return_dict=False):
    score_list = []
    score_dict = {}
    idx_done = []
    recsys = RecommendationSystems(data, metadata_col)
    recsys.fit(encoder)
    for i in tqdm(sample_list):
        while True:
            idx = np.random.randint(low=0, high=len(data))
            if idx not in idx_done:
                idx_done.append(idx)
                break
        recsys.recommender(index=idx, topn=topn)
        score = recsys.score()
        score_list.append(score)
        score_dict[idx] = score
    if return_dict:
        return score_dict, np.mean(score_list)
    return np.mean(score_list)

# ## 6.3 Model Bag-of-word

# In[91]:

dict_score_bow, score_bow = loop_rec(data=df_fix,
                                     metadata_col='metadata',
                                     encoder='bow',
                                     topn=5,
                                     sample_list=idx_sample_list,
                                     return_dict=True)

# In[92]:

score_bow

# In[93]:

dict_score_bow2, score_bow2 = loop_rec(data=df_fix,
                                     metadata_col='metadata',
                                     encoder='bow',
                                     topn=5,
                                     sample_list=idx_sample_list2,
                                     return_dict=True)

# In[94]:

score_bow2

# ## 6.4 Model Tf-idf

# In[95]:

dict_score_tfidf, score_tfidf = loop_rec(data=df_fix,
                                     metadata_col='metadata',
                                     encoder='tfidf',
                                     topn=5,
                                     sample_list=idx_sample_list,
                                     return_dict=True)

# In[96]:

score_tfidf

# In[97]:

dict_score_tfidf2, score_tfidf2 = loop_rec(data=df_fix,
                                     metadata_col='metadata',
                                     encoder='tfidf',
                                     topn=5,
                                     sample_list=idx_sample_list2,
                                     return_dict=True)

# In[98]:

score_tfidf2

# In[100]:

pd.DataFrame({
    'Tf-Idf':[score_tfidf, score_tfidf2],
    'BoW':[score_bow, score_bow2]
})

# In[ ]:


'''
jplot
Basic plot module for Colaboratory (Jupyter)
Compatible with Python 2.7 and 3.x
Includes funtions related for drawing curves

History:
  11/03/2018 : First version
  13/03/2018 : Add version string
'''

from __future__ import print_function

import numpy as np               # Import numpy for numeric calculations
import pylab as pl               # Import pylab
import matplotlib.pyplot as plt

# Version string
version = '13/3/2018'

#########################################################################################
# DRAWING CODE                                                                          #
#########################################################################################

# Internal functions ####################################################################

'''
_plotStart
Starts a new plot
Paramenters:
  title : Title of the plot (defaults to none)
  xt    : x label of the plot (defaults to none)
  yt    : y label of the plot (defaults to none)
  grid  : Determines if there is grid (defaults to True)
Returns:
  fig : Figure object
  ax  : Axes object
'''
def _plotStart(title="",xt="",yt="",grid=True):
    fig=plt.figure()
    ax = fig.add_subplot(111)
    ax.set_facecolor("white")
    ax.set_title(title)
    ax.set_xlabel(xt)
    ax.set_ylabel(yt)
    if (grid):
        plt.grid(True,color="lightgrey",linestyle='--')
    return fig,ax

'''
_plotEnd
Ends a previously started plot
Paramenters:
  fig      : Figure object obtained from plotStart
  ax       : Axes obtained from plotStart
  labels   : List of labels for the curves (defaults to none)
  location : Location for labels (defaults to 'best')
Returns nothing
'''
def _plotEnd(fig,ax,labels=[],location='best'):
    if not labels == []:
        pl.legend(loc=location)
    xmin, xmax = plt.xlim()
    ymin, ymax = plt.ylim()
    ax.axvline(x=xmin,linewidth=2, color='black')
    ax.axvline(x=xmax,linewidth=2, color='black')
    ax.axhline(y=ymin,linewidth=2, color='black')
    ax.axhline(y=ymax,linewidth=2, color='black')
    plt.show()

'''
_plotXY
Plot two magnitudes using log if needed
Used by the plot11, plot1n and plotnn commands
'''
def _plotXY(x,y,label="",logx=False,logy=False):
    if not logx and not logy:
        pl.plot(x,y,label=label)
        return
    if logx and not logy:
        pl.semilogx(x,y,label=label)
        return
    if logy and not logx:
        pl.semilogy(x,y,label=label)
        return
    if logx and logy:
        pl.loglog(x,y,label=label)
        return

# Public functions ######################################################################

'''
@plot11@
plot11(x,y,title,xt,yt,logx,logy)
Plot one input against one output
If x is an empty list [], a sequence number
will be used for the x axis

Required parameters:
  x : Horizontal vector
  y : Vertical vector

Optional parameters:
  title : Plot title (Defaults to none)
     xt : Label for x axis (Defaults to none)
     yt : Label for y axis (Defaults to none)
   logx : Use logarithmic x axis (Defaults to False)
   logy : Use logarithmic x axis (Defaults to False)
   grid : Use grid (Default to True)

Returns nothing
'''
def plot11(x,y,title="",xt="",yt="",logx=False,logy=False,grid=True):

    # Generate sequence if x is not provided
    if x == []:
        x = np.arange(0,len(y))

    fig,ax = _plotStart(title,xt,yt,grid)

    _plotXY(x,y,logx=logx,logy=logy)

    _plotEnd(fig,ax)

'''
@plot1n@
plot1n(x,ylist,title,xt,yt,labels,location,logx,logy)
Plot one input against several outputs
If x is an empty list [], a sequence number
will be used for the x axis

Required parameters:
      x : Horizontal vector
  ylist : List of vertical vectors

Optional parameters:
    title : Plot title (Defaults to none)
       xt : Label for x axis (Defaults to none)
       yt : Label for y axis (Defaults to none)
   labels : List of legend labels (Defaults to none)
 location : Location for legend (Defaults to 'best')
     logx : Use logarithmic x axis (Defaults to False)
     logy : Use logarithmic x axis (Defaults to False)
     grid : Use grid (Default to True)

Returns nothing
'''
def plot1n(x,ylist,title="",xt="",yt="",labels=[],location='best',logx=False,logy=False,grid=True):

    # Generate sequence is x is not provided
    if x == []:
        x = np.arange(0,len(ylist[0]))

    fig,ax=_plotStart(title,xt,yt,grid)

    if labels == []:
        for y in ylist:
            _plotXY(x,y,logx=logx,logy=logy)
    else:
        for y,lbl in zip(ylist,labels):
            _plotXY(x,y,label=lbl,logx=logx,logy=logy)

    _plotEnd(fig,ax,labels,location)

'''
@plotnn@
plotnn(xlist,ylist,title,xt,yt,labels,location,logx,logy)
Plot several curves with different inputs and outputs

Required parameters:
  xlist : List of horizontal vector
  ylist : List of vertical vectors

Optional parameters:
    title : Plot title (Defaults to none)
       xt : Label for x axis (Defaults to none)
       yt : Label for y axis (Defaults to none)
   labels : List of legend labels (Defaults to none)
 location : Location for legend (Defaults to 'best')
     logx : Use logarithmic x axis (Defaults to False)
     logy : Use logarithmic x axis (Defaults to False)
     grid : Use grid (Default to True)

Returns nothing
'''
def plotnn(xlist,ylist,title="",xt="",yt="",labels=[],location='best',logx=False,logy=False,grid=True):

    fig,ax=_plotStart(title,xt,yt,grid)

    if labels == []:
        for x,y in zip(xlist,ylist):
            _plotXY(x,y,logx=logx,logy=logy)
    else:
        for x,y,lbl in zip(xlist,ylist,labels):
            _plotXY(x,y,label=lbl,logx=logx,logy=logy)

    _plotEnd(fig,ax,labels,location)

'''
@plotHist@
plotHist(v,bins=10,title="",xt="",yt="",grid)
Plot an histagram from provided data

Required parameters:
  v : Data vector

Optional parameters:
     bins : Number of bins for the histogram (Defaults to 10)
    title : Plot title (Defaults to none)
       xt : Label for x axis (Defaults to none)
       yt : Label for y axis (Defaults to none)
     grid : Use grid (Default to True)

Returns nothing
'''
def plotHist(v,bins=10,title="",xt="",yt="",grid=True):

    fig,ax = _plotStart(title,xt,yt,grid)

    plt.hist(v,bins)

    _plotEnd(fig,ax)


#!/bin/python

from __future__ import division
from __future__ import print_function
from __future__ import absolute_import

import numpy as np
import sys

# Python 3 backwards compatibility tricks
if sys.version_info.major > 2:

    def xrange(*args, **kwargs):
        return iter(range(*args, **kwargs))

    def unicode(*args, **kwargs):
        return str(*args, **kwargs)

def textToTokens(text):
    """Converts input string to a corpus of tokenized sentences.

    Assumes that the sentences are divided by newlines (but will ignore empty sentences).
    You can use this to try out your own datasets, but is not needed for reading the homework data.
    """
    corpus = []
    sents = text.split("\n")
    from sklearn.feature_extraction.text import CountVectorizer
    count_vect = CountVectorizer()
    count_vect.fit(sents)
    tokenizer = count_vect.build_tokenizer()
    for s in sents:
        toks = tokenizer(s)
        if len(toks) > 0:
            corpus.append(toks)
    return corpus

def file_splitter(filename, seed = 0, train_prop = 0.7, dev_prop = 0.15,
    test_prop = 0.15):
    """Splits the lines of a file into 3 output files."""
    import random
    rnd = random.Random(seed)
    basename = filename[:-4]
    train_file = open(basename + ".train.txt", "w")
    test_file = open(basename + ".test.txt", "w")
    dev_file = open(basename + ".dev.txt", "w")
    with open(filename, 'r') as f:
        for l in f.readlines():
            p = rnd.random()
            if p < train_prop:
                train_file.write(l)
            elif p < train_prop + dev_prop:
                dev_file.write(l)
            else:
                test_file.write(l)
    train_file.close()
    test_file.close()
    dev_file.close()

def read_texts(tarfname, dname):
    """Read the data from the homework data file.

    Given the location of the data archive file and the name of the
    dataset (one of brown, reuters, or gutenberg), this returns a
    data object containing train, test, and dev data. Each is a list
    of sentences, where each sentence is a sequence of tokens.
    """
    import tarfile
    tar = tarfile.open(tarfname, "r:gz", errors = 'replace')
    train_mem = tar.getmember(dname + ".train.txt")
    train_txt = unicode(tar.extractfile(train_mem).read(), errors='replace')
    test_mem = tar.getmember(dname + ".test.txt")
    test_txt = unicode(tar.extractfile(test_mem).read(), errors='replace')
    dev_mem = tar.getmember(dname + ".dev.txt")
    dev_txt = unicode(tar.extractfile(dev_mem).read(), errors='replace')

    from sklearn.feature_extraction.text import CountVectorizer
    count_vect = CountVectorizer()
    count_vect.fit(train_txt.split("\n"))
    tokenizer = count_vect.build_tokenizer()
    class Data: pass
    data = Data()
    data.train = []
    for s in train_txt.split("\n"):
        toks = tokenizer(s)
        if len(toks) > 0:
            data.train.append(toks)
    data.test = []
    for s in test_txt.split("\n"):
        toks = tokenizer(s)
        if len(toks) > 0:
            data.test.append(toks)
    data.dev = []
    for s in dev_txt.split("\n"):
        toks = tokenizer(s)
        if len(toks) > 0:
            data.dev.append(toks)
    print(dname," read.", "train:", len(data.train), "dev:", len(data.dev), "test:", len(data.test))
    return data

def learn_unigram(data):
    """Learns a unigram model from data.train.

    It also evaluates the model on data.dev and data.test, along with generating
    some sample sentences from the model.
    """
    from lm import Unigram
    unigram = Unigram()
    unigram.fit_corpus(data.train)
    print("vocab:", len(unigram.vocab()))
    # evaluate on train, test, and dev
    print("train:", unigram.perplexity(data.train))
    print("dev  :", unigram.perplexity(data.dev))
    print("test :", unigram.perplexity(data.test))
    from generator import Sampler
    sampler = Sampler(unigram)
    print("sample: ", " ".join(str(x) for x in sampler.sample_sentence([])))
    print("sample: ", " ".join(str(x) for x in sampler.sample_sentence([])))
    print("sample: ", " ".join(str(x) for x in sampler.sample_sentence([])))
    return unigram

def print_table(table, row_names, col_names, latex_file = None):
    """Pretty prints the table given the table, and row and col names.

    If a latex_file is provided (and tabulate is installed), it also writes a
    file containing the LaTeX source of the table (which you can \\input into your report)
    """
    try:
        from tabulate import tabulate
        rows = [*map(lambda rt: [rt[0]] + rt[1], zip(row_names,table.tolist()))]
        print(tabulate(rows, headers = [""] + col_names))
        if latex_file is not None:
            latex_str = tabulate(rows, headers = [""] + col_names, tablefmt="latex")
            with open(latex_file, 'w') as f:
                f.write(latex_str)
                f.close()
    except ImportError as e:
        row_format ="{:>15} " * (len(col_names) + 1)
        print(row_format.format("", *col_names))
        for row_name, row in zip(row_names, table):
            print(row_format.format(row_name, *row))

if __name__ == "__main__":
    # Do no run, the following function was used to generate the splits
    # file_splitter("data/reuters.txt")

    dnames = ["brown", "reuters", "gutenberg"]
    datas = []
    models = []
    # Learn the models for each of the domains, and evaluate it
    for dname in dnames:
        print("-----------------------")
        print(dname)
        data = read_texts("data/corpora.tar.gz", dname)
        datas.append(data)
        model = learn_unigram(data)
        models.append(model)
    # compute the perplexity of all pairs
    n = len(dnames)
    perp_dev = np.zeros((n,n))
    perp_test = np.zeros((n,n))
    perp_train = np.zeros((n,n))
    for i in range(n):
        for j in range(n):
            perp_dev[i][j] = models[i].perplexity(datas[j].dev)
            perp_test[i][j] = models[i].perplexity(datas[j].test)
            perp_train[i][j] = models[i].perplexity(datas[j].train)

    print("-------------------------------")
    print("x train")
    print_table(perp_train, dnames, dnames, "table-train.tex")
    print("-------------------------------")
    print("x dev")
    print_table(perp_dev, dnames, dnames, "table-dev.tex")
    print("-------------------------------")
    print("x test")
    print_table(perp_test, dnames, dnames, "table-test.tex")


# -*- coding:utf-8 -*-
"""
Project   : numpy
File Name : 13_to_14_string_equal
Author    : Focus
Date      : 8/23/2021 9:19 AM
Keywords  :
Abstract  :
Param     :
Usage     : py 13_to_14_string_equal
Reference :
"""
import numpy as np
# import matplotlib.pyplot as plt
# import sys

print("Pass Case: ")
print(np.testing.assert_string_equal("NumPy", "NumPy"))
print("Fail Case:")
print(np.testing.assert_string_equal("Numpy", "numpy"))

#>> Traceback (most recent call last):
#  File "C:\work\code\python\numpy\8_assure_quality_with_testing\13_to_14_string_equal.py", line 21, in <module>
#     print(np.testing.assert_string_equal("Numpy", "numpy"))
#   File "C:\work\code\python\numpy\venv\lib\site-packages\numpy\testing\_private\utils.py", line 1206, in assert_string_equal
#     raise AssertionError(msg)
#  AssertionError: Differences in strings:
#  - Numpy? ^
#  + numpy? ^


r"""
Elements of posets, lattices, semilattices, etc.
"""
#*****************************************************************************
#       Copyright (C) 2008 Peter Jipsen <jipsen@chapman.edu>,
#                          Franco Saliola <saliola@gmail.com>
#
#  Distributed under the terms of the GNU General Public License (GPL)
#
#    This code is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#    General Public License for more details.
#
#  The full text of the GPL is available at:
#
#                  http://www.gnu.org/licenses/
#*****************************************************************************
from sage.structure.element import Element
from sage.structure.element import have_same_parent

class PosetElement(Element):

    def __init__(self, poset, element, vertex):
        r"""
        Establishes the parent-child relationship between ``poset``
        and ``element``, where ``element`` is associated to the
        vertex ``vertex`` of the Hasse diagram of the poset.

        INPUT:

        - ``poset`` - a poset object

        - ``element`` - any object

        - ``vertex`` - a vertex of the Hasse diagram of the poset

        TESTS::

            sage: from sage.combinat.posets.elements import PosetElement
            sage: P = Poset([[1,2],[4],[3],[4],[]], facade = False)
            sage: e = P(0)
            sage: e.parent() is P
            True
            sage: TestSuite(e).run()
        """
        Element.__init__(self, poset)
        if isinstance(element, self.parent().element_class):
            self.element = element.element
        else:
            self.element = element
        self.vertex = vertex

    def __hash__(self):
        r"""
        TESTS::

            sage: P = Poset([[1,2],[4],[3],[4],[]], facade = False)
            sage: e = P(0)
            sage: hash(e)
            0
        """
        return hash(self.element)

    def _repr_(self):
        """
        TESTS::

            sage: Poset([[1,2],[4],[3],[4],[]], facade = False)(0)._repr_()
            '0'
        """
        return "%s" %str(self.element)

    def _latex_(self):
        r"""
        Return the latex code of the poset element.

        EXAMPLES::

            sage: m = matrix(2,[1,2,3,4])
            sage: m.set_immutable()
            sage: P = Poset(([m],[]), facade = False)
            sage: [e] = P
            sage: type(e)
            <class 'sage.combinat.posets.elements.FinitePoset_with_category.element_class'>
            sage: latex(e)                 #indirect doctest
            \left(\begin{array}{rr}
            1 & 2 \\
            3 & 4
            \end{array}\right)
        """
        from sage.misc.latex import latex
        return latex(self.element)

    def __eq__(self,other):
        """
        TESTS::

            sage: P = Poset([["a","b"],["d"],["c"],["d"],[]], facade = False)
            sage: Q = Poset([["a","b"],["d"],["c"],[],[]], facade = False)
            sage: P(0).__eq__(P(4))
            False
            sage: from sage.combinat.posets.elements import PosetElement
            sage: PosetElement(P,0,"c") == PosetElement(P,0,"c")
            True
            sage: PosetElement(P,0,"c") == PosetElement(Q,0,"c")
            False
            sage: PosetElement(P,0,"b") == PosetElement(P,0,"c")
            False

        .. warning:: as an optimization, this only compares the parent
           and vertex, using the invariant that, in a proper poset
           element, ``self.element == other.element`` if and only
           ``self.vertex == other.vertex``::

            sage: PosetElement(P,1,"c") == PosetElement(P,0,"c")
            True

        Test that :trac:`12351` is fixed::

            sage: P(0) == int(0)
            False
        """
        # This should instead exploit unique representation, using
        # self is other, or best inherit __eq__ from there. But there
        # are issues around pickling and rich comparison functions.
        return have_same_parent(self, other) \
            and self.vertex == other.vertex

    def __ne__(self,other):
        r"""
        TESTS::

            sage: P = Poset([[1,2],[4],[3],[4],[]])
            sage: P = Poset([["a","b"],["d"],["c"],["d"],[]])
            sage: P(0).__ne__(P(4))
            True
            sage: from sage.combinat.posets.elements import PosetElement
            sage: PosetElement(P,0,"c") != PosetElement(P,0,"c")
            False
            sage: PosetElement(P,0,"b") != PosetElement(P,0,"c")
            True

        For this one, see comment in :meth:`__eq__`::

            sage: PosetElement(P,1,"c") != PosetElement(P,0,"c")
            False
        """
        return not self == other

    def _cmp(self,other):
        """
        TESTS::

            sage: P = Poset([[1,2],[4],[3],[4],[]], facade = False)
            sage: P(0)._cmp(P(4))
            -1
            sage: P(4)._cmp(P(0))
            1
            sage: P(0)._cmp(P(0))
            0
            sage: P(1)._cmp(P(2))

        """
        return self.parent().compare_elements(self,other)

    def __cmp__(self, other):
        r"""
        A default comparison of ``self`` with ``other``.

        .. note::

           The rich comparison methods have been implemented for poset
           elements, so when a user asks for ``x < y``, for example, rich
           comparison is used (that is, ``x.__lt__(y)`` is returned). This
           method is implemented because ``PosetElement`` inherits from
           ``Element``, which requires ``__cmp__`` to enable sorting by the
           ``cmp`` method.

           If both ``self`` and ``other`` have the same parent poset,
           then the comparison is done in the poset. If the elements
           are incomparable in the poset, then 0 is returned. Note that,
           in particular, ``cmp(a,b) == cmp(b,a)`` if ``a`` and ``b`` are
           equal or incomparable in the poset.

        TESTS::

            sage: P = Poset([[1,2],[4],[3],[4],[]], facade = False)
            sage: P(0).__cmp__(P(4))
            -1
            sage: P(4).__cmp__(P(0))
            1
            sage: P(0).__cmp__(P(0))
            0
            sage: P(1).__cmp__(P(2))
            0
            sage: cmp(P(0),P(4))
            -1
            sage: cmp(P(4),P(0))
            1
            sage: cmp(P(0),P(0))
            0
            sage: cmp(P(1),P(2))
            0
            sage: cmp(P(2),P(1))
            0
        """
        if isinstance(other, type(self)):
            r = self.parent().compare_elements(self,other)
            if r is None:
                return 0
            else:
                return r
        else:
            return cmp(type(other), type(self))

    def __lt__(self,other):
        """
        TESTS

        ::

            sage: dag = DiGraph({0:[2,3], 1:[3,4], 2:[5], 3:[5], 4:[5]})
            sage: P = Poset(dag, facade = False)
            sage: P(0) < P(1)
            False
            sage: P(4) < P(1)
            False
            sage: P(0) < P(0)
            False
        """
        return self._cmp(other) == -1 or False

    def __le__(self,other):
        """
        TESTS

        ::

            sage: dag = DiGraph({0:[2,3], 1:[3,4], 2:[5], 3:[5], 4:[5]})
            sage: P = Poset(dag, facade = False)
            sage: P(1) <= P(0)
            False
            sage: P(0) <= P(1)
            False
            sage: P(0) <= P(3)
            True
            sage: P(0) <= P(0)
            True
        """
        return self == other or self._cmp(other) == -1 or False

    def __gt__(self,other):
        """
        TESTS

        ::

            sage: dag = DiGraph({0:[2,3], 1:[3,4], 2:[5], 3:[5], 4:[5]})
            sage: P = Poset(dag)
            sage: P(0).__gt__(P(5))
            False
            sage: P(5).__gt__(P(0))
            True
            sage: P(0).__gt__(P(0))
            False
        """
        return self._cmp(other) == 1 or False

    def __ge__(self,other):
        """
        TESTS

        ::

            sage: dag = DiGraph({0:[2,3], 1:[3,4], 2:[5], 3:[5], 4:[5]})
            sage: P = Poset(dag)
            sage: P(0).__ge__(P(5))
            False
            sage: P(5).__ge__(P(0))
            True
            sage: P(0).__ge__(P(0))
            True
        """
        return self == other or self._cmp(other) == 1 or False

class MeetSemilatticeElement(PosetElement):
    def __mul__(self,other):
        r"""
        Return the meet of ``self`` and ``other`` in the lattice.

        EXAMPLES::

            sage: D = Posets.DiamondPoset(5,facade=False)
            sage: D(1) * D(2)
            0
            sage: D(1) * D(1)
            1
            sage: D(1) * D(0)
            0
            sage: D(1) * D(4)
            1
        """
        return self.parent().meet(self,other)

class JoinSemilatticeElement(PosetElement):
    def __add__(self,other):
        r"""
        Return the join of ``self`` and ``other`` in the lattice.

        EXAMPLES::

            sage: D = Posets.DiamondPoset(5,facade=False)
            sage: D(1) + D(2)
            4
            sage: D(1) + D(1)
            1
            sage: D(1) + D(4)
            4
            sage: D(1) + D(0)
            1
        """
        return self.parent().join(self,other)

class LatticePosetElement(MeetSemilatticeElement,JoinSemilatticeElement):
    pass


import numpy as np
import pandas as pd

#Check a dataframe for nulls, print/report them in a nice "pretty" format

def check_nulls(my_df):

    new_df = my_df.copy()

    return new_df.isna()

if __name__ == "__main__":
    #Creating a dictionary
    dict = {'Name': ['John', 'Peter', 'Sam', np.nan],
            'Age': [30, np.nan, np.nan, 45],
            'Income': [100000, 75000, np.nan, 50000]}

    #Creating a DataFrame from list
    df = pd.DataFrame(dict)
    print(df.head())

    df2 = check_nulls(df) # invoking a function
    print(df2.head())