from numpy.core.fromnumeric import shape
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import random
import tensorflow
from collections import Counter, defaultdict
from scipy import stats
from sklearn import preprocessing
from sklearn.preprocessing import MinMaxScaler, OneHotEncoder
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.neighbors import KNeighborsRegressor
import lightgbm
# Specific tf libraries
from tensorflow.keras import backend as K
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.layers import Dropout
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import KFold, StratifiedKFold, GroupKFold
from sklearn.ensemble import RandomForestRegressor

import math
import seaborn as sns
import xgboost

def root_mean_squared_log_error(y_true, y_pred):
    # Alternatively: sklearn.metrics.mean_squared_log_error(y_true, y_pred) ** 0.5
    #assert (y_true >= 0).all()
    #assert (y_pred >= 0).all()
    log_error = np.log1p(y_pred) - np.log1p(y_true)  # Note: log1p(x) = log(1 + x)
    return np.mean(log_error ** 2) ** 0.5

def categorical_to_numerical(data, features):
    le = LabelEncoder()
    for feature in features:
        data[feature] = le.fit_transform(data[feature])

def pre_process_numerical(features, numerical_features, train, test, metadata=[],
                    outliers_value=7, val_data=True, val_split=0.1, random_state=42, scaler="none",
                    add_R=False, add_rel_height=False, add_spacious=False, droptable=[],
                    one_hot_encode=True, cat_features=[], drop_old=True,categorical_unknown=False):
    """
    Pre processes pandas dataframe, returns split datasets with preprocessing applied
    to numerical data.

        args:
            - features, numerical features: list of all features and all numerical ones.
            - train and test: train and test dataset. Train also has target "price".
            - outliers_value: removes data outside the range of mean+outliers_value*std
            - val_data: bool to determine wheter or not you want validation data.
            - val_split: what percentage of data to use to validate.
            - scaler: none, minMax, or std. minMax scaled to range 1-0 and std scales around mean.
            - add_R if you want to add radius to dataset. add_rel_height to add rel height.
            - droptable: any features you want to drop at the end of preprocessing.
            Then for one-hot-encoding
            - Toogle it with "one_hot_encode"
            - Insert what cat-features you want to encode.
            - Drop_old is if you want to replace/delete the old categorical features,
            or keep them.
    """
    # Make a copy of data that is altered
    Numerical_features = numerical_features.copy()

    # Remove outlayers from training data
    no_outlayers = train[(np.abs(stats.zscore(train['price'])) < outliers_value)]

    # NAN PROBLEM - use mean for numerical, median for categorical.
    # Training and validation data preprocessing
    labels = no_outlayers[features]
    test_labels = test[features]

    if categorical_unknown:
        print("nan to extra unknown category")
        for feature in cat_features:
            unknown_category = len(list(train[feature].unique())) - 1
            labels[feature] = labels[feature].fillna(unknown_category)
            test_labels[feature] = test_labels[feature].fillna(unknown_category)

    labels = labels.fillna(labels.median())
    targets = no_outlayers['price']

    # Test data preprocessing
    test_labels = test_labels.fillna(test_labels.median())

    if one_hot_encode and len(metadata):
        print("Laure encoding!")
        oneHotFeatures(metadata, labels, cat_features)
        oneHotFeatures(metadata, test_labels, cat_features)

    elif one_hot_encode:
        print("Hot encoding")
        labels, test_labels = one_hot_encoder(labels, test_labels, cat_features, drop_old=drop_old)

    # Adding some new features
    # ADD R
    if add_R:
        labels, test_labels = polar_coordinates(labels, test_labels)
        Numerical_features.append("r")
    # ADD rel_height
    if add_rel_height:
        labels['rel_height'] = labels["floor"] / labels["stories"]
        test_labels['rel_height'] = test_labels["floor"] / test_labels["stories"]
        Numerical_features.append("rel_height")
    if add_spacious:
        labels['Spacious_rooms'] = labels['area_total'] /labels['rooms']
        test_labels['Spacious_rooms'] = test_labels['area_total'] /test_labels['rooms']
        Numerical_features.append("Spacious_rooms")

    # Split
    # TODO: dont split apartments of same building.

    if val_data:
        train_labels, val_labels, train_targets, val_targets = train_test_split(
            labels, targets, test_size=val_split, shuffle=True) #random_state=random_state
    else:
        train_labels = labels.copy()
        val_labels = labels.copy()
        train_targets = targets.copy()
        val_targets = targets.copy()

    # Only normalize/scale the numerical data. Categorical data is kept as is.
    train_labels_n = train_labels.filter(Numerical_features)
    val_labels_n = val_labels.filter(Numerical_features)
    test_labels_n = test_labels.filter(Numerical_features)

    # Scale it.
    if scaler=="minMax":
        print("minMax")
        scaler = MinMaxScaler(feature_range=(0, 1))
        train_labels_scaled = scaler.fit_transform(train_labels_n)
        val_labels_scaled = scaler.transform(val_labels_n)
        test_labels_scaled = scaler.transform(test_labels_n)
    elif scaler=="std":
        print("Std")
        std_scale = preprocessing.StandardScaler().fit(train_labels_n)
        train_labels_scaled = std_scale.transform(train_labels_n)
        val_labels_scaled = std_scale.transform(val_labels_n)
        test_labels_scaled = std_scale.transform(test_labels_n)
    elif scaler=="none":
        train_labels_scaled = train_labels_n
        val_labels_scaled = val_labels_n
        test_labels_scaled = test_labels_n
    else:
        assert ValueError, "Incorrect scaler"

    # Re-enter proceedure
    training_norm_col = pd.DataFrame(train_labels_scaled, index=train_labels_n.index, columns=train_labels_n.columns)
    train_labels.update(training_norm_col)

    val_norm_col = pd.DataFrame(val_labels_scaled, index=val_labels_n.index, columns=val_labels_n.columns)
    val_labels.update(val_norm_col)

    testing_norm_col = pd.DataFrame(test_labels_scaled, index=test_labels_n.index, columns=test_labels_n.columns)
    test_labels.update(testing_norm_col)

    # Drop the most correlated features.
    train_labels.drop(droptable, inplace=True, axis=1)
    val_labels.drop(droptable, inplace=True, axis=1)
    test_labels.drop(droptable, inplace=True, axis=1)

    if val_data:
        return train_labels, train_targets, val_labels, val_targets, test_labels
    else:
        return train_labels, train_targets, test_labels

def pre_process_2(train, test,
                    features, float_numerical_features, int_numerical_features, cat_features,
                    metadata=[], scaler="none", common_mean=True, professor_laures_fillna=False, log_it=[],
                    log_target=False, add_R=False, add_rel_height=False, add_spacious=False,
                    droptable=[], one_hot_encode=True, drop_old=True):
    """
    TODO: more outlayer detection?
    """
    float_numerical_features = float_numerical_features.copy()
    int_numerical_features = int_numerical_features.copy()
    all_numerical_features = float_numerical_features + int_numerical_features

    train_labels = train[features]
    test_labels = test[features]
    train_targets = train['price']

    if professor_laures_fillna:
        # Fill missing values based on heavy correlation
        # area_living
        train_labels = fillnaReg(train_labels, ['area_total'], 'area_living')
        test_labels = fillnaReg(test_labels, ['area_total'], 'area_living')
        # area_kitchen
        train_labels = fillnaReg(train_labels, ['area_total', 'area_living'], 'area_kitchen')
        test_labels = fillnaReg(test_labels, ['area_total', 'area_living'], 'area_kitchen')

    if not common_mean:
        # Train data preprocessing
        # Float
        train_labels[float_numerical_features] = train_labels[float_numerical_features].fillna(train_labels[float_numerical_features].mean())
        # Int
        train_labels[int_numerical_features] = train_labels[int_numerical_features].fillna(train_labels[int_numerical_features].median())
        # Cat
        train_labels[cat_features] = train_labels[cat_features].fillna(train_labels[cat_features].median())
        # Bool (The rest)
        train_labels = train_labels.fillna(train_labels.median()) # Boolean

        # Test data preprocessing
        # Float
        test_labels[float_numerical_features] = test_labels[float_numerical_features].fillna(test_labels[float_numerical_features].mean())
        # Int
        test_labels[int_numerical_features] = test_labels[int_numerical_features].fillna(test_labels[int_numerical_features].median())
        # Cat
        test_labels[cat_features] = test_labels[cat_features].fillna(test_labels[cat_features].median())
        # Bool (The rest)
        test_labels = test_labels.fillna(test_labels.median()) # Boolean
    else:
        # FLOAT
        train_float_mean = train_labels[float_numerical_features].mean()
        test_float_mean = test_labels[float_numerical_features].mean()
        tl = len(train_labels) + len(test_labels)
        total_mean = (train_float_mean*len(train_labels) + test_float_mean*len(test_labels)) / tl
        # Int
        int_median = train_labels[int_numerical_features].median()
        # Cat
        cat_median = train_labels[cat_features].median()
        # Bool (The rest)
        bool_median = train_labels.median()

        # Train
        # Float
        train_labels[float_numerical_features] = train_labels[float_numerical_features].fillna(total_mean)
        # Int
        train_labels[int_numerical_features] = train_labels[int_numerical_features].fillna(int_median)
        # Cat
        train_labels[cat_features] = train_labels[cat_features].fillna(cat_median)
        # Bool (The rest)
        train_labels = train_labels.fillna(bool_median)

        # TEST
        # Float
        test_labels[float_numerical_features] = test_labels[float_numerical_features].fillna(total_mean)
        # Int
        test_labels[int_numerical_features] = test_labels[int_numerical_features].fillna(int_median)
        # Cat
        test_labels[cat_features] = test_labels[cat_features].fillna(cat_median)
        # Bool (The rest)
        test_labels = test_labels.fillna(bool_median)

    if test_labels.isna().sum().sum():
        print("Test!")
        raise ValueError
    if train_labels.isna().sum().sum():
        print("TRAIN!")
        raise ValueError

    if log_it:
        for feature in log_it:
            # In case any features has 0, or negative numbers in them
            remove_zero = [row[feature] if row[feature] >= 1 else 1 for _, row in train_labels.iterrows()]
            train_labels[feature] = remove_zero
            remove_zero = [row[feature] if row[feature] >= 1 else 1 for _, row in test_labels.iterrows()]
            test_labels[feature] = remove_zero
            # Logging
            train_labels[feature] = np.log(train_labels[feature])
            test_labels[feature] = np.log(test_labels[feature])

    if log_target:
        train_targets = np.log(train_targets)

    if one_hot_encode and len(metadata):
        print("Laure encoding!")
        oneHotFeatures(metadata, train_labels, cat_features)
        oneHotFeatures(metadata, test_labels, cat_features)

    elif one_hot_encode:
        print("Hot encoding")
        train_labels, test_labels = one_hot_encoder(train_labels, test_labels, cat_features, drop_old=drop_old)

    # Adding some new features
    # ADD R
    if add_R:
        train_labels, test_labels = polar_coordinates(train_labels, test_labels)
        all_numerical_features.append("r")
        float_numerical_features.append("r")
    # ADD rel_height
    if add_rel_height:
        train_labels['rel_height'] = train_labels["floor"] / train_labels["stories"]
        test_labels['rel_height'] = test_labels["floor"] / test_labels["stories"]
        all_numerical_features.append("rel_height")
        float_numerical_features.append("rel_height")
    if add_spacious:
        train_labels['Spacious_rooms'] = train_labels['area_total'] /train_labels['rooms']
        test_labels['Spacious_rooms'] = test_labels['area_total'] /test_labels['rooms']
        all_numerical_features.append("Spacious_rooms")
        float_numerical_features.append("Spacious_rooms")

    # Only normalize/scale the numerical data. Categorical data is kept as is.
    train_labels_n = train_labels.filter(float_numerical_features)
    test_labels_n = test_labels.filter(float_numerical_features)

    # Scale it.
    if scaler=="minMax":
        print("minMax")
        scaler = MinMaxScaler(feature_range=(0, 1))
        train_labels_scaled = scaler.fit_transform(train_labels_n)
        test_labels_scaled = scaler.transform(test_labels_n)
    elif scaler=="std":
        print("Std")
        std_scale = preprocessing.StandardScaler().fit(train_labels_n)
        train_labels_scaled = std_scale.transform(train_labels_n)
        test_labels_scaled = std_scale.transform(test_labels_n)
    elif scaler=="none":
        train_labels_scaled = train_labels_n
        test_labels_scaled = test_labels_n
    else:
        assert ValueError, "Incorrect scaler"

    # Re-enter proceedure
    training_norm_col = pd.DataFrame(train_labels_scaled, index=train_labels_n.index, columns=train_labels_n.columns)
    train_labels.update(training_norm_col)

    testing_norm_col = pd.DataFrame(test_labels_scaled, index=test_labels_n.index, columns=test_labels_n.columns)
    test_labels.update(testing_norm_col)

    # Drop the most correlated features.
    train_labels.drop(droptable, inplace=True, axis=1)
    test_labels.drop(droptable, inplace=True, axis=1)

    return train_labels, train_targets, test_labels

def one_hot_encoder(train_df, test_df, cat_features, drop_old=True):
    '''
    Returns a copy of all three dfs, after one-hot encoding and !removing!
    their old cat_features.

    NB! pd.get_dummies() does pretty much the same job!
    https://stackoverflow.com/questions/36285155/pandas-get-dummies
    BUG! Some categories are only present in train not test or the other way around!
        - Then the encoding is made differently for the two!
    https://stackoverflow.com/questions/57946006/one-hot-encoding-train-with-values-not-present-on-test
    '''
    # TODO: use Laures one-hot encoder.

    #if(len(train_df.isna())!=0 or len(train_df.isna())!=0 or len(train_df.isna())!=0):
    #    assert ValueError

    train_labels = train_df.copy()
    test_labels = test_df.copy()

    encoded_features = []
    dfs=[train_labels, test_labels]
    for df in dfs:
        for feature in cat_features:
            encoded_feat = OneHotEncoder().fit_transform(df[feature].values.reshape(-1, 1)).toarray()
            n = df[feature].nunique()
            cols = ['{}_{}'.format(feature, n) for n in range(1, n + 1)]
            encoded_df = pd.DataFrame(encoded_feat, columns=cols)
            encoded_df.index = df.index
            encoded_features.append(encoded_df)

    n = len(cat_features)

    train_labels = pd.concat([train_labels, *encoded_features[ : n]], axis=1)
    test_labels = pd.concat([test_labels, *encoded_features[n : ]], axis=1)

    # Now drop the non-encoded ones!
    if drop_old:
        train_labels.drop(cat_features, inplace=True, axis=1)
        test_labels.drop(cat_features, inplace=True, axis=1)
    return train_labels, test_labels

def oneHotFeatures(metadata, data, features):
    values = []
    for feature in features:
        values.append(oneHotFeature(metadata, data, feature))
    return values

def oneHotFeature(metadata, data, feature):
    values = list(metadata.loc[metadata['name'] == feature]['cats'])[0]
    for i, value in enumerate(values):
        new_column = [1 if row == i else 0 for row in list(data[feature])]
        data[value] = new_column
    return values

def fillnaReg(df, X_features, y_feature):
    df = df.copy()
    df_temp = df[df[y_feature].notna()]
    if df_temp.shape[0] == 1: df_temp = df_temp.values.reshape(-1, 1)
    reg = LinearRegression().fit(df_temp[X_features], df_temp[y_feature])
    predict = reg.predict(df[X_features])
    df[y_feature] = np.where(df[y_feature]>0, df[y_feature], predict)
    return df

def get_cat_and_non_cat_data(metadata):
    categorical = []
    nonCategorical = []
    for _, feature in metadata.iterrows():
        if feature['type'] == 'categorical': categorical.append(feature['name'])
        else: nonCategorical.append(feature['name'])
    return nonCategorical, categorical

def polar_coordinates(train_labels, test_labels, use_city_centre=False):
    '''
    labels and input labels. Adds theta and also R to the dataframe copy.
    '''
    # Make a copy to manipulate
    labels1_normed_r = train_labels[["latitude", "longitude"]].copy()
    test1_normed_r = test_labels[["latitude", "longitude"]].copy()

    # City centre 55.755833, 37.617222 (lat/long)
    # Instead of mean, use city centre:
    if use_city_centre:
        labels1_normed_r['latitude'] = labels1_normed_r['latitude'] - 55.755833
        labels1_normed_r['longitude'] = labels1_normed_r['longitude'] - 37.617222
        test1_normed_r['latitude'] = test1_normed_r['latitude'] -  55.755833
        test1_normed_r['longitude'] = test1_normed_r['longitude'] -  37.617222
    else: # just use means
        # FLOAT
        train_float_mean = train_labels["longitude"].mean()
        test_float_mean = test_labels["longitude"].mean()
        tl = len(train_labels) + len(test_labels)
        total_mean_long = (train_float_mean*len(train_labels) + test_float_mean*len(test_labels)) / tl

        train_float_mean = train_labels["latitude"].mean()
        test_float_mean = test_labels["latitude"].mean()
        tl = len(train_labels) + len(test_labels)
        total_mean_lat = (train_float_mean*len(train_labels) + test_float_mean*len(test_labels)) / tl

        labels1_normed_r['latitude'] = labels1_normed_r['latitude'] - total_mean_lat
        labels1_normed_r['longitude'] = labels1_normed_r['longitude'] - total_mean_long
        test1_normed_r['latitude'] = test1_normed_r['latitude'] -  total_mean_lat
        test1_normed_r['longitude'] = test1_normed_r['longitude'] -  total_mean_long

    # Convert to polar coordinates
    labels1_normed_r['r'] =  np.sqrt(labels1_normed_r['latitude']**2 + labels1_normed_r['longitude']**2)
    labels1_normed_r['theta'] = np.arctan(labels1_normed_r['longitude']/labels1_normed_r['latitude'])
    test1_normed_r['r'] =  np.sqrt(test1_normed_r['latitude']**2 + test1_normed_r['longitude']**2)
    test1_normed_r['theta'] = np.arctan(test1_normed_r['longitude']/test1_normed_r['latitude'])

    # Add polar coordinates
    train_labels["r"] = labels1_normed_r['r']
    train_labels["theta"] = labels1_normed_r['theta']
    test_labels["r"] = test1_normed_r['r']
    test_labels["theta"] = test1_normed_r['theta']

    return train_labels, test_labels

def custom_asymmetric_eval(y_true, y_pred):
    loss = root_mean_squared_log_error(y_true,y_pred)
    return "custom_asymmetric_eval", np.mean(loss), False
class PrintDot(tensorflow.keras.callbacks.Callback):
    def on_epoch_end(self, epoch, logs):
        if epoch % 100 == 0: print('')
        print('.', end='')

def plot_history(hist):
    plt.figure()
    plt.xlabel('Epoch')
    plt.ylabel('MSLE')
    plt.yscale("log")
    plt.plot(hist['epoch'], hist['msle'], label='Train Error')
    plt.plot(hist['epoch'], hist['val_msle'], label = 'Val Error')
    plt.legend()

def rmsle_custom(y_true, y_pred):
    msle = tensorflow.keras.losses.MeanSquaredLogarithmicError()
    return K.sqrt(msle(y_true, y_pred))

def load_all_data(fraction_of_data=1, apartment_id='apartment_id',path=None):

    if path is not None:
        # Metadata
        metaData_apartment = pd.read_json(path+'../data/apartments_meta.json')
        metaData_building = pd.read_json(path+'../data/buildings_meta.json')
        metaData = pd.concat([metaData_apartment, metaData_building])

        # Train
        train_apartment = pd.read_csv(path+'../data/apartments_train.csv')
        train_building = pd.read_csv(path+'../data/buildings_train.csv')

        # Test
        test_apartment = pd.read_csv(path+'../data/apartments_test.csv')
        test_building = pd.read_csv(path+'../data/buildings_test.csv')

    else:
    # Metadata
        metaData_apartment = pd.read_json('../data/apartments_meta.json')
        metaData_building = pd.read_json('../data/buildings_meta.json')
        metaData = pd.concat([metaData_apartment, metaData_building])

        # Train
        train_apartment = pd.read_csv('../data/apartments_train.csv')
        train_building = pd.read_csv('../data/buildings_train.csv')

        # Test
        test_apartment = pd.read_csv('../data/apartments_test.csv')
        test_building = pd.read_csv('../data/buildings_test.csv')

    train = pd.merge(train_apartment, train_building, left_on='building_id', right_on='id')
    train.rename(columns={'id_x' : apartment_id}, inplace=True)
    train.drop('id_y', axis=1, inplace=True)
    train = train.head(int(train.shape[0] * fraction_of_data))

    test = pd.merge(test_apartment, test_building, left_on='building_id', right_on='id')
    test.rename(columns={'id_x' : apartment_id}, inplace=True)
    test.drop('id_y', axis=1, inplace=True)

    return train, test, metaData

def predict_and_store(model, test_labels, test_pd, path="default", exponential=False, price_per_sq = False, total_area_df = None, exponentialm1=False):
    '''
        Inputs
        - test_pd needs to be the original full test dataframe
    '''
    result = model.predict(test_labels)
    if exponential:
        result = np.exp(result)
    elif exponentialm1:
        print("expm1")
        result = np.expm1(result)
    if price_per_sq:
        result = result*total_area_df
    submission = pd.DataFrame()
    submission['id'] = test_pd['apartment_id']
    submission['price_prediction'] = result
    if len(submission['id']) != 9937:
        raise Exception("Not enough rows submitted!")
    submission.to_csv(path, index=False)

def create_ANN_model(dense_layers=[64, 64, 64], activation=tensorflow.nn.leaky_relu,
                     dropout=[False, False, False], dropout_rate=0.2, optimizer="adamn", #'adamn'
                      loss_function=rmsle_custom, metrics=['accuracy'], output_activation=True
                      ):
    # Model
    model = tensorflow.keras.Sequential()
    for i, n in enumerate(dense_layers):
        model.add(Dense(n, activation=activation))
        if dropout[i]:
            model.add(Dropout(dropout_rate))

    if output_activation:
        model.add(Dense(1, activation=tensorflow.nn.relu))
    else:
        model.add(Dense(1)) #Output

    # Optimized for reducing msle loss.
    model.compile(optimizer=optimizer,
                loss=loss_function, #'msle', 'rmse', RMSLETF, rmsle_custom
                metrics=metrics) # metrics=['mse', 'msle'] metrics=[tf.keras.metrics.Accuracy()]

    return model

def csv_bagging(kaggle_scores, csv_paths, submission_path):
    # Making the acc dataframe
    d = {}
    for i, score in enumerate(kaggle_scores):
        d[i] = score
    acc = pd.DataFrame(
    d,
    index=[0]
    )
    acc = acc.T
    acc.columns = ['RMSLE']
    acc

    # Read dataframes, sort and store
    pd_predictions = []
    for path in csv_paths:
        pd_predictions.append(
            pd.read_csv(path).sort_values(by="id")
            )
    # Cast to numpy
    np_predictions = []
    for pred in pd_predictions:
        np_predictions.append(
            pred["price_prediction"].to_numpy().T
        )

    # Bagging
    avg_prediction = np.average(
        np_predictions,
        weights = 1 / acc['RMSLE'] ** 4,
        axis=0
        )

    result = avg_prediction
    submission = pd.DataFrame()
    submission['id'] = pd_predictions[0]['id']
    submission['price_prediction'] = result
    if len(submission['id']) != 9937:
        raise Exception("Not enough rows submitted!")

    submission.to_csv(submission_path, index=False)

def tree_bagging(bags, submission_path, weight=[4]):
    submission = bag(bags, 0, weight)['data']
    submission.to_csv(submission_path, index=False)

def bag(bags, i, weight = [4]):
    if type(bags) != list:
        return bags
    else:
        new_bags = []
        for b in bags:
            new_bags.append(bag(b, i+1 if i < len(weight)-1 else i, weight))

    if 'data' not in new_bags[0].keys():
        for b in new_bags:
            b['data'] = pd.read_csv(b['path']).sort_values(by="id")

    np_predictions = []
    for b in new_bags:
        np_predictions.append(b['data']["price_prediction"].to_numpy().T)

    score = np.average([bag['score'] for bag in new_bags], weights = [1 / bag['score'] ** weight[i] for bag in new_bags])
    # Bagging
    result = np.average(
        np_predictions,
        weights = [1 / bag['score'] ** weight[i] for bag in new_bags],
        axis=0
        )

    submission = pd.DataFrame()
    submission['id'] = new_bags[0]['data']['id']
    submission['price_prediction'] = result
    return {'data': submission, 'score': score}

def get_oof_xgboost(clf, x_train, y_train, x_test, NFOLDS=5, eval_metric='rmse'):
    """
    NB! y should be logarithm of price.
    """
    ntrain = x_train.shape[0]
    ntest = x_test.shape[0]
    groups = x_train["building_id"]

    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    gkf = GroupKFold(
        n_splits=NFOLDS,
    )

    i=0
    scores = []

    for train_index, test_index in gkf.split(x_train, y_train, groups):
        x_tr, x_te = x_train.iloc[train_index], x_train.iloc[test_index]
        y_tr, y_te = y_train.iloc[train_index], y_train.iloc[test_index]

        x_tr = x_tr.drop(["building_id"], axis=1)
        x_te = x_te.drop(["building_id"], axis=1)
        x_test_no_id = x_test.drop(["building_id"], axis=1)

        clf.fit(x_tr, y_tr,
                eval_set=[(x_te, y_te)],
                eval_metric=eval_metric,
                early_stopping_rounds=15,   # To not overfit
                verbose=False,
               )

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test_no_id)

        i+=1

        # Just to get some idea of score
        prediction = clf.predict(x_te)
        score = root_mean_squared_log_error(np.exp(prediction), np.exp(y_te))
        scores.append(score)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1), scores

def get_oof_lgbm(clf, x_train, y_train, x_test, NFOLDS=5, eval_metric=custom_asymmetric_eval):
    """
    NB! y should be logarithm of price. Will also predict the log.
    """
    ntrain = x_train.shape[0]
    ntest = x_test.shape[0]
    groups = x_train["building_id"]

    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    gkf = GroupKFold(
        n_splits=NFOLDS,
    )

    i=0
    scores = []

    for train_index, test_index in gkf.split(x_train, y_train, groups):
        x_tr, x_te = x_train.iloc[train_index], x_train.iloc[test_index]
        y_tr, y_te = y_train.iloc[train_index], y_train.iloc[test_index]

        x_tr = x_tr.drop(["building_id"], axis=1)
        x_te = x_te.drop(["building_id"], axis=1)
        x_test_no_id = x_test.drop(["building_id"], axis=1)

        clf.fit(x_tr, y_tr,
                eval_set=[(x_te, y_te)],
                eval_metric=eval_metric,
                #early_stopping_rounds = 15,
                verbose=False,
                callbacks=[lightgbm.early_stopping(15, verbose=True)]
               )

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test_no_id)

        i+=1

        # Just to get some idea of score
        prediction = clf.predict(x_te)
        score = root_mean_squared_log_error(np.exp(prediction), np.exp(y_te))
        scores.append(score)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1), scores

def get_oof_gradientboost(clf, x_train, y_train, x_test, NFOLDS=5):
    """
    NB! y should be logarithm of price. Will also predict the log.
    """
    ntrain = x_train.shape[0]
    ntest = x_test.shape[0]
    groups = x_train["building_id"]

    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    gkf = GroupKFold(
        n_splits=NFOLDS,
    )

    i=0
    scores = []

    for train_index, test_index in gkf.split(x_train, y_train, groups):
        x_tr, x_te = x_train.iloc[train_index], x_train.iloc[test_index]
        y_tr, y_te = y_train.iloc[train_index], y_train.iloc[test_index]

        x_tr = x_tr.drop(["building_id"], axis=1)
        x_te = x_te.drop(["building_id"], axis=1)
        x_test_no_id = x_test.drop(["building_id"], axis=1)

        clf.fit(x_tr, y_tr,
               )

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test_no_id)

        i+=1

        # Just to get some idea of score
        prediction = clf.predict(x_te)
        score = root_mean_squared_log_error(np.exp(prediction), np.exp(y_te))
        scores.append(score)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1), scores

def get_oof_ann(model_params, x_train, y_train, x_test, NFOLDS=5, strict_restart=False):
    """
    Popular function on Kaggle. Adapted for ANN.

    """
    ntrain = x_train.shape[0]
    ntest = x_test.shape[0]
    groups = x_train["building_id"]

    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    gkf = GroupKFold(
        n_splits=NFOLDS,
    )
    early_stop = tensorflow.keras.callbacks.EarlyStopping(monitor='val_loss', verbose=1, mode='min', patience=40)

    i=0
    hists = []

    for train_index, test_index in gkf.split(x_train, y_train, groups):
        not_good_enough=True
        while not_good_enough:
            x_tr, x_te = x_train.iloc[train_index], x_train.iloc[test_index]
            y_tr, y_te = y_train.iloc[train_index], y_train.iloc[test_index]

            x_tr = x_tr.drop(["building_id"], axis=1)
            x_te = x_te.drop(["building_id"], axis=1)
            x_test_no_id = x_test.drop(["building_id"], axis=1)

            # Model
            ann_model = create_ANN_model(*model_params)

            # Fit
            history = ann_model.fit(x=x_tr, y=y_tr,
                validation_data=(x_te, y_te),
                verbose=0, epochs=1000,
                callbacks=[early_stop, PrintDot()]
                )

            if strict_restart and (history.history["val_loss"][-1]>0.4):
                print("Strict restart!")
                continue
            else:
                not_good_enough=False

            sample_pred = ann_model.predict(x_te)
            all_pred = ann_model.predict(x_test_no_id)

            oof_train[test_index] = sample_pred.reshape(sample_pred.shape[0],)
            oof_test_skf[i, :] = all_pred.reshape(all_pred.shape[0],)

            i+=1
            hists.append(history)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1), hists

def get_oof_rf(x_train, y_train, x_test, NFOLDS=5):
    """
    """
    ntrain = x_train.shape[0]
    ntest = x_test.shape[0]
    groups = x_train["building_id"]

    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    gkf = GroupKFold(
        n_splits=NFOLDS,
    )

    i=0
    scores = []

    for train_index, test_index in gkf.split(x_train, y_train, groups):
        x_tr, x_te = x_train.iloc[train_index], x_train.iloc[test_index]
        y_tr, y_te = y_train.iloc[train_index], y_train.iloc[test_index]

        x_tr = x_tr.drop(["building_id"], axis=1)
        x_te = x_te.drop(["building_id"], axis=1)
        x_test_no_id = x_test.drop(["building_id"], axis=1)

        clf = RandomForestRegressor(
            n_estimators=2000,
            max_depth=400,
            min_samples_split=2,
            min_samples_leaf= 2,
            min_weight_fraction_leaf=0.00008,
            max_features='auto',
            max_leaf_nodes=None,
            min_impurity_decrease=1100,
            bootstrap=True,
            oob_score=False,
            n_jobs=None,
            random_state=None,
            verbose=0,
            warm_start=True,
            ccp_alpha=20000,
            max_samples=None
        )

        clf.fit(x_tr, y_tr)

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test_no_id)

        i+=1

        # Just to get some idea of score
        prediction = clf.predict(x_te)
        score = root_mean_squared_log_error(prediction, y_te)
        scores.append(score)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1), scores

def get_oof_knn(x_train, y_train, x_test, NFOLDS=5):
    """
    """
    ntrain = x_train.shape[0]
    ntest = x_test.shape[0]
    groups = x_train["building_id"]

    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    gkf = GroupKFold(
        n_splits=NFOLDS,
    )

    i=0
    scores = []

    for train_index, test_index in gkf.split(x_train, y_train, groups):
        x_tr, x_te = x_train.iloc[train_index], x_train.iloc[test_index]
        y_tr, y_te = y_train.iloc[train_index], y_train.iloc[test_index]

        x_tr = x_tr.drop(["building_id"], axis=1)
        x_te = x_te.drop(["building_id"], axis=1)
        x_test_no_id = x_test.drop(["building_id"], axis=1)

        clf = KNeighborsRegressor(
            n_neighbors=20,
            weights='uniform',
            algorithm='ball_tree',
            leaf_size=30,
            p=1,
            metric='minkowski',
            metric_params=None,
            n_jobs=None
            )

        clf.fit(x_tr, y_tr)

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test_no_id)

        i+=1

        # Just to get some idea of score
        prediction = clf.predict(x_te)
        score = root_mean_squared_log_error(prediction, y_te)
        scores.append(score)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1), scores

def bestRFPredict(
    train, test,
    features = ["area_total", "latitude", "longitude", "floor", "district", "stories", 'condition']
    ):
    isTest = 'price' in test.colmuns

    train.fillna(train.mean(), inplace = True)
    test.fillna(test.mean(), inplace = True)

    X_train, y_train = train[features], train['price']
    X_test = test[features]
    if isTest: y_test = test['price']

    model = RandomForestRegressor(
        n_estimators=2000,
        max_depth=400,
        min_samples_split=2,
        min_samples_leaf= 2,
        min_weight_fraction_leaf=0.00008,
        max_features='auto',
        max_leaf_nodes=None,
        min_impurity_decrease=1100,
        bootstrap=True,
        oob_score=False,
        n_jobs=None,
        random_state=None,
        verbose=0,
        warm_start=True,
        ccp_alpha=20000,
        max_samples=None
    )

    if isTest:
        model.fit(X_train, y_train)
        y_predict = model.predict(X_test)
        return root_mean_squared_log_error(y_predict, y_test)
    else:
        model.fit(X_train, y_train)
        y_predict = model.predict(X_test)
        return y_predict

def XGB_groupKFold(number_of_splits, model_params, X_train, y_train,
    eval_metric=None):
    ''' y_train needs to be log. Model trains to predict logs now!'''
    X_train = X_train.copy()
    y_train = y_train.copy()

    scores = []
    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]
    best_score = 1
    i = 0
    models=[]
    for train_index, test_index in gkf.split(X_train, y_train, groups):
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]

        # We don't need it anymore.
        X_train2 = X_train2.drop(["building_id"], axis=1)
        X_test = X_test.drop(["building_id"], axis=1)

        model = xgboost.XGBRegressor(
            max_depth=model_params[0], min_child_weight=model_params[1], gamma=model_params[2], subsample=model_params[3], colsample_bytree=model_params[4],
             reg_alpha=model_params[5], reg_lambda=model_params[6], learning_rate=model_params[7], n_estimators=model_params[8])

        model.fit(
            X_train2,
            y_train2,
            eval_set=[(X_test, y_test)],
            eval_metric=eval_metric,
            early_stopping_rounds=15,   # To not overfit
            verbose=False,
        )
        prediction = np.exp(model.predict(X_test))
        score = root_mean_squared_log_error(prediction, np.exp(y_test))
        if score <  best_score:
            best_score = score
            best_model = model
            best_index = i
        scores.append(score)
        i += 1
        models.append(model)
    return scores, np.average(scores), best_model, models

def ANN_groupKFold(number_of_splits, model_params, X_train, y_train, strict_restart=False):
    ''' y_train needs not to be log: as we got the RMSLE loss function for ANN!'''
    X_train = X_train.copy()
    y_train = y_train.copy()

    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]
    best_score = 10
    i = 0
    ann_scores = []
    hists=[]
    models=[]
    for train_index, test_index in gkf.split(X_train, y_train, groups):
        not_good_enough=True
        while not_good_enough:
            X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
            y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]

            # We don't need it anymore.
            X_train2 = X_train2.drop(["building_id"], axis=1)
            X_test = X_test.drop(["building_id"], axis=1)

            model = create_ANN_model(*model_params)
            early_stop = tensorflow.keras.callbacks.EarlyStopping(monitor='val_loss', verbose=1, mode='min', patience=40)

            history = model.fit(x=X_train2, y=y_train2.values,
            validation_data=(X_test, y_test),
            verbose=0, epochs=1000, callbacks=[early_stop, PrintDot()]
            )

            if strict_restart and (history.history["val_loss"][-1]>0.4):
                print("strict restart!")
                continue
            else:
                not_good_enough=False

            ann_score = history.history["val_loss"][-1]
            ann_scores.append(ann_score)
            hists.append(history)
            models.append(model)
            if ann_score <  best_score:
                print("New best model!")
                best_score = ann_score
                best_model = model
            i += 1

    return ann_scores, models, best_model, hists

def lgbm_groupKFold(number_of_splits, model, X_train, y_train,
    eval_metric=None):
    # y_train is log!!
    X_train = X_train.copy()
    y_train = y_train.copy()

    scores = []
    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]
    best_score = 1
    i = 0

    for train_index, test_index in gkf.split(X_train, y_train, groups):
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]
        model.fit(
            X_train2,
            y_train2,
            eval_set=[(X_test, y_test)],
            eval_metric=eval_metric,
            verbose=False,
        )
        prediction = np.exp(model.predict(X_test))
        score = root_mean_squared_log_error(prediction, np.exp(y_test))
        if score <  best_score:
            best_score = score
            best_model = model
            best_index = i
        scores.append(score)
        i += 1
    return scores, np.average(scores), best_model, best_index

def lgbm_groupKFold_not_log_input(number_of_splits, model, X_train, y_train,
    eval_metric=None):
    # y_train is NOT log!!
    X_train = X_train.copy()
    y_train = y_train.copy()

    scores = []
    best_model = ""
    best_index = -1
    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]
    best_score = 1
    i = 0

    for train_index, test_index in gkf.split(X_train, y_train, groups):
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]
        model.fit(
            X_train2,
            y_train2,
            eval_set=[(X_test, y_test)],
            eval_metric=eval_metric,
            verbose=False,
        )
        prediction = model.predict(X_test)
        score = root_mean_squared_log_error(prediction, y_test)
        if score <  best_score:
            best_score = score
            best_model = model
            best_index = i
        scores.append(score)
        i += 1
    return scores, np.average(scores), best_model, best_index

def lgbm_groupKFold_price_sq_logged(number_of_splits, model, X_train, y_train, y_target,
    eval_metric=None):
    # y_train is log price / area_total!!
    # y_Target is price

    x_area = X_train['area_total']

    X_train = X_train.drop(['area_total'],axis=1).copy()
    y_train = y_train.copy()

    scores = []
    best_model = ""
    best_index = -1
    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]
    best_score = 1
    i = 0

    for train_index, test_index in gkf.split(X_train, y_train, groups):
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]
        y_target2, area_test = y_target.iloc[test_index], x_area.iloc[test_index]

        model.fit(
            X_train2,
            y_train2,
            eval_set=[(X_test, y_test)],
            eval_metric=eval_metric,
            verbose=False,
        )
        prediction = model.predict(X_test)*area_test
        score = root_mean_squared_log_error(np.exp(prediction), y_target2)
        if score <  best_score:
            best_score = score
            best_model = model
            best_index = i
        scores.append(score)
        i += 1
    return scores, np.average(scores), best_model, best_index

def XGB_groupKFold_new_log(number_of_splits, model_params, X_train, y_train,
    eval_metric=None):
    ''' y_train is log1p'''
    X_train = X_train.copy()
    y_train = y_train.copy()

    scores = []
    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]
    best_score = -1
    i = 0
    models=[]
    for train_index, test_index in gkf.split(X_train, y_train, groups):
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]

        # We don't need it anymore.
        X_train2 = X_train2.drop(["building_id"], axis=1)
        X_test = X_test.drop(["building_id"], axis=1)

        model = xgboost.XGBRegressor(
            max_depth=model_params[0], min_child_weight=model_params[1], gamma=model_params[2], subsample=model_params[3], colsample_bytree=model_params[4],
             reg_alpha=model_params[5], reg_lambda=model_params[6], learning_rate=model_params[7], n_estimators=model_params[8])

        model.fit(
            X_train2,
            y_train2,
            eval_set=[(X_test, y_test)],
            eval_metric=eval_metric,
            early_stopping_rounds=15,   # To not overfit
            verbose=False,
        )
        prediction = np.expm1(model.predict(X_test))
        score = root_mean_squared_log_error(prediction, np.expm1(y_test))
        if score <  best_score or best_score==-1:
            best_score = score
            best_model = model
        scores.append(score)
        i += 1
        models.append(model)
    return scores, np.average(scores), best_model, models

def gradient_boost_groupKFold(number_of_splits, model, X_train, y_train):
    X_train = X_train.copy()
    y_train = y_train.copy()

    scores = []
    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]
    best_score = 1
    i = 0

    for train_index, test_index in gkf.split(X_train, y_train, groups):
        print('starting on split ',i+1,' of cross validation')
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]
        model.fit(
            X_train2,
            y_train2,
        )
        prediction = np.exp(model.predict(X_test))
        score = root_mean_squared_log_error(prediction, np.exp(y_test))
        if score <  best_score:
            best_score = score
            best_model = model
            best_index = i
        scores.append(score)
        i += 1
    return scores, np.average(scores), best_model, best_index

def catboost_groupKFold(number_of_splits, model, X_train, y_train, categorical_features=[],text_features=[],random_state=1):
    scores = []

    cv = GroupKFold(n_splits=number_of_splits)

    groups = X_train["building_id"]
    best_score = 1
    i = 1

    for train_index, test_index in cv.split(X_train, y_train, groups):
        print("starting on split ",i)
        i+=1
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]
        X_train2.drop(['building_id'], axis=1,inplace=True)
        X_test.drop(['building_id'], axis=1,inplace=True)

        model.fit(
            X_train2,
            y_train2,
            eval_set=[(X_test, y_test)],
            verbose=False,
            early_stopping_rounds=100,
        )
        prediction = np.exp(model.predict(X_test))
        score = root_mean_squared_log_error(prediction, np.exp(y_test))
        if score <  best_score:
            best_score = score
            best_model = model
            best_index = i
        scores.append(score)
    return scores, np.average(scores), best_model, best_index

def RF_groupKFold(number_of_splits, model, X_train, y_train):
    ''' y_train needs to be log. Model trains to predict logs now!'''
    X_train = X_train.copy()
    y_train = y_train.copy()

    scores = []
    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]

    best_score = 1
    i = 0
    best_model = model
    best_index = 0

    for train_index, test_index in gkf.split(X_train, y_train, groups):
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]
        model.fit( X_train2, y_train2)
        prediction =model.predict(X_test)
        score = root_mean_squared_log_error(prediction, y_test)
        if score <  best_score:
            best_score = score
            best_model = model
            best_index = i
        scores.append(score)
        i += 1
    return scores, np.average(scores), best_model, best_index

def KNN_groupKFold(number_of_splits, model, X_train, y_train):
    ''' y_train needs to be log. Model trains to predict logs now!'''
    X_train = X_train.copy()
    y_train = np.log(y_train.copy())

    scores = []
    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]

    best_score = 1
    i = 0
    best_model = model
    best_index = 0

    for train_index, test_index in gkf.split(X_train, y_train, groups):
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]
        model.fit( X_train2, y_train2)
        prediction =model.predict(X_test)
        score = root_mean_squared_log_error(np.exp(prediction), np.exp(y_test))
        if score <  best_score:
            best_score = score
            best_model = model
            best_index = i
        scores.append(score)
        i += 1
    return scores, np.average(scores), best_model, best_index

def plot_feature_importance(importance,names,model_type, figsize=(10,20)):

    #Create arrays from feature importance and feature names
    feature_importance = np.array(importance)
    feature_names = np.array(names)

    #Create a DataFrame using a Dictionary
    data={'feature_names':feature_names,'feature_importance':feature_importance}
    fi_df = pd.DataFrame(data)

    #Sort the DataFrame in order decreasing feature importance
    fi_df.sort_values(by=['feature_importance'], ascending=False,inplace=True)

    #Define size of bar plot
    plt.figure(figsize=figsize)
    #Plot Searborn bar chart
    sns.barplot(x=fi_df['feature_importance'], y=fi_df['feature_names'])
    #Add chart labels
    plt.title(model_type + 'Feature Importance')
    plt.xlabel('FEATURE IMPORTANCE')
    plt.ylabel('FEATURE NAMES')

def SVR_groupKFold(number_of_splits, model, X_train, y_train):
    ''' y_train needs to be log. Model trains to predict logs now!'''
    X_train = X_train.copy()
    y_train = np.log(y_train.copy())

    scores = []
    gkf = GroupKFold(n_splits=number_of_splits)
    groups = X_train["building_id"]

    best_score = 1
    i = 0
    best_model = model
    best_index = 0

    for train_index, test_index in gkf.split(X_train, y_train, groups):
        X_train2, X_test = X_train.iloc[train_index], X_train.iloc[test_index]
        y_train2, y_test = y_train.iloc[train_index], y_train.iloc[test_index]
        model.fit( X_train2, y_train2)
        prediction =model.predict(X_test)
        score = root_mean_squared_log_error(np.exp(prediction), np.exp(y_test))
        if score <  best_score:
            best_score = score
            best_model = model
            best_index = i
        scores.append(score)
        i += 1
    return scores, np.average(scores), best_model, best_index

def closest_district(lat, long, district_centre_long, district_centre_lat):
    best_distance = -1
    closest_dist = -1
    for i in range(len(district_centre_long)):
        long_dist =  district_centre_long[i][0]
        lat_dist = district_centre_lat[i][0]
        total_dist = np.sqrt((long-long_dist)**2 + (lat-lat_dist)**2)
        if total_dist < best_distance or best_distance==-1:
            best_distance = total_dist
            closest_dist = i
    return closest_dist

def clean_data(train, test,
                 features, float_numerical_features, int_numerical_features, cat_features,
                 log_targets=True, log_area=True, fillNan=True, log1_area=False, log1_targets=False):
    '''Clean the data according to best knowledge so far...

    '''

    # Extract
    train_labels = train[features]
    test_labels = test[features]
    train_targets = train['price']

    # Log targets
    if log_targets:
        train_targets = np.log(train_targets)
    elif log1_targets:
        print("log1p targets!")
        train_targets = np.log1p(train_targets)

    ## ------------------------------------------------------------------------------------------------ ##
    # TODO: Shoul we use thise at all?
    # Fill nans using correlated features
    train_labels = fillnaReg(train_labels, ['area_total'], 'area_living')
    test_labels = fillnaReg(test_labels, ['area_total'], 'area_living')
    # Area_kitchen
    train_labels = fillnaReg(train_labels, ['area_total', 'area_living'], 'area_kitchen')
    test_labels = fillnaReg(test_labels, ['area_total', 'area_living'], 'area_kitchen')
    # ceiling
    is_outlier = ((train_labels["ceiling"] > 10) | (train_labels["ceiling"] < 0.5))
    outliers = train_labels.copy()[is_outlier]
    inliers_index=[]
    for index in train_labels.index:
        if index not in outliers.index:
            inliers_index.append(index)
    train_labels.loc[outliers.index,['ceiling']] = train_labels.loc[inliers_index,['ceiling']].mean()

    is_outlier = ((test_labels["ceiling"] > 10) | (test_labels["ceiling"] < 0.5))
    outliers = test_labels.copy()[is_outlier]
    for index in test_labels.index:
        if index not in outliers.index:
            inliers_index.append(index)
    test_labels.loc[outliers.index,['ceiling']] = test_labels.loc[test_labels.index.intersection(inliers_index),['ceiling']].mean()

    ## ------------------------------------------------------------------------------------------------ ##

    # Remove zero area living
    # TODO: why not > 0?
    remove_zero = [row["area_living"] if row["area_living"] >= 1 else row["area_total"]*(train_labels["area_living"].mean() / train_labels["area_total"].mean()) for _, row in train_labels.iterrows()]
    train_labels["area_living"] = remove_zero

    remove_zero = [row["area_living"] if row["area_living"] >= 1 else row["area_total"]*(test_labels["area_living"].mean() / test_labels["area_total"].mean()) for _, row in test_labels.iterrows()]
    test_labels["area_living"] = remove_zero

    # Log the areas
    fs = ["area_total", "area_living", "area_kitchen"]
    if log_area:
        for feature in fs:
            # Logging
            train_labels[feature] = np.log(train_labels[feature])
            test_labels[feature] = np.log(test_labels[feature])
    elif log1_area:
        print("log1p area!!")
        for feature in fs:
            # Logging
            train_labels[feature] = np.log1p(train_labels[feature])
            test_labels[feature] = np.log1p(test_labels[feature])

    # remove upper Strip
    remove_upper_stripe = [row["area_living"] if row["area_living"] < row["area_total"] else row["area_total"]*(train_labels["area_living"].mean() / train_labels["area_total"].mean()) for _, row in train_labels.iterrows()]
    train_labels["area_living"] = remove_upper_stripe

    remove_upper_stripe = [row["area_living"] if row["area_living"] < row["area_total"] else row["area_total"]*(test_labels["area_living"].mean() / test_labels["area_total"].mean()) for _, row in test_labels.iterrows()]
    test_labels["area_living"] = remove_upper_stripe
    ## ------------------------------------------------------------------------------------------------ ##

    # Fillnans of the two lat/log.
    # Insert median district
    #unknown_index = test_labels[["district", "latitude", "longitude"]][test_labels["latitude"].isna()==True].index
    #test_labels.loc[unknown_index,['district']] = test_labels["district"].median()
    ## Mean the long/lat
    #test_labels["longitude"] = test_labels["longitude"].fillna(test_labels["longitude"].mean())
    #test_labels["latitude"] = test_labels["latitude"].fillna(test_labels["latitude"].mean())

    # Both nans in test-data is in same known street.
    # (37.470959, 55.570540)
    is_na = (test_labels["longitude"].isna())
    nas = test_labels.copy()[is_na]
    test_labels.loc[nas.index,['longitude']] = 37.470959
    test_labels.loc[nas.index,['latitude']] = 55.570540

    # Fix houses on the "north pole".
    is_outlier = (test_labels["longitude"] > 39) | (test_labels["longitude"] < 35)
    outliers = test_labels.copy()[is_outlier]
    for index in outliers.index:
        if test_labels.loc[index,['street']][0]=="Бунинские Луга ЖК":
            test_labels.loc[index,['longitude']] = 37.482297
            test_labels.loc[index,['latitude']] = 55.543673
        elif test_labels.loc[index,['street']][0]=="улица Центральная":
            test_labels.loc[index,['longitude']] = 37.640383
            test_labels.loc[index,['latitude']] = 55.566131
        else:
            test_labels.loc[index,['longitude']] = train_labels["longitude"].mean()
            test_labels.loc[index,['latitude']] = train_labels["latitude"].mean()
            test_labels.loc[index,['street']] = "Ленинский проспект"

    # Fill districts using long/lat
    district_centre_long = np.array(train_labels[["district", "longitude"]].groupby(['district']).mean())
    district_centre_lat = np.array(train_labels[["district", "latitude"]].groupby(['district']).mean())
    remove_nan_districts = [closest_district(row["latitude"],
     row["longitude"], district_centre_long, district_centre_lat) if math.isnan(row["district"]) else row["district"] for _, row in train_labels.iterrows()]
    train_labels["district"] = remove_nan_districts
    remove_nan_districts = [closest_district(row["latitude"],
     row["longitude"], district_centre_long, district_centre_lat) if math.isnan(row["district"]) else row["district"] for _, row in test_labels.iterrows()]
    test_labels["district"] = remove_nan_districts

    # Fix the seller with unknown category
    unknown_category = len(list(train_labels["seller"].unique())) - 1
    train_labels["seller"] = train_labels["seller"].fillna(unknown_category)
    test_labels["seller"] = test_labels["seller"].fillna(unknown_category)
    ## --------------------------------------------------------------------------------------------------------------- ##
    if fillNan:
        # Taking care of the string features first... NB! no nans in string data.
        # Fill rest with median or mean
        # Float
        train_labels[float_numerical_features] = train_labels[float_numerical_features].fillna(train_labels[float_numerical_features].mean())
        # Int
        train_labels[int_numerical_features] = train_labels[int_numerical_features].fillna(train_labels[int_numerical_features].median())
        # Cat
        train_labels[cat_features] = train_labels[cat_features].fillna(train_labels[cat_features].median())
        # Bool (The rest)
        train_labels = train_labels.fillna(train_labels.median()) # Boolean

        # Float
        test_labels[float_numerical_features] = test_labels[float_numerical_features].fillna(test_labels[float_numerical_features].mean())
        # Int
        test_labels[int_numerical_features] = test_labels[int_numerical_features].fillna(test_labels[int_numerical_features].median())
        # Cat
        test_labels[cat_features] = test_labels[cat_features].fillna(test_labels[cat_features].median())
        # Bool (The rest)
        test_labels = test_labels.fillna(test_labels.median()) # Boolean

    if "constructed" in features:
        # Is supposed to be integer.
        train["constructed"] = np.asarray(train["constructed"]).astype("int")
        test["constructed"] = np.asarray(test["constructed"]).astype("int")

    return train_labels, train_targets, test_labels

def feature_engineering(train_labels, test_labels,
    add_base_features=True,
    add_bool_features=True,
    add_weak_features=False,
    add_dist_to_metro=False,
    add_close_to_uni=False,
    add_dist_to_hospital=False,
    add_floor_features=False,
    add_street_info=False,
    add_some_more_features=False,
    add_district_information=False,
    ):

    if add_base_features:
        added_features = []
        # Add R and theta
        train_labels, test_labels = polar_coordinates(train_labels, test_labels)
        added_features.append("r")
        added_features.append("theta")

        # Add "Spacious_rooms": area per room
        train_labels['spacious_rooms'] = train_labels['area_total'] / train_labels['rooms']
        test_labels['spacious_rooms'] = test_labels['area_total'] / test_labels['rooms']
        added_features.append('spacious_rooms')

        # Newly_built
        is_new = [1 if row["constructed"] >= 2000 else 0 for _, row in train_labels.iterrows()]
        train_labels["actually_new"] = is_new
        is_new = [1 if row["constructed"] >= 2000 else 0 for _, row in test_labels.iterrows()]
        test_labels["actually_new"] = is_new
        added_features.append("actually_new")

        train_labels["rel_living"] = np.asarray(train_labels["area_living"] / train_labels["area_total"]).astype("float32")
        test_labels["rel_living"] = np.asarray(test_labels["area_living"] / test_labels["area_total"]).astype("float32")
        added_features.append("rel_living")

        train_labels["total_bathrooms"] = np.asarray(train_labels["bathrooms_private"] + train_labels["bathrooms_shared"]).astype("int")
        test_labels["total_bathrooms"] = np.asarray(test_labels["bathrooms_private"] + test_labels["bathrooms_shared"]).astype("int")
        added_features.append("total_bathrooms")

    if add_bool_features:
        # Some boolean features
        train_labels["multiple_balconies"] = np.asarray((train_labels["balconies"]>1)).astype("int")
        test_labels["multiple_balconies"] = np.asarray((test_labels["balconies"]>1)).astype("int")
        added_features.append("multiple_balconies")

        train_labels["multiple_loggias"] = np.asarray((train_labels["loggias"]>1)).astype("int")
        test_labels["multiple_loggias"] = np.asarray((test_labels["loggias"]>1)).astype("int")
        added_features.append("multiple_loggias")

        train_labels["both_windows"] = np.asarray((train_labels["windows_court"]==True) & (train_labels["windows_street"]==True)).astype("int")
        test_labels["both_windows"] = np.asarray((test_labels["windows_court"]==True) & (test_labels["windows_street"]==True)).astype("int")
        added_features.append("both_windows")

    if add_weak_features:
        # Weakly correlated ones.
        train_labels["rel_kitchen"] = np.asarray(train_labels["area_kitchen"] / train_labels["area_total"]).astype("float32")
        test_labels["rel_kitchen"] = np.asarray(test_labels["area_kitchen"] / test_labels["area_total"]).astype("float32")
        added_features.append("rel_kitchen")

        train_labels['rel_height'] = np.asarray(train_labels["floor"] / train_labels["stories"]).astype("float32")
        test_labels['rel_height'] = np.asarray(test_labels["floor"] / test_labels["stories"]).astype("float32")
        added_features.append('rel_height')

    if add_dist_to_metro:
        # Pre-calculated
        dist_to_metro_train = np.loadtxt("metro_distances_train.csv")
        dist_to_metro_test = np.loadtxt("metro_distances_test.csv")
        # Meter variant
        train_labels["dist_to_metro_m"] = dist_to_metro_train * (2*np.pi*(6371000) / 360)
        test_labels["dist_to_metro_m"] = dist_to_metro_test * (2*np.pi*(6371000) / 360)
        added_features.append('dist_to_metro_m')
        # Walking distance
        train_labels["metro_walking_distance"] = np.asarray(train_labels["dist_to_metro_m"]<train_labels["dist_to_metro_m"].median()).astype("int")
        test_labels["metro_walking_distance"] = np.asarray(test_labels["dist_to_metro_m"]<test_labels["dist_to_metro_m"].median()).astype("int")
        added_features.append('metro_walking_distance')

    if add_close_to_uni:
        # Uni location
        uni_location = (37.5286, 55.7039)
        ## Distance to state university
        dist_to_uni_train = np.zeros(len(train_labels))
        for i, row_t in train_labels.iterrows():
            apartment = (row_t["longitude"], row_t["latitude"])
            dist_to_uni_train[i] = eucledian_distance(apartment, uni_location)
        dist_to_uni_test = np.zeros(len(test_labels))
        for i, row_t in test_labels.iterrows():
            apartment = (row_t["longitude"], row_t["latitude"])
            dist_to_uni_test[i] = eucledian_distance(apartment, uni_location)
        # Meter edition
        dist_to_uni_train_meters = dist_to_uni_train*(2*np.pi*(6371000) / 360)
        dist_to_uni_test_meters = dist_to_uni_test*(2*np.pi*(6371000) / 360)
        # Close to uni
        train_labels["close_to_uni"] = np.asarray((dist_to_uni_train_meters<2000)).astype("int")
        test_labels["close_to_uni"] = np.asarray((dist_to_uni_test_meters<2000)).astype("int")
        added_features.append('close_to_uni')

    if add_dist_to_hospital:
        # Location of state hospital
        hosp_location = (37.389167, 55.746389)
        ## Distance to state university
        dist_to_hosp_train = np.zeros(len(train_labels))
        for i, row_t in train_labels.iterrows():
            apartment = (row_t["longitude"], row_t["latitude"])
            dist_to_hosp_train[i] = eucledian_distance(apartment, hosp_location)
        dist_to_hosp_test = np.zeros(len(test_labels))
        for i, row_t in test_labels.iterrows():
            apartment = (row_t["longitude"], row_t["latitude"])
            dist_to_hosp_test[i] = eucledian_distance(apartment, hosp_location)

        dist_to_hosp_train_meters = dist_to_hosp_train*(2*np.pi*(6371000) / 360)
        train_labels["dist_to_hospital_m"] = dist_to_hosp_train_meters
        dist_to_hosp_test_meters = dist_to_hosp_test*(2*np.pi*(6371000) / 360)
        test_labels["dist_to_hospital_m"] = dist_to_hosp_test_meters
        added_features.append('dist_to_hospital_m')

    if add_floor_features:
        train_labels["lives_in_highrise"] = np.asarray((train_labels["stories"]>30)).astype("int")
        test_labels["lives_in_highrise"] = np.asarray((test_labels["stories"]>30)).astype("int")
        added_features.append('lives_in_highrise')

        train_labels["first_floor"] = np.asarray(train_labels["floor"]==1).astype("int")
        test_labels["first_floor"] = np.asarray(test_labels["floor"]==1).astype("int")
        added_features.append('first_floor')

        train_labels["floor_inverse"] = train_labels["stories"]-train_labels["floor"]
        median_floor_inverse = np.median(train_labels["floor_inverse"])
        train_labels["floor_inverse"]=train_labels["floor_inverse"].where(train_labels["floor_inverse"]>=0, other=median_floor_inverse)
        test_labels["floor_inverse"] = test_labels["stories"]-test_labels["floor"]
        test_labels["floor_inverse"]=test_labels["floor_inverse"].where(test_labels["floor_inverse"]>=0, other=median_floor_inverse)
        added_features.append('floor_inverse')

    if add_street_info:
        # Seafront = набережная.
        # This was pretty much the only one correlated to price.
        on_type = [1 if "набережная" in row["street"] else 0 for _, row in train_labels.iterrows()]
        train_labels["on_seafront"] = on_type
        on_type = [1 if "набережная" in row["street"] else 0 for _, row in test_labels.iterrows()]
        test_labels["on_seafront"] = on_type
        added_features.append('on_seafront')
        # Also, these streets seemed good to live in.
        good_streets = ["Мосфильмовская улица", "набережная Пресненская",
         "улица Ефремова", "Казарменный переулок"]
        for street_name in good_streets:
            on_street = [1 if street_name in row["street"] else 0 for _, row in train_labels.iterrows()]
            train_labels["in_"+street_name] = on_street
        for street_name in good_streets:
            on_street = [1 if street_name in row["street"] else 0 for _, row in test_labels.iterrows()]
            test_labels["in_"+street_name] = on_street
            added_features.append("in_"+street_name)

    if add_some_more_features:
        # This is moved to data_cleaning.
        #train_labels = train_labels.astype({'constructed':'int'})
        #test_labels = test_labels.astype({'constructed':'int'})

        train_labels["area_floor"] = train_labels["area_total"] / train_labels["floor"]
        train_labels["area_stories"] = train_labels["area_total"] / train_labels["stories"]
        train_labels["area_rooms"] = train_labels["area_total"] / np.average(train_labels["rooms"]) # is just the invert of spacious rooms
        train_labels["old_building"] = (train_labels["constructed"]<1950)
        train_labels["cold_war_building"] = (train_labels["constructed"]>1955) & (train_labels["constructed"]<2000)
        train_labels["modern_but_not_too_modern"] = (train_labels["constructed"]>2000) & (train_labels["constructed"]<2018)
        train_labels["bathroom_area"] = (train_labels["bathrooms_private"] + train_labels["bathrooms_shared"])/train_labels["area_total"]
        train_labels['bathrooms_per_room'] = (train_labels["total_bathrooms"])/train_labels["rooms"]

        test_labels["area_floor"] = test_labels["area_total"] / test_labels["floor"]
        test_labels["area_stories"] = test_labels["area_total"] / test_labels["stories"]
        test_labels["area_rooms"] = test_labels["area_total"] / np.average(test_labels["rooms"]) # is just the invert of spacious rooms
        test_labels["old_building"] = (test_labels["constructed"]<1950)
        test_labels["cold_war_building"] = (test_labels["constructed"]>1955) & (test_labels["constructed"]<2000)
        test_labels["modern_but_not_too_modern"] = (test_labels["constructed"]>2000) & (test_labels["constructed"]<2018)
        test_labels["bathroom_area"] = (test_labels["bathrooms_private"] + test_labels["bathrooms_shared"])/test_labels["area_total"]
        test_labels['bathrooms_per_room'] = (test_labels["total_bathrooms"])/test_labels["rooms"]

        # Calculate district average areas
        arr_train = train_labels[["district","area_total","area_living","area_kitchen",]].to_numpy()
        arr_test = test_labels[["district","area_total","area_living","area_kitchen",]].to_numpy()

        arr_full = np.concatenate((arr_train,arr_test),axis=0)

        average_area_total_district = {}
        average_area_living_district = {}
        average_area_kitchen_district = {}

        for x in sorted(np.unique(arr_full[...,0])):
            average_area_total_district[x] = np.average(arr_full[np.where(arr_full[...,0]==x)][...,1])
            average_area_living_district[x] = np.average(arr_full[np.where(arr_full[...,0]==x)][...,2])
            average_area_kitchen_district[x] = np.average(arr_full[np.where(arr_full[...,0]==x)][...,3])

        # Calculate floor average areas
        arr_train = train_labels[["floor","area_total"]].to_numpy()
        arr_test = test_labels[["floor","area_total"]].to_numpy()

        arr_full = np.concatenate((arr_train,arr_test),axis=0)

        average_area_total_floor = {}
        for x in sorted(np.unique(arr_full[...,0])):
            average_area_total_floor[x] = np.average(arr_full[np.where(arr_full[...,0]==x)][...,1])

        # Calculate construction year average areas
        arr_train = train_labels[["constructed","area_total","area_living","area_kitchen",]].to_numpy()
        arr_test = test_labels[["constructed","area_total","area_living","area_kitchen",]].to_numpy()

        arr_full = np.concatenate((arr_train,arr_test),axis=0)

        average_area_total_constructed = {}
        for x in sorted(np.unique(arr_full[...,0])):
            average_area_total_constructed[x] = np.average(arr_full[np.where(arr_full[...,0]==x)][...,1])

        total_district = []
        living_district = []
        kitchen_district = []

        total_floor = []
        total_constructed = []

        for _,row in train_labels.iterrows():
            total_district.append(row['area_total']/average_area_total_district[row['district']])
            living_district.append(row['area_living']/average_area_living_district[row['district']])
            kitchen_district.append(row['area_kitchen']/average_area_kitchen_district[row['district']])
            total_floor.append(row['area_total']/average_area_total_floor[row['floor']])
            total_constructed.append(row['area_total']/average_area_total_constructed[row['constructed']] if (pd.notnull(row['area_total']) and pd.notnull(row['constructed'])) else None)

        total_district_test = []
        living_district_test = []
        kitchen_district_test = []

        total_floor_test = []
        total_constructed_test = []

        for _,row in test_labels.iterrows():
            total_district_test.append(row['area_total']/average_area_total_district[row['district']])
            living_district_test.append(row['area_living']/average_area_living_district[row['district']])
            kitchen_district_test.append(row['area_kitchen']/average_area_kitchen_district[row['district']])
            total_floor_test.append(row['area_total']/average_area_total_floor[row['floor']])
            total_constructed_test.append(row['area_total']/average_area_total_constructed[row['constructed']] if (pd.notnull(row['area_total']) and pd.notnull(row['constructed'])) else None)

        train_labels['average_area_total_district'] = total_district
        train_labels['average_area_living_district'] = living_district
        train_labels['average_area_kitchen_district'] = kitchen_district
        train_labels['average_area_total_floor'] = total_floor
        train_labels['average_area_year_constructed'] = total_constructed

        test_labels['average_area_total_district'] = total_district_test
        test_labels['average_area_living_district'] = living_district_test
        test_labels['average_area_kitchen_district'] = kitchen_district_test
        test_labels['average_area_total_floor'] = total_floor_test
        test_labels['average_area_year_constructed'] = total_constructed_test

    if add_district_information:
        density = {
            0: 701353/66.1755,
            1: 1112846/109.9,
            2: 1240062/101.889,
            3: 1394497/154.6,
            4: 1116924/117.6,
            5: 1593065/132,
            6: 1179211/111.4,
            7: 1049104/153,
            8: 779965/93.281,
            9: 215727/37.22,
            10:86752/1084.3,
            11:113569/361.4
          }
        population = {
            0: 701353,
            1: 1112846,
            2: 1240062,
            3: 1394497,
            4: 1116924,
            5: 1593065,
            6: 1179211,
            7: 1049104,
            8: 779965,
            9: 215727,
            10:86752,
            11:113569
                }
        district_area ={
            0: 66.1755,
            1: 109.9,
            2: 101.889,
            3: 154.6,
            4: 117.6,
            5: 132,
            6: 111.4,
            7: 153,
            8: 93.281,
            9: 37.22,
            10:1084.3,
            11:361.4
                }

        district_popularity_train_set = train_labels[['district','building_id']].groupby(['district']).count()
        district_popularity_test_set = test_labels[['district','building_id']].groupby(['district']).count()
        population_df = pd.DataFrame.from_dict(population,orient='index')
        total = len(train_labels) + len(test_labels)
        popularity_data_df = (district_popularity_train_set+district_popularity_test_set)/total
        popularity_data = popularity_data_df.to_dict()['building_id']
        popularity_population_df = pd.DataFrame.from_dict(popularity_data,orient='index')/population_df
        popularity_population = popularity_population_df.to_dict()[0]

        density_district = [density[row['district']] for _,row in train_labels.iterrows()]
        population_district = [population[row['district']] for _,row in train_labels.iterrows()]
        area_district = [district_area[row['district']] for _,row in train_labels.iterrows()]
        outside_MKAD = [1 if (row["district"] in [9,10,11]) else 0 for _,row in train_labels.iterrows()]
        popularity_district_data = [popularity_data[row['district']] for _,row in train_labels.iterrows()]
        popularity_district_population = [popularity_population[row['district']] for _,row in train_labels.iterrows()]

        train_labels["density_district"] = density_district
        train_labels["population_district"] = population_district
        train_labels["area_district"] = area_district
        train_labels["outside_MKAD"] = outside_MKAD
        train_labels["popularity_district_data"] = popularity_district_data
        train_labels["popularity_district_population"] = popularity_district_population

        density_district_test = [density[row['district']] for _,row in test_labels.iterrows()]
        population_district_test = [population[row['district']] for _,row in test_labels.iterrows()]
        area_district_test = [district_area[row['district']] for _,row in test_labels.iterrows()]
        outside_MKAD_test = [1 if (row["district"] in [9,10,11]) else 0 for _,row in test_labels.iterrows()]
        popularity_district_data_test = [popularity_data[row['district']] for _,row in test_labels.iterrows()]
        popularity_district_population_test = [popularity_population[row['district']] for _,row in test_labels.iterrows()]

        test_labels["density_district"] = density_district_test
        test_labels["population_district"] = population_district_test
        test_labels["area_district"] = area_district_test
        test_labels["outside_MKAD"] = outside_MKAD_test
        test_labels["popularity_district_data"] = popularity_district_data_test
        test_labels["popularity_district_population"] = popularity_district_population_test

    return train_labels, test_labels, added_features

def normalize(train_labels, test_labels, features, scaler="minMax"):
    # Only normalize/scale the numerical data. Categorical data is kept as is.
    train_labels_n = train_labels.filter(features)
    test_labels_n = test_labels.filter(features)

    # Scale it.
    if scaler=="minMax":
        print("minMax")
        scaler = MinMaxScaler(feature_range=(0, 1))
        train_labels_scaled = scaler.fit_transform(train_labels_n)
        test_labels_scaled = scaler.transform(test_labels_n)
    elif scaler=="std":
        print("Std")
        std_scale = preprocessing.StandardScaler().fit(train_labels_n)
        train_labels_scaled = std_scale.transform(train_labels_n)
        test_labels_scaled = std_scale.transform(test_labels_n)
    else:
        assert ValueError, "Incorrect scaler"

    # Re-enter proceedure
    training_norm_col = pd.DataFrame(train_labels_scaled, index=train_labels_n.index, columns=train_labels_n.columns)
    train_labels.update(training_norm_col)

    testing_norm_col = pd.DataFrame(test_labels_scaled, index=test_labels_n.index, columns=test_labels_n.columns)
    test_labels.update(testing_norm_col)

    return train_labels, test_labels

def eucledian_distance(point1, point2):
    return np.sqrt((point1[0]-point2[0])**2 + (point1[1]-point2[1])**2)

def root_mean_squared_error(y_true, y_pred):
        return K.sqrt(K.mean(K.square(y_pred - y_true)))


import re, unicodedata, nltk, json, ssl
import pandas as uwu
import numpy as np
from nltk.corpus import stopwords
from flask import Flask, request, jsonify
from tensorflow.keras.models import model_from_yaml
import tensorflow as tf
from flask import jsonify

app = Flask(__name__)
try:
    _create_unverified_https_context = ssl._create_unverified_context
except AttributeError:
    pass
else:
    ssl._create_default_https_context = _create_unverified_https_context

print('Instalando librerias de NLTK')
nltk.download('punkt')
nltk.download('stopwords')

def remove_stop_words(list_tokens):
    stop = stopwords.words('spanish')
    for token in list_tokens:
        if token.lower() in stop:
            list_tokens.pop(list_tokens.index(token))
    return list_tokens

def remove_void_elements(tokens):
    for token in tokens:
        if token == '' or token == " " or token == "  ":
            tokens.pop(tokens.index(token))
    return tokens

def normalize_text(tokens):
    for token in tokens:
        pos = tokens.index(token)
        if token.isdigit() or token.isalpha() == False:
            tokens.pop(pos)
        else:
            token = re.sub(r'[^\w\s]', ' ', token)
            if token != ' ' or token != '':
                tokens[pos] = unicodedata.normalize('NFKD', token.lower()).encode('ascii', 'ignore').decode('utf-8','ignore')
            else:
                tokens.pop(pos)
    return tokens

def create_lemma_dict():
    lemmaDiccionario = {}
    with open('/app/Herramientas/lemma.txt', 'rb') as fichero:
        datos = (fichero.read().decode('utf8').replace(u'\r', u'').split(u'\n'))
        datos = ([avance.split(u'\t') for avance in datos])
    for avance in datos:
        if len(avance) > 1:
            lemmaDiccionario[avance[1]] = avance[0]
    return lemmaDiccionario

def lemmatize(lemmaDiccionario, word):
    return lemmaDiccionario.get(word, word + u'')

def lemmatize_words(words):
    lemmaDiccionario = create_lemma_dict()
    new_words = []
    for palabra in words:
        new_word = lemmatize(lemmaDiccionario, palabra)
        new_words.append(new_word)
    return new_words

def wordsToNumbers(tokens, vocabulary):
    number_array = np.zeros(10, dtype=int)
    for i in range(len(tokens)):
        if i < 10 and tokens[i] in vocabulary:
            number_array[i] = vocabulary.index(i) + 1
    return number_array

def processText(Text):
    list_text = nltk.word_tokenize(Text)
    list_text = remove_stop_words(list_text)
    list_text = remove_void_elements(list_text)
    list_text = normalize_text(list_text)
    list_text = lemmatize_words(list_text)
    if len(list_text) > 10:
        return list_text[:10]
    else:
        return list_text

def get_dimension_of_text(text, area):
    if not isinstance(text, str) or len(text) < 5:
        return 'Texto a clasificar mal ingresado o muy corto (minimo 5 palabras)', 200
    if area not in ['Gobierno Corporativo', 'Medio Ambiente', 'Social Externo', 'Social Interno']:
        return 'El area ingresada no coincide, puede ser Gobierno Corporativo, Medio Ambiente, Social Externo o Social Interno', 200
    list_words = processText(text)
    vocab = list(uwu.read_csv('./Herramientas/vocabulary.csv'))
    list_words_numeric = wordsToNumbers(list_words, vocab)
    if area == 'Gobierno Corporativo':
        df = uwu.read_csv('./Herramientas/dims_gob.csv', sep=";")
        RNNmodel = tf.keras.models.load_model('model_gob.h5')
    elif area == 'Medio Ambiente':
        df = uwu.read_csv('./Herramientas/dims_amb.csv', sep=";")
        RNNmodel = tf.keras.models.load_model('./model_amb.h5')
    elif area == 'Social Externo':
        df = uwu.read_csv('./Herramientas/dims_soce.csv', sep=";")
        RNNmodel = tf.keras.models.load_model('./model_soce.h5')
    elif area == 'Social Interno':
        df = uwu.read_csv('./Herramientas/dims_soci.csv', sep=";")
        RNNmodel = tf.keras.models.load_model('./model_soci.h5')

    print("\n\n\nello, como estas")

    dim = RNNmodel.predict_classes(np.array([list_words_numeric]))[0]
    print("\n\n\nello, como estas")

    dim_str = df.Dimension[dim]
    return dim, dim_str, list_words

@app.route('/classify/v1', methods=["POST"])
def classify_text():
    body = request.get_json()
    if 'area' in body:
        area = body['area']
    else:
        return 'Area no encontrada', 200
    if 'text' in body:
        text = body['text']
    else:
        return 'Texto no encontrado', 200
    dimension, dimension_string, palabras = get_dimension_of_text(text, area)
    response = {
            'dim_cod': int(dimension),
            'dim_str': dimension_string,
            'palabras': palabras
            }
    return response, 200

@app.route('/')
def presentation():
    return '''
    Desarrollo de Proyecto de tesis de Ricardo Alvarez Zambrano

    Consiste en un clasificador de texto de dimensiones ESG para ESG Compass

    Para clasificar texto se debe hacer una request "POST" a la ruta < /classify/v1 > y en el body incluir un json los siguientes campos

    {

        'area':'Gobierno Corporativo',

        'text':'El texto que se desea clasificar'

    }

    Las areas posibles que acepta el modelo son:

    - Gobierno Corporativo

    - Social Externo

    - Social Interno

    - Medio Ambiente

    '''

if __name__ == "__main__":
    app.run(host="0.0.0.0", debug=False, port=80)


import os
import random
import time
import torch
import torch.backends.cudnn as cudnn
import models
from utils.logger import Logger
import yaml
import argparse
from utils import utils
import sys

def add_learner_params():
    parser = argparse.ArgumentParser(description='ML')
    parser.add_argument('--problem', default='finetune',
                        help='The problem to train',
                        choices=models.REGISTERED_MODELS,
                        )
    parser.add_argument('--name', default='demo',
                        help='Name for the experiment')
    parser.add_argument('--nodes', default='',
                        help='slurm nodes for the experiment')
    parser.add_argument('--slurm_partition', default='',
                        help='slurm partitions for the experiment')
    # optimizer params
    parser.add_argument('--lr_schedule', default='warmup-const')
    parser.add_argument('--loss', default='ce')
    parser.add_argument('--opt', default='sgd',
                        help='Optimizer to use', choices=['sgd', 'adam', 'lars'])
    parser.add_argument('--iters', default=-1, type=int,
                        help='The number of optimizer updates')
    parser.add_argument('--warmup', default=0, type=float,
                        help='The number of warmup iterations in proportion to \'iters\'')
    parser.add_argument('--lr', default=0.1, type=float,
                        help='Base learning rate')
    parser.add_argument('--inner_lr', default=0.1, type=float,
                        help='inner learning rate')
    parser.add_argument('--q', default=0.66, type=float,
                        help='generalized ce loss q value')
    parser.add_argument('--wd', '--weight_decay',
                        default=1e-4, type=float, dest='weight_decay')
    # noise params
    parser.add_argument('--corruption_prob', type=float, default=0.9,
                        help='label noise')
    parser.add_argument('--corruption_type', '-ctype', type=str, default='unif',
                        help='Type of corruption ("unif" or "flip" or "flip2" or "asym").')

    # trainer params
    parser.add_argument('--save_freq', default=1000000000000,
                        type=int, help='Frequency to save the model')
    parser.add_argument('--log_freq', default=100,
                        type=int, help='Logging frequency')
    parser.add_argument('--eval_freq', default=100000000000,
                        type=int, help='Evaluation frequency')
    parser.add_argument('-j', '--workers', default=4, type=int,
                        help='The number of data loader workers')
    parser.add_argument('--seed', default=222, type=int, help='Random seed')
    # parallelizm params:
    parser.add_argument('--test_bs', default=256, type=int)
    parser.add_argument('--encoder_ckpt', default='',
                        help='Path to the encoder checkpoint')
    parser.add_argument('--encoder_type', default='simclr',
                        help='pretrained "imagenet" or "simclr" for simclr')
    parser.add_argument('--config', default='configs/cifar10.yaml', type=str,
                        help='the number of nodes (scripts launched)',
                        )
    #
    parser.add_argument('--cuda', action='store_true')
    parser.add_argument('--neptune', action='store_true')

    params = parser.parse_args()
    d = vars(params)
    if params.config:
        with open(params.config, 'r') as stream:
            try:
                params_2 = yaml.safe_load(stream)
            except yaml.YAMLError as exc:
                print(exc)
    for k, v in params_2.items():
        d[k] = v
    return params

def main():
    args = add_learner_params()
    if args.seed != -1:
        random.seed(args.seed)
        torch.manual_seed(args.seed)
        cudnn.deterministic = True
    args.root = 'logs/'+args.name+'/'

    if args.neptune:
        import neptune
        project = "arighosh/pretrain_noisy_label"
        neptune.init(project_qualified_name=project,
                     api_token=os.environ["NEPTUNE_API_TOKEN"])
        neptune.create_experiment(
            name=args.name, send_hardware_metrics=False, params=vars(args))
    fmt = {
        'train_time': '.3f',
        'val_time': '.3f',
        'train_epoch': '.1f',
        'lr': '.1e',
    }
    logger = Logger('logs', base=args.root, fmt=fmt)
    if torch.cuda.is_available():
        device = torch.device('cuda')
    else:
        device = torch.device('cpu')
    if args.cuda:
        assert device.type == 'cuda', 'no gpu found!'

    with open(args.root+'config.yml', 'w') as outfile:
        yaml.dump(vars(args), outfile, default_flow_style=False)

    # create model
    model = models.REGISTERED_MODELS[args.problem](args, device=device)
    cur_iter = 0
    # Data loading code
    model.prepare_data()

    continue_training = cur_iter < args.iters
    data_time, it_time = 0, 0
    best_acc = 0.
    best_valid_acc, best_acc_with_valid = 0, 0

    while continue_training:
        train_loader, test_loader, valid_loader, meta_loader = model.dataloaders(
            iters=args.iters)
        train_logs = []
        model.train()
        start_time = time.time()
        for _, batch in enumerate(train_loader):
            cur_iter += 1
            batch = [x.to(device) for x in batch]
            data_time += time.time() - start_time
            logs = {}
            if args.problem not in {'finetune'}:
                meta_batch = next(iter(meta_loader))
                meta_batch = [x.to(device) for x in meta_batch]
                logs = model.train_step(batch, meta_batch, cur_iter)
            else:
                logs = model.train_step(batch, cur_iter)

            # save logs for the batch
            train_logs.append({k: utils.tonp(v) for k, v in logs.items()})
            if cur_iter % args.eval_freq == 0 or cur_iter >= args.iters:
                test_start_time = time.time()
                test_logs, valid_logs = [], []
                model.eval()
                with torch.no_grad():
                    for batch in test_loader:
                        batch = [x.to(device) for x in batch]
                        logs = model.test_step(batch)
                        test_logs.append(logs)
                    for batch in valid_loader:
                        batch = [x.to(device) for x in batch]
                        logs = model.test_step(batch)
                        valid_logs.append(logs)
                model.train()
                test_logs = utils.agg_all_metrics(test_logs)
                valid_logs = utils.agg_all_metrics(valid_logs)
                best_acc = max(best_acc, float(test_logs['acc']))
                test_logs['best_acc'] = best_acc
                if float(valid_logs['acc']) > best_valid_acc:
                    best_valid_acc = float(valid_logs['acc'])
                    best_acc_with_valid = float(test_logs['acc'])
                test_logs['best_acc_with_valid'] = best_acc_with_valid
                #

                if args.neptune:
                    for k, v in test_logs.items():
                        neptune.log_metric('test_'+k, float(v))
                    for k, v in valid_logs.items():
                        neptune.log_metric('valid_'+k, float(v))
                    test_it_time = time.time()-test_start_time
                    neptune.log_metric('test_it_time', test_it_time)
                    neptune.log_metric('test_cur_iter', cur_iter)
                logger.add_logs(cur_iter, test_logs, pref='test_')
            it_time += time.time() - start_time
            if (cur_iter % args.log_freq == 0 or cur_iter >= args.iters):
                train_logs = utils.agg_all_metrics(train_logs)
                if args.neptune:
                    for k, v in train_logs.items():
                        neptune.log_metric('train_'+k, float(v))
                    neptune.log_metric('train_it_time', it_time)
                    neptune.log_metric('train_data_time', data_time)
                    neptune.log_metric(
                        'train_lr', model.optimizer.param_groups[0]['lr'])
                    neptune.log_metric('train_cur_iter', cur_iter)
                logger.add_logs(cur_iter, train_logs, pref='train_')
                logger.add_scalar(
                    cur_iter, 'lr', model.optimizer.param_groups[0]['lr'])
                logger.add_scalar(cur_iter, 'data_time', data_time)
                logger.add_scalar(cur_iter, 'it_time', it_time)
                logger.iter_info()
                logger.save()
                data_time, it_time = 0, 0
                train_logs = []
            if cur_iter >= args.iters:
                continue_training = False
                break
            start_time = time.time()

if __name__ == '__main__':
    main()


from lib2to3.pytree import Base

class Character(Base):
      def __init__(self, name,year_born, year_died, bonded_with, birthplace,abilities,species,titles
                   ,profession,groups,nationality,planet,universe,books,ethnicity,spouse,relatives,
                   acestors,alias,residence,children):
        super().__init__(name,year_born,year_died)
        self.bonded_with = bonded_with
        self.birthplace =birthplace
        self.abilities = abilities
        self.species = species
        self.titles = titles
        self.profession = profession
        self.groups = groups
        self.nationality = nationality
        self.planet = planet
        self.universe = universe
        self.books = books
        self.ethnicity=ethnicity
        self.spouse=spouse
        self.relatives =relatives
        self.acestors =acestors
        self.alias =alias
        self.residence =residence
        self.children = children


from django.conf import settings
from django.urls import include, path
from rest_framework_nested.routers import DefaultRouter, NestedSimpleRouter, SimpleRouter

from todo_app.checklists.api.views import ChecklistItemViewSet, ChecklistViewSet
from todo_app.users.api.views import UserViewSet

if settings.DEBUG:
    router = DefaultRouter()
else:
    router = SimpleRouter()

router.register("users", UserViewSet)
router.register("checklists", ChecklistViewSet)
checklists_router = NestedSimpleRouter(router, r"checklists", lookup="checklist")
checklists_router.register(r"items", ChecklistItemViewSet, basename="checklist-items")

app_name = "api"
urlpatterns = (
    router.urls
    + checklists_router.urls
    + [
        path("auth/", include("dj_rest_auth.urls")),
        path("auth/registration/", include("dj_rest_auth.registration.urls")),
    ]
)


from gym import spaces
from robot_sf.robot_env import RobotEnv, OBS_DRIVE_STATE, OBS_RAYS

def test_can_create_env():
    env = RobotEnv()
    assert env is not None

def test_can_return_valid_observation():
    env = RobotEnv()
    drive_state_spec: spaces.Box = env.observation_space[OBS_DRIVE_STATE]
    lidar_state_spec: spaces.Box = env.observation_space[OBS_RAYS]

    obs = env.reset()

    assert isinstance(obs, dict)
    assert OBS_DRIVE_STATE in obs and OBS_RAYS in obs
    assert drive_state_spec.shape == obs[OBS_DRIVE_STATE].shape
    assert lidar_state_spec.shape == obs[OBS_RAYS].shape

def test_can_simulate_with_pedestrians():
    total_steps = 1000
    env = RobotEnv()
    env.reset()
    for _ in range(total_steps):
        rand_action = env.action_space.sample()
        _, _, done, _ = env.step(rand_action)
        if done:
            env.reset()


import cCommonGame
import json
import numpy
import sys

class WosInterface(object):
    def __init__(self):
        self.actions = None
        self.battlefield = None
        # Server config, server_cfg not in use to be refractored
        self.cfg = dict()
        self.client_cfg = dict()
        self.console = None
        self.is_debug = False
        self.loading_widget = None
        self.player_info = None
        self.server_cfg = None
        self.window = None
        self.read_client_config()
        self.overwrite_client_config()

    def clean_up(self):
        self.loading_widget.deleteLater()

    def client_config(self):
        return self.client_cfg

    def read_client_config(self):
        try:
            with open('client/game_client.cfg') as data_file:
                self.client_cfg = json.load(data_file)
                random_seed = self.client_cfg['random_seed']
                numpy.random.seed(seed=random_seed)
        except FileNotFoundError:
            print('No such file or directory: \'client/game_client2.cfg\'')

    def overwrite_client_config(self):
        if len(sys.argv) > 1:
            try:
                self.client_cfg.update(json.loads(sys.argv[1]))
            except TypeError:
                print("Second argument must be a valid JSON string")

    def log(self, text, type=cCommonGame.LogType.GAME):
        self.console.log_simple(text, type)

    def main_window(self):
        return self.window

    def toggle_overlay(self, is_show, text="Please wait..."):
        if self.loading_widget is None:
            return
        if is_show:
            self.loading_widget.set_text(text)
            self.loading_widget.show()
        else:
            self.loading_widget.close()

    def server_config(self):
        return self.server_cfg