"""This module implements classes and independent functions related to feature extraction
module of our work.
To be specific, this module helps identify handful of best features out of humongous number
of features; created from raw data """

import numpy as np
import pandas as pd
from namedlist import namedlist
from sklearn.metrics import mean_squared_error
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedShuffleSplit as split
from sklearn.ensemble import BaggingRegressor
# from sklearn.ensemble import RandomForestRegressor
import warnings
warnings.filterwarnings('ignore')

def rmse(labels, predictions):
    """
    This function returns the root mean squared error.
    Note: If predictions is an integer, we interpret it as a constant prediction.
    """

    """ If predictions is an integer, we make it a array to comply with sklearn API """
    if isinstance(predictions, int):
         # create an array same as labels and fill it with constant prediction
        predictions = np.full(labels.shape, predictions)

    """ mean_squared_error is an error metric; imported from sklearn. """
    mse = mean_squared_error(labels, predictions)
    return np.sqrt(mse)

def train_test_split(data, test_ratio = 0.5, n_splits=10, best_split = True):
    """
    This function splits the data into two using stratified sampling in ratios as determined by test_ratio.
    The strata is constructed out of creating quartile splits on the target variable, i.e., sales_volume.
    If the best_split is True, The split which yields the minimum differen in the means of target is
    returned. Else, the last split is returned as it is.
    Note the number of splits are determined by n_splits"""

    # Object for stratified sampling
    split_obj = split(n_splits=n_splits, test_size=test_ratio, random_state=180)
    # Discretizing the target volume to guide stratified sampling
    data['categories'] = pd.qcut(data['sales_volume'], 4, labels=["low", "low mid",'high mid',"high"])

    # best split is one that yields least difference in mean sales_volume of both folds
    least_diff_in_means = None
    best_split = None, None
    # Looping over each split
    for idx_train, idx_test in split_obj.split(data, data['categories']):
        train = data.iloc[idx_train]
        test = data.iloc[idx_test]

        diff_in_means = abs(train.sales_volume.mean() - test.sales_volume.mean())
        """ Update the best split if best_split=True and
        either the current split is the first split or the best split.
        """
        if best_split and ((least_diff_in_means is None) or (least_diff_in_means > diff_in_means)):
            least_diff_in_means = diff_in_means
            best_split = idx_train, idx_test

    if best_split[0] is None:
        best_split = idx_train, idx_test

    del data['categories']
    idx_train, idx_test = best_split

    train = data.iloc[idx_train]
    test = data.iloc[idx_test]

    return train, test

class IterVarModel:
    """ This class iteratively find best features one by one iteratively; starting from no features """

    """
    At a particular iteration, all the candidate features are evaluated and the features that
    yields the best 2-fold cross validation performance are added to the model (best features).
    Number of best features to extract is determined by max_features_to_extract.
    If none of the features improves performance beyond already obtained in the previous
    iteration, The feature search process stops even before finding max_features_to_extract features.
    """

    """
    This class maintains two folds for performance evaluation and comparison.
    """

    "Train fold 1 and evaluate fold 2 and call it performance over fold 2"
    "Train fold 2 and evaluate fold 1 and call it performance over fold 1"

    "Note that a specified model is used for all kind of training, testing purposes."

    class RMSEFolds:
        """ A nested class that we define to maintain and compare RMSE results over both folds """
        def __init__(self, rmse_1, rmse_2):
            self.fold_1 = rmse_1 # RMSE over fold 1
            self.fold_2 = rmse_2 # RMSE over fold 2

        def __lt__(self,other):
            """
            __lt__ is a Special method that can define < operator on class instances.
            We define RMSE1 < RMSE2 if and only if the RMSE1 is strictly lower than RMSE2
            in both the folds.
            """
            # defining < condition.
            # Condition 1 - RMSE_1 < RMSE_2 if results of both folds in RMSE_1 are less than that in RMSE_2
            # cond_1 = (self.fold_1 < other.fold_1) and (self.fold_2 < other.fold_2)
            # Condition 2 - RMSE_1 < RMSE_2 if the sum of rmse in both folds of RMSE_1 is less than that in RMSE_2
            # cond = (self.fold_1 ** 2 + self.fold_2 ** 2) < (other.fold_1 ** 2 + other.fold_2 ** 2)
            cond = (self.fold_1 < other.fold_1) and (self.fold_2 < other.fold_2)
            # RMSE_1 < RMSE_2 if either condition is true

            return cond# _1 or cond_2

    # Special method that gets run on object instantiation.
    def __init__(self,  data, model, max_features_to_extract):
        # data over which we create folds and extract best features.
        self.data = data
        # maximum feautres to extract
        self.max_features_to_extract = max_features_to_extract
        # model to be used in feature evaluations
        self.model = model
        # input columns are all the columns in the dataframe data except the target
        self.input_variables = [col for col in self.data.columns if col not in ['sales_volume']]
        # maintaining data for the folds. This attribute holds data related to folds.
        self.folds = None

        # Maintains a list of useful features
        self.extracted_features = []
        # Stops the feature extraction process if it becomes True
        self.stop_feature_extraction = False
        # create 2 folds out of all the data. Basically, split data into folds and also create additional variables.
        self.create_folds()

    def standardize_folds_inputs(self):
        """ Standardize inputs in fold 1 from parameters obtained in fold 2 and
        Standardize inputs in fold 2 from parameters obtained in fold 1
        Logic: test data cannot know her own mean and variance;
        hence has to be standardize with training set parameters
        """

        fold_1_X = self.folds[1].input
        fold_2_X = self.folds[2].input

        # get parameters from fold 1 and standardize and update fold 2
        model = StandardScaler() # standard scalar
        model.fit(fold_1_X) # get parameters from inputs in fold 1
        self.folds[2]._update(input=pd.DataFrame(model.transform(fold_2_X), columns=fold_1_X.columns)) # transform inputs in fold 2

        # get parameters from fold 2 and standardize and update fold 1
        model = StandardScaler() # standard scalar
        model.fit(fold_2_X) # get parameters from inputs in fold 2
        self.folds[1]._update(input=pd.DataFrame(model.transform(fold_1_X), columns=fold_1_X.columns)) # transform inputs in fold 1

    def add_data_in_folds(self):
        data = self.data
        # We use stratified sampling to split the data; see function train_test_split() for details
        fold_1_data, fold_2_data = train_test_split(data, test_ratio = 0.5, n_splits=10, best_split = True)
        # inputs
        input_features = self.input_variables

        ## Now we add inputs and outputs to each fold.
        # update inputs
        self.folds[1]._update(input=fold_1_data[input_features])
        self.folds[2]._update(input=fold_2_data[input_features])

        # update outputs
        self.folds[1]._update(output=fold_1_data['sales_volume'])
        self.folds[2]._update(output=fold_2_data['sales_volume'])

    def create_folds(self):
        """
        This function uses stratified sampling to split data into two equal-sized folds
        and maintains these folds using class attribute of folds.
        We use namedlist; one for each fold to hold its data
        """

        """
        namedlist is a factory function for creating mutable collections of list items;
        it is similar to python's list but enables us to name each component and access using
        dot notation.
        """
        Fold = namedlist('Fold', 'input output rmse')

        """
        class attribute folds is a dictionary with 2 keys;
        key=1, refers to namedlist that holds data related to fold 1
        key=2, refers to namedlist that holds data related to fold 2
        """
        self.folds = dict()
        for i in [1,2]:
            self.folds[i] = Fold(input=None, output=None,rmse=None)

        # add inputs and outputs to the folds by intelligently splitting data; see class method add_data_in_folds()
        self.add_data_in_folds()
        # Standardize inputs in the folds for better ML performance; see class method standardize_folds_inputs()
        self.standardize_folds_inputs()

        """
        Now after having inputs and outputs in both folds, we update RMSE.
        As of now, we have not extracted any feature.
        Hence, we consider a base model i.e., one that spits out mean of its training target.
        """
        # predictions of base model over fold 1 is a constant; mean of target variable in fold 2
        # predictions of base model over fold 2 is a constant; mean of target variable in fold 1
        # updating RMSE based on this logic.

        self.folds[1]._update(rmse=rmse(np.abs(self.folds[1].output - self.folds[2].output.mean()), 0))
        self.folds[2]._update(rmse=rmse(np.abs(self.folds[2].output - self.folds[1].output.mean()), 0))

    def eval_fold(self, eval_fold_number, features):
        """
        This function evaluates a fold specified by eval_fold_number based on features
        and returns RMSE
        """
        "fold 1 is evaluated by training over fold 2 and evaluating over fold 1"
        train_fold = 1 if eval_fold_number == 2 else 2
        test_fold = 2 if eval_fold_number == 2 else 1

        model = self.model
        # training data from train_fold
        X, Y = self.folds[train_fold].input[features], self.folds[train_fold].output
        # learning
        model.fit(X, Y)

        # test data
        test_X, test_Y = self.folds[test_fold].input[features], self.folds[test_fold].output
        # prediction
        test_predict = model.predict(test_X)
        # evaluate predictions and compute rmse
        tmp_rmse = rmse(test_Y, test_predict)

        return tmp_rmse

    def is_new_feature_good(self, features):
        """
        This function evaluates fold 1 and fold 2 with features
        and determines if features leads to better performance
        compared to extracted best features.
        """
        # class method eval_fold() is used to evaluate a fold and returns RMSE. see eval_fold()
        rmse_1 = self.eval_fold(1, features)
        rmse_2 = self.eval_fold(2, features)

        # Construct an RMSE object comprising RMSEs of folds resulted from current features.
        RMSE = self.RMSEFolds(rmse_1, rmse_2)

        ## Construct an RMSE object comprising RMSEs of folds resulted best features obtained so far
        RMSE_current = self.RMSEFolds(self.folds[1].rmse, self.folds[2].rmse)

        result = False
        """if RMSE is better than RMSE_current, then set of variables in features are better
        than ones in best features extracted so far
        """
        if RMSE < RMSE_current:
            # do advanced analysis on residuals.
            result = True

        return result, RMSE

    def add_var(self):
        """
        This method search for the variable; if such a variable exists that when
        added can improve performance
        """
        #
        # We define best_RMSE to be RMSE reached in previous iteration.
        best_RMSE = self.RMSEFolds(self.folds[1].rmse, self.folds[2].rmse)
        # Initially a None, best_var indicates the candidate variable that can be included.
        best_var = None # maintain the best variable found in this iteration

        # Looping over the candidate variables
        for col in self.input_variables:
            # candidate variable should not be already in the best extracted features.
            if col not in self.extracted_features:
                # make a temporary list of features by adding candidate feature to the existing best features.
                tmp_features = self.extracted_features + [col]
                """
                Evaluate the goodness of candidate feature;
                see class method is_new_feature_good() for further details.
                is_good=True indicates that the candidate variable can improve the performance.
                """
                is_good, RMSE = self.is_new_feature_good(tmp_features)
                # Update the best_var if is_good=True and RMSE is better than best RMSE so far.
                if (is_good and (RMSE < best_RMSE)):
                    best_RMSE = RMSE
                    best_var = col

        # If we find a variable that can improve performance.
        if best_var is not None:
            print('adding_variable: {}'.format(best_var))
            self.extracted_features.append(best_var)

            # Update rmse and residuals
            self.folds[1]._update(rmse=best_RMSE.fold_1)
            self.folds[2]._update(rmse=best_RMSE.fold_2)

        # If we cannot find a variable that can improve performance.
        else:
            # Turning this to True stops feature extraction.
            self.stop_feature_extraction = True
            # print('new features cannot be added')

    def extract_features(self):
        """
        This function runs feature extraction routines until either
        the maximum allowed features is reached or when none of the
        variables can improve the performance by getting added.
        """
        for _ in range(self.max_features_to_extract):
            # running until stop_feature_extraction=False
            if not self.stop_feature_extraction:
                # add_var() is a function that looks for best variable to add; see method add_var() for details.
                self.add_var()

        return self.extracted_features

def extract_model_features(model, df, max_features=5):
    """
    This function extracts features in the dataframe df based on specified model;
    this function initializes instance of class IterVarModel with specified model
    and the number of features to extract.
    It uses class method extract_features() to extract best features and returns
    these features
    """
    # class instance.
    tmp = IterVarModel(df, model, max_features)
    return tmp.extract_features()

def bagging_feature_extraction(model, df, max_features):
    """
    This function firstly defines a bagging model with specified base model
    and uses this bagging model to extract features using function extract_model_features()
    Bagging helps shortlisting best features and avoids unstable features.
    base model is subsequently used over the extracted features to extract features.
    """
    # Bagging model with each estimator utilizing 60% of the data; adding some randomness.
    bagging_model = BaggingRegressor(base_estimator=model, max_samples=0.6, random_state=25)
    # Extract bagging features
    print('-'*18)
    print('Bagging Features')
    print('-'*18)
    bagging_features = extract_model_features(bagging_model, df, max_features=15) # 15
    # Modify the dataframe to include only the bagging features.
    features_to_retain = bagging_features + ['sales_volume']
    df = df[features_to_retain]
    # feature extraction from the remaining features using base model
    print('-'*18)
    print('Final Features')
    print('-'*18)
    base_model_features = extract_model_features(model, df, max_features=max_features)
    return base_model_features

def feature_extraction(models, df, max_features = 5, precede_bagging=False):
    """
    This function uses multiple base models; stored in dictionary models and runs feature extract for each.
    It may or may not include bagging based on precede_bagging.
    The record for each model is stored in a namedlist.
    The function returns a list of namedlists; each holding a record for a model.
    """
    # A named list to maintain data for each model
    """
    namedlist is a factory function for creating mutable collections of list items;
    it is similar to python's list but enables us to name each component and access using
    dot notation.
    """
    Model = namedlist('Model', 'name sklearn_form extracted_features')
    # a list of model features is created in which each component correspond to a model.
    # list to hold namedlists.
    models_features = []
    # iterate over each model in models
    for model_name, model in models.items():
        print('\n')
        print('*'*18)
        print(model_name)
        print('*'*18)
        # If the features list should be made smaller with bagging
        if precede_bagging:
            tmp_features = bagging_feature_extraction(model, df, max_features)
        else:
            tmp_features = extract_model_features(model, df, max_features=max_features)

        # creating namedlist to store records for the model.
        tmp_model = Model(name=model_name, sklearn_form=model, extracted_features=tmp_features)
        # adding record to the list
        models_features.append(tmp_model)

    return models_features

if __name__ == '__main__':
    print('This file is not run as a module')


# import tenTo8
import encdec_8b10b
import veri

# encdec_8b10b.trans10to8(ten)

def negate(Str):
    X = Str.replace('0','x')
    X = X.replace('1','0')
    X = X.replace('x','1')
    return X

K280_m = '0011110100'
K280_p = negate(K280_m)

K281_m = '0011111001'
K281_p = negate(K281_m)

K283_m = '0011110011'
K283_p = negate(K283_m)

K284_m = '0011110010'
K284_p = negate(K284_m)

K285_m = '0011111010'
K285_p = negate(K285_m)

K287_m = '0011111000'
K287_p = negate(K287_m)

import logs

class lvdsMonitor(logs.driverClass):
    def __init__(self,Path,Monitors):
        logs.driverClass.__init__(self,Path,Monitors)
        self.Pin0 = 'lvds0'
        self.Pin1 = 'lvds1'
        self.LVDS0 = ''
        self.LVDS1 = ''
        self.state0 = 'idle'
        self.state1 = 'idle'
        self.CRC = []
        self.Ones0 = 0
        self.Ones1 = 0
        self.txc_0 = -1
        self.txd_0 = -1
        self.txc_1 = -1
        self.txd_1 = -1
        self.Stream = [[],[]]
        self.Chars0,self.Chars1 = [],[]
        self.syncState = 'idle'
        self.dins = []
        self.State = 'idle'
        self.Frame = []
        self.FrameId = 0
        self.Bytes = 0
        self.Indx = 0
        self.keep = False

    def run(self):
#        self.run1()
        self.run0()

    def run1(self):
        if veri.peek('tb.dut.dig_top.usat_tx.enable')!='1': return
        cnt = logs.peek('tb.dut.dig_top.usat_tx.serializer.cnt')
        if cnt!=11: return
        din0 = veri.peek('tb.dut.dig_top.usat_tx.serializer.data_in_0')
        din1 = veri.peek('tb.dut.dig_top.usat_tx.serializer.data_in_1')
        Chr0 = encdec_8b10b.trans10to8(din0)
        Chr1 = encdec_8b10b.trans10to8(din1)
        Chr0 = nice(Chr0)
        Chr1 = nice(Chr1)
        self.Stream[0].append(Chr0)
        self.Stream[1].append(Chr1)
        self.dins.append(('%03x'%(logs.intx(din1)),'%03x'%(logs.intx(din0))))
        if len(self.dins)>8: self.dins.pop(0)
        if len(self.Stream[0])>8: self.Stream[0].pop(0)
        if len(self.Stream[1])>8: self.Stream[1].pop(0)
#        if len(self.Stream[0])==8:
#            logs.log_info('STREAM side0 %d %s %s'%(len(self.Stream[0]),self.Stream[0],self.Stream[1]))
#            logs.log_info('STREAM side1 %d %s %s'%(len(self.Stream[0]),self.Stream[0],self.Stream[1]))

        if syncAll(self.Stream[0]) and syncAll(self.Stream[1]):
            self.syncState = 'idle'
        if self.syncState == 'idle':
            if sync8(self.Stream[0]) and sync8(self.Stream[1]):
                self.syncState = 'sync8'
                self.Stream[0]=[]
                self.Stream[1]=[]
        elif self.syncState == 'sync8':
            if len(self.Stream[0])==8:
                logs.log_info('STREAM8   %s %s'%(self.Stream[1],self.Stream[0]))
#                logs.log_info('STREAM8 X  %s'%str(self.dins))
                self.Stream[0]=[]
                self.Stream[1]=[]

        txc_0 = logs.peek('tb.dut.dig_top.usat_tx.txc_0')
        txd_0 = logs.peek('tb.dut.dig_top.usat_tx.txd_0')
        txc_1 = logs.peek('tb.dut.dig_top.usat_tx.txc_1')
        txd_1 = logs.peek('tb.dut.dig_top.usat_tx.txd_1')

#        logs.log_info('side0 %s %s  %s %02x     side1 %s %s    %s %02x'%(din0,Chr0,self.txc_0,self.txd_0,din1,Chr1,self.txc_1,self.txd_1))

        self.txc_0 = txc_0
        self.txd_0 = txd_0
        self.txc_1 = txc_1
        self.txd_1 = txd_1

    def run0(self):
        lvds0 = self.peekbin(self.Pin0)
        lvds1 = self.peekbin(self.Pin1)
        self.run_lvds(lvds0,lvds1)

    def run_lvds(self,lvds0,lvds1):
        if lvds0 == '1': self.Ones0 += 1
        if lvds0 == '0': self.Ones0 -= 1
        if lvds1 == '1': self.Ones1 += 1
        if lvds1 == '0': self.Ones1 -= 1

        self.LVDS0 += lvds0
        if len(self.LVDS0)>80: self.LVDS0 = self.LVDS0[-80:]
        self.LVDS1 += lvds1
        if len(self.LVDS1)>80: self.LVDS1 = self.LVDS1[-80:]

        if self.state0 == 'idle':
            if len(self.LVDS0)>60:
                if  idleChar(self.LVDS0[:10]) and idleChar(self.LVDS0[10:20]) and idleChar(self.LVDS0[20:30]):
                    if  idleChar(self.LVDS0[30:40]) and idleChar(self.LVDS0[40:50]) and idleChar(self.LVDS0[50:60]):
                        self.state0 = 'sync8'
                        self.LVDS0 = self.LVDS0[60:]
        elif self.state0 == 'sync8':
            if len(self.LVDS0)>=10:
                Chr0 = encdec_8b10b.trans10to8(self.LVDS0[:10])
                Chr0 = nice(Chr0)
                self.Chars0.append(Chr0)
                logs.log_info('CHARS0 %s %s'%(str(self.Chars0),str(self.LVDS0)),2)
                self.LVDS0 = self.LVDS0[10:]

        if self.state1 == 'idle':
            if len(self.LVDS1)>60:
                if  idleChar(self.LVDS1[:10]) and idleChar(self.LVDS1[10:20]) and idleChar(self.LVDS1[20:30]):
                    if  idleChar(self.LVDS1[30:40]) and idleChar(self.LVDS1[40:50]) and idleChar(self.LVDS1[50:60]):
                        self.state1 = 'sync8'
                        self.LVDS1 = self.LVDS1[60:]
        elif self.state1 == 'sync8':
            if len(self.LVDS1)>=10:
                Chr1 = encdec_8b10b.trans10to8(self.LVDS1[:10])
                Chr1 = nice(Chr1)
                self.Chars1.append(Chr1)
                logs.log_info('CHARS1 %s %s'%(str(self.Chars1),str(self.LVDS1)),2)
                self.LVDS1 = self.LVDS1[10:]

        if self.State == 'idle':
#            if self.keep:
#                logs.log_info('CCCC 1=%s 0=%s'%(self.Chars1,self.Chars0))
            if len(self.Chars1)>=10:
                if (self.Chars1[0] in ['K285','K283','K280'])and(self.Chars1[1] == '00')and(self.Chars1[2]=='K281'):
                    if (self.Chars0[0] in ['K285','K283','K280'])and(self.Chars0[1] == '00'):
                        self.State = 'inframe'
                        self.Frame = self.Chars1[2:10]+ self.Chars0[2:10]
                        self.Chars1 = self.Chars1[10:]
                        self.Chars0 = self.Chars0[10:]
                        logs.log_info('FRAMEStart1 %s'%str(self.Frame))
            elif len(self.Chars1)==8:
                if (self.Chars1[0]=='K281'):
                    self.State = 'inframe'
                    self.Frame = self.Chars1[:8]+ self.Chars0[:8]
                    self.Chars1 = self.Chars1[8:]
                    self.Chars0 = self.Chars0[8:]
                    logs.log_info('FRAMEStart2 %s'%str(self.Frame))

        elif self.State == 'inframe':
            if len(self.Chars1)>=8:
                self.Frame += self.Chars1[:8]+ self.Chars0[:8]
                self.Chars1 = self.Chars1[8:]
                self.Chars0 = self.Chars0[8:]

                if 'K287' in self.Frame:
                    FRM = self.Frame[:self.Frame.index('K287')]
                    FRM = FRM[self.Frame.index('K281'):]
                    self.Frame = self.Frame[self.Frame.index('K287'):]
                    Len = len(FRM)-8
                    self.Bytes += Len
                    logs.log_info('FRAME #%d bytes=%d / %d : %s'%(self.FrameId,Len,self.Bytes,FRM))
                    if Len>100:
                      self.assembleData2(FRM[1:-1])
                    self.FrameId += 1
                    self.State = 'idle'
                    self.keep =True

        if len(self.Chars1)>10: self.Chars1.pop(0)
        if len(self.Chars0)>10: self.Chars0.pop(0)

    def assembleData2(self,Bytes):
        while Bytes[-1] == 'K284': Bytes.pop(-1)
        state = 0
        while Bytes!=[]:
            P0 = self.sngl(Bytes.pop(0))
            P1 = self.sngl(Bytes.pop(0))
            state = self.runPair(P0,P1,state)

    def sngl(self,X):
        try:
            return int(X,16)
        except:
            return X

    def assembleData(self,Bytes):
        while Bytes[-1] == 'K284': Bytes.pop(-1)
        state = 0
        while Bytes!=[]:
            Seq0 = Bytes[:8]
            Seq1 = Bytes[8:16]
            logs.log_info('SEQ 1=%s 0=%s'%(Seq0,Seq1))
            while Seq1!=[]:
                P1 = int(Seq1.pop(0),16)
                P0 = int(Seq0.pop(0),16)
                state = self.runPair(P0,P1,state)
            Bytes  = Bytes[16:]

    def report__(self,St):
        Exp1,Exp0,Indx = extract(self.Pix1,self.Pix0)
        Lefts = len(list(PIXELS[3].keys()))
        if Indx<0:
            logs.log_wrong('#%d %s %d (left %d) self.Pix1 = %x  self.Pix0 = %x (E1=%x E0=%x)'%(self.Indx,St,Indx,Lefts,self.Pix1,self.Pix0,Exp1,Exp0))
        else:
            logs.log_info('#%d %s %d self.Pix1 = %x  self.Pix0 = %x (E1=%x E0=%x)'%(self.Indx,St,Indx,self.Pix1,self.Pix0,Exp1,Exp0))
        self.Indx += 1
        if self.Indx>2368:
            for Key in PIXELS[3]:
                logs.log_info('leftovers ind=%d pix %04x %04x'%(PIXELS[3][Key],Key[0],Key[1]))
    def report(self,St):
        Gr1e = gray_encode(self.Pix1)
        Gr1d = gray_decode(self.Pix1)

        Gr0e = gray_encode(self.Pix0)
        Gr0d = gray_decode(self.Pix0)

        logs.log_info('REPORT %s %04x %04x en(%04x %04x)'%(St,self.Pix0,self.Pix1,Gr0e,Gr1e))

    def runPair(self,P0,P1,state):
        if state==0:
            self.Pix1 = (P1>>6)& 3
            self.Pix0 = ((P1&0x3f)<<8)+P0
            state = 1
        elif state==1:
            self.Pix1 = self.Pix1 + (P0<<2) + ((P1 & 0xf)<<10)
            self.report('st1')
            self.Pix0 = (P1>>4)&0xf
            state = 2
        elif state==2:
            self.Pix0 = self.Pix0 + (P0<<4) + ((P1 & 3)<<12)
            self.Pix1 = P1>>2
            state = 3
        elif state==3:
            self.Pix1 = self.Pix1 + (P0<<6)
            self.report('st3')
            self.Pix0 = P1
            state = 4
        elif state==4:
            self.Pix0 = self.Pix0 + ((P0 & 0x3f)<<8)
            self.Pix1 = ((P0 >>6) & 0x3) + (P1<<2)
            state = 5
        elif state==5:
            self.Pix1 = self.Pix1 + ((P0 & 0xf)<<10)
            self.report('st5')
            self.Pix0 = (P1<<4)+ ((P0>>4) & 0xf)
            state = 6
        elif state==6:
            self.Pix1 = (P1<<6)+(P0>>2)
            self.Pix0 = self.Pix0 + ((P0&3)<<12)
            self.report('st6')
            state = 0
        return state

    def trace(self,LVDS,state,Side):
        if len(LVDS)<10:
            return state,LVDS
        if state=='idle':
            LVDS = self.idles(LVDS)
            if LVDS:
                return 'sync0',LVDS
            return False
        elif state=='sync0':
            Chr = encdec_8b10b.trans10to8(LVDS[5:10]+LVDS[0:5])
            self.Stream[Side].append(Chr)
            if len(self.Stream[Side])>8:
                self.Stream[Side].pop(0)
#            if len(self.Stream[Side])==8:
            logs.log_info('STREAM %s %d %s'%(Side,len(self.Stream[Side]),self.Stream[Side]))

        elif state=='sync':
            Chr = encdec_8b10b.trans10to8(LVDS[5:10]+LVDS[0:5])
            logs.log_info('%s sync %s   %s'%(Side,Chr,state),2)
            Str = self.cutIt(LVDS,[K281_m,K281_p])
            if Str:
                return 'sof',Str
            else:
                Str = self.cutIt(LVDS,[K280_m,K280_p,K283_m,K283_p,K285_m,K285_p])
                if not Str:
                    What = encdec_8b10b.trans10to8(LVDS[5:10]+LVDS[0:5])
                    logs.log_info('token? %s    %s '%(LVDS[:10],What),2)
                    return 'sync',LVDS[10:]
                else:
                    return 'sync',Str
        elif state=='sof':
            Str = self.cutIt(LVDS,[K287_m,K287_p])
            if Str:
                return 'eof',Str

            Chr = LVDS[:10]
            What = encdec_8b10b.trans10to8(LVDS[5:10]+LVDS[0:5])
            logs.log_info('header %s    %s'%(Chr,What),3)
            return 'sof',LVDS[10:]
        elif state=='eof':
            Chr = LVDS[:10]
            Str = LVDS[10:]
            logs.log_info('reserved %s'%(Chr),3)
            return 'res',LVDS
        elif state=='res':
            Chr = LVDS[:10]
            Str = LVDS[10:]
            logs.log_info('crc %s'%(Chr),3)
            self.CRC.append(Chr)
            if (len(self.CRC)==4):
                return 'crc',Str
            return 'res',LVDS
        elif state=='crc':
            Chr = LVDS[:10]
            Str = LVDS[10:]
            logs.log_info('token %s'%(Chr),3)
            return 'sync',Str

    def idles(self,LVDS):
        Str = self.cutIt(LVDS,[K280_m,K280_p,K283_m,K283_p,K285_m,K285_p])
        if not Str: return False
        Str = Str[10:]
        Str = self.cutIt(Str,[K280_m,K280_p,K283_m,K283_p,K285_m,K285_p])
        if not Str: return False
        Str = Str[10:]
        Str = self.cutIt(Str,[K280_m,K280_p,K283_m,K283_p,K285_m,K285_p])
        if not Str: return False
        Str = Str[10:]
        return Str

    def cutIt(self,Str,Chars):
        for Char in Chars:
            Chrx = Char
            if ((Chrx) in Str) and (Str.index(Chrx)<10):
                Str = Str[Str.index(Chrx):]
                return Str
        return False

def extract(Pix0,Pix1):
    Key = (Pix0,Pix1)
    if Key in PIXELS[2]:
        Ind = PIXELS[2].index(Key)
        Exp0,Exp1 = PIXELS[2][Ind]
        if Key in PIXELS[3]: PIXELS[3].pop(Key)
        return Exp1,Exp0,Ind
    return (0x9999,0x9999,-9999)

def nice(Char):
    if type(Char) is tuple: return str(Char)

    if Char[0] in '01': return '%02x'%int(Char,2)
    return Char

def sync8(Seq):
    if len(Seq)<8: return False
    if (Seq[7][0]=='K'): return False
    for II in range(7):
        if (Seq[II][0]!='K'): return False
    return True

def syncAll(Seq):
    if len(Seq)<8: return False
    for II in range(8):
        if (Seq[II][0]!='K'): return False
    return True

def idleChar(Str):
    return Str in [K280_m,K280_p,K283_m,K283_p,K285_m,K285_p]

def gray_encode(n):
    return n ^ n >> 1

def gray_decode(n):
    m = n >> 1
    while m:
        n ^= m
        m >>= 1
    return n


import pandas as pd
import numpy as np
import os
import cv2
import json

from sklearn.model_selection import train_test_split
from trainer import Trainer
from sklearn.metrics import accuracy_score, mean_absolute_error, mean_squared_error
from collections import Counter

# read the images in the same order specified by df["id"]
def read(corpus_path, df):
    file_paths = [os.path.join(corpus_path, name) for name in df["id"]]

    data = []
    for file_path in file_paths:
        image = cv2.imread(file_path)
        data.append(image)
    data = np.asarray(data, dtype=np.float32)

    # collect the labels as well (if exist)
    labels = None
    if "label" in df:
        labels = np.asarray(df["label"], dtype=np.uint8)
        print("#### Histogram: {}".format(Counter(labels)))

    return data, labels

# plot images per group (to better undestand the data)
def plot_per_group(data, labels, target_label):
    import matplotlib.pyplot as plt

    i, curr_idx = 0, 0
    plt.figure(figsize=(10, 10))

    while (i < len(data) and curr_idx < 9):
        image = (data[i] * 255).astype("uint8")
        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
        label = labels[i]

        if label == target_label:
            plt.subplot(3, 3, curr_idx + 1)
            plt.imshow(image)
            plt.title(label)
            plt.axis("off")
            curr_idx += 1

        i += 1

if __name__ == "__main__":
    with open('config_file.json') as json_file:
        config_file = json.load(json_file)

    os.environ["CUDA_VISIBLE_DEVICES"] = str(config_file["device"])
    PROJECT_PATH = os.path.join(config_file["training_path"], "{}/".format(config_file["device"]))

    if os.path.exists(PROJECT_PATH):
        raise Exception("{} already exists. Try another path!".format(PROJECT_PATH))
    os.makedirs(PROJECT_PATH, exist_ok=True)

    config_file["checkpoint_args"]["filepath"] = os.path.join(PROJECT_PATH, config_file["checkpoint_args"]["filepath"])

    with open(os.path.join(PROJECT_PATH, "config_file.json"), 'w+') as f:
        json.dump(config_file, f, indent=4)

    df_train = pd.read_csv("corpus/train.csv")
    df_test = pd.read_csv("corpus/test.csv")

    all_train_data, all_train_labels = read(corpus_path="corpus/train/", df=df_train) # <--- collect all the images from the training dataset

    df_train = df_train.sample(frac=1).reset_index(drop=True) # <--- Shuffle the training data. Because of the augmentation step, we cannot shuffle during the `fit` call.
    df_train, df_validation = train_test_split(df_train, test_size=0.2) # <--- split into train + validation

    train_data, train_labels = read(corpus_path="corpus/train/", df=df_train) # <--- load the corresponding train images
    validation_data, validation_labels = read(corpus_path="corpus/train/", df=df_validation) # <--- load the corresponding validation images
    test_data, _ = read(corpus_path="corpus/test/", df=df_test) # <--- load the corresponding test images

    print("##### [Train]: Initial: {} -> {}".format(len(df_train), train_data.shape))
    print("##### [Validation]: Initial: {} -> {}".format(len(df_validation), validation_data.shape))
    print("##### [Test]: Initial: {} -> {}".format(len(df_test), test_data.shape))

    # for label in range(1, 6):
    #    plot_per_group(data=train_data, labels=train_labels, target_label=label)

    # instantiate a trainer object
    trainer = Trainer(all_train_data=all_train_data,
                      all_train_labels=all_train_labels,
                      train_data=train_data,
                      train_labels=train_labels,
                      validation_data=validation_data,
                      validation_labels=validation_labels,
                      loss_name=config_file["loss_name"],
                      model_type=config_file["model_type"],
                      learning_rate_decay=config_file["learning_rate_decay"],
                      enable_batchnorm=config_file["enable_batchnorm"],
                      batch_size=config_file["batch_size"],
                      epochs=config_file["epochs"],
                      early_stopping_args=config_file["early_stopping_args"],
                      checkpoint_args=config_file["checkpoint_args"],
                      task_type=config_file["task_type"],
                      learning_rate=config_file["learning_rate"],
                      weight_decay=config_file["weight_decay"],
                      num_classes=5)

    # Cross Validation case
    if config_file["type"] == "cross_validation":
        mae_list, mse_list, accuracy_list = trainer.cross_validation(epochs=75)
        history_cross_validation = {}
        history_cross_validation["mae_list"] = str(mae_list)
        history_cross_validation["mse_list"] = str(mse_list)
        history_cross_validation["accuracy_list"] = str(accuracy_list)
        with open(os.path.join(PROJECT_PATH, "history_cross_validation.json"), 'w+') as f:
            json.dump(history_cross_validation, f, indent=4)
    # Training case
    elif config_file["type"] == "train":
        history = trainer.train()
        for key in history:
            history[key] = str(history[key])

        train_predicted_labels, train_predictions = trainer.get_predictions(train_data)
        validation_predicted_labels, validation_predictions = trainer.get_predictions(validation_data)
        test_predicted_labels, test_predictions = trainer.get_predictions(test_data)

        # Accuracy score for train + validation
        accuracy_train = accuracy_score(train_labels, train_predicted_labels)
        accuracy_validation = accuracy_score(validation_labels, validation_predicted_labels)

        print("#### [Train] Accuracy for the best model: {}".format(accuracy_train))
        print("#### [Validation] Accuracy for the best model: {}".format(accuracy_validation))

        history["train_accuracy"] = str(accuracy_train)
        history["validation_accuracy"] = str(accuracy_validation)

        # For regression, add MAE and MSE too
        if config_file["task_type"] == "regression":
            mae_train = mean_absolute_error(train_labels, train_predictions)
            mae_validation = mean_absolute_error(validation_labels, validation_predictions)

            mse_train = mean_squared_error(train_labels, train_predictions)
            mse_validation = mean_squared_error(validation_labels, validation_predictions)

            history["train_mae"] = str(mae_train)
            history["validation_mae"] = str(mae_validation)

            history["train_mse"] = str(mse_train)
            history["validation_mse"] = str(mse_validation)

            print("#### [Train] Mae for the best model: {}".format(mae_train))
            print("#### [Validation] Mae for the best model: {}".format(mae_validation))

            print("#### [Train] Mse for the best model: {}".format(mse_train))
            print("#### [Validation] Mse for the best model: {}".format(mse_validation))

        df = pd.DataFrame.from_dict({"id": df_test["id"], "label": list(test_predicted_labels)})
        df.to_csv(os.path.join(PROJECT_PATH, "final_predictions.csv"), index=False)

        df_train["prediction"] = list(train_predictions) # <--- add predictions
        df_validation["prediction"] = list(validation_predictions) # <--- add predictions
        df_test["prediction"] = list(test_predictions) # <--- add predictions

        # merge back train + validation
        merged_train_df = pd.concat([df_train, df_validation], ignore_index=True) # <--- merge train + validation predictions
        merged_train_df = merged_train_df.sort_values(by=["id"]) # <--- sort by id

        merged_train_df.to_csv(os.path.join(PROJECT_PATH, "train_validation_predictions.csv"), index=False)
        df_test.to_csv(os.path.join(PROJECT_PATH, "test_predictions.csv"), index=False)

        with open(os.path.join(PROJECT_PATH, "history.json"), 'w+') as f:
            json.dump(history, f, indent=4)

        print(df)


import abc
import os.path

from oslo_config import cfg
from oslo_log import log
import requests
import requests.certs
import six

from atrope import exception
from atrope import ovf
from atrope import paths
from atrope import utils

opts = [
    cfg.StrOpt('download_ca_file',
               default=paths.state_path_def('atrope-ca-bundle.pem'),
               help='Atrope will build a CA bundle for verifying the '
                    'HTTP servers when it is downloading the image, '
                    'concatenating the default OS CA bundle and the '
                    'CAs present in the $ca_path directory. This '
                    'is done as there may be certificates signed by '
                    'CAs that are trusted by the provider, but untrusted '
                    'by the default bundle and we need to trust both.'),
]

CONF = cfg.CONF
CONF.register_opts(opts)
CONF.import_opt("ca_path", "atrope.smime")

LOG = log.getLogger(__name__)

@six.add_metaclass(abc.ABCMeta)
class BaseImage(object):
    @abc.abstractmethod
    def __init__(self, image_info):
        self.uri = None
        self.sha512 = None
        self.identifier = None
        self.location = None
        self.verified = False

    @abc.abstractmethod
    def download(self, dest):
        """Download the image.

        :param dest: destionation directory.
        """

    def get_file(self, mode="rb"):
        """Return a File object containing the downloaded file."""
        return open(self.location, mode)

    def get_kernel(self):
        raise NotImplementedError()

    def get_ramdisk(self):
        raise NotImplementedError()

    def get_disk(self):
        """Return the format and a 'ro' File-like object containing the disk.

        Images can be stored in containers like OVA, this method will return a
        tuple (format, fd) being 'format' a string containing the image disk
        format and 'fd' File-like object containing the original image disk as
        extracted from the container.

        We assume that containers only store one image disk. We scan the file
        in reverse order, as OVF specification states that files can be
        appended so as to update the OVF file.
        """
        if self.format.lower() != "ova":
            return self.format, self.get_file()

        ovf_file = ovf.get_ovf(self.location)

        fmt, disk_filename = ovf.get_disk_name(ovf_file)
        disk_fd = ovf.extract_file(self.location, disk_filename)
        return fmt, disk_fd

    def verify_checksum(self, location=None):
        """Verify the image's checksum."""
        LOG.info("Image '%s' present in '%s', verifying checksum",
                 self.identifier, location)

        location = location or self.location
        if location is None:
            raise exception.ImageNotFoundOnDisk(location=location)

        sha512 = utils.get_file_checksum(location)
        if sha512.hexdigest() != self.sha512:
            raise exception.ImageVerificationFailed(
                id=self.identifier,
                expected=self.sha512,
                obtained=sha512.hexdigest()
            )
        LOG.info("Image '%s' present in '%s', checksum OK",
                 self.identifier, location)
        self.verified = True

class HepixImage(BaseImage):
    field_map = {
        "ad:group": "group",
        "ad:mpuri": "mpuri",
        "ad:user:fullname": "user_fullname",
        "ad:user:guid": "user_guid",
        "ad:user:uri": "user_uri",
        "dc:description": "description",
        "dc:identifier": "identifier",
        "dc:title": "title",
        "hv:hypervisor": "hypervisor",
        "hv:format": "format",
        "hv:size": "size",
        "hv:uri": "uri",
        "hv:version": "version",
        "sl:arch": "arch",
        "sl:checksum:sha512": "sha512",
        "sl:comments": "comments",
        "sl:os": "os",
        "sl:osname": "osname",
        "sl:osversion": "osversion",
    }
    required_fields = field_map.keys()

    def __init__(self, image_info):
        super(HepixImage, self).__init__(image_info)

        image_dict = image_info.get("hv:image", {})

        utils.ensure_ca_bundle(CONF.download_ca_file,
                               [requests.certs.where()],
                               CONF.ca_path)

        for i in self.required_fields:
            value = image_dict.get(i, None)
            if value is None:
                reason = "Invalid image definition, missing '%s'" % i
                raise exception.InvalidImageList(reason=reason)

            attr = self.field_map.get(i)
            setattr(self, attr, value)
        # add everything from hepix as 'extra', so it can be queried in glance
        self.appliance_attributes = image_dict

    def _download(self, location):
        LOG.info("Downloading image '%s' from '%s' into '%s'",
                 self.identifier, self.uri, location)
        with open(location, 'wb') as f:
            try:
                response = requests.get(self.uri, stream=True,
                                        verify=CONF.download_ca_file)
            except Exception as e:
                LOG.error(e)
                raise exception.ImageDownloadFailed(code=e.errno,
                                                    reason=e)

            if not response.ok:
                LOG.error("Cannot download image: (%s) %s",
                          response.status_code, response.reason)
                raise exception.ImageDownloadFailed(code=response.status_code,
                                                    reason=response.reason)

            for block in response.iter_content(1024):
                if block:
                    f.write(block)
                    f.flush()
        try:
            self.verify_checksum(location=location)
        except exception.ImageVerificationFailed as e:
            LOG.error(e)
            raise
        else:
            LOG.info("Image '%s' stored as '%s'",
                     self.identifier, location)

    def download(self, basedir):
        # The image has been already downloaded in this execution.
        if self.location is not None:
            raise exception.ImageAlreadyDownloaded(location=self.location)

        location = os.path.join(basedir, self.identifier)

        if not os.path.exists(location):
            self._download(location)
        else:
            # Image exists, is it checksum valid?
            try:
                self.verify_checksum(location=location)
            except exception.ImageVerificationFailed:
                LOG.warning("Image '%s' present in '%s' is not valid, "
                            "downloading again",
                            self.identifier, location)
                self._download(location)

        self.location = location


# -*- coding: utf-8 -*-
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics.scorer import make_scorer
from sklearn import model_selection
from sklearn.base import BaseEstimator, ClassifierMixin
from sklearn.linear_model import LogisticRegression
from sklearn.metrics.scorer import make_scorer
from sklearn import model_selection
from sklearn.ensemble import GradientBoostingRegressor  #GBM algorithm
from sklearn import cross_validation, metrics   #Additional scklearn functions
from sklearn.grid_search import GridSearchCV   #Perforing grid search

"""
Created on Sat Nov 24 13:28:11 2018

@author: Muhammad Shahraiz Khan Niazi and Muhammad Daniyal Saqib
"""
'''
#Start:Kaggle.com
#Explanation: Plot a bar graph against SalePrice
#End '''
def barPlot(df, var):

    plt.xticks(rotation =90)
    sns.barplot(df.loc[:,'SalePrice'], df.loc[:, var])
    plt.title('SalePrice vs ' + var)

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------

def colDrop(df, colName):
    return df.drop(colName, axis = 1)

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
'''
#Start:Kaggle.com
#Explanation: to figure out the division/scattering of the data's values
#End '''

def checkRange(df, var):
    labels = df.loc[:,var].unique()
    sizes = df.loc[:,var].value_counts().values
    percent = 100.*sizes/sizes.sum()
    labels = ['{0} - {1:1.1f} %'.format(i,j) for i,j in zip(labels, percent)]
    print(labels)

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
'''
#Start:Kaggle.com
#Explanation: Plot a historgram graph
#End '''

def histogram(df, var):
    sns.distplot(df.loc[:, var]);
    fig = plt.figure()

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------

def join(df1, df2):
    return pd.concat([df1, df2], axis = 1)

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------
'''
#Start:Kaggle.com
#Explanation: Plot a scatter graph against SalePrice
#End '''

def scatterPlot(df, var):
    data = pd.concat([df.loc[:, 'SalePrice'], df.loc[:,var]], axis=1)
    data.plot.scatter(x=var, y='SalePrice', ylim=(0,800000));

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------

#This method implements binary encoding on particular columns in the dataframe and returns that dataframe
def encode(ser):
    return pd.get_dummies(ser)

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------

#This function checks percentage of the number of NA values in a coloumn
def checkMissingValues(df):
    return df.apply(lambda col: (col.isnull().sum()/ df.shape[0]) *100 )

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------

#This function reads the data from the file and returns the entire dataset, a list of input cols and outputCol
def readData_1(numRows = None):

    inputCols = ['MSSubClass', 'MSZoning', 'LotFrontage',
                 'LotArea', 'Street', 'Alley', 'LotShape', 'LandContour',
                 'Utilities', 'LotConfig', 'LotConfig',
                 'Neighborhood', 'Condition1', 'Condition2', 'BldgType'
                 'HouseStyle', 'OverallQual', 'OverallCond', 'YearBuilt'
                 'YearRemodAdd', 'RoofStyle', 'RoofMatl',
                 'Exterior1st', 'Exterior2nd', 'MasVnrType', 'MasVnrArea',
                 'ExterQual', 'ExterCond', 'Foundation', 'BsmtQual', 'BsmtCond',
                 'BsmtExposure', 'BsmtFinType1', 'BsmtFinSF1', 'BsmtFinType2',
                 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF', 'Heating', 'HeatingQC',
                 'CentralAir', 'Electrical','1stFlrSF',  '2ndFlrSF', 'LowQualFinSF',
                 'GrLivArea','BsmtFullBath','BsmtHalfBath','FullBath','HalfBath','Bedroom',
                 'Kitchen','KitchenQual','TotRmsAbvGrd','Functional',
                 'Fireplaces','FireplaceQu','GarageType','GarageYrBlt',
                 'GarageFinish','GarageCars','GarageArea',
                 'GarageQual','GarageCond','PavedDrive',
                 'WoodDeckSF','OpenPorchSF','EnclosedPorch',
                '3SsnPorch', 'ScreenPorch','PoolArea',
                'PoolQC','Fence','MiscFeature', 'MiscVal',
                'MoSold','YrSold','SaleType','SaleCondition']

    outputCol = ['SalePrice']

    trainHouseDF = pd.read_csv('Data/train.csv')
    testHouseDF = pd.read_csv('Data/test.csv')

    houseDF = pd.concat([trainHouseDF, testHouseDF], sort=True)
    #houseDF = houseDF.sample(frac = 1, random_state = 99).reset_index(drop = True)

    return houseDF, inputCols, outputCol

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------

#This function reads the data from the file and returns the two datasets -
# training and testing,  a list of input cols and outputCol

def readData(numRows = None):
    trainHouseDF = pd.read_csv('Data/train.csv')
    testHouseDF = pd.read_csv('Data/test.csv')
    outputCol = ['SalePrice']

    return trainHouseDF, testHouseDF, outputCol

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------

def c2(df, colName):
    return df.loc[:, colName].corr(df.loc[:, 'SalePrice'])

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------

#This function finds out the correlation between the numerical columns and the SalePrice
def corrRelationForNumerical():

    df, b, c = readData_1()
    corr=df.corr()["SalePrice"]
    print(corr[np.argsort(corr, axis=0)[::-1]])

    inputCols = ['OverallQual', 'GrLivArea', 'GarageCars', 'GarageArea', 'TotalBsmtSF', '1stFlrSF',
                 'FullBath', 'TotRmsAbvGrd', 'YearBuilt', 'YearRemodAdd', 'GarageYrBlt', 'MasVnrArea',
                 'Fireplaces']

    corrMatrix= df[inputCols].corr()
    sns.set(font_scale=1.10)
    plt.figure(figsize=(10, 10))
    sns.heatmap(corrMatrix, vmax=.8, linewidths=0.01, square=True,annot=True,cmap='viridis',linecolor="white")
    plt.title('Correlation between features')

    '''
    According to the correlation that we found with respect to SalePrice, we think:
    values that are above close to 0.5 or greater than 0.5 would affect SalePrice and values that are less than -0.1.
    Hence, our predictors list till now is: OverallQual, GrLiveArea, GarageCars, GarageArea
    TotalBsmtSF, 1stFlrSF, FullBath, TotRmsAbvGrd, YearBuilt, YearRemodAdd, GarageYrBl
    '''

#----------------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------

#This function would preprocess the nonNumerical data and we would evaluate which column to keep as a parameter
#for the final algorithm
def dataProcessing():

    df, b, c = readData()

    #____________________________________________________________________________________________
    #While going through the Utilities - there is only one entry that is other than AllPub (Id = 945)
    #Hence dropping ID 945 and then the entire col would not affect the TargetPrice - Sale price
    df = df.drop(labels = 944, axis = 0)

    '''

    From: Kaggle.com
    Explanation: After trying different plotting techniques, such as boxPlot, scatterplot and few others,
    We think this describes the neighborhood perfectly.
    '''

    #barPlot(df, 'Neighborhood')
    df = colDrop(df, 'Neighborhood')

    #End: Kaggle.com
    '''Other than few extra spikes, the SalePrice is not really affected that much by Neighborhood, as it is all between
    100000 - 200000. Shahraiz and I believe, the Neighborhood really doesn't matter. Hence, we would drop this column'''

    #----------------------------------------------------------------------------------------------------------
    #MSZoning Attribute
    #plt.xticks(rotation =90)
    #plt.scatter(df.loc[:,'MSZoning'], df.loc[:, 'SalePrice'])
    #plt.title('SalePrice vs MSZoning')

    '''This graph clearly shows that they majority of the data is in RL; however, to confirm it further we would use Classifcation graph
    to plot it'''

    #labels = df.loc[:,"MSZoning"].unique()
    #sizes = df.loc[:,"MSZoning"].value_counts().values
    #explode=[0.1,0,0,0,0]
    #parcent = 100.*sizes/sizes.sum()
    #labels = ['{0} - {1:1.1f} %'.format(i,j) for i,j in zip(labels, parcent)]
    #colors = ['yellowgreen', 'gold', 'lightblue', 'lightcoral','blue']
    #patches = plt.pie(sizes, colors=colors,explode=explode, shadow=True,startangle=90)

    #plt.title("Zoning Classification")
    #plt.show()
    #print(labels)
    #print()

    '''Clearly a large part of the pie is yellow green which is RL - and the second most occuring
    is gold which is RM. Therefore, we would keep the values that are RL and RM and remove the rows
    that are any other value.'''
    '''We would discreditize the data in MSZoning: 1 - RL, 0 - RM and then find the correlation'''

    df = df.loc[(df.loc[:,'MSZoning'] == 'RL') | (df.loc[:,'MSZoning']=='RM') | (df.loc[:,'MSZoning']=='C')]
    df.loc[:,'MSZoning'] = df.loc[:,'MSZoning'].map(lambda val: 0 if(val=='RL') else val)
    df.loc[:,'MSZoning'] = df.loc[:,'MSZoning'].map(lambda val: 1 if(val=='RM') else val)
    df.loc[:,'MSZoning'] = df.loc[:,'MSZoning'].map(lambda val: 2 if(val=='C') else val)
    #print(corrRelation2Cols(df, ['MSZoning', 'SalePrice']))
    '''
    The correlation between SalePrice and MSZoning is 0.29556497792 - that is below the threshold of 0.5; hence, we
    would not use this as well.
    '''
    df = colDrop(df, 'MSZoning')

    #----------------------------------------------------------------------------------------------------------
    #Street

    #plt.xticks(rotation =90)
    #plt.scatter(df.loc[:,'Street'], df.loc[:, 'SalePrice'])
    #plt.title('SalePrice vs Street')

    df = df.loc[(df.loc[:, 'Street'] == 'Pave')]

    '''
    The graph shows that majority of the values are Pave.
    Therefore we would keep all the values that are Pave and get rid of the column.
    '''

    df = colDrop(df, 'Street')

    #----------------------------------------------------------------------------------------------------------
    #dropping 'Ally', MiscFeature, PoolQC because there high percentage of Na values, Uncomment the print line below to see that.
    #print(checkMissingValues(df))
    df = df.drop(['Alley', 'MiscFeature', 'PoolQC', 'Utilities', 'Fence', 'FireplaceQu', 'LotFrontage'], axis = 1)

    #----------------------------------------------------------------------------------------------------------
    # LotShape
    #checkRange(df, 'LotShape')

    '''['Reg - 62.2 %', 'IR1 - 34.4 %', 'IR2 - 2.6 %', 'IR3 - 0.7 %'] - the percentage of
    the values show that it is all Reg and IR1. Hence we would keep all those values that
    are Reg and IR1 otherwise, we would drop those rows.'''

    #df = df.loc[(df.loc[:,'LotShape'] == 'Reg') | (df.loc[:,'LotShape']=='IR1') | (df.loc[:,'LotShape']=='IR2')]
    #df.loc[:,'LotShape'] = df.loc[:,'LotShape'].map(lambda val: 0 if(val=='Reg') else val)
    #df.loc[:,'LotShape'] = df.loc[:,'LotShape'].map(lambda val: 1 if(val=='IR1') else val)
    #df.loc[:,'LotShape'] = df.loc[:,'LotShape'].map(lambda val: 2 if(val=='IR2') else val)

    #scatterPlot(df, 'LotShape')

    '''Now we would discreditize the data into 0,1 or 2 and find the correlation between Lot Shape
    and SalePrice'''

    #print(c2(df, 'LotShape'))

    #Hence we would drop this too.
    df = colDrop(df, 'LotShape')

    #_____________________________________________________________________________________________
    #This is LandContour
    #checkRange(df, 'LandContour')
    #df = df.loc[(df.loc[:,'LandContour'] == 'Lvl') | (df.loc[:,'LandContour'] == 'Bnk') | (df.loc[:,'LandContour'] == 'Low') | (df.loc[:,'LandContour'] == 'HLS')]
    #df.loc[:,'LandContour'] = df.loc[:,'LandContour'].map(lambda val: 0 if(val=='Lvl') else val)
    #df.loc[:,'LandContour'] = df.loc[:,'LandContour'].map(lambda val: 1 if(val=='Bnk') else val)
    #df.loc[:,'LandContour'] = df.loc[:,'LandContour'].map(lambda val: 2 if(val=='Low') else val)
    #df.loc[:,'LandContour'] = df.loc[:,'LandContour'].map(lambda val: 3 if(val=='HLS') else val)
    #histogram(df, 'LandContour')
    #print(c2(df, 'LandContour'))
    '''Since, LandContour and LotShape basically is providing the same information - we would use only
    one of it - the one with the higher correlation with SalePrice, if it exceeds 0.5'''
    df = colDrop(df, 'LandContour')

    #____________________________________________________________________________________________
    #This is LotConfig

    #checkRange(df, 'LotConfig')
    df = df.loc[(df.loc[:,'LotConfig'] == 'Inside') | (df.loc[:,'LotConfig'] == 'FR2') |(df.loc[:,'LotConfig'] == 'Corner') | (df.loc[:,'LotConfig'] == 'CulDsac') ]

    df.loc[:,'LotConfig'] = df.loc[:,'LotConfig'].map(lambda val: 0 if(val=='Inside') else val)
    df.loc[:,'LotConfig'] = df.loc[:,'LotConfig'].map(lambda val: 1 if(val=='FR2') else val)
    df.loc[:,'LotConfig'] = df.loc[:,'LotConfig'].map(lambda val: 2 if(val=='Corner') else val)
    df.loc[:,'LotConfig'] = df.loc[:,'LotConfig'].map(lambda val: 3 if(val=='CulDsac') else val)

    #print(c2(df, 'LotConfig'))
    #Removed Landconfig as well because the correlation is very less
    df = colDrop(df, 'LotConfig')

    #__________________________________________________________________________________________________
    #LandSlope

    #checkRange(df, 'LandSlope')
    df = df.loc[(df.loc[:,'LandSlope'] == 'Gtl') | (df.loc[:, 'LandSlope'] == 'Mod')]
    df.loc[:, 'LandSlope'] = df.apply(lambda row: 1 if row.loc['LandSlope'] == 'Gtl' else 0, axis = 1)
    #print(c2(df, 'LandSlope'))
    #It shows a high percentage of Gtl values, therefore - we would just keep those and remove the others
    #and this column - It also shows a very high -negative correlation - therewould we would keep this column

    #_____________________________________________________________________________________________
    #Condition1 Condition2

    df.loc[:,'Condition1'] = df.loc[:,'Condition1'].map(lambda val: 0 if(val=='Artery') else val)
    df.loc[:,'Condition1'] = df.loc[:,'Condition1'].map(lambda val: 1 if(val=='Feedr') else val)
    df.loc[:,'Condition1'] = df.loc[:,'Condition1'].map(lambda val: 2 if(val=='Norm') else val)
    df.loc[:,'Condition1'] = df.loc[:,'Condition1'].map(lambda val: 3 if(val=='RRNn') else val)
    df.loc[:,'Condition1'] = df.loc[:,'Condition1'].map(lambda val: 4 if(val=='PosN') else val)
    df.loc[:,'Condition1'] = df.loc[:,'Condition1'].map(lambda val: 5 if(val=='PosA') else val)
    df.loc[:,'Condition1'] = df.loc[:,'Condition1'].map(lambda val: 6 if(val=='RRNe') else val)
    df.loc[:,'Condition1'] = df.loc[:,'Condition1'].map(lambda val: 7 if(val=='RRAe') else val)
    df.loc[:,'Condition1'] = df.loc[:,'Condition1'].map(lambda val: 8 if(val=='RRAn') else val)

    #print(c2(df, 'Condition1'))

    df.loc[:,'Condition2'] = df.loc[:,'Condition2'].map(lambda val: 0 if(val=='Artery') else val)
    df.loc[:,'Condition2'] = df.loc[:,'Condition2'].map(lambda val: 1 if(val=='Feedr') else val)
    df.loc[:,'Condition2'] = df.loc[:,'Condition2'].map(lambda val: 2 if(val=='Norm') else val)
    df.loc[:,'Condition2'] = df.loc[:,'Condition2'].map(lambda val: 3 if(val=='RRNn') else val)
    df.loc[:,'Condition2'] = df.loc[:,'Condition2'].map(lambda val: 4 if(val=='PosN') else val)
    df.loc[:,'Condition2'] = df.loc[:,'Condition2'].map(lambda val: 5 if(val=='PosA') else val)
    df.loc[:,'Condition2'] = df.loc[:,'Condition2'].map(lambda val: 6 if(val=='RRNe') else val)
    df.loc[:,'Condition2'] = df.loc[:,'Condition2'].map(lambda val: 7 if(val=='RRAe') else val)
    df.loc[:,'Condition2'] = df.loc[:,'Condition2'].map(lambda val: 8 if(val=='RRAn') else val)

    #print(c2(df, 'Condition2'))

    ''' Clearly the correlation between these two coloumns and SalePrice is very low; therefore, we
    would drop these two columns'''
    df = colDrop(df, 'Condition1')
    df = colDrop(df, 'Condition2')

    #____________________________________________________________________________________________
    #BldgType
    #labels = df.loc[:,'BldgType'].unique()
    #sizes = df.loc[:,"BldgType"].value_counts().values
    #explode=[0.1,0,0,0,0]
    #parcent = 100.*sizes/sizes.sum()
    #labels = ['{0} - {1:1.1f} %'.format(i,j) for i,j in zip(labels, parcent)]
    #print(labels)

    '''1 Fam and Duplex are more than 90% of the values for this column therefore we would,
    binary encode them and see there correlation with the SalePrice'''

    a = encode(df.loc[:, 'BldgType'])
    #df.loc[:, '1Fam'] = a.loc[:, '1Fam']
    df.loc[:, 'Duplex'] = a.loc[:, 'Duplex']
    #print(c2(df, '1Fam'))
    #print(c2(df, 'Duplex'))

    '''
    1Fam is not highly correlated with the SalePrice; therefore, which can be confirmed
    by there scatter plots, we won't keep that. However, we would keep the Duplex.
    '''

    #data = pd.concat([df.loc[:, 'SalePrice'], df.loc[:,'1Fam']], axis=1)
    #data.plot.scatter(x='1Fam', y='SalePrice', ylim=(0,800000));

    #data1 = pd.concat([df.loc[:, 'SalePrice'], df.loc[:,'Duplex']], axis=1)
    #data1.plot.scatter(x='Duplex', y='SalePrice', ylim=(0,800000));

    '''In both the cases - SalePrice, is barely affected by 1Fam or DuPlex as it is scattered all over the
    place. Hence we would also not consider BldgType. '''
    df = colDrop(df, 'BldgType')

    #____________________________________________________________________________________________
    #RoofMatl
    #checkRange(df, 'RoofMatl')

    ''' 99.6 percent of the values are CompShg therefore we would ignore the other values and ignore
    this column'''

    df = df.loc[df.loc[:, 'RoofMatl'] == 'CompShg']
    df = colDrop(df, 'RoofMatl')

    #____________________________________________________________________________________________
    #MasVnrType
    #checkRange(df, 'MasVnrType')
    #scatterPlot(df, 'MasVnrArea')

    df.loc[:, 'MasVnrType'] == df.loc[:, 'MasVnrType'].fillna('None')
    df.loc[:, 'MasVnrArea'] == df.loc[:, 'MasVnrArea'].fillna(0)

    a = df.loc[(df.loc[:,'MasVnrType'] == 'None') & (df.loc[:, 'MasVnrArea'] == 0)]

    #print(len(a))
    #print(len(df[df.loc[:, 'MasVnrType'] == 'None']))
    #print(len(df[df.loc[:, 'MasVnrArea'] == 0]))

    '''This shows a relationship between the two and even from the names we can figure it out that we just need one of it;
    therefore, we would keep the one that is more easier to use (the one that is numeric) and figure out it's corr'''
    df = colDrop(df, 'MasVnrType')
    #print(c2(df, 'MasVnrArea'))
    '''We would keep this because it's correlation with SalePrice is 0.499 which is approximately 0.5 - among the threshold
    we are considering values.'''

    #____________________________________________________________________________________________
    #ExteriorQuality

    #checkRange(df, 'ExterQual')
    '''From this we figure out that Gd, TA, and Ex makes the entire data therefore we would remove the rest'''

    df = df.loc[(df.loc[:, 'ExterQual'] == 'Gd') | (df.loc[:, 'ExterQual'] == 'TA') | (df.loc[:, 'ExterQual'] == 'Ex')]

    df.loc[:,'ExterQual'] = df.loc[:,'ExterQual'].map(lambda val: 1 if(val=='Gd') else val)
    df.loc[:,'ExterQual'] = df.loc[:,'ExterQual'].map(lambda val: 2 if(val=='TA') else val)
    df.loc[:,'ExterQual'] = df.loc[:,'ExterQual'].map(lambda val: 3 if(val=='Ex') else val)

    #histogram(df, 'ExterQual')
    #checkRange(df, 'ExterCond')
    #print(c2(df, 'ExterQual'))

    '''Since, it's corr relation is negative and  far from zero, therefore, we would keep this. However, we would not
    use exterior condition because both of them are almost the same thing because in both the cases - TA and Gd make up a large
    portion of the data.'''
    df = colDrop(df, 'ExterCond')

    #____________________________________________________________________________________________
    #Foundation

    #checkRange(df, 'Foundation')

    df = df.loc[(df.loc[:, 'Foundation'] == 'PConc') | (df.loc[:, 'Foundation'] == 'CBlock') | (df.loc[:, 'Foundation'] == 'CBlock')]
    df.loc[:,'Foundation'] = df.loc[:,'Foundation'].map(lambda val: 1 if(val=='PConc') else val)
    df.loc[:,'Foundation'] = df.loc[:,'Foundation'].map(lambda val: 2 if(val=='CBlock') else val)
    df.loc[:,'Foundation'] = df.loc[:,'Foundation'].map(lambda val: 3 if(val=='CBlock') else val)

    #histogram(df, 'Foundation')
    #scatterPlot(df, 'Foundation')

    #print(c2(df, 'Foundation'))

    '''The corr is very high negative - therefore we will take this into consideration either.'''

    #____________________________________________________________________________________________
    #Basement Features

    '''There is high correlation between basement features (BsmtFinType2, BsmtQual, BsmtCond, BsmtExposure, BsmtFinType1);
    therefore, we would only use BsmtCond - correlation shown below'''

    #checkRange(df, 'BsmtQual')
    #checkRange(df, 'BsmtCond')

    df.loc[:, 'BsmtQual'] = df.loc[:, 'BsmtQual'].fillna('None')
    df.loc[:, 'BsmtCond'] = df.loc[:, 'BsmtCond'].fillna('None')

    df.loc[:,'BsmtQual'] = df.loc[:,'BsmtQual'].map(lambda val: 1 if(val=='Gd') else val)
    df.loc[:,'BsmtQual'] = df.loc[:,'BsmtQual'].map(lambda val: 2 if(val=='TA') else val)
    df.loc[:,'BsmtQual'] = df.loc[:,'BsmtQual'].map(lambda val: 2 if(val== 'Ex') else val)
    df.loc[:,'BsmtQual'] = df.loc[:,'BsmtQual'].map(lambda val: 4 if(val== 'Fa') else val)
    df.loc[:,'BsmtQual'] = df.loc[:,'BsmtQual'].map(lambda val: 5 if(val==  'None') else val)
    df.loc[:,'BsmtQual'] = df.loc[:,'BsmtQual'].map(lambda val: 6 if(val== 'Po') else val)

    df.loc[:,'BsmtCond'] = df.loc[:,'BsmtCond'].map(lambda val: 1 if(val=='Gd') else val)
    df.loc[:,'BsmtCond'] = df.loc[:,'BsmtCond'].map(lambda val: 2 if(val=='TA') else val)
    df.loc[:,'BsmtCond'] = df.loc[:,'BsmtCond'].map(lambda val: 3 if(val== 'Ex') else val)
    df.loc[:,'BsmtCond'] = df.loc[:,'BsmtCond'].map(lambda val: 4 if(val== 'Fa') else val)
    df.loc[:,'BsmtCond'] = df.loc[:,'BsmtCond'].map(lambda val: 5 if(val==  'None') else val)
    df.loc[:,'BsmtCond'] = df.loc[:,'BsmtCond'].map(lambda val: 6 if(val== 'Po') else val)

    #print(df.loc[:, 'BsmtCond'].corr(df.loc[:,'BsmtQual']))
    '''This shows a high correlation  between BsmtCond and BsmtQual - therefore, we would keep BsmtCond'''
    df = colDrop(df, 'BsmtQual')

    #____________________________________________________________________________________________
    #Gas

    #checkRange(df, 'Heating') #Show 99.6% GasA - therefore keeping only data that is GasA
    df = df.loc[df.loc[:, 'Heating'] == 'GasA']
    df = colDrop(df, 'Heating')

    #checkRange(df, 'HeatingQC')
    df.loc[:,'HeatingQC'] = df.loc[:,'HeatingQC'].map(lambda val: 1 if(val=='Gd') else val)
    df.loc[:,'HeatingQC'] = df.loc[:,'HeatingQC'].map(lambda val: 2 if(val=='TA') else val)
    df.loc[:,'HeatingQC'] = df.loc[:,'HeatingQC'].map(lambda val: 3 if(val== 'Ex') else val)
    df.loc[:,'HeatingQC'] = df.loc[:,'HeatingQC'].map(lambda val: 4 if(val== 'Fa') else val)
    df.loc[:,'HeatingQC'] = df.loc[:,'HeatingQC'].map(lambda val: 5 if(val== 'Po') else val)

    #print(c2(df, 'HeatingQC'))
    #scatterPlot(df, 'HeatingQC')

    '''The graph shows that Heating Quality definitely affects the price range towards the upper end.'''

    #____________________________________________________________________________________________
    #Central Air
    #checkRange(df, 'CentralAir')
    df.loc[:,'CentralAir'] = df.loc[:,'CentralAir'].map(lambda val: 0 if(val=='N') else val)
    df.loc[:,'CentralAir'] = df.loc[:,'CentralAir'].map(lambda val: 1 if(val=='Y') else val)

    #scatterPlot(df, 'CentralAir')
    #histogram(df, 'CentralAir')
    #print(c2(df, 'CentralAir'))
    df = colDrop(df, 'CentralAir')

    #checkRange(df, 'Electrical')
    df = df.loc[(df.loc[:, 'Electrical'] == 'SBrkr') | (df.loc[:, 'Electrical'] == 'FuseF')] #98.8 of the total values make this up
    df.loc[:,'Electrical'] = df.loc[:,'Electrical'].map(lambda val: 0 if(val=='SBrkr') else 1)
    #print(df.loc[:, 'Electrical'])
    #print(c2(df, 'Electrical'))
    '''This is a negative number close to 0; hence, a low corr between Electricity and SalePrice.
    therefore - we would not keep this.'''
    df = colDrop(df, 'Electrical')

    #KitchenQuality
    #checkRange(df, 'KitchenQual')
    ''' 98% of the data is made up of Gd, TA and Ex hence we would only keep, discretize it
    and then find the correlation between KitcenQual and SalePrice'''

    df = df.loc[(df.loc[:, 'KitchenQual'] == 'Gd') | (df.loc[:, 'KitchenQual'] == 'Ex') | (df.loc[:, 'KitchenQual'] == 'TA')]
    df.loc[:,'KitchenQual'] = df.loc[:,'KitchenQual'].map(lambda val: 0 if(val=='Ex') else val)
    df.loc[:,'KitchenQual'] = df.loc[:,'KitchenQual'].map(lambda val: 1 if(val=='TA') else val)
    df.loc[:,'KitchenQual'] = df.loc[:,'KitchenQual'].map(lambda val: 2 if(val=='Gd') else val)

    #print(c2(df, 'KitchenQual'))

    '''It really has a low negative corr with SalePrice - therefore we would not keep this.'''
    df = colDrop(df, 'KitchenQual')

    #function
    #checkRange(df, 'Functional')
    df = df.loc[(df.loc[:, 'Functional'] == 'Typ') | (df.loc[:, 'Functional'] == 'Min1') | (df.loc[:, 'Functional'] == 'Maj1')]
    df.loc[:,'Functional'] = df.loc[:,'Functional'].map(lambda val: 0 if(val=='Typ') else val)
    df.loc[:,'Functional'] = df.loc[:,'Functional'].map(lambda val: 1 if(val=='Min1') else val)
    df.loc[:,'Functional'] = df.loc[:,'Functional'].map(lambda val: 2 if(val=='Maj1') else val)

    #print(c2(df, 'Functional'))

    '''This has negative corr very close to zero; therefore, we would drop it.'''
    df = colDrop(df, 'Functional')

    #----------------------------------------------------------------------------------------------------------
    #HouseStyle
    #checkRange(df, 'HouseStyle')
    df = df.loc[(df.loc[:, 'HouseStyle'] == '2Story') | (df.loc[:, 'HouseStyle'] == '1Story') | (df.loc[:, 'HouseStyle'] == '1.5Unf')]
    df.loc[:,'HouseStyle'] = df.loc[:,'HouseStyle'].map(lambda val: 0 if(val=='2Story') else val)
    df.loc[:,'HouseStyle'] = df.loc[:,'HouseStyle'].map(lambda val: 1 if(val=='1Story') else val)
    df.loc[:,'HouseStyle'] = df.loc[:,'HouseStyle'].map(lambda val: 2 if(val=='1.5Unf') else val)

    #print( df.loc[:,'HouseStyle'].dtype)
    '''It is a high negative correlation with SalePrice - therefore we would keep it.'''

    #----------------------------------------------------------------------------------------------------------
    #RoofStyle
    #checkRange(df, 'RoofStyle')
    '''Data represents a lot like HouseStyle; therefore, assuming a high correlation between
    roofstyle and housestyle - we would drop this as well.'''
    df = colDrop(df, 'RoofStyle')

    #_____________________________________________________________
    #Exterior1st

    df = colDrop(df, 'Exterior1st')
    df = colDrop(df, 'Exterior2nd')

    df = colDrop(df, 'BsmtExposure')
    df = colDrop(df, 'BsmtFinType1')
    df = colDrop(df, 'BsmtFinType2')
    df = colDrop(df, 'GarageType')

    #checkRange(df, 'GarageFinish')
    a = encode(df.loc[:, 'GarageFinish'])
    df = join(df, a)

    df = colDrop(df,'GarageFinish')

    #checkRange(df, 'PavedDrive')
    df = df.loc[(df.loc[:, 'PavedDrive'] == 'Y') | (df.loc[:, 'PavedDrive'] == 'N')]
    df.loc[:,'PavedDrive'] = df.loc[:,'PavedDrive'].map(lambda val: 0 if(val=='N') else 1)
    #print(c2(df, 'PavedDrive'))
    '''Very low correlation - hence we would drop this.'''
    df = colDrop(df, 'PavedDrive')
    df = colDrop(df, 'GarageCond')
    df = colDrop(df, 'GarageQual')

    #checkRange(df, 'SaleType')
    df = df.loc[(df.loc[:, 'SaleType'] == 'WD') | (df.loc[:, 'SaleType'] == 'New') | (df.loc[:, 'SaleType'] == 'COD')]
    df.loc[:,'SaleType'] = df.loc[:,'SaleType'].map(lambda val: 0 if(val=='WD') else val)
    df.loc[:,'SaleType'] = df.loc[:,'SaleType'].map(lambda val: 1 if(val=='New') else val)
    df.loc[:,'SaleType'] = df.loc[:,'SaleType'].map(lambda val: 2 if(val=='COD') else val)
    df = colDrop(df, 'SaleType')

    #checkRange(df, 'SaleCondition')
    df = df.loc[(df.loc[:, 'SaleCondition'] == 'Normal') | (df.loc[:, 'SaleCondition'] == 'Partial') | (df.loc[:, 'SaleCondition'] == 'Abnormal')]
    df.loc[:,'SaleCondition'] = df.loc[:,'SaleCondition'].map(lambda val: 0 if(val=='Normal') else val)
    df.loc[:,'SaleCondition'] = df.loc[:,'SaleCondition'].map(lambda val: 1 if(val=='Partial') else val)
    df.loc[:,'SaleCondition'] = df.loc[:,'SaleCondition'].map(lambda val: 2 if(val=='Abnormal') else val)

    #print(c2(df, 'SaleCondition'))

    #_handling the NA values
    df.loc[:,'MasVnrArea'] = df.loc[:,'MasVnrArea'].fillna(0)
    #df.loc[:,'BsmtExposure'] = df.loc[:,'BsmtExposure'].fillna(df.loc[:,'BsmtExposure'].mode()[0])
    #df.loc[:,'BsmtFinType1'] = df.loc[:,'BsmtFinType1'].fillna(df.loc[:,'BsmtFinType1'].mode()[0])
    #df.loc[:,'BsmtFinType2'] = df.loc[:,'BsmtFinType2'].fillna(df.loc[:,'BsmtFinType2'].mode()[0])
    #df.loc[:,'GarageType'] = df.loc[:,'GarageType'].fillna(df.loc[:,'GarageType'].mode()[0])
    df.loc[:,'GarageYrBlt'] = df.loc[:,'GarageYrBlt'].fillna(df.loc[:,'GarageYrBlt'].mean())
    #df.loc[:,'GarageFinish'] = df.loc[:,'GarageFinish'].fillna(df.loc[:,'GarageFinish'].mode()[0])
    #df.loc[:,'GarageQual'] = df.loc[:,'GarageQual'].fillna(df.loc[:,'GarageQual'].mode()[0])
    #df.loc[:,'GarageCond'] = df.loc[:,'GarageCond'].fillna(df.loc[:,'GarageCond'].mode()[0])

    #corr=df.corr()["SalePrice"]
    #corr[np.argsort(corr, axis=0)[::-1]]
    #print(corr)

    '''Cols that we would keep because the corr relation with SalePrice are relatively higher

    'LotArea', 'HouseStyle', 'OverallQual', 'OverallCond', 'OverallCond',
                  'YearRemodAdd', 'MasVnrArea', 'ExterQual', 'Foundation', 'BsmtCond',
                  'TotalBsmtSF', '1stFlrSF', 'GrLivArea', 'FullBath', 'KitchenAbvGr',
                  'TotRmsAbvGrd', 'GarageYrBlt', 'GarageCars', 'Unf'
     '''

    df = colDrop(df, 'MSSubClass')
    df = colDrop(df, 'Fireplaces')
    df = colDrop(df, 'LandSlope')
    df = colDrop(df, 'BsmtFinSF1')
    df = colDrop(df, 'BsmtFinSF2')
    df = colDrop(df, 'BsmtUnfSF')
    df = colDrop(df, 'HeatingQC')
    df = colDrop(df, '2ndFlrSF')
    df = colDrop(df, 'LowQualFinSF')
    df = colDrop(df, 'BsmtHalfBath')
    df = colDrop(df, 'FullBath')
    df = colDrop(df, 'HalfBath')
    df = colDrop(df, 'BedroomAbvGr')
    df = colDrop(df, 'KitchenAbvGr')
    df = colDrop(df, 'TotRmsAbvGrd')
    df = colDrop(df, 'GarageCars')
    df = colDrop(df, 'WoodDeckSF')
    df = colDrop(df, 'OpenPorchSF')
    df = colDrop(df, '3SsnPorch')
    df = colDrop(df, 'ScreenPorch')
    df = colDrop(df, 'PoolArea')
    df = colDrop(df, 'MiscVal')
    df = colDrop(df, 'MoSold')
    df = colDrop(df, 'YrSold')
    df = colDrop(df, 'Fin')
    df = colDrop(df, 'RFn')
    df = colDrop(df, 'EnclosedPorch')

    return df

def standardize(df, ls):

    df.loc[:, ls] =  (df.loc[:, ls] - df.loc[:, ls].mean())
    df.loc[:, ls] = (df.loc[:, ls])/df.loc[:, ls].std()

def test10():
    df  = dataProcessing()
    inputCols = ['Id', 'LotArea', 'HouseStyle', 'OverallQual', 'OverallCond', 'YearBuilt',
                  'YearRemodAdd', 'MasVnrArea', 'ExterQual', 'Foundation', 'BsmtCond',
                  'TotalBsmtSF', '1stFlrSF', 'GrLivArea', 'BsmtFullBath',
                  'GarageYrBlt', 'GarageArea', 'SaleCondition', 'Duplex', 'Unf']

    outputCol = ['SalePrice']

    #print(df.dtypes)

    standardize(df, inputCols + outputCol)

    df1 = df.loc[:,inputCols]
    outputSeries = df.loc[:,outputCol]

    alg = GradientBoostingRegressor()
    alg.fit(df1, outputSeries)
    cvScores = model_selection.cross_val_score(alg, df1, outputSeries ,cv=10, scoring='r2')
    print(cvScores.mean())

    '''
    alg =  LogisticRegression()

    df1 = df.loc[:, inputCols]
    df2 = df.loc[:, outputCol]

    standardize(df1, inputCols)
    standardize(df2, outputCol)

    #FROM:https://stackoverflow.com/questions/34165731/a-column-vector-y-was-passed-when-a-1d-array-was-expected
    #Explanation: casts flaot to int types. As an error came forward
    df2=df2.astype('int')
    alg.fit(df1,df2.values.ravel())
    #END: https://stackoverflow.com/questions/34165731/a-column-vector-y-was-passed-when-a-1d-array-was-expected

    cvScores = model_selection.cross_val_score(alg, df1, df2 ,cv=10, scoring='accuracy')
    orginal_cvScore_mean = cvScores.mean()
    print(orginal_cvScore_mean)

    '''


#Интеграл от неявнозаданной функции
from math import *

def func(x, y): #Неявнозаданная функция
    f = exp(x**3)-(exp(y))*(x**6 - 2*(x**3)*y - 2*(x**3) + y**2 + 2*y +2)
    return f

def found_y(x, eps = 1e-8):
    y_left = -0.1
    y_right = 0.1
    while (func(x, y_left)*func(x, y_right) > 0):
        y_left = y_left-0.1
        y_right = y_right+0.1
    y_midle = y_left + (y_right - y_left)/2
    while ( (y_right-y_left) > eps*abs(y_midle) + eps):
        if (func(x, y_left)*func(x, y_midle) > 0):
            y_left = y_midle
            y_midle = y_left + (y_right - y_left)/2
        elif (func(x, y_left)*func(x, y_midle) < 0):
            y_right = y_midle
            y_midle = y_left + (y_right - y_left)/2
    return y_midle

def integrall(x_left, x_right, eps = 1e-8):
    S1 = ((found_y(x_left, eps)+found_y(x_right, eps))/2*(x_right - x_left))
    step = 1
    while 1:
        step *= 2
        h = (x_right - x_left)/step
        left = x_left
        S2 = (found_y(x_left,eps)+found_y(x_right,eps))/2
        for i in range(1, step-1):
            left += h
            S2 += found_y(left,eps)
        S2 = S2*h
        if (abs(S2-S1) < eps*S1 + eps):
            break
        S1 = S2
    print(step)
    return S2

############ Основная программа ############
MIN_LEFT = 0
MAX_RIGHT = 2

eps = 1e-4

print(integrall(MIN_LEFT, MAX_RIGHT, eps))

Вычислительные алгоритмы/lab1.integral/ВА_Задача1.py

ВА_Задача1.py


# __init__.py

__version__ = "1.2.2"  # Be sure to update version in setup.py as well

from difflib import SequenceMatcher
from sh3ll.command import command
from art import tprint

class IS(object):
    def __init__(self, name="", font="", prefix="CLA>"):
        self.name = name
        self.font = font
        self.prefix = prefix
        self.commands = []
        self.categories = []
        """
        Param: -variable value
        """

    def run(self):
        if self.font != "":
            tprint(self.name, font=self.font)
        else:
            tprint(self.name)

        while True:
            try:
                line = input(self.prefix).lower()
            except KeyboardInterrupt:
                exit()
            inputted_command = line.split()[0]

            # Parse command into command and arguments
            args = []
            tmp = line.split()[1:]
            count = 0
            for arg in range(len(tmp)):
                if count == 0:
                    if tmp[arg][0] == "'":
                        for arg2 in range(len(tmp[arg + 1:])):
                            if tmp[arg + 1 + arg2][-1] == "'":
                                args.append(''.join([s.strip("'") for s in tmp[arg:arg + arg2 + 2]]))
                                count = len(tmp[arg:arg + arg2 + 1])
                    else:
                        if tmp[arg][-1] != "'":
                            args.append(tmp[arg])
                else:
                    count -= 1

            cmds = [cmd.name for cmd in self.commands]
            categories = [cmd.category for cmd in self.commands]

            aliases = {}
            for command in self.commands:
                aliases[command.name] = command.aliases

            if inputted_command != "help" and inputted_command != "exit" and inputted_command != "q":
                if inputted_command in cmds and self.commands[cmds.index(inputted_command)].category == "":
                    self.commands[cmds.index(inputted_command)].execute(args)
                elif inputted_command in categories:
                    if line.split()[1] in cmds:
                        if self.commands[cmds.index(line.split()[1])].category == line.split()[0]:
                            self.commands[cmds.index(line.split()[1])].execute(args[1:])
                    else:
                        for command in self.commands:
                            if line.split()[1] in command.aliases:
                                command.execute(args[1:])
                else:
                    highestSimilarity = 0
                    mostSimilarCommand = ""
                    mostSimilarCommandCategory = ""

                    for command in self.commands:
                        similarity = SequenceMatcher(None, command.name, inputted_command).ratio()
                        if similarity > highestSimilarity:
                            highestSimilarity = SequenceMatcher(None, command.name, inputted_command).ratio()
                            mostSimilarCommand = command.name
                            mostSimilarCommandCategory = command.category

                    print(f"Command not recognized.\nDid you mean: '{mostSimilarCommandCategory} {mostSimilarCommand}'?")
            else:
                self.help() if inputted_command == "help" else exit()

    def help(self):
        print("help\tDisplays this menu")
        print("exit OR q\tExits the program")
        for command in self.commands:
            if command.category == "":
                print(f"{command.name}\t{command.help}")
        print()

        for category in self.categories:
            if category != "":
                print(f"\"{category}\" Commands:\n" + ("-" * (len(category) + 12)))
                cmds = []
                for command in self.commands:
                    if command.category == category:
                        cmds.append(command)

                longest_name = max([len(cmd.name) for cmd in cmds])
                longest_aliases = max([len(str(cmd.aliases)) for cmd in cmds])
                longest_help = max([len(cmd.help) for cmd in cmds])

                print("\tCommand" + (" " * (abs((len(category) + 1 + longest_name) - 7) + 4)) + "Aliases" + (
                            " " * (abs(longest_aliases - 7) + 4)) + "Help" + " " * (abs(longest_help - 4) + 4))
                print("\t" + ("-" * 7) + (" " * (abs((len(category) + 1 + longest_name) - 7) + 4)) + ("-" * 8) + (
                            " " * (abs(longest_aliases - 8) + 4)) + ("-" * 4))

                for command in cmds:
                    if abs(longest_name - len(command.name)) == 0:
                        print(f"\t{category} {command.name}" + (" " * (abs((len(category) + 1 + longest_name) - (
                                    len(category) + len(command.name) + 1)) + 4)), end="")
                    else:
                        print(f"\t{category} {command.name}" + (" " * (
                                    abs((len(category) + 1 + longest_name) - len(f"{category} {command.name}")) + 4)),
                              end="")
                    if abs(longest_aliases - len(str(command.aliases))) == 0:
                        print(f"{command.aliases}    ", end="")
                    else:
                        print(f"{command.aliases}" + (" " * (abs(longest_aliases - len(str(command.aliases))) + 4)),
                              end="")
                    print(f"{command.help}" + (" " * abs(longest_help - len(command.help))))
                print()

    def command(self, name="Unknown command", aliases=[], help="No help given", category="", progress=()):
        def wrap(function):
            if category not in self.categories:
                self.categories.append(category)  # Auto register cats
            self.commands.append(command(function, name=name, aliases=aliases, help=help, category=category, progress=progress))

            def wrapped_function(*args):
                return function(*args)

            return wrapped_function

        return wrap


#!/usr/bin/env python3
import ast

f = open("in.txt", "r").read().strip().split("\n\n")
xy = [(-1, 0), (0, 1), (1, 0), (0, -1)]

def compare(first, second):
    if type(first) is not list and type(second) is not list:
        return 0 if first == second else (-1 if first < second else 1)

    if type(first) is int and type(second) is list:
        first = [first]

    if type(first) is list and type(second) is int:
        second = [second]

    for firstc, secondc in zip(first, second):
        r = compare(firstc, secondc)
        if r != 0:
            return r

    return -1 if len(first) < len(second) else (0 if len(first) == len(second) else 1)

seznam = []

for x in f:
    prvi = x.split("\n")[0]
    drugi = x.split("\n")[1]
    seznam.append((ast.literal_eval(prvi), ast.literal_eval(drugi)))

packets = []

for a, b in seznam:
    packets.append(a), packets.append(b)

packets.append([[2]]), packets.append([[6]])

a = len(packets)
for i in range(0, a):
    for j in range(0, a - 1):
        if compare(packets[j], packets[j + 1]) > 0:
            temp = packets[j + 1]
            packets[j + 1] = packets[j]
            packets[j] = temp

zmnozek = 1
for i in range(a):
    if packets[i] == [[2]] or packets[i] == [[6]]:
        zmnozek *= i + 1

print(zmnozek)


import time

class Timer:

    def __init__(self):
        self.dt = -1.0
        self.et = 0.0
        self.currentTime = 0.0
        self.lastTime = 0.0
        self.timerRunning = False

    def isTimerOn(self):
        if self.dt > 0:
            return True
            return False

    def start(self):
        if self.timerRunning is not True:
            self.currentTime = time.clock()
            self.lastTime = self.currentTime

        self.timerRunning = True

    def stop(self):
        self.timerRunning = False

    def reset(self):
        self.dt = -1.0
        self.et = 0.0
        self.currentTime = 0.0
        self.lastTime = 0.0
        self.timerRunning = False

    def getDt(self):
        if self.timerRunning is not True :
            return 0.0

        self.currentTime = time.clock()
        self.dt = self.currentTime - self.lastTime
        self.lastTime = self.currentTime
        self.et += self.dt
        return self.dt

    def getEt(self):
        self.getDt()
        return self.et


from django.conf.urls import url, include
from django.contrib.auth.decorators import login_required

from frontend.views import SPAView, UserConfigSPAWebService

# spa view
VIEWS_PATTERNS = [
    url(regex=r'$',
        view=login_required(SPAView.as_view()),
        name='spa'),

]

# config endpoint
API_PATTERNS = [
    url(regex=r'user-config/$',
        view=login_required(UserConfigSPAWebService.as_view()),
        name='user-config'),
]

urlpatterns = [

    url(r'^api/', include(API_PATTERNS)),
    url(r'^', include(VIEWS_PATTERNS)),
]


import struct
import os

def reverse_string(string):
	string_out =""
	for i in range(len(string)-1,-1,-1):
		string_out+=string[i]
	return string_out

def print_hex(number):
	num = number
	string = ""
	if num>0:
		while num>=1:
			ost = num%16
			if ost<10:
				string+=str(int(ost))
			elif ost==10:
				string+="A"
			elif ost==11:
				string+="B"
			elif ost==12:
				string+="C"
			elif ost==13:
				string+="D"
			elif ost==14:
				string+="E"
			elif ost==15:
				string+="F"
			else:
				print("Error")
				break
			num = int(num/16)
		print (reverse_string(string))
		return True
	else:
		return False

num = 1234567890
print_hex(num)
print (hex(num))

numb =hex(num)
print (numb)


# A simple MLP network structure for point clouds,
#
# Added by Jiadai Sun

import torch
import torch.nn as nn
import torch.nn.functional as F

class PointRefine(nn.Module):

    def __init__(self, n_class=3,
                 in_fea_dim=35,
                 out_point_fea_dim=64):
        super(PointRefine, self).__init__()

        self.n_class = n_class
        self.PPmodel = nn.Sequential(
            nn.BatchNorm1d(in_fea_dim),

            nn.Linear(in_fea_dim, 64),
            nn.BatchNorm1d(64),
            nn.ReLU(),

            nn.Linear(64, 128),
            nn.BatchNorm1d(128),
            nn.ReLU(),

            nn.Linear(128, 256),
            nn.BatchNorm1d(256),
            nn.ReLU(),

            nn.Linear(256, out_point_fea_dim)
        )

        self.logits = nn.Sequential(
            nn.Linear(out_point_fea_dim, self.n_class)
        )

    def forward(self, point_fea):
        # the point_fea need with size (b, N, c) e.g.  torch.Size([1, 121722, 35])
        # process feature
        # torch.Size([124668, 9]) --> torch.Size([124668, 256])
        processed_point_fea = self.PPmodel(point_fea)
        logits = self.logits(processed_point_fea)
        point_predict = F.softmax(logits, dim=1)
        return point_predict

if __name__ == '__main__':

    import time
    device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
    model = PointRefine()
    model.train()

    # t0 = time.time()
    # pred = model(cloud)
    # t1 = time.time()
    # print(t1-t0)

    total = sum([param.nelement() for param in model.parameters()])
    print("Number of PointRefine parameter: %.2fM" % (total/1e6))
    # Number of PointRefine parameter: 0.04M


# Multiples of 3 and 5 return 'Fizz' and 'Buzz' respectively while 15 returns 'FizzBuzz'

def fizzBuzz():
	# Solution 1
	'''
	for i in range(1, 16):
		 if i in [3, 6, 9, 12]:
			 i='Fizz'
		 if i in [5, 10]:
			 i='Buzz'
		 if i==15:
			 i='FizzBuzz'
		 print(i)
	'''
	# Solution 2
	for ii in range(1, 16):
                if ii == 15:
                        ii='FizzBuzz'
                elif ii % 3 == 0:
                        ii='Fizz'
                elif ii % 5 == 0:
                        ii='Buzz'
                print(ii)

if __name__ == "__main__":
	fizzBuzz()


from PySide import QtCore, QtGui

class UserDialog(QtGui.QDialog):
    def __init__(self, cl, text, parent=None):
        super(UserDialog, self).__init__(parent)

        textLabel = QtGui.QLabel(str('%s' % text))
        mainbox = QtGui.QVBoxLayout()
        mainbox.addWidget(textLabel)

        vbox = QtGui.QVBoxLayout()
        entry = 1

        for client in cl:
            userLabel = QtGui.QLabel(str('%d. %s' % (entry, client.username())))
            vbox.addWidget(userLabel)
            entry += 1

        widget = QtGui.QWidget()
        widget.setLayout(vbox)

        scrollArea = QtGui.QScrollArea()
        scrollArea.setWidget(widget)
        mainbox.addWidget(scrollArea)

        closeButton = QtGui.QPushButton('&Close')
        closeButton.clicked.connect(self.closeDialog)
        mainbox.addWidget(closeButton)

        self.setLayout(mainbox)
        self.resize(270, 400)

        self.setWindowTitle(text)

    def closeDialog(self):
        self.done(0)


from . import views
from django.urls import path, include

urlpatterns = [
    path('', views.index, name="index"),
    path('about/', views.about, name="about"),
    path('contact/', views.contact, name="contact"),
    path('services/', views.services, name="services"),
    path('skill/', views.skill, name="skill"),
]