import os
import warnings
warnings.filterwarnings('ignore')

# data handling and numerical analysis
import uproot
import awkward as ak
import numpy as np
import pandas as pd
from coffea import processor, hist

import scipy

# Plotting / histogramming
from yahist import Hist1D
import matplotlib.pyplot as plt
import mplhep
plt.style.use(mplhep.style.CMS)

# Machine learning packages
import tensorflow as tf
from keras.utils.vis_utils import plot_model

from sklearn.model_selection import GridSearchCV
from keras.wrappers.scikit_learn import KerasClassifier

from keras.utils import np_utils
import onnxruntime as rt
from sklearn.utils import resample
from sklearn.metrics import roc_curve, roc_auc_score, auc, confusion_matrix
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder, RobustScaler
import joblib

# private modules and functions
from Tools.dataCard import dataCard
from Tools.helpers import finalizePlotDir, mt
from Tools.limits import makeCardFromHist
from processor.default_accumulators import dataset_axis
from plots.helpers import makePlot
from ML.multiclassifier_tools import get_one_hot, get_class_weight, get_sob,\
            load_onnx_model, predict_onnx, dump_onnx_model,\
            store_model, load_model,\
            store_transformer, load_transformer
from ML.models import baseline_model, create_model

def test_train(test, train, y_test, y_train, labels=[], bins=25, node=0, plot_dir=None, weight_test=None, weight_train=None):
    ks = {}

    fig, ax = plt.subplots(1,1,figsize=(10,10))

    h = {}
    for i, label in enumerate(labels):

        _ks, _p = scipy.stats.kstest(
            train[:,node][(y_train==i)],
            test[:,node][(y_test==i)]
        )

        ks[label] = (_p, _ks)

        h[label+'_test'] = Hist1D(test[:,node][(y_test==i)], bins=bins, weights=weight_test[(y_test==i)]).normalize()
        h[label+'_train'] = Hist1D(train[:,node][(y_train==i)], bins=bins, label=label+' (p=%.2f, KS=%.2f)'%(_p, _ks), weights=weight_train[(y_train==i)]).normalize()

        h[label+'_test'].plot(color=colors[i], histtype="step", ls='--', linewidth=2)
        h[label+'_train'].plot(color=colors[i], histtype="step", linewidth=2)

    if plot_dir:
        finalizePlotDir(plot_dir)
        fig.savefig("{}/score_node_{}.png".format(plot_dir, node))
        fig.savefig("{}/score_node_{}.pdf".format(plot_dir, node))

    return ks

def test_train_cat(test, train, y_test, y_train, labels=[], n_cat=5, plot_dir=None, weight_test=None, weight_train=None):
    ks = {}
    bins = [x-0.5 for x in range(n_cat+1)]

    fig, ax = plt.subplots(1,1,figsize=(10,10))

    h = {}
    for i, label in enumerate(labels):

        _ks, _p = scipy.stats.kstest(
            train.argmax(axis=1)[(y_train==i)],
            test.argmax(axis=1)[(y_test==i)]
        )

        ks[label] = (_p, _ks)

        h[label+'_test'] = Hist1D(test.argmax(axis=1)[(y_test==i)], bins=bins, weights=weight_test[(y_test==i)]).normalize()
        h[label+'_train'] = Hist1D(train.argmax(axis=1)[(y_train==i)], bins=bins, label=label+' (p=%.2f, KS=%.2f) train'%(_p, _ks), weights=weight_train[(y_train==i)]).normalize()

        h[label+'_test'].plot(color=colors[i], histtype="step", ls='--', linewidth=2)
        h[label+'_train'].plot(color=colors[i], histtype="step", linewidth=2)

    ax.set_ylabel('a.u.')
    ax.set_xlabel('category')

    ax.set_ylim(0,1/n_cat*5)

    if plot_dir:
        finalizePlotDir(plot_dir)
        fig.savefig("{}/categories.png".format(plot_dir))
        fig.savefig("{}/categories.pdf".format(plot_dir))

    return ks

def get_cat_plot(X, y, labels=[], n_cat=5, plot_dir=None, weight=None):
    ks = {}
    bins = [x-0.5 for x in range(n_cat+1)]

    fig, ax = plt.subplots(1,1,figsize=(10,10))

    h = {}
    for i, label in enumerate(labels):

        h[label+'_train'] = Hist1D(X.argmax(axis=1)[(y==i)], bins=bins, label=label, weights=weight[(y==i)])

        h[label+'_train'].plot(color=colors[i], histtype="step", linewidth=2)

    ax.set_ylabel('a.u.')
    ax.set_xlabel('category')

    ax.set_ylim(0,200)

    if plot_dir:
        finalizePlotDir(plot_dir)
        fig.savefig("{}/abs_categories.png".format(plot_dir))
        fig.savefig("{}/abs_categories.pdf".format(plot_dir))

def get_ROC(test, train, y_test, y_train, node=0):

    y_test_binary = (y_test!=node)*0 + (y_test==node)*1

    fpr_test, tpr_test, thresholds_test = roc_curve( y_test_binary, test[:,node] )
    auc_val_test = auc(fpr_test, tpr_test)

    plt.plot( tpr_test, 1-fpr_test, 'b', label= 'AUC NN (test)=' + str(round(auc_val_test,4) ))

    y_train_binary = (y_train!=node)*0 + (y_train==node)*1

    fpr_train, tpr_train, thresholds_test = roc_curve( y_train_binary, train[:,node]  )
    auc_val_train = auc(fpr_train, tpr_train)

    plt.plot( tpr_train, 1-fpr_train, 'r', label= 'AUC NN (train)=' + str(round(auc_val_train,4) ))

    plt.xlabel('$\epsilon_{Sig}$', fontsize = 20) # 'False positive rate'
    plt.ylabel('$1-\epsilon_{Back}$', fontsize = 20) #  '1-True positive rate'
    plt.legend(loc ='lower left')

if __name__ == '__main__':

    import argparse

    argParser = argparse.ArgumentParser(description = "Argument parser")
    argParser.add_argument('--load', action='store_true', default=None, help="Load weights?")
    argParser.add_argument('--cat', action='store_true', default=None, help="Use categories?")
    argParser.add_argument('--fit', action='store_true', default=None, help="Do combine fit?")
    argParser.add_argument('--version', action='store', default='v1', help="Version number")
    argParser.add_argument('--year', action='store', default='2018', help="Which year?")
    argParser.add_argument('--seed', action='store', default=-1, type=int, help="Fix the random seed in numpy?")
    argParser.add_argument('--optimize', action='store_true', help="Run grid search for hyper parameters?")
    # NOTE: need to add the full Run2 training option back
    args = argParser.parse_args()

    load_weights = args.load
    version = "_".join([args.year, args.version])
    is_cat = args.cat

    if args.seed>0:
        np.random.seed(args.seed)

    plot_dir = os.path.expandvars("/home/users/$USER/public_html/tW_scattering/ML/%s/"%(version))

    # Load the input data.
    # This data frame is produced with the SS_analysis processor:
    # ipython -i SS_analysis.py -- --dump
    #df = pd.read_hdf('/hadoop/cms/store/user/dspitzba/ML/multiclass_input_2018_v2.h5')

    sample_list =  ['DY', 'topW_lep', 'top', 'TTW', 'TTZ', 'TTH', 'XG', 'rare', 'diboson']
    years = ['2016APV', '2016', '2017', '2018'] if args.year == '2019' else [args.year]
    #years = ['2018']
    df = pd.DataFrame()
    for year in years:
        print(f"Loading files for year {year}")
        tmp = pd.concat([pd.read_hdf(f"../processor/multiclass_input_{sample}_{year}.h5") for sample in sample_list])
        df = pd.concat([df, tmp])
    del tmp

    variables = [
        ## best results with all variables, but should get pruned at some point...
        'n_jet',
        ##'n_central',
        'n_fwd',
        'n_b',
        'n_tau', ## important for ttZ
        #'n_track', ## not so important, and very bad data/MC agreement
        'st',
        ##'ht',
        'met',
        'mjj_max',
        'delta_eta_jj',
        'lead_lep_pt',
        'lead_lep_eta',
        'sublead_lep_pt',
        'sublead_lep_eta',
        'dilepton_mass',
        'dilepton_pt',
        'fwd_jet_pt',
        'fwd_jet_p',
        'fwd_jet_eta',
        'lead_jet_pt',
        'sublead_jet_pt',
        'lead_jet_eta',
        'sublead_jet_eta',
        'lead_btag_pt',
        'sublead_btag_pt',
        'lead_btag_eta',
        'sublead_btag_eta',
        'min_bl_dR',
        'min_mt_lep_met',
    ]

    baseline = (df['n_fwd']>=0)
    #baseline = (df['n_fwd']>0)

    df['label_orig'] = df['label']

    # Take input dataframe, rearrange the categories, calculate the correct weights, and relabel
    # Signal is easy. Just take events that are in the SS category: df['SS']==1, passing the baseline selection
    # Asigned label: 0
    df_signal       = df[((df['label']==0)&(df['SS']==1)&baseline)]
    df_signal['label'] = np.ones(len(df_signal))*0

    n_max = 200000

    # Prompt backgrounds from the various processes that contribute.
    # We only take events from the SS category that are not labeled as containing a lost lepton (LL).
    # Asigned label: 1
    rescaler        = len(df[((df['label']<7)&(df['label']>0)&(df['label']!=4)&(df['LL']==0)&(df['SS'])&baseline)])/n_max
    df_prompt       = resample(df[((df['label']<7)&(df['label']>0)&(df['label']!=4)&(df['LL']==0)&(df['SS'])&baseline)], n_samples=n_max)
    #df_prompt       = df[((df['label']<7)&(df['label']>0)&(df['label']!=4)&(df['LL']==0)&(df['SS'])&baseline)] # every prompt background except ttbar (which shouldn't have prompt anyway)
    df_prompt['weight'] = df_prompt['weight']*rescaler
    df_prompt['label'] = np.ones(len(df_prompt))*1

    # Lost lepton backgrounds from the various processes that contribute.
    # Very similar to above, but now requiring a lost lepton
    # Asigned label: 2
    rescaler        = len(df[((df['label']<7)&(df['label']>0)&(df['label']!=4)&(df['LL']>0)&(df['SS'])&baseline)])/n_max
    df_LL           = resample(df[((df['label']<7)&(df['label']>0)&(df['label']!=4)&(df['LL']>0)&(df['SS'])&baseline)], n_samples=n_max)
    df_LL['weight'] = df_LL['weight']*rescaler
    df_LL['label']  = np.ones(len(df_LL))*2

    # Nonprompt leptons, taken from top quark process (input label 4).
    # We use the data driven background estimate, so we use events from the AR region, and adjust the weight accordingly
    # Asigned label: 3
    df_NP           = df[((df['label']==4)&(df['AR']==1)&baseline)]
    df_NP['weight'] = df_NP['weight']*df_NP['weight_np']
    df_NP['label']  = np.ones(len(df_NP))*3

    # Charge flip category. We don't use all the events to train because they are just too many
    # We adjust the weight according to our background prediction, the rescaling, and only take events from the OS category: df['OS']==1
    # Asigned label: 4
    rescaler        = len(df[((df['label']==4)&(df['OS']==1)&baseline)])/n_max
    df_CF           = resample(df[((df['label']==4)&(df['OS']==1)&baseline)], n_samples=n_max)
    df_CF['weight'] = df_CF['weight']*df_CF['weight_cf']*rescaler
    df_CF['label']  = np.ones(len(df_CF))*4

    # These data frames are currently not used in training, but just for visualizations
    df_TTW          = df[(((df['label']==1)|(df['label']==3)|(df['label']==5))&(df['SS']==1)&baseline)]  # assume that rares (4-top, VVV) is prompt, too
    df_TTW['label'] = np.ones(len(df_TTW))
    df_TTZ          = df[((df['label']==2)&(df['SS']==1)&baseline)]
    df_TTH          = df[((df['label']==3)&(df['SS']==1)&baseline)]

    print ()
    print ("Yields after training preselection (baseline selection):")
    df_list = [\
        ('signal',      df_signal),
        ('TTW/TTH',     df_TTW),
        ('TTZ',         df_TTZ),
        ('TTH',         df_TTH),
        ('prompt',      df_prompt),
        ('LL',          df_LL),
        ('nonprompt',   df_NP),
        ('charge flip', df_CF),
    ]

    print ("{:30}{:>10}{:>10}".format("Name", "Weighted", "Raw"))
    for name, df in df_list:
        print ("{:30}{:10.2f}{:10}".format(name, sum(df['weight']), len(df)))

    del df_TTW, df_TTZ, df_TTH, df
    print ()

    # Now, merge all the separate dataframes into one again
    df_in = pd.concat([df_signal, df_prompt, df_LL, df_NP, df_CF])
    df_in = df_in[(df_in['event']%2 == 1)]  # NOTE: only use events with odd event number.
    labels = df_in['label'].values
    df_train, df_test, y_train_int, y_test_int = train_test_split(df_in, labels, train_size= int( 0.9*labels.shape[0] ), random_state=42 )

    X_train = df_train[variables].values
    X_test  = df_test[variables].values

    y_train = get_one_hot(y_train_int.astype(int))
    y_test = get_one_hot(y_test_int.astype(int))

    input_dim = len(variables)
    out_dim = len(y_train[0])

    # Adjust the weights of every category so that they have equal importance
    # NOTE: We can try to increase e.g. the importance of prompt or lost lepton backgrounds vs others by multiplying
    # their weight by a constant factor. get_class_weight is imported from ML.multiclassifier_tools
    class_weight = get_class_weight(df_train, dim=out_dim)

    '''
    # NOTE: Can't use pipelines, unfortunately.
    pipeline = Pipeline([
        ('scaler', RobustScaler()),
        ('NN', baseline_model()),
    ])
    '''

    if not load_weights:

        epochs = 50  # 50 -> 200 # NOTE 10 was found to be best?
        batch_size = 100 #5120
        validation_split = 0.2

        scaler = RobustScaler()
        X_train_scaled = scaler.fit_transform(X_train)
        params = scaler.get_params()

        if args.optimize:
            model = KerasClassifier(build_fn=create_model,
                                    input_dim=input_dim, out_dim=out_dim,
                                    epochs = epochs,
                                    batch_size = batch_size,
                                    verbose = 0,
                                    #class_weight = class_weight,
                                    #sample_weight = np.abs(df_train['weight'].values),  # this doesn't work here.
                                    )
            print ("Input dim: %s"%input_dim)
            # define the grid search parameters
            neurons = [1, 10, 20, 50, 75, 100, 150]
            #param_grid = dict(neurons=neurons)
            # Best: 0.499523 using {'neurons': 100}
            dropout_rate = [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
            #param_grid = dict(dropout_rate=dropout_rate)
            # Best: 0.506186 using {'dropout_rate': 0.5}
            activation = ['softmax', 'softplus', 'softsign', 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear']
            #param_grid = dict(activation=activation)
            # Best: 0.648132 using {'activation': 'relu'} NOTE: this was without sample_weight
            # Best: 0.498214 using {'activation': 'relu'} NOTE: this was with sample_weight
            batch_size = [100, 1000]
            epochs = [10, 100]
            #param_grid = dict(batch_size=batch_size, epochs=epochs)
            # Best: 0.516183 using {'batch_size': 100, 'epochs': 10}
            learning_rates = [0.0005, 0.001, 0.005, 0.01, 0.05, 0.1]
            param_grid = dict(learning_rate = learning_rates)
            # Best: 0.497346 using {'learning_rate': 0.0005}
            grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=5, cv=3)
            grid_result = grid.fit(
                X_train_scaled,
                y_train,
                class_weight = class_weight,
                sample_weight = np.abs(df_train['weight'].values),
            )
            # summarize results
            print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
            means = grid_result.cv_results_['mean_test_score']
            stds = grid_result.cv_results_['std_test_score']
            params = grid_result.cv_results_['params']
            for mean, stdev, param in zip(means, stds, params):
                print("%f (%f) with: %r" % (mean, stdev, param))

            model = create_model(input_dim=input_dim, out_dim=out_dim,
                                 dropout_rate=0.5,
                                 neurons=100,  # FIXME
                                 learning_rate=0.001,
                                 activation='relu',
                                 optimizer='RMSprop',
                                 )
        else:
            model = baseline_model(input_dim, out_dim)
            #model = create_model(input_dim=input_dim, out_dim=out_dim,
            #                     dropout_rate=0.5,
            #                     neurons=100,
            #                     learning_rate=0.001,
            #                     activation='relu',
            #                     optimizer='RMSprop',
            #                     )

        history = model.fit(
            X_train_scaled,
            y_train,
            epochs = epochs,
            batch_size = batch_size,
            verbose = 0,
            class_weight = class_weight,
            sample_weight = df_train['weight'].values,
        )

        store_model(model, scaler, version=version)

        plot_model(model, to_file='./model_plot.png', show_shapes=True, show_layer_names=True)

    else:
        #model, scaler = load_model(version=version)
        model, scaler = load_onnx_model(version=version)

        X_train_scaled = scaler.transform(X_train)
        print ("Loaded weights.")

    X_all = df_in[variables].values

    X_all_scaled  = scaler.transform(X_all)
    X_test_scaled = scaler.transform(X_test)

    # Evaluate the model for the entire data frame (pred_all), just the training set (pred_train) or the test set (pred_test)
    if not load_weights:
        pred_all    = model.predict( X_all_scaled )
        pred_train  = model.predict( X_train_scaled )
        pred_test   = model.predict( X_test_scaled )
    else:
        # always use ONNX for inference
        pred_all    = predict_onnx(model, X_all_scaled )
        pred_train  = predict_onnx(model, X_train_scaled )
        pred_test   = predict_onnx(model, X_test_scaled )

    # We can now evaluate the performance
    df_in['score_topW'] = pred_all[:,0]
    df_in['score_prompt'] = pred_all[:,1]
    df_in['score_ll'] = pred_all[:,2]
    df_in['score_np'] = pred_all[:,3]
    df_in['score_cf'] = pred_all[:,4]
    df_in['score_best'] = pred_all.argmax(axis=1)

    # For quantile transformation on the top-W node:
    from sklearn.preprocessing import QuantileTransformer
    qt = QuantileTransformer(n_quantiles=40, random_state=0)
    qt.fit(df_in[((df_in['label']==0)&(df_in['score_best']==0))]['score_topW'].values.reshape(-1, 1))

    store_transformer(qt, version=version)

    df_in['score_topW_transform'] = qt.transform(df_in['score_topW'].values.reshape(-1, 1))
    df_in['score_prompt_transform'] = qt.transform(df_in['score_prompt'].values.reshape(-1, 1))

    for i in range(3):
        print ("Checking assignment for cat %s"%i)
        for x in range(5):
            print (x, round(sum(df_in[((df_in['SS']==1)&(df_in['label']==i)&(df_in['score_best']==x))]['weight'])/sum(df_in[((df_in['SS']==1)&(df_in['label']==i))]['weight']), 3))

    print ("NP assignment")
    for x in range(5):
        print (x, round(sum(df_in[((df_in['AR']==1)&(df_in['label']==3)&(df_in['score_best']==x))]['weight'])/sum(df_in[((df_in['AR']==1)&(df_in['label']==3))]['weight']), 3))

    print ("CF assignment")
    for x in range(5):
        print (x, round(sum(df_in[((df_in['OS']==1)&(df_in['label']==4)&(df_in['score_best']==x))]['weight'])/sum(df_in[((df_in['OS']==1)&(df_in['label']==4))]['weight']), 3))

    def get_bkg(x):
        bkg = sum(df_in[((df_in['SS']==1)&(df_in['label']==1)&(df_in['score_best']==0)&(df_in['score_topW']>x))]['weight']) + \
                sum(df_in[((df_in['SS']==1)&(df_in['label']==2)&(df_in['score_best']==0)&(df_in['score_topW']>x))]['weight']) + \
                sum(df_in[((df_in['AR']==1)&(df_in['label']==3)&(df_in['score_best']==0)&(df_in['score_topW']>x))]['weight']) + \
                sum(df_in[((df_in['OS']==1)&(df_in['label']==4)&(df_in['score_best']==0)&(df_in['score_topW']>x))]['weight'])
        return bkg

    def get_sig(x):
        return sum(df_in[((df_in['SS']==1)&(df_in['label']==0)&(df_in['score_best']==0)&(df_in['score_topW']>x))]['weight'])

    print ("Signal yield in node 0: %.2f"%get_sig(0))
    print ("Baseline S/b: %.3f"%(get_sig(0)/get_bkg(0)))

    # find the cut where there are only 9 bkg events left (arbitrary threshold)
    for i in range(0, 500, 1):
        bkg = get_bkg(i/500)
        if bkg < 9:
            break
    thresh = i/500

    print ("S/B for bkg=9: %.3f"%(get_sig(thresh)/get_bkg(thresh)))

    for i in range(0, 500, 1):
        sig = get_sig(i/500)
        if sig < 1:
            break
    thresh = i/500

    print ("S/B for sig=1: %.3f"%(get_sig(thresh)/get_bkg(thresh)))

    print ("Checking for overtraining in max node asignment...")

    colors = ['gray', 'blue', 'red', 'green', 'orange']
    hist_labels = ['top-W', 'prompt', 'LL', 'NP', 'CF']

    ks = test_train_cat(
        pred_test,
        pred_train,
        y_test_int,
        y_train_int,
        labels = hist_labels,
        n_cat = len(hist_labels),
        plot_dir = plot_dir,
        weight_test = df_test['weight'].values,
        weight_train = df_train['weight'].values,
    )

    for label in ks:
        if ks[label][0]<0.05:
            print ("- !! Found small p-value for process %s: %.2f"%(label, ks[label][0]))

    get_cat_plot(
        pred_all,
        df_in['label'].values,
        labels = hist_labels,
        n_cat = len(hist_labels),
        plot_dir = plot_dir,
        weight = df_in['weight'].values,
    )

    print ("Checking for overtraining in the different nodes...")

    bins = [x/20 for x in range(21)]

    for node in [0,1,2,3,4]:
        ks = test_train(
            pred_test,
            pred_train,
            y_test_int,
            y_train_int,
            labels=hist_labels,
            node=node,
            bins=bins,
            plot_dir=plot_dir,
            weight_test = df_test['weight'].values,
            weight_train = df_train['weight'].values,
        )
        for label in ks:
            if ks[label][0]<0.05:
                print ("- !! Found small p-value for process %s in node %s: %.2f"%(label, node, ks[label][0]))

    if not load_weights:
        dump_onnx_model(model, version=version)

    # Correlations
    from ML.multiclassifier_tools import get_correlation_matrix, get_confusion_matrix
    get_correlation_matrix(
        df_in[(df_in['label']==0)][(variables+['score_topW', 'score_prompt', 'score_ll', 'score_np', 'score_cf'])],
        f_out=plot_dir+'/correlation.png'
    )

    # make this a function for two histograms with ratio, for a certain binning

    def shape_comparison(array1, array2, bins=20, weight1=[], weight2=[], normalize=True, labels=["first", "second"], save=False):

        import mplhep as hep
        plt.style.use(hep.style.CMS)

        hist1 = Hist1D(array1, bins=np.arange(0,bins+1)/bins, weights=weight1)
        hist2 = Hist1D(array2, bins=np.arange(0,bins+1)/bins, weights=weight2)

        if normalize:
            hist1 = hist1.normalize()
            hist2 = hist1.normalize()

        ratio = hist1.divide(hist2)

        fig, (ax, rax) = plt.subplots(2,1,figsize=(10,10), gridspec_kw={"height_ratios": (3, 1), "hspace": 0.05}, sharex=True)

        hep.cms.label(
            "Preliminary",
            data=False,
            lumi=60.0,
            loc=0,
            ax=ax,
        )

        hep.histplot(
            [ hist1.counts, hist2.counts ],
            hist1.edges,
            yerr=[ hist1.errors, hist2.errors ],
            histtype="step",
            stack=False,
            label=labels,
            ax=ax)

        hep.histplot(
            ratio.counts,
            ratio.edges,
            yerr=ratio.errors,
            histtype="errorbar",
            color='black',
            ax=rax)

        rax.set_ylim(0,1.99)
        rax.set_xlabel(r'$score$')
        rax.set_ylabel(r'Ratio')
        ax.set_ylabel(r'Events')

        #add_uncertainty(total_mc, rax, ratio=True)
        #add_uncertainty(total_mc, ax)

        ax.legend()

        plt.show()

        if save:
            fig.savefig("{}.png".format(save))
            fig.savefig("{}.pdf".format(save))

    shape_comparison(
        df_in[((df_in['label']==0)&(df_in['score_best']==0))]['score_topW_transform'].values,
        df_in[((df_in['label']==1)&(df_in['score_best']==0))]['score_topW_transform'].values,
        bins=8,
        weight1=df_in[((df_in['label']==0)&(df_in['score_best']==0))]['weight'].values,
        weight2=df_in[((df_in['label']==1)&(df_in['score_best']==0))]['weight'].values,
        normalize = False,
        save = "{}/score_transformed".format(plot_dir),
    )

    ## Confusion matrix
    get_confusion_matrix(
        y_train_int,
        np.argmax(pred_train, axis=1),
        f_out=plot_dir+'/confusion_train.pdf',
    )
    get_confusion_matrix(
        y_test_int,
        np.argmax(pred_test, axis=1),
        f_out=plot_dir+'/confusion_test.pdf',
    )
    # normalize='true' essentially does the same thing as
    # CM_test / (np.ones_like(CM_test) * np.sum(CM_test, axis=1)[:, np.newaxis])
    # which of course is ugly.


# Copyright (c) 2011-2020 Eric Froemling
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
# -----------------------------------------------------------------------------
"""A simple cloud caching system for making built binaries/assets available.

The basic idea here is the ballistica-internal project can flag file targets
in its Makefiles as 'cached', and the public version of those Makefiles will
be filtered to contain cache downloads in place of the original build commands.
Cached files are gathered and uploaded as part of the pubsync process.
"""

from __future__ import annotations

import os
import subprocess
import json
from typing import TYPE_CHECKING
from multiprocessing import cpu_count
from concurrent.futures import ThreadPoolExecutor

from efro.terminal import Clr

if TYPE_CHECKING:
    from typing import List, Dict, Tuple, Set

BASE_URL = 'https://files.ballistica.net/cache/ba1/'

TARGET_TAG = '#__EFROCACHE_TARGET__'
STRIP_BEGIN_TAG = '#__EFROCACHE_STRIP_BEGIN__'
STRIP_END_TAG = '#__EFROCACHE_STRIP_END__'

CACHE_DIR_NAME = '.efrocache'
CACHE_MAP_NAME = '.efrocachemap'

UPLOAD_STATE_CACHE_FILE = '.cache/efrocache_upload_state'

def get_file_hash(path: str) -> str:
    """Return the hash used for caching.

    This incorporates the file contents as well as its path.
    """
    import hashlib
    md5 = hashlib.md5()
    with open(path, 'rb') as infile:
        md5.update(infile.read())
    md5.update(path.encode())
    return md5.hexdigest()

def get_target(path: str) -> None:
    """Fetch a target path from the cache, downloading if need be."""
    from efro.error import CleanError
    from efrotools import run
    with open(CACHE_MAP_NAME) as infile:
        efrocachemap = json.loads(infile.read())
    if path not in efrocachemap:
        raise RuntimeError(f'Path not found in efrocache: {path}')
    url = efrocachemap[path]
    subpath = '/'.join(url.split('/')[-3:])
    local_cache_path = os.path.join(CACHE_DIR_NAME, subpath)
    local_cache_path_dl = local_cache_path + '.download'
    hashval = ''.join(subpath.split('/'))

    # First off: if there's already a file in place, check its hash.
    # If it matches the cache, we can just update its timestamp and
    # call it a day.
    if os.path.isfile(path):
        existing_hash = get_file_hash(path)
        if existing_hash == hashval:
            os.utime(path, None)
            print(f'Refreshing from cache: {path}')
            return

    # Ok there's not a valid file in place already.
    # Clear out whatever is there to start with.
    if os.path.exists(path):
        os.unlink(path)

    # Now if we don't have this entry in our local cache,
    # download it.
    if not os.path.exists(local_cache_path):
        os.makedirs(os.path.dirname(local_cache_path), exist_ok=True)
        print(f'Downloading: {Clr.BLU}{path}{Clr.RST}')
        result = subprocess.run(
            f'curl --fail --silent {url} --output {local_cache_path_dl}',
            shell=True,
            check=False)

        # We prune old cache files on the server, so its possible for one to
        # be trying to build something the server can no longer provide.
        # try to explain the situation.
        if result.returncode == 22:
            raise CleanError('Server gave an error.'
                             ' Old build files may no longer be available;'
                             ' make sure you are using a recent commit.')
        if result.returncode != 0:
            raise CleanError('Download failed; is your internet working?')

        run(f'mv {local_cache_path_dl} {local_cache_path}')

    # Ok we should have a valid .tar.gz file in our cache dir at this point.
    # Just expand it and it get placed wherever it belongs.

    # Strangely, decompressing lots of these simultaneously leads to occasional
    # "File does not exist" errors when running on Windows Subsystem for Linux.
    # There should be no overlap in files getting written, but perhaps
    # something about how tar rebuilds the directory structure causes clashes.
    # It seems that just explicitly creating necessary directories first
    # prevents the problem.
    os.makedirs(os.path.dirname(path), exist_ok=True)

    print(f'Extracting: {path}')
    try:
        subprocess.run(['tar', '-zxf', local_cache_path], check=True)
    except Exception:
        # If something goes wrong, try to make sure we don't leave a half
        # decompressed file lying around or whatnot.
        print(f"Error expanding cache archive for '{local_cache_path}'.")
        if os.path.exists(local_cache_path):
            os.remove(local_cache_path)
        raise

    # The file will wind up with the timestamp it was compressed with,
    # so let's update its timestamp or else it will still be considered
    # dirty.
    run(f'touch {path}')
    if not os.path.exists(path):
        raise RuntimeError(f'File {path} did not wind up as expected.')

def filter_makefile(makefile_dir: str, contents: str) -> str:
    """Filter makefile contents to use efrocache lookups."""

    if makefile_dir:
        # Assuming just one level deep at the moment; can revisit later.
        assert '/' not in makefile_dir
        to_proj_root = '..'
    else:
        to_proj_root = ''

    cachemap = os.path.join(to_proj_root, CACHE_MAP_NAME)
    lines = contents.splitlines()
    snippets = 'tools/snippets'

    # Strip out parts they don't want.
    while STRIP_BEGIN_TAG in lines:
        index = lines.index(STRIP_BEGIN_TAG)
        endindex = index
        while lines[endindex] != STRIP_END_TAG:
            endindex += 1

        # If the line after us is blank, include it too to keep spacing clean.
        if not lines[endindex + 1].strip():
            endindex += 1

        del lines[index:endindex + 1]

    # Replace cachable targets with cache lookups
    while TARGET_TAG in lines:
        index = lines.index(TARGET_TAG)
        endindex = index
        while lines[endindex].strip() != '':
            endindex += 1
        tname = lines[index + 1].split(':')[0]
        del lines[index:endindex]
        lines.insert(index, tname + ': ' + cachemap)
        target = (makefile_dir + '/' + '$@') if makefile_dir else '$@'
        pre = f'cd {to_proj_root} && ' if makefile_dir else ''
        lines.insert(index + 1, f'\t@{pre}{snippets} efrocache_get {target}')
    return '\n'.join(lines) + '\n'

def update_cache(makefile_dirs: List[str]) -> None:
    """Given a list of directories containing Makefiles, update caches."""

    import multiprocessing
    cpus = multiprocessing.cpu_count()
    fnames1: List[str] = []
    fnames2: List[str] = []
    for path in makefile_dirs:
        cdp = f'cd {path} && ' if path else ''

        # First, make sure all cache files are built.
        mfpath = os.path.join(path, 'Makefile')
        print(f'Building efrocache targets for {Clr.SBLU}{mfpath}{Clr.RST}...')
        subprocess.run(f'{cdp}make -j{cpus} efrocache-build',
                       shell=True,
                       check=True)

        rawpaths = subprocess.run(f'{cdp}make efrocache-list',
                                  shell=True,
                                  check=True,
                                  capture_output=True).stdout.decode().split()

        # Make sure the paths they gave were relative.
        for rawpath in rawpaths:
            if rawpath.startswith('/'):
                raise RuntimeError(f'Invalid path returned for caching '
                                   f'(absolute paths not allowed): {rawpath}')

        # Break these into 2 lists, one of which will be included in the
        # starter-cache.
        for rawpath in rawpaths:
            fullpath = os.path.join(path, rawpath)

            # The main reason for this cache is to reduce round trips to
            # the staging server for tiny files, so let's include small files
            # only here. For larger stuff its ok to have a request per file..
            if os.path.getsize(fullpath) < 100000:
                fnames1.append(fullpath)
            else:
                fnames2.append(fullpath)

    # Ok, we've got 2 lists of filenames that we need to cache in the cloud.
    # First, however, let's look up modtimes for everything and if everything
    # is exactly the same as last time we can skip this step.
    hashes = _gen_hashes(fnames1 + fnames2)
    if os.path.isfile(UPLOAD_STATE_CACHE_FILE):
        with open(UPLOAD_STATE_CACHE_FILE) as infile:
            hashes_existing = infile.read()
    else:
        hashes_existing = ''
    if hashes == hashes_existing:
        print(
            f'{Clr.SBLU}Efrocache state unchanged;'
            f' skipping cache push.{Clr.RST}',
            flush=True)
    else:
        _upload_cache(fnames1, fnames2, hashes, hashes_existing)

    print(f'{Clr.SBLU}Efrocache update successful!{Clr.RST}')

    # Write the cache state so we can skip the next run if nothing changes.
    os.makedirs(os.path.dirname(UPLOAD_STATE_CACHE_FILE), exist_ok=True)
    with open(UPLOAD_STATE_CACHE_FILE, 'w') as outfile:
        outfile.write(hashes)

def _upload_cache(fnames1: List[str], fnames2: List[str], hashes_str: str,
                  hashes_existing_str: str) -> None:
    from efrotools import run

    # First, if we've run before, print the files causing us to re-run:
    if hashes_existing_str != '':
        changed_files: Set[str] = set()
        hashes = json.loads(hashes_str)
        hashes_existing = json.loads(hashes_existing_str)
        for fname, ftime in hashes.items():
            if ftime != hashes_existing.get(fname, ''):
                changed_files.add(fname)

        # We've covered modifications and additions; add deletions:
        for fname in hashes_existing:
            if fname not in hashes:
                changed_files.add(fname)
        print(f'{Clr.SBLU}Updating efrocache due to'
              f' {len(changed_files)} changes:{Clr.RST}')
        for fname in sorted(changed_files):
            print(f'  {Clr.SBLU}{fname}{Clr.RST}')

    # Now do the thing.
    staging_dir = 'build/efrocache'
    mapping_file = 'build/efrocachemap'
    run(f'rm -rf {staging_dir}')
    run(f'mkdir -p {staging_dir}')

    _write_cache_files(fnames1, fnames2, staging_dir, mapping_file)

    print(f'{Clr.SBLU}Starter cache includes {len(fnames1)} items;'
          f' excludes {len(fnames2)}{Clr.RST}')

    # Sync all individual cache files to the staging server.
    print(f'{Clr.SBLU}Pushing cache to staging...{Clr.RST}', flush=True)
    run('rsync --progress --recursive build/efrocache/'
        ' ubuntu@ballistica.net:files.ballistica.net/cache/ba1/')

    # Now generate the starter cache on the server..
    run('ssh -oBatchMode=yes -oStrictHostKeyChecking=yes ubuntu@ballistica.net'
        ' "cd files.ballistica.net/cache/ba1 && python3 genstartercache.py"')

def _gen_hashes(fnames: List[str]) -> str:
    import hashlib

    def _get_file_hash(fname: str) -> Tuple[str, str]:
        md5 = hashlib.md5()
        with open(fname, mode='rb') as infile:
            md5.update(infile.read())
        return fname, md5.hexdigest()

    # Now use all procs to hash the files efficiently.
    with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
        hashes = dict(executor.map(_get_file_hash, fnames))

    return json.dumps(hashes, separators=(',', ':'))

def _write_cache_files(fnames1: List[str], fnames2: List[str],
                       staging_dir: str, mapping_file: str) -> None:
    import functools
    fhashes1: Set[str] = set()
    fhashes2: Set[str] = set()
    mapping: Dict[str, str] = {}
    call = functools.partial(_write_cache_file, staging_dir)

    # Do the first set.
    with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
        results = executor.map(call, fnames1)
    for result in results:
        mapping[result[0]] = BASE_URL + result[1]
        fhashes1.add(result[1])

    # Now finish up with the second set.
    with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
        results = executor.map(call, fnames2)
    for result in results:
        mapping[result[0]] = BASE_URL + result[1]
        fhashes2.add(result[1])

    # We want the server to have a startercache.tar.xz file which contains
    # the entire first set. It is much more efficient to build that file
    # on the server than it is to build it here and upload the whole thing.
    # ...so let's simply write a script to generate it and upload that.

    # Also let's have the script touch both sets of files so we can use
    # mod-times to prune older files. (otherwise files that never change
    # might have very old mod times)
    script = (
        'import os\n'
        'import pathlib\n'
        'import subprocess\n'
        'fnames = ' + repr(fhashes1) + '\n'
        'fnames2 = ' + repr(fhashes2) + '\n'
        'subprocess.run(["rm", "-rf", "efrocache"], check=True)\n'
        'print("Copying starter cache files...", flush=True)\n'
        'for fname in fnames:\n'
        '    dst = os.path.join("efrocache", fname)\n'
        '    os.makedirs(os.path.dirname(dst), exist_ok=True)\n'
        '    subprocess.run(["cp", fname, dst], check=True)\n'
        'print("Touching full file set...", flush=True)\n'
        'for fname in list(fnames) + list(fnames2):\n'
        '    fpath = pathlib.Path(fname)\n'
        '    assert fpath.exists()\n'
        '    fpath.touch()\n'
        'print("Compressing starter cache archive...", flush=True)\n'
        'subprocess.run(["tar", "-Jcf", "tmp.tar.xz", "efrocache"],'
        ' check=True)\n'
        'subprocess.run(["mv", "tmp.tar.xz", "startercache.tar.xz"],'
        ' check=True)\n'
        'subprocess.run(["rm", "-rf", "efrocache", "genstartercache.py"])\n'
        'print("Starter cache generation complete!", flush=True)\n')

    with open('build/efrocache/genstartercache.py', 'w') as outfile:
        outfile.write(script)

    with open(mapping_file, 'w') as outfile:
        outfile.write(json.dumps(mapping, indent=2, sort_keys=True))

def _write_cache_file(staging_dir: str, fname: str) -> Tuple[str, str]:
    import hashlib
    from efrotools import run
    print(f'Caching {fname}')
    if ' ' in fname:
        raise RuntimeError('Spaces in paths not supported.')

    # Just going with ol' md5 here; we're the only ones creating these so
    # security isn't a concern.
    md5 = hashlib.md5()
    with open(fname, 'rb') as infile:
        md5.update(infile.read())
    md5.update(fname.encode())
    finalhash = md5.hexdigest()
    hashpath = os.path.join(finalhash[:2], finalhash[2:4], finalhash[4:])
    path = os.path.join(staging_dir, hashpath)
    os.makedirs(os.path.dirname(path), exist_ok=True)

    # Fancy pipe stuff which will give us deterministic tar.gz files
    # with no embedded timestamps.
    # Note: The 'COPYFILE_DISABLE' prevents mac tar from adding
    # file attributes/resource-forks to the archive as as ._filename.
    run(f'COPYFILE_DISABLE=1 tar cf - {fname} | gzip -n > {path}')
    return fname, hashpath

def _check_warm_start_entry(entry: Tuple[str, str]) -> None:
    import hashlib
    fname, filehash = entry
    md5 = hashlib.md5()
    with open(fname, 'rb') as infile:
        md5.update(infile.read())
    md5.update(fname.encode())
    finalhash = md5.hexdigest()

    # If the file still matches the hash value we have for it,
    # go ahead and update its timestamp.
    if finalhash == filehash:
        os.utime(fname, None)

def _check_warm_start_entries(entries: List[Tuple[str, str]]) -> None:
    with ThreadPoolExecutor(max_workers=cpu_count()) as executor:
        # Converting this to a list pulls results and propagates errors)
        list(executor.map(_check_warm_start_entry, entries))

def warm_start_cache() -> None:
    """Run a pre-pass on the efrocache to improve efficiency."""
    from efrotools import run

    # We maintain a starter-cache on the staging server, which
    # is simply the latest set of cache entries compressed into a single
    # compressed archive. If we have no local cache yet we can download
    # and expand this to give us a nice head start and greatly reduce
    # the initial set of individual files we have to fetch.
    # (downloading a single compressed archive is much more efficient than
    # downloading thousands)
    if not os.path.exists(CACHE_DIR_NAME):
        print('Downloading asset starter-cache...', flush=True)
        run(f'curl --fail {BASE_URL}startercache.tar.xz'
            f' --output startercache.tar.xz')
        print('Decompressing starter-cache...', flush=True)
        run('tar -xf startercache.tar.xz')
        run(f'mv efrocache {CACHE_DIR_NAME}')
        run('rm startercache.tar.xz')
        print('Starter-cache fetched successfully!'
              ' (should speed up asset builds)')

    # In the public build, let's scan through all files managed by
    # efrocache and update any with timestamps older than the latest
    # cache-map that we already have the data for.
    # Otherwise those files will update individually the next time
    # they are 'built'. Even though that only takes a fraction of a
    # second per file, it adds up when done for thousands of assets
    # each time the cache map changes. It is much more efficient to do
    # it in one go here.
    cachemap: Dict[str, str]
    with open(CACHE_MAP_NAME) as infile:
        cachemap = json.loads(infile.read())
    assert isinstance(cachemap, dict)
    cachemap_mtime = os.path.getmtime(CACHE_MAP_NAME)
    entries: List[Tuple[str, str]] = []
    for fname, url in cachemap.items():

        # File hasn't been pulled from cache yet = ignore.
        if not os.path.exists(fname):
            continue

        # File is newer than the cache map = ignore.
        if cachemap_mtime < os.path.getmtime(fname):
            continue

        # Don't have the cache source file for this guy = ignore.
        cachefile = CACHE_DIR_NAME + '/' + '/'.join(url.split('/')[-3:])
        if not os.path.exists(cachefile):
            continue

        # Ok, add it to the list of files we can potentially update timestamps
        # on once we check its hash.
        filehash = ''.join(url.split('/')[-3:])
        entries.append((fname, filehash))

    if entries:
        # Now fire off a multithreaded executor to check hashes and update
        # timestamps.
        _check_warm_start_entries(entries)


#! /usr/bin/env python
#
def i4vec_frac ( n, a, k ):

#*****************************************************************************80
#
#% I4VEC_FRAC searches for the K-th smallest entry in an N-vector.
#
#  Discussion:
#
#    Hoare's algorithm is used.
#
#  Licensing:
#
#    This code is distributed under the GNU LGPL license.
#
#  Modified:
#
#    26 May 2015
#
#  Author:
#
#    John Burkardt.
#
#  Parameters:
#
#    Input, integer N, the number of elements of A.
#
#    Input, integer A(N), the array to search.
#
#    Input, integer K, the fractile to be sought.  If K = 1, the minimum
#    entry is sought.  If K = N, the maximum is sought.  Other values
#    of K search for the entry which is K-th in size.  K must be at
#    least 1, and no greater than N.
#
#    Output, integer FRAC, the value of the K-th fractile of A.
#
  from sys import exit

  if ( n <= 0 ):
    print ( '' )
    print ( 'I4VEC_FRAC - Fatal error!' )
    print ( '  Illegal nonpositive value of N = %d' % ( n ) )
    exit ( 'I4VEC_FRAC - Fatal error!' )

  if ( k <= 0 ):
    print ( '' )
    print ( 'I4VEC_FRAC - Fatal error!' )
    print ( '  Illegal nonpositive value of K = %d' % ( k ) )
    exit ( 'I4VEC_FRAC - Fatal error!' )

  if ( n < k ):
    print ( '' )
    print ( 'I4VEC_FRAC - Fatal error!' )
    print ( '  Illegal N < K, K = %d' % ( k ) )
    exit ( 'I4VEC_FRAC - Fatal error!' )

  left = 1
  iryt = n

  while ( True ):

    if ( iryt <= left ):
      frac = a[k-1]
      break

    x = a[k-1]
    i = left
    j = iryt

    while ( True ):

      if ( j < i ):
        if ( j < k ):
          left = i
        if ( k < i ):
          iryt = j
        break
#
#  Find I so that X <= A(I)
#
      while ( a[i-1] < x ):
        i = i + 1
#
#  Find J so that A(J) <= X
#
      while ( x < a[j-1] ):
        j = j - 1

      if ( i <= j ):

        temp   = a[i-1]
        a[i-1] = a[j-1]
        a[j-1] = temp

        i = i + 1
        j = j - 1

  return frac

def i4vec_frac_test ( ):

#*****************************************************************************80
#
#% I4VEC_FRAC_TEST tests I4VEC_FRAC
#
#  Licensing:
#
#    This code is distributed under the GNU LGPL license.
#
#  Modified:
#
#    26 May 2015
#
#  Author:
#
#    John Burkardt
#
  import platform
  from i4vec_print import i4vec_print
  from i4vec_uniform_ab import i4vec_uniform_ab

  n = 10
  b = 1
  c = 2 * n
  seed = 123456789

  print ( '' )
  print ( 'I4VEC_FRAC_TEST' )
  print ( '  Python version: %s' % ( platform.python_version ( ) ) )
  print ( '  I4VEC_FRAC: K-th smallest integer vector entry.' )
  print ( '  Using initial random number seed = %d' % ( seed ) )

  a, seed = i4vec_uniform_ab ( n, b, c, seed )

  i4vec_print ( n, a, '  The array to search:' )

  print ( '' )
  print ( '  Fractile    Value' )
  print ( '' )

  nh = ( n // 3 )

  for k in range ( 1, n + 1, nh ):

    afrac = i4vec_frac ( n, a, k )

    print ( '  %6d  %6d' % ( k, afrac ) )
#
#  Terminate.
#
  print ( '' )
  print ( 'I4VEC_FRAC_TEST' )
  print ( '  Normal end of execution.' )
  return

if ( __name__ == '__main__' ):
  from timestamp import timestamp
  timestamp ( )
  i4vec_frac_test ( )
  timestamp ( )


import asyncio
import datetime
import os
import time
from slack_sdk.webhook.async_client import AsyncWebhookClient
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

import info

# setup slack url
SLACK_URL = os.getenv("SLACK_URL")
if SLACK_URL is None:
    print("Missing environment variable SLACK_URL. Example setup:")
    print('$ export SLACK_URL="https://hooks.slack.com/services/foo/bar/baz"')
    print("Exiting")
    exit()

# make sure this path is correct
PATH = os.path.dirname(os.path.realpath(__file__)) + "/opt/chromedriver"

driver = webdriver.Chrome(PATH)

# urls of bestbuy items
RTX3080_EVGA = "https://www.bestbuy.com/site/evga-geforce-rtx-3080-xc3-ultra-gaming-10gb-gddr6-pci-express-4-0-graphics-card/6432400.p?skuId=6432400"
RTX3070LINK1 = "https://www.bestbuy.com/site/nvidia-geforce-rtx-3070-8gb-gddr6-pci-express-4-0-graphics-card-dark-platinum-and-black/6429442.p?skuId=6429442"
RTX3070LINK2 = "https://www.bestbuy.com/site/gigabyte-geforce-rtx-3070-8g-gddr6-pci-express-4-0-graphics-card-black/6437912.p?skuId=6437912"
XBOXONETEST = "https://www.bestbuy.com/site/microsoft-xbox-one-s-1tb-console-bundle-white/6415222.p?skuId=6415222"
GAMING_CHAIR = "https://www.bestbuy.com/site/akracing-core-series-sx-gaming-chair-black/6215628.p?skuId=6215628"

# set the item to scan for
BOT_URL = RTX3080_EVGA

# method to send a slack message
async def send_message_via_webhook(msg: str, url: str):
    webhook = AsyncWebhookClient(SLACK_URL)
    response = await webhook.send(text="{} {}".format(msg, url))
    assert response.status_code == 200
    assert response.body == "ok"

driver.get(BOT_URL)

asyncio.run(send_message_via_webhook("Bot started for", BOT_URL))

# main loop
while True:
    # find add to cart button
    try:
        atcBtn = WebDriverWait(driver, 10).until(
            EC.element_to_be_clickable((By.CSS_SELECTOR, ".add-to-cart-button"))
        )
    except:
        print("{} Retry in 50 sec".format(datetime.datetime.now()))
        time.sleep(50)
        driver.refresh()
        continue

    print("{} Found! Sending slack message.".format(datetime.datetime.now()))
    asyncio.run(send_message_via_webhook("Availability found!", BOT_URL))
    time.sleep(50)


import sys
sys.path.append('..')
import PyQt4.QtGui as QtGui
import PyQt4.QtCore as QtCore
from gui.widgetitem import WidgetItem

class MainWindow(QtGui.QDialog):
    def __init__(self, parent=None):
        super(MainWindow, self).__init__(parent)
        self.item1 = WidgetItem(1)
        self.item2 = WidgetItem(2)
        self.stackedWidget = QtGui.QStackedWidget()
        self.stackedWidget.addWidget(self.item1)
        self.stackedWidget.addWidget(self.item2)
        self.stackedWidget.setCurrentIndex(1)
        self.pushbutton = QtGui.QPushButton()
        self.mainlayout = QtGui.QHBoxLayout(self)
        self.mainlayout.addWidget(self.stackedWidget)
        self.mainlayout.addWidget(self.pushbutton)
        self.pushbutton.clicked.connect(self.onOk)

    def onOk(self):
        index = self.stackedWidget.currentIndex()
        if index == 0:
            self.stackedWidget.setCurrentIndex(1)
            self.stackedWidget.widget(1).ui.lEVerfiyCode.setText('134')
        else:
            self.stackedWidget.setCurrentIndex(0)

def main():
    app = QtGui.QApplication(sys.argv)
    spliterH = QtGui.QSplitter(QtCore.Qt.Horizontal, 0)
    item = WidgetItem(spliterH)

    spliterV = QtGui.QSplitter(QtCore.Qt.Vertical, spliterH)
    item2 = WidgetItem(spliterV)

    spliter = QtGui.QSplitter(QtCore.Qt.Vertical, spliterV)
    item3 = WidgetItem(spliter)
    spliterH.setStretchFactor(1,1)
    spliterH.show()

    app.exec_()

main()


import os

# import from the 21 Bitcoin Developer Library
from two1.wallet import Wallet
from two1.bitserv.flask import Payment

# import from Flask library
from flask import Flask, jsonify, request
from flask import send_from_directory
from werkzeug import secure_filename

# import for pep8
import pep8

app = Flask(__name__)

# set up the bitcoin wallet
wallet = Wallet()
payment = Payment(app, wallet)

# configure uploads
UPLOAD_FOLDER = './uploads/'
ALLOWED_EXTENSIONS = set(['py'])
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

class CustomReport(pep8.StandardReport):
    """
    Collect report into an array of results.
    """
    results = []
    def get_file_results(self):
        if self._deferred_print:
            self._deferred_print.sort()
            for line_number, offset, code, text, _ in self._deferred_print:
                self.results.append({
                    'path': self.filename,
                    'row': self.line_offset + line_number,
                    'col': offset + 1,
                    'code': code,
                    'text': text,
                })
        return self.file_errors

# verify filetype
def allowed_file(filename):
    return '.' in filename and \
       filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

# upload file handler
@app.route('/upload', methods=['POST'])
def upload_file():
    if request.method == 'POST':
        file = request.files['file']
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            print("filename = {0}".format(filename))
            file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
    return 'OK'

# cleanup file handler
def cleanup(file_name):
    os.remove(file_name)
    return None

# perform pep8 linting
@app.route('/pep8', methods=['POST'])
@payment.required(1000)
def pep8linter():
    file_name = request.args.get('file')
    file_name = "./uploads/" + file_name
    pep8style = pep8.StyleGuide(reporter=CustomReport)
    sources = [file_name]
    report = pep8style.init_report()
    pep8style.check_files(sources)
    cleanup(file_name)
    return jsonify(errors=report.results)

# set up and run the server
if __name__ == '__main__':
        app.debug = True
        app.run(host='::', port=5001)


import copy
import re
import subprocess

class Reservoir(object):
    def __init__(self):
        self._clay_tiles = []
        self._map = []
        self.x_min = self.x_max = self.y_min = self.y_max = None
        self._check_nodes = []

    @property
    def x_boundary(self):
        return len(self._map[0])

    def add_clay(self, line):
        match = re.match(r"x=(\d+), y=(\d+)\.\.(\d+)", line)
        if match:
            x = int(match.group(1))
            for y in range(int(match.group(2)), int(match.group(3)) + 1):
                self._clay_tiles.append((x, y))
        else:
            match = re.match(r"y=(\d+), x=(\d+)\.\.(\d+)", line)
            assert(match)

            y = int(match.group(1))
            for x in range(int(match.group(2)), int(match.group(3)) + 1):
                self._clay_tiles.append((x, y))

    def generate_map(self):
        self.x_min = min([x for x, y in self._clay_tiles]) - 1
        self.x_max = max([x for x, y in self._clay_tiles]) + 1
        self.y_min = min([y for x, y in self._clay_tiles])
        self.y_max = max([y for x, y in self._clay_tiles])
        self._map = [["." for x in range(self.x_min, self.x_max + 1)] for y in range(self.y_min, self.y_max + 1)]
        for point in self._clay_tiles:
            x, y = point
            self._set_element(x - self.x_min, y - self.y_min, "#")

    def count_water(self):
        count = 0
        for line in self._map:
            for tile in line:
                if tile == "|" or tile == "~":
                    count += 1
        return count

    def spawn_water(self):
        position = (500, self.y_min)

        x, y = position
        x -= self.x_min
        y -= self.y_min
        # Mark this position as having water
        self._map[y][x] = "|"

        self._check_nodes = [(x, y)]
        while self._check_nodes:
            self._add_water(*self._check_nodes.pop())

    def _add_water(self, x, y):
        element = self._element_at_position(x, y)
        if element == "#" or element == "~":
            return False

        element_under = self._element_under_position(x, y)
        if element_under == None:
            return self._set_element(x, y, "|")

        elif element_under == "." or element_under == "|":
            self._set_element(x, y, "|")
            self._check_nodes += [(x, y+1)]

        elif element_under == "#" or element_under == "~":
            left_x, left_bounded = self._find_left_boundary(x, y)
            right_x, right_bounded = self._find_right_boundary(x, y)

            if left_bounded and right_bounded:
                made_change = False
                for sub_x in range(left_x, right_x + 1):
                    made_change |= self._set_element(sub_x, y, "~")
                if made_change:
                    self._check_nodes += [(x, y - 1)]
            else:
                made_change = False
                for sub_x in range(left_x, right_x + 1):
                    made_change |= self._set_element(sub_x, y, "|")
                if made_change:
                    if not left_bounded:
                        self._check_nodes += [(left_x, y+1)]
                    if not right_bounded:
                        self._check_nodes += [(right_x, y+1)]

    def _find_left_boundary(self, x, y):
        while True:
            x -= 1
            if x < 0:
                return 0, False

            if self._element_at_position(x, y) == "#":
                return x+1, True

            if self._element_under_position(x, y) != "#" and self._element_under_position(x, y) != "~":
                return x, False

    def _find_right_boundary(self, x, y):
        while True:
            x += 1
            if x >= self.x_boundary:
                return self.x_boundary-1, False

            if self._element_at_position(x, y) == "#":
                return x-1, True

            if self._element_under_position(x, y) != "#" and self._element_under_position(x, y) != "~":
                return x, False

    def _set_element(self, x, y, element):
        if self._map[y][x] == element:
            return False
        self._map[y][x] = element
        return True

    def _element_at_position(self, x, y):
        return self._map[y][x]

    def _element_under_position(self, x, y):
        y += 1
        if y >= len(self._map):
            return None

        return self._map[y][x]

    def print(self):
        subprocess.call("cls", shell=True)
        for line in self._map:
            print("".join(line))

    def print_file(self, filename, check_nodes):
        map_copy = copy.deepcopy(self._map)
        for x, y in check_nodes:
            map_copy[y][x] = "X"

        with open(filename, "w") as file:
            for line in map_copy:
                file.write("".join(line) + "\n")

r = Reservoir()
with open("input_file.txt", "r") as file:
    for line in file:
        r.add_clay(line)

r.generate_map()
r.spawn_water()
r.print_file("output.txt", check_nodes=[])
print(r.count_water())


#!/usr/bin/env python
import os
from google.appengine.ext import webapp
from google.appengine.ext.webapp import template
from google.appengine.ext.webapp.util import run_wsgi_app

feeds = {
    'bbc'      : 'http://news.bbc.co.uk/weather/forecast/',
    'guardian' : 'http://content.guardianapis.com/search?format=json&use-date=last-modified&show-fields=headline,trailText&ids=',
    'yahoo'    : 'http://query.yahooapis.com/v1/public/yql?format=json&q='
}

class Newyork(webapp.RequestHandler):
    def get(self):
        path = os.path.join(os.path.dirname(__file__),'templates/mashup.html')
        vars = {
            'name'     : 'New York',
            'bbc'      : '101',
            'news'     : feeds['guardian']+'travel/newyork',
            'weather'  : feeds['yahoo']+'select * from weather.forecast where location=10118',
        }
        self.response.out.write(template.render(path,vars))

application = webapp.WSGIApplication([
        ('/',Newyork)
    ],debug=True)

if __name__ == '__main__':
    run_wsgi_app(application)


## Copyright (c) 2012-2015 Aldebaran Robotics. All rights reserved.
## Use of this source code is governed by a BSD-style license that can be
## found in the COPYING file.

import os
import py

import qibuild.cmake

from qisys.test.conftest import skip_on_win

def test_get_cmake_qibuild_dir_no_worktree():
    res = qibuild.cmake.get_cmake_qibuild_dir()
    assert os.path.exists(os.path.join(res, "qibuild/general.cmake"))

def test_pip_std_install(tmpdir):
    python_dir = tmpdir.join("lib", "python2.7", "site-packages", "qibuild")
    python_dir.ensure("__init__.py", file=True)
    cmake_dir = tmpdir.join("share", "cmake")
    cmake_dir.ensure("qibuild", "qibuild-config.cmake", file=True)
    res = qibuild.cmake.find_installed_cmake_qibuild_dir(python_dir.strpath)
    assert res == cmake_dir.strpath

@skip_on_win
def test_pip_debian_install(tmpdir):
    local = tmpdir.mkdir("local")
    lib = tmpdir.mkdir("lib")
    local.join("lib").mksymlinkto(lib)
    python_dir = tmpdir.join("local", "lib", "python2.7", "site-packages", "qibuild")
    python_dir.ensure("__init__.py", file=True)
    cmake_dir = tmpdir.join("share", "cmake")
    cmake_dir.ensure("qibuild", "qibuild-config.cmake", file=True)
    res = qibuild.cmake.find_installed_cmake_qibuild_dir(python_dir.strpath)
    assert res == cmake_dir.strpath


#!/usr/bin/python
import sys

num = 0
oldword = None

for line in sys.stdin:
    date = line.strip().split('\t')
    word, id, index = date

    if oldword is None:
        oldword = word
    if oldword == word:
        num += 1
        print('%s : index : %s' % (oldword, id + '-' + index))
    if oldword != word:
        print('%s : count : %s' % (oldword, num))
        oldword = word
        num += 1

if oldword is not None:
    print('%s : index : %s' % (oldword, id + '-' + index))
    print('%s : count : %s' % (oldword, num))


import qcfractal.interface as ptl
import itertools
import qcengine
from tqdm import tqdm

client = ptl.FractalClient.from_file()

def is_semiempirical(method):
    return method.lower() in {"hf-3c", "pbe-3c", "pm6", "xtb"}

bases = ["sto-3g", "def2-svp", "def2-tzvp", "aug-cc-pvdz", "aug-cc-pvtz"]
methods = [
    "hf-3c",
    "pbe-3c",
    "hf",
    "b3lyp",
    "b2plyp",
    "pbe",
    "m06-2x",
    "m06-l",
    "pbe0",
    "wb97",
    "wb97x",
    "scan",
    "mp2",
]
ed_suffixes = ["", "-d3", "-d3m", "-d3bj", "-d3mbj"]

ds = client.get_collection("ReactionDataset", "ASCDB")
for method in tqdm(methods):
    try:
        if is_semiempirical(method):
            ds.compute(
                program="psi4", method=method, basis=None, tag="molssi_small",
            )
        else:
            for basis, ed_suffix in itertools.product(bases, ed_suffixes):
                d3method = method + ed_suffix
                if ed_suffix != "":
                    try:
                        qcengine.programs.empirical_dispersion_resources.from_arrays(
                            d3method
                        )
                    except qcengine.exceptions.InputError:
                        continue
                if basis == "sto-3g":
                    tag = "molssi_small"
                elif basis in {"def2-svp", "aug-cc-pvdz"}:
                    tag = "molssi_medium"
                else:
                    tag = "molssi_large"
                ds.compute(
                    program="psi4", method=d3method, basis=basis, tag=tag,
                )
    except KeyError:
        continue


"""

- By JIAYI CHEN
- (1) The code, including how to build, train and test a cnn, was written with the help of the code from cnn Labs by Professor Tom Gedeon, ANU
- (2) The main technique of the code: Autoencoder was inspired by the paper: Image Compression using Shared Weights and Bidirectional Networks
- (3) Thanks to Xuanyu Chen, I have become clear about how to implement a cnn autoencoder, that is to build a cnn, where convolutional layers and full connection layers are opposite,
   the dimension of convolutional layers should increase then decrease, and dimension of full connection layers should decrease then increase
- (4) Thanks to Ao Feng, I have become clear about how to use view function to flatten or unflatten the cnn in the AutoEncoder, which is swapping in 4D and 2D.
   Moreover, the size of the input data would affect the parameter that view function takes, so it's better to include it in the definition of AutoEncoder.

"""

import os
import torch
import torch.nn as nn
import torch.utils.data as Data
import torch.optim as op
import torchvision.datasets as dt
import torch.nn.functional as F
from torch.autograd import Variable
import torchvision.transforms as trans
import matplotlib.pyplot as plt

learning_rate = 0.01
batch_size_ae = 1000
batch_size = 100
num_epochs_ae = 10
num_epochs = 100

# load dataset
train_set = dt.MNIST('./data', train=True, transform=trans.Compose([trans.ToTensor(), trans.Normalize((0,), (1.0,))]))
test_set = dt.MNIST('./data', train=False, transform=trans.Compose([trans.ToTensor(), trans.Normalize((0,), (1.0,))]))

# set train loader
train_loader_c = torch.utils.data.DataLoader(dataset=train_set,batch_size=batch_size_ae,shuffle=True)

# define AutoEncoder
class AutoEncoder(nn.Module): #(2)(3)
    def __init__(self):
        super().__init__()
        # Encoder
        self.cnn_1 = nn.Conv2d(1, 16, 5)
        self.cnn_2 = nn.Conv2d(16, 32, 5)
        self.linear_1 = nn.Linear(4 * 4 * 32, 60)
        self.linear_2 = nn.Linear(60, 16)     # 28*28 -> 16
        # Decoder                             # can be encoded to different size by changing these two 16 to the preferred size
        self.linear_3 = nn.Linear(16, 200)    # 16 -> 28*28
        self.linear_4 = nn.Linear(200, 28*28)

    def forward(self, data):
        # Encoder
        encoded = self.cnn_1(data)
        encoded = F.max_pool2d(encoded, 2)
        encoded = F.rrelu(encoded)
        encoded = self.cnn_2(encoded)
        encoded = F.max_pool2d(encoded, 2)
        encoded = F.rrelu(encoded)
        encoded = encoded.view([data.size(0), -1]) #(4)
        encoded = self.linear_1(encoded)
        encoded = F.rrelu(encoded)
        encoded = self.linear_2(encoded)
        # Decoder
        decoded = self.linear_3(encoded)
        decoded = F.rrelu(decoded)
        decoded = self.linear_4(decoded)
        decoded = F.rrelu(decoded)
        decoded = decoded.view([encoded.size(0), 1, 28, 28]) #(4)
        return decoded, encoded

# Training
autoencoder = AutoEncoder()
if torch.cuda.is_available():
    autoencoder = autoencoder.cuda()
loss_fn = nn.MSELoss()
optimizer = op.Adam(autoencoder.parameters(), lr=learning_rate)

for epoch in range(num_epochs_ae):
    for batch, (data, target) in enumerate(train_loader_c):
        if torch.cuda.is_available():
            data, target = data.cuda(), target.cuda()
        data = Variable(data)
        decoded, encoded = autoencoder(data)
        target = Variable(target)
        optimizer.zero_grad()
        loss = loss_fn(decoded, data)
        loss.backward()
        optimizer.step()
        #if (batch+1) == len(train_loader_c):
        print ('autoencoder training - ','epoch:',epoch, 'batch:', batch+1, 'loss:', loss.data[0])

"""
#show input image
image = data.cpu().data
image = image / 2 + 0.5
image = image.clamp(0,1)
image = image.view(image.size(0),1,28,28)
img = trans.ToPILImage()(image[0])
plt.imshow(img)
plt.show()

#show processed image image
image = decoded.cpu().data
image = image / 2 + 0.5
image = image.clamp(0,1)
image = image.view(image.size(0),1,28,28)
img = trans.ToPILImage()(image[0])
plt.imshow(img)
plt.show()
"""

# set train loader and test loader
dataset_C = torch.utils.data.TensorDataset(data_tensor=decoded.data, target_tensor=target.data)
train_loader = torch.utils.data.DataLoader(dataset_C, batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset=test_set,batch_size=batch_size,shuffle=False)

# define classification network
class Net(nn.Module): #(1)
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 20, 5)
        self.conv2 = nn.Conv2d(20, 50, 5)
        self.fc1 = nn.Linear(4*4*50, 500)
        self.fc2 = nn.Linear(500, 10)

    def forward(self, x):
        x = F.relu(F.max_pool2d(self.conv1(x), 2))
        x = F.relu(F.max_pool2d(self.conv2(x), 2))
        x = x.view(-1, 4*4*50)
        x = F.relu(self.fc1(x))
        x = self.fc2(x)
        return x

# training
net = Net()
if torch.cuda.is_available():
    net = net.cuda()
criterion = nn.CrossEntropyLoss()
optimizer = op.SGD(net.parameters(), lr=learning_rate, momentum=0.5)

for epoch in range(num_epochs):
    for batch, (data, target) in enumerate(train_loader):
        if torch.cuda.is_available():
            data, target = data.cuda(), target.cuda()
        data, target = Variable(data), Variable(target)
        optimizer.zero_grad()
        output = net(data)
        loss = criterion(output, target)
        loss.backward()
        optimizer.step()
        if (batch+1) == len(train_loader):
            print ('classification cnn training - ','epoch:',epoch, 'loss:', loss.data[0])

# testing
correct = total = 0
for batch, (data, target) in enumerate(test_loader):
    if torch.cuda.is_available():
        data, target = data.cuda(), target.cuda()
    data, target = Variable(data), Variable(target)
    output = net(data)
    loss = criterion(output, target)
    _, predicted = torch.max(output.data, 1)
    total += data.data.size()[0]
    correct += (predicted == target.data).sum()
    if (batch+1) == len(test_loader):
        print ('testing - ','Loss:', loss.data[0],'accuracy:', correct * 1.0 * 100/ total,'%' )


# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2008 Deltatech All Rights Reserved
#                    Dorin Hongu <dhongu(@)gmail(.)com
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as
#    published by the Free Software Foundation, either version 3 of the
#    License, or (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from openerp import models, fields, api, _
from openerp.exceptions import except_orm, Warning, RedirectWarning
from openerp.tools import float_compare
from openerp.addons.product import _common
import openerp.addons.decimal_precision as dp
from dateutil.relativedelta import relativedelta
from datetime import datetime, date, timedelta
import logging
import time
from openerp.tools import DEFAULT_SERVER_DATE_FORMAT
from operator import attrgetter
from itertools import groupby

_logger = logging.getLogger(__name__)

ITEM_CATEG = [('primary','Primary'),
              ('normal','Normal'),
              ('optional','Optional'),
              ('service','Service'),
              ('opt_serv','Optional Service')]

class mrp_bom(models.Model):
    _inherit = 'mrp.bom'

    article_list = fields.Boolean(string='Article List')

    @api.model
    def _factor(self, factor, product_efficiency, product_rounding):
        factor = factor / (product_efficiency or 1.0)
        factor = _common.ceiling(factor, product_rounding)
        return factor

    @api.model
    def _prepare_consume_line(self, bom_line, quantity, factor=1):
        res = super(mrp_bom, self)._prepare_consume_line(bom_line, quantity, factor)
        res['item_categ'] = bom_line.item_categ
        return res

    def _check_attribute_in_list(self, check_attribs, component_attribs):
        """ Check if component is suitable for given attributes
        @param check_attribs: Attribute id list
        @param component_attribs: Component defined attributes to check
        @return: Component validity
        """
        getattr = attrgetter('attribute_id')
        for key, group in groupby(component_attribs, getattr):
            if set(check_attribs).intersection([x.id for x in group]):
                return True
        return False

    # metoda a fost redefinita pentru a selecta o linie daca exista cel putin un atribut
    def _skip_bom_line(self, line, product):
        today = fields.Date.context_today(self)
        if (line.date_start and
                line.date_start > today or
                line.date_stop and (line.date_stop < today)):
            return True
        # all bom_line_id variant values must be in the product
        if line.attribute_value_ids:
            production_attr_values = []
            if not product and self.env.context.get('production'):
                production = self.env.context['production']
                for attr_value in production.product_attributes:
                    production_attr_values.append(attr_value.value.id)
                if not self._check_attribute_in_list(
                        production_attr_values,
                        line.attribute_value_ids):
                    return True
            elif not product or not self._check_product_suitable(
                    product.attribute_value_ids.ids,
                    line.attribute_value_ids):
                return True
        if not line.product_id:
            if not product and self.env.context.get('production'):
                production = self.env.context['production']
                product_attributes = (
                    line.product_template._get_product_attributes_inherit_dict(
                        production.product_attributes))
                comp_product = self.env['product.product']._product_find(
                    line.product_template, product_attributes)
                if not comp_product:
                    return True
        return False

    """
    @api.multi
    def _bom_explode_variants_categ(
            self,  product, factor, properties=None, level=0,
            routing_id=False, previous_products=None, master_bom=None, product_attributes=None):

        self.ensure_one()
        bom = self
        routing_id = bom.routing_id.id or routing_id
        uom_obj = self.env["product.uom"]

        master_bom = master_bom or bom

        def _factor(factor, product_efficiency, product_rounding):
            factor = factor / (product_efficiency or 1.0)
            factor = _common.ceiling(factor, product_rounding)
            #if factor < product_rounding:
            #    factor = product_rounding
            return factor

        factor = _factor(factor, bom.product_efficiency, bom.product_rounding)

        result = []
        production_attr_values = []
        for attr_value in product_attributes:
            if attr_value.value:
                production_attr_values.append(attr_value.value.id)

        for bom_line_id in bom.bom_line_ids:

            if bom_line_id.date_start and  (bom_line_id.date_start > fields.Date.context_today(self)) or \
               bom_line_id.date_stop  and  (bom_line_id.date_stop < fields.Date.context_today(self)):
                continue
            # all bom_line_id variant values must be in the product
            if bom_line_id.attribute_value_ids:
                if not product and product_attributes:

                    if not self._check_attribute_in_list(    production_attr_values,
                                                            bom_line_id.attribute_value_ids):

                        continue
                elif not product or not self._check_product_suitable(   product.attribute_value_ids.ids,
                                                                        bom_line_id.attribute_value_ids):

                    continue

            if previous_products and (bom_line_id.product_id.product_tmpl_id.id
                                      in previous_products):
                raise exceptions.Warning(
                    _('Invalid Action! BoM "%s" contains a BoM line with a'
                      ' product recursion: "%s".') %
                    (master_bom.name, bom_line_id.product_id.name_get()[0][1]))

            quantity = _factor(bom_line_id.product_qty * factor,
                               bom_line_id.product_efficiency,
                               bom_line_id.product_rounding)

            if not bom_line_id.product_id:
                if not bom_line_id.type != "phantom":
                    bom_id = self._bom_find( product_tmpl_id=bom_line_id.product_template.id,
                                             properties=properties)
                else:
                    bom_id = False
            else:
                bom_id = self._bom_find(product_id=bom_line_id.product_id.id,
                                        properties=properties)

            if (bom_line_id.type != "phantom" and
                    (not bom_id or self.browse(bom_id).type != "phantom")):

                if not bom_line_id.product_id  and product_attributes:
                    item_product_attributes = ( bom_line_id.product_template._get_product_attributes_inherit_dict(product_attributes))
                    comp_product = self.env['product.product']._product_find(  bom_line_id.product_template, item_product_attributes)
                else:
                    comp_product = bom_line_id.product_id
                    item_product_attributes = (  bom_line_id.product_id. _get_product_attributes_values_dict())

                result.append({
                    'name': (bom_line_id.product_id.name or
                             bom_line_id.product_template.name),
                    'product_id': comp_product and comp_product.id,
                    'product_template': (
                        bom_line_id.product_template.id or
                        bom_line_id.product_id.product_tmpl_id.id),
                    'product_qty': quantity,
                    'product_uom': bom_line_id.product_uom.id,
                    'product_uos_qty': (
                        bom_line_id.product_uos and
                        _factor((bom_line_id.product_uos_qty * factor),
                                bom_line_id.product_efficiency,
                                bom_line_id.product_rounding) or False),
                    'product_uos': (bom_line_id.product_uos and
                                    bom_line_id.product_uos.id or False),
                    'product_attributes': map(lambda x: (0, 0, x),
                                              item_product_attributes),
                    #'level':level,
                    'item_categ':bom_line_id.item_categ,
                    #'row': bom_id is False
                })

            elif bom_id:
                all_prod = [bom.product_tmpl_id.id] + (previous_products or [])
                bom2 = self.browse(bom_id)
                # We need to convert to units/UoM of chosen BoM
                factor2 = uom_obj._compute_qty(
                    bom_line_id.product_uom.id, quantity, bom2.product_uom.id)
                quantity2 = factor2 / bom2.product_qty
                res = bom2._bom_explode_variants_categ(  bom_line_id.product_id, quantity2,
                    properties=properties, level=level + 1,
                    previous_products=all_prod, master_bom=master_bom, product_attributes = product_attributes)
                result = result + res
            else:
                if not bom_line_id.product_id:
                    name = bom_line_id.product_template.name_get()[0][1]
                else:
                    name = bom_line_id.product_id.name_get()[0][1]
                raise exceptions.Warning(
                    _('Invalid Action! BoM "%s" contains a phantom BoM line'
                      ' but the product "%s" does not have any BoM defined.') %
                    (master_bom.name, name))

        return result
    """

    def _bom_find(self, cr, uid, product_tmpl_id=None, product_id=None, properties=None, context=None):
        """ Finds BoM for particular product and product uom.
        @param product_tmpl_id: Selected product.
        @param product_uom: Unit of measure of a product.
        @param properties: List of related properties.
        @return: False or BoM id.
        """
        if not context:
            context = {}
        if properties is None:
            properties = []
        if product_id:
            if not product_tmpl_id:
                product_tmpl_id = self.pool['product.product'].browse(cr, uid, product_id, context=context).product_tmpl_id.id
            domain = [
                '|',
                    ('product_id', '=', product_id),
                    '&',
                        ('product_id', '=', False),
                        ('product_tmpl_id', '=', product_tmpl_id)
            ]
        elif product_tmpl_id:
            #domain = [('product_id', '=', False), ('product_tmpl_id', '=', product_tmpl_id)]
            domain = [ ('product_tmpl_id', '=', product_tmpl_id)]
        else:
            # neither product nor template, makes no sense to search
            return False
        if context.get('company_id'):
            domain = domain + [('company_id', '=', context['company_id'])]
        domain = domain + [ '|', ('date_start', '=', False), ('date_start', '<=', time.strftime(DEFAULT_SERVER_DATE_FORMAT)),
                            '|', ('date_stop', '=', False), ('date_stop', '>=', time.strftime(DEFAULT_SERVER_DATE_FORMAT))]
        # order to prioritize bom with product_id over the one without
        ids = self.search(cr, uid, domain, order='sequence, product_id', context=context)
        # Search a BoM which has all properties specified, or if you can not find one, you could
        # pass a BoM without any properties with the smallest sequence
        bom_empty_prop = False
        for bom in self.pool.get('mrp.bom').browse(cr, uid, ids, context=context):
            if not set(map(int, bom.property_ids or [])) - set(properties or []):
                if not properties or bom.property_ids:
                    return bom.id
                elif not bom_empty_prop:
                    bom_empty_prop = bom.id
        return bom_empty_prop

class mrp_bom_line(models.Model):
    _inherit = 'mrp.bom.line'

    item_categ = fields.Selection(ITEM_CATEG, default='normal', string='Item Category')

    child_bom_id = fields.Many2one('mrp.bom',string="Child BOM", compute="_compute_child_bom")

    @api.multi
    @api.depends('product_template','product_id')
    def _compute_child_bom(self):
        for bom_line in self:
           bom_id = self.env['mrp.bom']._bom_find(product_tmpl_id=bom_line.product_template.id,
                                                   product_id=bom_line.product_id.id,
                                                   properties=bom_line.property_ids )
           bom_line.child_bom_id = self.env['mrp.bom'].browse( bom_id )

    @api.multi
    @api.depends('product_template','product_id')
    def _calculate_standard_price(self):
        for bom_line in self:
            if bom_line.product_id:
                bom_line.standard_price = bom_line.product_id.standard_price
            else:
                bom_line.standard_price = bom_line.product_template.standard_price

    @api.multi
    @api.depends('product_template','product_id','child_bom_id')
    def _calculate_price(self):
        for bom_line in self:
            if bom_line.child_bom_id:
                price = bom_line.child_bom_id.calculate_price
            else:
                if bom_line.product_id:
                    price = bom_line.product_id.standard_price
                else:
                   price = bom_line.product_template.standard_price

            bom_line.calculate_price = price

    def _get_child_bom_lines(self, cr, uid, ids, field_name, arg, context=None):
        """If the BOM line refers to a BOM, return the ids of the child BOM lines"""
        bom_obj = self.pool['mrp.bom']
        res = {}
        for bom_line in self.browse(cr, uid, ids, context=context):
            bom_id = bom_obj._bom_find(cr, uid,
                product_tmpl_id=bom_line.product_template.id,
                product_id=bom_line.product_id.id, context=context)
            if bom_id:
                child_bom = bom_obj.browse(cr, uid, bom_id, context=context)
                res[bom_line.id] = [x.id for x in child_bom.bom_line_ids]
            else:
                res[bom_line.id] = False
        return res

    _sql_constraints = [
        ('bom_qty_zero', 'CHECK (1=1)', 'Error'),
    ]

    @api.multi
    def open_bom(self):
        self.ensure_one()
        if self.child_bom_id:
            #print "Deschid sublista de materiale"
            return {
                'res_id': self.child_bom_id.id,
                'domain': "[('id','=', "+str(self.child_bom_id.id)+")]",
                'name': _('BOM'),
                'view_type': 'form',
                'view_mode': 'form,tree',
                'res_model': 'mrp.bom',
                'view_id': False,
                'target': 'current',
                'nodestroy': True,
               'type': 'ir.actions.act_window'
            }

    @api.multi
    def check_line(self):
        for line in self:
            if not line.product_id:
                if line.product_template.product_variant_count == 1:
                    for product in line.product_template.product_variant_ids:
                        line.write({'product_id':product.id})

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:


import random
import numpy as np
n = int(input("Nhap so luong sample: "))
numbers = random.sample(range(1, 20), n)
arr = [[0 for x in range(n)] for y in range(n)]
# print(arr)
arr = np.random.randint(1,10,size=(n,n))
for i in range(n):
    for j in range(n):
        arr[j][i] = arr[i][j]
        arr[i][i] = 0
print(arr)

with open('data.txt', 'w') as f:
    f.write(str(n))
    for i in range(n):
        f.write("\n")
        for j in range(n):
            f.write("%d" % arr[i][j])
            f.write(" ")


#!/usr/bin/python3

import argparse
import os
import json
import sys
from tableschema import Table
from DIPSeparator import *

datapackageJson = json.load(sys.stdin)
parser = argparse.ArgumentParser()
parser.add_argument("uploadDir", help="The file upload directory which the metadata file is about")
parser.add_argument("uploadsDir", help="The top level file upload directory")

args = parser.parse_args()
uploadDir = args.uploadDir
uploadsDir = args.uploadsDir

resourcesJson = datapackageJson["resources"]
for resourceJson in resourcesJson:

    tblSchema = resourceJson.get("schema")
    srcFile = uploadsDir + uploadDir + resourceJson.get('path')
    print(srcFile)
    table = Table(srcFile, schema=tblSchema)

    seperator = DIPSeparator()
    startIndex = srcFile.rfind(os.sep)
    endIndex = srcFile.rfind(".")

    if ( startIndex == -1 ):
        startIndex = 0
    else:
        startIndex += 1

    if ( endIndex == -1 ):
        endIndex = len(srcFile)

    fileNameNoExt = srcFile[startIndex:endIndex]
    outputFiles = seperator.separate(table, fileNameNoExt, uploadDir)
    print(outputFiles, end='')


from theatre import buyer,seller

while True:
	print("press 1 to select new theatre layout")
	print("press 2 to book todays show tickets")
	print("press 3 to select all ticket buyer information list")
	print("press 4 to close this application")
	out = int(input())
	if out == 1:
		seller.setup()

	elif out == 2:
		x=0
		while x!=5 :
			print("==========***========")
			print("1. show seats")
			print("2. buy a ticket")
			print("3. statistics")
			print("4. show booked tickets user info")
			print("0. Exit")
			print("==========***========")
			i = int(input())
			if i == 1:
				seller.show_seats()
			elif i==2:
				buyer.buy_ticket()
			elif i==3:
				seller.statistics()
			elif i==4:
				seller.booked_ticket_buyer_info()
			if i==0:
				buyer.exit()
				x = 5

	elif out == 3:
		seller.all_ticket_buyer_info()

	elif out == 4:
		break


import matplotlib.pyplot as plt

"""
We have 3 elements to a pie graph that we need to populate.

Elements
--------
labels:
    Labels describe the names for the pie slices and are rendered on the
    outermost perimeter.
Sizes:
    The sizes variable dictates the percentage split of the graph.
    NOTE* These must add up to 100 percent.
Explode:
    Explode seperates certain slices of the graph and shifts them away
    from the remaining slices. The default value for explode is 0.

Note: Labels and Sized must have the same amount of entries. If you don't want
to have labels for a slice, replace the element with and empty string. E.G.
("")
"""
labels = 'Frogs', 'Hogs', 'Dogs', 'Logs'
sizes = [15, 30, 45, 10]
explode = (0, 0.5, 0, 0) #Change these values. What does this do to the graph?

fig1, ax1 = plt.subplots() #Creates the pie graph
ax1.set_color_cycle(['pink','blue','yellow','red']) #Change or add more colours
"""
set_color_cycle allows us to set custom colours for the pie graph. There is no
minimum and their is no limit to the amount of colours specified but if their
are more slices than there are colors defined, they will be reused. To add more
colours, simply follow the syntax above.
"""
ax1.pie(sizes,
        explode=explode,
        labels=labels,
        autopct='%1.1f%%',
        shadow=True,
        startangle=0) #startangle rotates the whole graph.
ax1.axis('equal')

plt.show()


#имя файла: Task 29.py
#номер версии: 1.0
#автор и его учебная группа: Александровский А.П., ЭУ-142
#дата создания: 22.05.2019
#связанные файлы: пакет numpy
#версия Python: 3.6
#ОПИСАНИЕ:  Создать прямоугольную матрицу A, имеющую N строк и M столбцов со
            #случайными элементами. Добавить к матрице столбец чисел и вставить его
            #под номером K.

# Подключение библиотеки Numpy и Random
import numpy as np
import random

# Число строк и столбцов, число L
N = random.randint(2, 10)
M = random.randint(2, 10)
K = random.randint(1, M)

# Так как матрица должна быть прямоугольной, то N не может быть равно M
while N == M:
    N = random.randint(1, 10)
    M = random.randint(1, 10)

# Создание матрицы
A = np.random.randint(0, 10, (N, M))
print(str(A) + "\n")

# Создание дополнительной столбца
a = np.random.randint(0, 10, N)
a = a[: , np.newaxis]
print("Допонлительный столбец: \n" + str(a))

# Добавление столбца под номером K
print("\n K = " + str(K))

if K == M:
    X = np.hstack((A, a))
elif K != M:
    X = np.array(A[:, 0])
    X = X[: , np.newaxis]
    for i in range (1, M):
        if i == K:
            X = np.hstack((X, a))
            Y = A[:, i]
            Y = Y[: , np.newaxis]
            X = np.hstack((X, Y))
        elif i != K:
            Y = A[:, i]
            Y = Y[:, np.newaxis]
            X = np.hstack((X, Y))

print("\nНовая матрица: \n" + str(X))

2 Часть курсовой работы/Task 29.py


class Solution:
    def getPermutation(self, n: int, k: int) -> str:
        """
        选择路径
        https://leetcode-cn.com/problems/permutation-sequence/solution/hui-su-jian-zhi-python-dai-ma-java-dai-ma-by-liwei/
        :param n:
        :param k:
        :return:
        """
        nums = [str(i) for i in range(1, n + 1)]

        fact = [1 for i in range(n + 1)]
        for i in range(1, n):
            fact[i] = fact[i - 1] * i

        k -= 1
        res = ""
        for i in range(n - 1, -1, -1):
            choice = k // fact[i]
            k -= fact[i] * choice
            res += nums.pop(choice)

        return res


import cv2
import numpy as np
from stack_images import stackImages
from get_contours import getContours
from resize import resize
from picamera import PiCamera
from time import sleep
from motor import start, stop, right, left, clear

camera = PiCamera()
stopped = False

def empty(a):
    pass

def classifyForm(form, mode, setLabelSignal):
    print(mode)
    if mode == 1:
        byForm(form, setLabelSignal)
    if mode == 2:
        bySize(form, setLabelSignal)
    if mode == 3:
        byColor(form, setLabelSignal)

    sleep(4)

def byForm(form, setLabelSignal):
    if form.corners == 4:
        print("Patrulater")
        sleep(1)
        left()
        sleep(1)
        start()
        setLabelSignal.emit("Patrulater")

    if form.corners == 3:
        print("Triunghi")
        sleep(1)
        right()
        sleep(1)
        start()
        setLabelSignal.emit("Triunghi")

    if form.corners == 6:
        print("Hexagon")
        setLabelSignal.emit("Hexagon")

def bySize(form, setLabelSignal):
    print(form.area)
    if (form.area <= 10000):
        print("Small")
        sleep(1)
        right()
        sleep(1)
        start()
        setLabelSignal.emit("Small")

    if (form.area > 10000):
        print("Big")
        sleep(1)
        right()
        sleep(1)
        start()
        setLabelSignal.emit("Big")

def byColor(form, setLabelSignal):
    # TODO: By color
    print(form.color)
    h = form.color[0]
    s = form.color[1]
    v = form.color[2]

    if (h >50 and h<80 and s>50 and s<90 and v>50 and v<80):
        print("Galben")
        setLabelSignal.emit("Galben")
    if (h >150 and h<180 and s>25 and s<55 and v>50 and v<80):
        print("Verde")
        setLabelSignal.emit("Verde")

def run(updateImageSignal, mode, setLabelSignal):
    camera.capture('image.jpg')
    ogImg = cv2.imread('image.jpg')
    colorImg = ogImg.copy()
    img = resize(ogImg, 50)
    imgContour = img.copy()

    img = img[20:200, 0:500]
    imgContour = imgContour[20:200, 0:500]

    cv2.imwrite('imageCrop.jpg', imgContour)

    imgBlur = cv2.GaussianBlur(img, (7, 7), 1)
    imgGray = cv2.cvtColor(imgBlur, cv2.COLOR_BGR2GRAY)

    threshold1 = 40
    threshold2 = 40

    imgCanny = cv2.Canny(imgGray, threshold1, threshold2)
    kernel = np.ones((5, 5))
    imgDil = cv2.dilate(imgCanny, kernel, iterations=1)

    form = getContours(imgDil, imgContour, img)
    print(form.corners)
    imgContour = resize(imgContour, 50)
    cv2.imwrite('output.jpg', imgContour)
    updateImageSignal.emit()
    classifyForm(form, mode, setLabelSignal)