'''
prepares data
'''

import numpy as np
import sparse

'''
global variable, used when preprocessing
'''
deltat = 250
T = 3500

'''
reads file and turns to numpy array
'''
def get_file(file_name, dim):
    f = open(file_name, 'r')
    c = f.read()
    c = c[1:]
    c = c.replace('\n', ',')
    c = c.split(',')
    c = np.array(c)
    c = c[:-1]
    c = c.reshape((-1,dim))
    return c

'''
normalize data
'''
def normalize(data):
    mins = np.amin(data, axis=0)
    maxs = np.amax(data, axis=0)

    dims = data.shape
    mins = np.tile(mins, (dims[0], 1))
    maxs = np.tile(maxs, (dims[0], 1))

    ranges = maxs - mins
    ranges[ranges == 0] = 1

    data[:, :] = (data[:,:] - mins[:, :]) / (ranges)
    return data

'''
load time invariant data
'''
def load_time_invar(data_source):
    raw_data = np.load('placeholder' + data_source + '/s.npz')
    data = sparse.COO(raw_data['coords'], raw_data['data'], tuple(raw_data['shape']))
    data = data.todense()

    pat_trajectories = get_file('placeholder', 2)
    pats = pat_trajectories[:, 0]
    pat_sort_i = np.argsort(pats)

    data = data[pat_sort_i, :]

    return data

'''
flip time variant data
'''
def flip_data(data, traj_bins):
    count = 0
    for bin_num in traj_bins:
        data[count, :bin_num + 1, :] = np.flip(data[count, :bin_num + 1, :], axis=1)
        count += 1
    return data

'''
load time variant data
mod_type determines whether to use summary of sequence data
data_types can be summary of sequential
    summary: concatenates sum of first 2 elements of time sequence and last 8 (values clipped at 1)
    sequential: leaves data as is and returns a sequence of length 10
'''
def load_time_var(data_source, data_type):
    raw_data = np.load('placeholder' + data_source + '/X.npz')
    data = sparse.COO(raw_data['coords'], raw_data['data'], tuple(raw_data['shape']))
    data = np.array(data.todense())

    pat_trajectories = get_file('placeholder', 2)
    traj_lens = pat_trajectories[:, 1].astype(int)
    traj_lens[np.where(traj_lens > T)[0]] = T
    traj_bins = (traj_lens // deltat).astype(int)
    pat_sort_i = np.argsort(pat_trajectories[:, 0])
    traj_bins = traj_bins[pat_sort_i]

    if data_type == 'summary':
        #process variable lengths patient trajectories
        num_segs = 4 #number of segments per feature vector
        seg_size = 1 #number of bins per segment
        ret_data = np.zeros((data.shape[0], 0))
        for i in range(num_segs):
            segment = np.sum(data[:, i*seg_size:(i*seg_size) + seg_size, :], axis=1)
            ret_data = np.concatenate((ret_data, segment), axis=1)
    else:
        ret_data = flip_data(data, traj_bins) #prepare_sequences(data)

    ret_data = normalize(ret_data)
    return ret_data

'''
prepare sequences for input
'''
def prepare_sequences(data):
    traj_lens = get_file('placeholder', 2)
    traj_lens = traj_lens[:, 1].astype(int)
    traj_bins = (traj_lens / deltat).astype(int)

    inputs = []

    for i in range(traj_bins.shape[0]):
        inputs.append(data[i, :traj_bins[i] + 1, :])

    sorted_order = np.flip(np.argsort(traj_bins))
    traj_bins = traj_bins[sorted_order]
    inputs = [inputs[i] for i in sorted_order]

    return inputs

'''
combine time in/variant data
'''
def combine_data(data_source, data_type, var_len=False):
    time_invar = load_time_invar(data_source)
    time_var = load_time_var(data_source, data_type)

    #if using sequence data, need to tile time_invar
    if len(time_var.shape) == 3:
        temp = np.zeros((time_invar.shape[0], time_var.shape[1], time_invar.shape[1]))
        #loop over sequence length
        for i in range(time_var.shape[1]):
            temp[:, i, :] = time_invar
        time_invar = temp
        all_data = np.concatenate((time_invar, time_var), axis=2)
        if var_len:
            all_data = prepare_sequences(all_data)
    #using summary data, concatenate on features
    else:
        #all_data = time_var
        all_data = np.concatenate((time_invar, time_var), axis=1) #use everything
        '''dems = time_invar #demographics only
        icds = time_var[:, 1416:2720] #icds only
        cpts = time_var[:, 162:1395] #cpts only
        labs = time_var[:, 2721:3832] #labs only
        util = np.concatenate((time_var[:, 0].reshape(-1, 1), time_var[:, 3935:3939], time_var[:, 3948:3964]), axis=1) #util
        age = np.concatenate((time_var[:, 0].reshape(-1, 1), time_var[:, 3948:3964]), axis=1)
        vist = time_var[:, 3935:3939] #vistype
        vits = np.concatenate((time_var[:, 11:41], time_var[:, 122:127], \
                   time_var[:, 134:139], time_var[:, 1407:1412]), axis=1) #vitals
        meds = np.concatenate((time_var[:, 1:11], time_var[:, 41:122], \
                   time_var[:, 127:134], time_var[:, 139:162], time_var[:, 1395:1407], \
                   time_var[:, 1412:1416], time_var[:, 2720].reshape(-1, 1), \
                   time_var[:, 3832:3935], time_var[:, 3939:3948]), axis=1) #meds
        if time_var.shape[1] > 3964:
            num_segs = int(time_var.shape[1] / 3964)
            for i in range(1, num_segs):
                icds = np.concatenate((icds, time_var[:, (i*3964)+1416:(i*3964)+2720]), axis=1)
                cpts = np.concatenate((cpts, time_var[:, (i*3964)+162:(i*3964)+1395]), axis=1)
                labs = np.concatenate((labs, time_var[:, (i*3964)+2721:(i*3964)+3832]), axis=1)
                util = np.concatenate((util, time_var[:, (i*3964)+0].reshape(-1, 1), \
                           time_var[:, (i*3964)+3935:(i*3964)+3939], time_var[:, (i*3964)+3948:(i*3964)+3964]), axis=1)
                age = np.concatenate((util, time_var[:, (i*3964)+0].reshape(-1, 1), \
                           time_var[:, (i*3964)+3948:(i*3964)+3964]), axis=1)
                vist = np.concatenate((vist, time_var[:, (i*3964)+3935:(i*3964)+3939]), axis=1)
                vits = np.concatenate((vits, time_var[:, (i*3964)+11:(i*3964)+41], time_var[:, (i*3964)+122:(i*3964)+127], \
                           time_var[:, (i*3964)+134:(i*3964)+139], time_var[:, (i*3964)+1407:(i*3964)+1412]), axis=1)
                meds = np.concatenate((meds, time_var[:, (i*3964)+1:(i*3964)+11], time_var[:, (i*3964)+41:(i*3964)+122], \
                           time_var[:, (i*3964)+127:(i*3964)+134], time_var[:, (i*3964)+139:(i*3964)+162], \
                           time_var[:, (i*3964)+1395:(i*3964)+1407], \
                           time_var[:, (i*3964)+1412:(i*3964)+1416], time_var[:, (i*3964)+2720].reshape(-1, 1), \
                           time_var[:, (i*3964)+3832:(i*3964)+3935], time_var[:, (i*3964)+3939:(i*3964)+3948]), axis=1)
        #all_data = np.concatenate((cpts, labs, util, icds, vits, dems), axis=1)
        all_data = dems'''

    return all_data

'''
gets patient labels
first column is patient id
second columns is the time from alignment to conversion (if they convert, otherwise -1)
third column is label
'''
def get_labels():
    labels_file = 'placeholder'
    labels = get_file(labels_file, 4)
    labels_sort_i = np.argsort(labels[:, 0])
    labels = labels[labels_sort_i, :]

    #remove extra patients
    pats = np.sort(get_file('placeholder', 1)[1:].reshape(-1))
    excluded_patients = np.where(np.logical_not(np.isin(labels[:, 0], pats)))[0]
    labels = np.delete(labels, excluded_patients, axis=0)
    print(labels.shape)

    excluded_patients2 = np.where(np.logical_not(np.isin(pats, labels[:, 0])))[0]
    print(excluded_patients2.shape)
    return labels, excluded_patients2

'''
split data into training and test set
in addition to training and test split, returns indexes of test patients
'''
def split_data(data_source, data_type):
    #setup
    data = combine_data(data_source, data_type)
    labels_raw, exclude_pats = get_labels()#[:, 2].astype(int)
    labels = np.logical_and(labels_raw[:, 2].astype(int) == 1, labels_raw[:, 1].astype(int) <= 120)
    data = np.delete(data, exclude_pats, axis=0)
    print(np.sum(labels), labels.shape, data.shape)

    prop_training = 0.8
    num_features = data.shape[1] if len(data.shape) != 3 else data.shape[2]
    pos_pats = np.where(labels == 1)[0]
    neg_pats = np.where(labels == 0)[0]
    training_labs = np.zeros((0,))
    test_labs = np.zeros((0,))
    test_pat_i = np.zeros((0,))

    if len(data.shape) != 3:
        training_data = np.zeros((0, num_features))
        test_data = np.zeros((0, num_features))
    else:
        training_data = np.zeros((0, data.shape[1], num_features))
        test_data = np.zeros((0, data.shape[1], num_features))

    #stratified split
    for pat_set in [pos_pats, neg_pats]:
        num_pats = pat_set.shape[0]
        rand_perm = np.random.permutation(num_pats)
        num_training = int(prop_training * num_pats)
        pat_labs = labels[pat_set][rand_perm]
        training_labs = np.append(training_labs, pat_labs[:num_training])
        test_labs = np.append(test_labs, pat_labs[num_training:])
        test_pat_i = np.append(test_pat_i, pat_set[rand_perm][num_training:])

        if len(data.shape) != 3:
            pat_data = data[pat_set, :][rand_perm, :]
            training_data = np.append(training_data, pat_data[:num_training, :], axis=0)
            test_data = np.append(test_data, pat_data[num_training:, :], axis=0)
        else:
            pat_data = data[pat_set, :, :][rand_perm, :, :]
            training_data = np.append(training_data, pat_data[:num_training, :, :], axis=0)
            test_data = np.append(test_data, pat_data[num_training:, :, :], axis=0)

    #randomize order of positive/negative examples
    train_perm = np.random.permutation(training_labs.shape[0])
    test_perm = np.random.permutation(test_labs.shape[0])

    training_labs = training_labs[train_perm]
    test_labs = test_labs[test_perm]
    test_pat_i = test_pat_i[test_perm]

    if len(data.shape) != 3:
        training_data = training_data[train_perm, :]
        test_data = test_data[test_perm, :]
    else:
        training_data = training_data[train_perm, :, :]
        test_data = test_data[test_perm, :, :]

    return (training_data, training_labs, test_data, test_labs, test_pat_i)

'''
splits squential_data
'''
def split_data_sequential(data_source):
    #setup
    data = combine_data(data_source, 'sequential', var_len=True)
    labels = get_labels()[:, 2].astype(int)

    prop_training = 0.8
    pos_pats = np.where(labels == 1)[0]
    neg_pats = np.where(labels == 0)[0]
    training_labs = np.zeros((0,))
    test_labs = np.zeros((0,))
    test_pat_i = np.zeros((0,))

    training_data = []
    test_data = []

    #stratified split
    for pat_set in [pos_pats, neg_pats]:
        num_pats = pat_set.shape[0]
        rand_perm = np.random.permutation(num_pats)
        num_training = int(prop_training * num_pats)
        pat_labs = labels[pat_set][rand_perm]
        training_labs = np.append(training_labs, pat_labs[:num_training])
        test_labs = np.append(test_labs, pat_labs[num_training:])
        test_pat_i = np.append(test_pat_i, pat_set[rand_perm][num_training:])

        pat_data = [data[i] for i in pat_set]
        pat_data = [pat_data[i] for i in rand_perm]
        training_data = training_data + pat_data[:num_training]
        test_data = test_data + pat_data[num_training:]

    #randomize order of positive/negative examples
    train_perm = np.random.permutation(training_labs.shape[0])
    test_perm = np.random.permutation(test_labs.shape[0])

    training_labs = training_labs[train_perm]
    test_labs = test_labs[test_perm]

    training_data = [training_data[i] for i in train_perm]
    test_data = [test_data[i] for i in test_perm]

    print('done splitting')
    return (training_data, training_labs, test_data, test_labs, test_pat_i)

'''
main block
'''
if __name__ == '__main__':
    pass
    #find_mem_prob()
    #data_type = 'summary'
    #split_data_sequential('first_align')


import datetime
from datetime import date
from datetime import timedelta

import requests

def tri_rendez_vous(liste_creneaux, date_alignement):
    """
    :param liste_creneaux: liste dictionnaire des creneaux horaire
    :param date_alignement: datetime.datetime date à laquelle les créneaux horaire vont être disponible
    :return: liste du type de rendez-vous avec nombre de personne
    """
    return [x for x in liste_creneaux["data"] if date_alignement >= datetime.datetime.strptime(x["date"], "%Y-%m-%d")]

try:
    headers = {'Accept': 'application/json'}
    # args[0] == l'url des créneaux horaire
    # args[1] == date départ de l'alignement
    # args[2] == le nombre de jour d'alignement max
    liste_creneaux = requests.get(args[0], headers=headers).json()
    date_debut = datetime.datetime.strptime(args[1], "%d/%m/%Y")
    date_alignement = date_debut + timedelta(days=args[2])
    date_alignement = datetime.datetime.fromordinal(date_alignement.toordinal())
    result = tri_rendez_vous(liste_creneaux, date_alignement)
except Exception as e:
    None


from django.core.management.base import BaseCommand
from django.contrib.auth.models import User
from ask_mogilnikov.models import Question, Answer, Tag
import random

class Command(BaseCommand):
    tags = Tag.objects.all()
    users = User.objects.all()

    def create_questions(self, n):
        for i in range(n):
            question = Question(title='Can you solve this?',
                content="VWDQ LV QRW ZKDW KH VHHPV",
                author=random.choice(self.users),
                likes_count=0,
                answers_count=0)
            question.save()

            tags_num = random.randint(1,3)
            for i in range(tags_num):
                question.tags.add(random.choice(self.tags))
            question.save()

    def handle(self, *args, **options):
        self.create_questions(n=30)


def dzielniki(n):
    t = [1]
    for i in range(2, n//2 + 1):
        if n % i == 0:
            t.append(i)
    t.append(n)
    return t

def NWD(a, b):
    while b != 0:
        a, b = b, a % b
    return a

def wzgl_prw(n, arr):
    for L in arr:
        if n != L:
            if NWD(n, L) != 1:
                return False
    return True

dane = []
with open("liczby.txt", "r") as file:
    for line in file:
        line = line.strip()
        dane.append(int(line))

# 1, 2
odp1 = ""
odp2 = ""
l1 = 0
for L in dane[::-1]:
    if L < 1000:
        l1 += 1
        if l1 <= 2:
            odp1 += str(L) + "\n"
    if len(dzielniki(L)) == 18:
        odp2 += f'{L}\t{dzielniki(L)}\n'

# 3
najw = 0
for x in dane:
    if wzgl_prw(x, dane):
        if x > najw:
            najw = x

with open("wyniki1.txt", "w") as odp:
    odp.write("1.\n" + str(l1) + "\n" + odp1)
    odp.write("\n\n2.\n" + odp2)
    odp.write("\n\n3.\n" + str(najw))


from datasets.main import load_dataset
from networks.main import build_network
from datasets.main import DigitConDataset
from loss import HLoss,mixup
from tqdm import tqdm
from util import plot_confusion_matrix, sim_matrix
import torch
import torch.nn.functional as F
from tensorboardX import SummaryWriter
import numpy as np
import torchvision
import math
from sklearn.manifold import TSNE
from matplotlib import pyplot as plt
import faiss
from faiss import normalize_L2
import time
import scipy
from sklearn.metrics import confusion_matrix
from sklearn.neighbors.kde import KernelDensity
import os
from torch.autograd import Variable
from sklearn.cluster import KMeans
import pandas as pd
from draw import *
class CDA(object):
    """A class for the Curriculum Domain Adaptation method.

    Attributes:
        self.experiment           = str   --experiment name for log file
        self.pseudo_label_method  = str   "label propagation" or "nearest center" or "NN prediction"
        self.strategy             = str   "decrementedthreshold" or "equallysegmented" or "hard2easy" or "all"
        self.schedule             = str   "log" or "linear" or "exp"
        self.subdomains           = int   --curriculum steps
        self.alpha                = float --parameter alpha for label propagation
        self.k                    = int   --parameter k for label propagation
        self.init                 = float --init threshold
        self.src_w                = float --parameter for source loss
        self.tar_w                = float --parameter for target loss
        self.mft_w                = float --parameter for mixup feature loss
        self.epoch                = int   --parameter for training epoch in each easy target adaptation
        build_default_parameter:  build network, batchsize, and optimizer according dataset
    """

    def __init__(self, src = "amazon", tar = "webcam", pseudo_label_method = "lp", strategy = "easy2hard",schedule = None, subdomains = 5, experiment = "", epoch = 10, alpha = 0.5, src_w = 1, tar_w = 1, init = 0.1, mft_w = 0.1,k=50,record=True,device="cuda:0"):

        ## parameter
        self.experiment = experiment
        self.pseudo_label_method = pseudo_label_method
        self.strategy = strategy
        self.subdomains = subdomains
        self.k = k
        self.alpha = alpha
        self.src = src
        self.tar = tar
        self.src_w = src_w
        self.tar_w = tar_w
        self.mft_w = mft_w
        self.init = init
        self.device = device
        self.record = record
        self.epoch = epoch
        self.schedule = schedule
        ## record pseudo label
        self.weight = None
        self.pseudo_label = None
        self.src_feature = None
        self.tar_feature = None
        self.seg_point = None
        self.globalstep = 0

        ## data
        self.src_dataloader = None
        self.tar_dataloader = None
        self.src_dataloader_shuffle = None
        self.tar_dataloader_shuffle = None
        self.src_dataset = load_dataset(src)
        self.tar_dataset = load_dataset(tar)
        self.easy_tar_dataset = load_dataset(tar)
        ## model
        self.build_default_parameter()
        self.load_dataloader()

        ## log files
        self.writer = SummaryWriter("../log/"+experiment)
        fp = open('experiments.txt', "r")
        for i in iter(fp):
            if not os.path.exists('../exp/exp_fig/'+i+'/'):
                os.makedirs('../exp/exp_fig/'+i+'/')
        fp.close()
        ## loss
        self.ce = torch.nn.NLLLoss()
        self.ce_noreduce = torch.nn.NLLLoss(reduce=False)
        self.ent = HLoss()

    def train(self):
        easy_tar_dataloader = None
        if self.pseudo_label_method == "entropy":
            self.pretrain()
            self.sub_da_testing()
            return

        for self.sub in range(self.subdomains):
            self.update_pseudo_label()
            draw_feature_distribution(self)
#             self.draw_lp_vs_k()
            if self.record:
                draw_pseudolabel_correctness_distribution(self)
                draw_pl_confidience_feature_distribution_partialclass(self)
                draw_pl_confidience_feature_distribution(self)
                draw_pl_portions_class_distribution(self)
            easy_tar_dataloader = self.easy_target_selection()
            if easy_tar_dataloader == None:
                return
            if self.record:
                draw_weight_segmented_distribution(self)

            for e in range(self.epoch):
                self.easy_target_adaptation(easy_tar_dataloader,self.src_w,self.tar_w,self.mft_w)
                self.adapted_network_testing()
            if self.record:
                draw_prediction_entropy_portions_accuracy(self)
                draw_prediction_portions_accuracy(self)
                draw_prediction_confusion(self)

    def build_default_parameter(self):
        if self.src_dataset.domain=="office31":
            self.batchsize = 25
            lr=0.001
            momentum=0.9
            l2_decay=5e-4
            self.net = build_network("office_ResNet50").to(self.device)
            self.optimizer = torch.optim.SGD([
            {'params': self.net.backbone.parameters(), 'lr': lr},
            {'params': self.net.classifier1.parameters(), 'lr': lr*10}
            ], momentum=momentum, weight_decay=l2_decay)
        elif self.src_dataset.domain=="visda":
            self.batchsize = 50
            lr=0.001
            momentum=0.9
            l2_decay=5e-4
            self.net = build_network("visda_ResNet50")
            self.optimizer = torch.optim.SGD([
            {'params': self.net.backbone.parameters(), 'lr': lr},
            {'params': self.net.classifier1.parameters(), 'lr': lr*10}
            ], momentum=momentum, weight_decay=l2_decay)

    def load_dataloader(self):
        self.src_dataloader_shuffle = self.src_dataset.loaders(batch_size=self.batchsize, num_workers=8)
        self.tar_dataloader_shuffle = self.tar_dataset.loaders(batch_size=self.batchsize, num_workers=8)
        self.src_dataloader = self.src_dataset.loaders(batch_size=self.batchsize, shuffle=False, num_workers=8)
        self.tar_dataloader = self.tar_dataset.loaders(batch_size=self.batchsize, shuffle=False, num_workers=8)

    def update_pseudo_label(self):
        if self.pseudo_label_method == "label propagation":
            self.weight, self.pseudo_label ,self.src_feature, self.tar_feature = self._plabels_label_propagation(alpha = self.alpha, k = self.k, max_iter = 20,layer = 0)
        elif self.pseudo_label_method == "nearest center":
            self.weight, self.pseudo_label, self.src_feature, self.tar_feature = self._plabels_nearest_center()
        elif self.pseudo_label_method == "NN prediction":
            self.weight, self.pseudo_label, self.src_feature, self.tar_feature = self._plabels_NN_prediction()
        self.pl_correct_idx = np.int64(self.tar_dataset.train_set.trainy == self.pseudo_label)
        self.writer.add_scalar('pseudo label acc', 100.*self.pl_correct_idx.mean(), self.sub)
        print("pseudo label accuracy",100.*self.pl_correct_idx.mean())
        return 100.*self.pl_correct_idx.mean()

    def easy_target_selection(self):
        if self.sub!=0: ### record label propagation imformation before after adaptation
            hardcorrect_idx = np.int64(self.tar_dataset.train_set.trainy[self.hard_index] == self.pseudo_label[self.hard_index])
            wrong = np.where((hardcorrect_idx-self.hardcorrect_idx)<0)[0].shape[0]
            correct = np.where((hardcorrect_idx-self.hardcorrect_idx)>0)[0].shape[0]
            easycertainty = self.weight[self.easy_index]
            self.writer.add_scalar('wrong hard pseudo label acc after', wrong, self.sub)
            self.writer.add_scalar('correct hard pseudo label acc after', correct, self.sub)
            self.writer.add_scalar('easy certainty after', 100.*easycertainty.mean(), self.sub)

        portions = []

        if self.strategy=="equallysegmented":
            for i in range(self.subdomains):
                portions.append(np.argsort(-self.weight)[int(self.weight.shape[0]*((i)/self.subdomains)):int(self.weight.shape[0]*((i+1)/self.subdomains))].reshape(-1))
            easy_index = np.argsort(-self.weight)[:int(self.weight.shape[0]*((self.sub+1)/self.subdomains))].reshape(-1)
            c = []
            for i in range(self.subdomains-1):
                c.append(-np.sort(-self.weight)[int(self.weight.shape[0]*((i+1)/self.subdomains))].reshape(-1))
            self.easy_index = easy_index
            self.hard_index = np.argsort(-self.weight)[int(self.weight.shape[0]*((self.sub+1)/self.subdomains)):].reshape(-1)
            self.seg_point = np.array(c)
        elif self.strategy == "decrementedthreshold":
            if "log" == self.schedule:
                t = 1-((1-math.exp((-self.sub/self.subdomains)*5))*(self.init)+(1-self.init))
            elif "linear" == self.schedule:
                t = 1-(self.sub/self.subdomains*(self.init)+(1-self.init))
            elif "exp" == self.schedule:
                t = 1-(math.exp((self.sub/self.subdomains-1)*5)*(self.init)+(1-self.init))
            print(t)
            easy_index = np.where(self.weight>t)
            self.easy_index = easy_index
            self.hard_index = np.where(self.weight<t)
            self.seg_point = np.array([t])
        elif self.strategy=="hard2easy":
            for i in range(self.subdomains):
                portions.append(np.argsort(self.weight)[int(self.weight.shape[0]*((i)/self.subdomains)):int(self.weight.shape[0]*((i+1)/self.subdomains))].reshape(-1))
            easy_index = np.argsort(self.weight)[:int(self.weight.shape[0]*((self.sub+1)/self.subdomains))].reshape(-1)
            c = []
            for i in range(self.subdomains-1):
                c.append(np.sort(self.weight)[int(self.weight.shape[0]*((i+1)/self.subdomains))].reshape(-1))
            self.seg_point = np.array(c)
        elif self.strategy == "all":
            easy_index = np.argsort(-self.weight)[:].reshape(-1)
            self.easy_index = easy_index
            self.hard_index = easy_index
            self.seg_point = np.array([0])
        self.easy_tar_dataset.train_set.conf = self.weight[easy_index]
        self.easy_tar_dataset.train_set.ps = self.pseudo_label[easy_index]
        self.easy_tar_dataset.train_set.img = self.tar_dataset.train_set.img[easy_index]
        self.easy_tar_dataset.train_set.trainy = self.tar_dataset.train_set.trainy[easy_index]
        try:
            easy_tar_dataloader = self.easy_tar_dataset.loaders(batch_size=self.batchsize, num_workers=0)
        except:
            return None
        easycorrect_idx = np.int64(self.tar_dataset.train_set.trainy[easy_index] == self.pseudo_label[easy_index])
        self.writer.add_scalar('easy pseudo label acc', 100.*easycorrect_idx.mean(), self.sub)
        print("easy pseudo label accuracy",100.*easycorrect_idx.mean(),"numbers ",easycorrect_idx.shape)
        self.hardcorrect_idx = np.int64(self.tar_dataset.train_set.trainy[self.hard_index] == self.pseudo_label[self.hard_index])
        easycertainty = self.weight[self.easy_index]
        self.writer.add_scalar('hard pseudo label acc before', 100.*self.hardcorrect_idx.sum(), self.sub)
        self.writer.add_scalar('easy certainty before', 100.*easycertainty.mean(), self.sub)

        return easy_tar_dataloader
    def pretrain(self):
#         self.net.load_state_dict(torch.load("../model/pretrain_"+self.src))
        iteration=int(len(self.tar_dataset)/self.batchsize)
        epoch=10
        lr=0.001
        momentum=0.9
        l2_decay=5e-4
        optimizer = torch.optim.SGD([
                {'params': self.net.classifier1.parameters(), 'lr': lr*10}
                ], momentum=momentum, weight_decay=l2_decay)
        self.net.eval()
        for name, p in self.net.named_parameters():
            if "backbone" in name:
                p.requires_grad = False
#             param.requires_grad = False
        for e in range(epoch):
            print(e)

            for src_data, src_label, _, _, _ in self.src_dataloader_shuffle:
                src_data, src_label = src_data.to(self.device), src_label.to(self.device)
                feature = self.net.backbone(src_data)
                src_logit = self.net.classifier1(feature)
                src_out_NLL = F.log_softmax(src_logit, 1)
                source_distance_loss=self.ce(src_out_NLL,src_label)
                source_distance_loss.backward()
                optimizer.step()
                optimizer.zero_grad()
        for name, p in self.net.named_parameters():
            if "backbone" in name:
                p.requires_grad = True

    def easy_target_adaptation(self,easy_tar_dataloader,src_w,tar_w,mft_w):
        iteration=int(len(self.tar_dataset)/self.batchsize)
        self.net.train()

        for i in range(1, iteration+1):
            try:
                src_data, src_label, _, _, _ = src_iter.next()
            except Exception as err:
                src_iter=iter(self.src_dataloader_shuffle)
                src_data, src_label, _, _, _  = src_iter.next()

            try:
                tar_data_easy, tar_label_easy, weight, _, plabel = tar_iter_easy.next()
            except Exception as err:
                tar_iter_easy = iter(easy_tar_dataloader)
                tar_data_easy, tar_label_easy, weight, _, plabel = tar_iter_easy.next()

            try:
                tar_data, tar_label, _, _, _ = tar_iter.next()
            except Exception as err:
                tar_iter=iter(self.tar_dataloader_shuffle)
                tar_data, tar_label, _, _, _ = tar_iter.next()

            src_data, src_label = src_data.to(self.device), src_label.to(self.device)
            tar_data,tar_label = tar_data.to(self.device),tar_label.to(self.device)
            tar_data_easy,tar_label_easy = tar_data_easy.to(self.device),tar_label_easy.to(self.device)
            weight = weight.to(self.device)
            plabel = plabel.to(self.device)

            ### source task specific loss
            feature1 = self.net.backbone(src_data)
            src_logit = self.net.classifier1(feature1)
            srcprob = F.softmax(src_logit,1)
            src_out_NLL = F.log_softmax(src_logit, 1)
            source_distance_loss=self.ce(src_out_NLL,src_label)

            ### weighted pseudo-labeling loss
            tar_feature_easy1 = self.net.backbone(tar_data_easy)
            tar_logit_easy = self.net.classifier1(tar_feature_easy1)
            tar_out_NLL = F.log_softmax(tar_logit_easy, 1)
            target_distance_loss = torch.sum(weight*self.ce_noreduce(tar_out_NLL,plabel))/(torch.sum(weight)+0.00001)

            ### mixup-feature training loss
            tar_feature1 = self.net.backbone(tar_data)
            sn,sd = feature1.shape
            tn,td = tar_feature1.shape
            mix = mixup( torch.cat((src_data[:int(sn/2)],tar_data[:int(tn/2)]),0), torch.cat((feature1[:int(sn/2)],tar_feature1[:int(tn/2)]),0).detach(),self.net.backbone,0.1)

            ### total loss
            loss_total = 0.5 * source_distance_loss + target_distance_loss + mft_w*mix

            loss_total.backward()
            self.optimizer.step()
            self.optimizer.zero_grad()

            if self.globalstep % 20 == 0:
                self.writer.add_scalar('source_distance_loss', source_distance_loss.item(), self.globalstep)
                self.writer.add_scalar('target_distance_loss', target_distance_loss.item(),  self.globalstep)
                self.writer.add_scalar('mix', mix.item(),  self.globalstep)
                self.writer.add_scalar('total_loss', loss_total.item(),  self.globalstep)
            self.globalstep+=1

    def adapted_network_testing(self):
        self.net.eval()
        src_dataset_len = len(self.src_dataset)
        src_correct=0
        with torch.no_grad():
            for i,(src_data, src_label,_,_,_) in enumerate(self.src_dataloader):
                src_data, src_label = src_data.to(self.device), src_label.to(self.device)
                feature = self.net.backbone(src_data)
                logit = self.net.classifier1(feature)
                prob = F.softmax(logit, 1)
                pred = prob.data.max(1)[1] # get the index of the max log-probability
                src_correct += pred.eq(src_label.data.view_as(pred)).cpu().sum()
        print("src acc",100. * src_correct/src_dataset_len)

        test_loss = 0
        tar_correct = 0
        tar_dataset_len = len(self.tar_dataset)
        correct_idx,preds,ent_weight = [],[],[]
        with torch.no_grad():
            for tar_data, tar_label,_,_,_ in self.tar_dataloader:
                tar_data, tar_label = tar_data.to(self.device), tar_label.to(self.device)
                feature = self.net.backbone(tar_data)
                logit = self.net.classifier1(feature)
                test_loss += F.nll_loss(F.log_softmax(logit, dim = 1), tar_label, reduction='sum').item()
                prob = F.softmax(logit, 1)
                pred = prob.data.max(1)[1] # get the index of the max log-probability
                ent_weight.append(self.ent(logit).data.cpu())
                correct_idx.append(pred.eq(tar_label.data.view_as(pred)).data.cpu())
                tar_correct += pred.eq(tar_label.data.view_as(pred)).cpu().sum()
                preds.append(pred.data.cpu())
        self.ent_weight = np.asarray(torch.cat(ent_weight).numpy())
        self.prediction_correct_idx = np.asarray(torch.cat(correct_idx).numpy())
        self.prediction = np.asarray(torch.cat(preds).numpy())
        print(self.tar+" Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)"
              .format(test_loss/tar_dataset_len, tar_correct, tar_dataset_len,100. * tar_correct / tar_dataset_len))
        self.writer.add_scalar('acc_'+str(self.src)+"_to_"+str(self.tar), 100. * tar_correct / tar_dataset_len, self.globalstep)

    def _plabels_NN_prediction(self):
        self.net.eval()
        embeddings_src,embeddings_tar,preds,ent_weight=[],[],[],[]
        with torch.no_grad():
            for src_input, _, _, _, _ in self.src_dataloader:
                src_input = src_input.to(self.device)
                feature = self.net.backbone(src_input)
                embeddings_src.append(feature.data.cpu())
        with torch.no_grad():
            for tar_data, tar_label,_,_,_ in self.tar_dataloader:
                tar_data, tar_label = tar_data.to(self.device), tar_label.to(self.device)
                feature = self.net.backbone(tar_data)
                embeddings_tar.append(feature.data.cpu())
                logit = self.net.classifier1(feature)
                prob = F.softmax(logit, 1)
                pred = prob.data.max(1)[1] # get the index of the max log-probability
                ent_weight.append((1-self.ent(logit)).data.cpu())
                preds.append(pred.data.cpu())
        pss = np.asarray(torch.cat(preds).numpy())
        coss = np.asarray(torch.cat(ent_weight).numpy())

        coss = (coss-coss.min()) /(coss.max()-coss.min())

        src_X = np.asarray(torch.cat(embeddings_src).numpy())
        tar_X = np.asarray(torch.cat(embeddings_tar).numpy())

        return coss,pss,src_X,tar_X

    def _plabels_nearest_center(self):
        c = self._init_center(self.src_dataloader,self.src_dataset.n_classes)
        self.net.eval()
        embeddings_src,embeddings_tar,pss,coss=[],[],[],[]
        for src_input, _, _, _, _ in self.src_dataloader:
            src_input = src_input.to(self.device)
            feature = self.net.backbone(src_input)
            embeddings_src.append(feature.data.cpu())
        for tar_input, _, _, _, _ in self.tar_dataloader:
            tar_input = tar_input.to(self.device)
            feature = self.net.backbone(tar_input)
            embeddings_tar.append(feature.data.cpu())
            cosine, cos_psuedo_label = torch.max(sim_matrix(feature,c),axis=1)
            coss.append(cosine.data.cpu())
            pss.append(cos_psuedo_label.data.cpu())

        pss = np.asarray(torch.cat(pss).numpy())
        coss = np.asarray(torch.cat(coss).numpy())
        coss = (coss-coss.min()) /(coss.max()-coss.min())

        src_X = np.asarray(torch.cat(embeddings_src).numpy())
        tar_X = np.asarray(torch.cat(embeddings_tar).numpy())

        return coss,pss,src_X,tar_X

    def _init_center(self,src_dataloader,classnum):
        self.net.eval()
        center = torch.zeros((classnum,self.net.rep_dim)).to(self.device)
        size = torch.zeros((classnum)).to(self.device)

        for step,(src_input, src_label,_,_,_) in enumerate(src_dataloader):
            src_input = src_input.to(self.device)
            src_label = src_label.to(self.device).reshape(-1)
            feature = self.net.backbone(src_input)
            size = size.scatter_add_(0, src_label, torch.ones_like(size).data)
            center = center.scatter_add_(0, src_label.unsqueeze(1).expand(feature.size()), feature.data)
        return center / size.reshape(-1, 1)
    def _plabels_label_propagation(self,alpha = 0.50, k = 50, max_iter = 100,layer=0):

        embeddings_src,embeddings_tar,labels_src=[],[],[]
        embeddings_tar_adv = []
        self.net.eval()
        with torch.no_grad():
            if layer==0:
                model = self.net.backbone
            else:
                model = torch.nn.Sequential(*list(self.net.backbone.children())[:-layer])
            for src_input, src_label,_,_,_ in tqdm(self.src_dataloader):
                src_input = src_input.to(self.device,non_blocking=True)
                src_label = src_label.to(self.device,non_blocking=True)
                feature = model(src_input).reshape(src_input.shape[0],-1)
                embeddings_src.append(feature.data.cpu())
                labels_src.append(src_label.data.cpu())
            for tar_input, _,_,_,_ in tqdm(self.tar_dataloader):
                tar_input = tar_input.to(self.device,non_blocking=True)
                feature = model(tar_input).reshape(tar_input.shape[0],-1)
                embeddings_tar.append(feature.data.cpu())

            src_X = np.asarray(torch.cat(embeddings_src).numpy())
            labels = np.asarray(torch.cat(labels_src).numpy())
            tar_X = np.asarray(torch.cat(embeddings_tar).numpy())

        embeddings_src_len = src_X.shape[0]
        embeddings_tar_len = tar_X.shape[0]
        X = np.concatenate((src_X,tar_X),axis=0)
        print('Updating pseudo-labels...')
        labeled_idx = np.asarray([a for a in range(embeddings_src_len)])
        unlabeled_idx = np.asarray([embeddings_src_len+a for a in range(embeddings_tar_len)])

        with torch.cuda.device(self.device):
            torch.cuda.empty_cache()
        # kNN search for the graph
        d = X.shape[1]
        res = faiss.StandardGpuResources()
        flat_config = faiss.GpuIndexFlatConfig()
        flat_config.device = int(self.device[-1])
        index = faiss.GpuIndexFlatIP(res,d,flat_config)   # build the index

        normalize_L2(X)
        index.add(X)
        N = X.shape[0]
        Nidx = index.ntotal

        c = time.time()
        D, I = index.search(X, k + 1)
        elapsed = time.time() - c
        print('kNN Search done in %d seconds' % elapsed)

        # Create the graph
        D = D[:,1:] ** 3
        I = I[:,1:]
        row_idx = np.arange(N)
        row_idx_rep = np.tile(row_idx,(k,1)).T
        W = scipy.sparse.csr_matrix((D.flatten('F'), (row_idx_rep.flatten('F'), I.flatten('F'))), shape=(N, N))
        W = W + W.T

        # Normalize the graph
        W = W - scipy.sparse.diags(W.diagonal())
        S = W.sum(axis = 1)
        S[S==0] = 1
        D = np.array(1./ np.sqrt(S))
        D = scipy.sparse.diags(D.reshape(-1))
        Wn = D * W * D

        # Initiliaze the y vector for each class (eq 5 from the paper, normalized with the class size) and apply label propagation
        Z = np.zeros((N,self.src_dataset.n_classes))
        A = scipy.sparse.eye(Wn.shape[0]) - alpha * Wn

        for i in range(self.src_dataset.n_classes):
            cur_idx = labeled_idx[np.where(labels ==i)]
            y = np.zeros((N,))
            y[cur_idx] = 1.0 / cur_idx.shape[0]
            f, _ = scipy.sparse.linalg.cg(A, y, tol=1e-6, maxiter=max_iter)
            Z[:,i] = f

        # Handle numberical errors
        Z[Z < 0] = 0
        # Compute the weight for each instance based on the entropy (eq 11 from the paper)
        probs_l1 = F.normalize(torch.tensor(Z),1).numpy()
        probs_l1[probs_l1 <0] = 0
        sortprob = -np.sort(-probs_l1,axis=1)
        probsum = np.sum(probs_l1,axis=1)
        entropy = scipy.stats.entropy(probs_l1.T)
        entropy[np.where(probsum==0)]=np.log(self.src_dataset.n_classes)
        weights = 1 - entropy / np.log(self.src_dataset.n_classes)
        weights = weights / np.max(weights)
        p_labels = np.argmax(probs_l1,1)
        conf = weights[unlabeled_idx]
        conf = (conf-np.min(conf)) /(np.max(conf)-np.min(conf))
        pss = p_labels[unlabeled_idx]

        return conf,pss,src_X,tar_X


import re
import os
from operator import itemgetter # sort lists using items in list
import datetime

# universal parameters
from RJdef import base_paths
#base_paths = [r"Y:", r"W:\HP", r"T:\HP_BAK"]

tags = [r"C[0-9]{2}",
        r"COMIC[0-9]+[^\)\]0-9]*[0-9]+",
        r"コミティア[0-9]+",
        r"シンデレラメモリーズ[0-9]+",
        r"(ふた|ぷに)(けっと|ケット)\s*[0-9\.]+",
        r"サンクリ[0-9]+",
        r"CT[0-9]+",
        r"SC[0-9]+",
        r"とら祭り[0-9]+",
        r"例大祭[0-9]+",
        r"紅楼夢[0-9]+",
        r"(春|夏|秋|冬)コミ[0-9]+",
        r"大⑨州東方祭[0-9]+"
        ]
logfile = r"R:\reorder_C00.log"

# create regex representation
pttn = ""
i = 0
i_f = tags.__len__()
while i<i_f:
    pttn = pttn +"(" + tags[i] + r")"
    if i != (i_f-1):
        pttn += r"|"
    i+=1
else: # name the tag
    pttn = r"\s*[\(\[]\s*" + r"(?P<tag>" + pttn + r")" + r"\s*[\)\]]\s*"

# (1) directory: (C00) body
pttn_dir = pttn + r"(?P<body>.*)$"
# (2) file: (C00) body.ext   , (CG is optional & non-capturing)
pttn_file = pttn + r"(?P<body>.*)\.(?P<ext>\w+)$"

########## 1. get files or dirs matching the pattern ##########

ls_match = [] # a list of [oldname, newname]
for base_path in base_paths:
    for root, dirs, files in os.walk(base_path):
        for d in dirs:
            tmp_match = re.match(pttn_dir, d, flags=re.IGNORECASE)
            if tmp_match is not None:
                # original name
                tmp_old = os.path.join(root, d)
                # new name
                tmp_new = tmp_match.group('body') + "(" + \
                          tmp_match.group('tag') + ")"
                tmp_new = os.path.join(root, tmp_new)
                ls_match.append([tmp_old, tmp_new])
        for file in files:
            tmp_match = re.match(pttn_file, file, flags=re.IGNORECASE)
            if tmp_match is not None:
                # old name
                tmp_old = os.path.join(root, file)
                # new name
                tmp_new = tmp_match.group('body') + "(" + \
                          tmp_match.group('tag') + r")." + \
                          tmp_match.group('ext')
                tmp_new = os.path.join(root, tmp_new)
                ls_match.append([tmp_old, tmp_new])
else:
    # reversely sorted: deeper files first!
    ls_match.sort(key=itemgetter(0), reverse=True)
#     print("\nls_match = ")
#     print(ls_match)

################ 2. rename ###################

with open(logfile, mode="w", encoding="utf-8") as logf:
    for ls in ls_match:
        try:
            os.rename(ls[0], ls[1])
        except Exception as e:
            print(e)
            dtobj = datetime.datetime.now()
            dtstr = dtobj.strftime("%y%m%d-%H:%M:%S ")
            logf.write(dtstr + str(e) + "\n")
        else:
            print('renamed: ' + ls[0])
            print('      to ' + ls[1])

print(r'Job done!')


from argparse import ArgumentParser
from collections import namedtuple
from datetime import timedelta, datetime
from typing import Tuple, List
from warnings import simplefilter

import torch
import torch.multiprocessing as mp
from gym import make
from ignite.contrib.handlers import tensorboard_logger as tb_logger
from ignite.engine import Engine
from ignite.metrics import RunningAverage
from ptan.actions import EpsilonGreedyActionSelector
from ptan.agent import DQNAgent, TargetNet
from ptan.common.wrappers import wrap_dqn
from ptan.experience import ExperienceSourceFirstLast, ExperienceReplayBuffer
from ptan.ignite import EpisodeFPSHandler, EndOfEpisodeHandler, EpisodeEvents, PeriodicEvents, PeriodEvents
from torch import device
from torch.optim import Adam

from Chapter8.lib import common, dqn_model
from Chapter9.lib import common

BATCH_MUL = 4
EpisodeEnded = namedtuple("EpisodeEnded", field_names="reward steps epsilon")
NAME = "03_parallel"

def play_func(params, net, cuda, exp_queue):
    env = make(params.env_name)
    env = wrap_dqn(env)
    env.seed(common.SEED)
    device = torch.device("cuda" if cuda else "cpu")
    selector = EpsilonGreedyActionSelector(epsilon=params.epsilon_start)
    epsilon_tracker = common.EpsilonTracker(selector, params)
    agent = DQNAgent(net, selector, device=device)
    exp_source = ExperienceSourceFirstLast(env, agent, gamma=params.gamma)

    for frame_idx, exp in enumerate(exp_source):
        epsilon_tracker.frame(frame_idx / BATCH_MUL)
        exp_queue.put(exp)
        for reward, steps in exp_source.pop_rewards_steps():
            exp_queue.put(EpisodeEnded(reward, steps, selector.epsilon))

class BatchGenerator:
    def __init__(self, buffer: ExperienceReplayBuffer, exp_queue: mp.Queue, fps_handler: EpisodeFPSHandler,
                 initial: int, batch_size: int):
        self.buffer = buffer
        self.exp_queue = exp_queue
        self.fps_handler = fps_handler
        self.initial = initial
        self.batch_size = batch_size
        self._rewards_steps = []
        self.epsilon = None

    def pop_rewards_steps(self) -> List[Tuple[float, int]]:
        res = list(self._rewards_steps)
        self._rewards_steps.clear()
        return res

    def __iter__(self):
        while True:
            while self.exp_queue.qsize() > 0:
                exp = self.exp_queue.get()
                if isinstance(exp, EpisodeEnded):
                    self._rewards_steps.append((exp.reward, exp.steps))
                    self.epsilon = exp.epsilon
                else:
                    self.buffer._add(exp)
                    self.fps_handler.step()
            if len(self.buffer) < self.initial:
                continue
            yield self.buffer.sample(self.batch_size * BATCH_MUL)

if __name__ == "__main__":
    simplefilter("ignore", category=UserWarning)

    mp.set_start_method("spawn")
    params = common.HYPERPARAMS["pong"]
    parser = ArgumentParser()
    parser.add_argument("--cuda", default=True, action="store_true", help="Enable cuda")
    args = parser.parse_args()
    device = device("cuda" if args.cuda else "cpu")

    env = make(params.env_name)
    env = wrap_dqn(env)

    net = dqn_model.DQN(env.observation_space.shape, env.action_space.n).to(device)
    tgt_net = TargetNet(net)

    buffer = ExperienceReplayBuffer(experience_source=None, buffer_size=params.replay_size)
    optimizer = Adam(net.parameters(), lr=params.learning_rate)

    # start subprocess and experience queue
    exp_queue = mp.Queue(maxsize=BATCH_MUL * 2)
    play_proc = mp.Process(target=play_func, args=(params, net, args.cuda, exp_queue))
    play_proc.start()

    fps_handler = EpisodeFPSHandler()
    batch_generator = BatchGenerator(buffer, exp_queue, fps_handler, params.replay_initial, params.batch_size)

    def process_batch(engine, batch):
        optimizer.zero_grad()
        loss = common.calc_loss_dqn(batch, net, tgt_net.target_model, gamma=params.gamma, device=device)
        loss.backward()
        optimizer.step()
        if engine.state.iteration % params.target_net_sync == 0:
            tgt_net.sync()
        return {"loss": loss.item(), "epsilon": batch_generator.epsilon}

    engine = Engine(process_batch)
    EndOfEpisodeHandler(batch_generator, bound_avg_reward=17.0).attach(engine)
    fps_handler.attach(engine, manual_step=True)

    @engine.on(EpisodeEvents.EPISODE_COMPLETED)
    def episode_completed(trainer: Engine):
        print("Episode %d: reward=%s, steps=%s, speed=%.3f frames/s, elapsed=%s" % (
            trainer.state.episode, trainer.state.episode_reward,
            trainer.state.episode_steps, trainer.state.metrics.get('fps', 0),
            timedelta(seconds=trainer.state.metrics.get('time_passed', 0))))

    @engine.on(EpisodeEvents.BOUND_REWARD_REACHED)
    def game_solved(trainer: Engine):
        print("Game solved in %s, after %d episodes and %d iterations!" % (
            timedelta(seconds=trainer.state.metrics['time_passed']),
            trainer.state.episode, trainer.state.iteration))
        trainer.should_terminate = True

    now = datetime.now().isoformat(timespec="minutes").replace(":", "-")
    logdir = f"runs/{now}-{params.run_name}-{NAME}"
    tb = tb_logger.TensorboardLogger(log_dir=logdir)
    RunningAverage(output_transform=lambda v: v['loss']).attach(engine, "avg_loss")

    episode_handler = tb_logger.OutputHandler(tag="episodes", metric_names=['reward', 'steps', 'avg_reward'])
    tb.attach(engine, log_handler=episode_handler, event_name=EpisodeEvents.EPISODE_COMPLETED)

    # write to tensorboard every 100 iterations
    PeriodicEvents().attach(engine)
    handler = tb_logger.OutputHandler(tag="train", metric_names=['avg_loss', 'avg_fps'],
                                      output_transform=lambda a: a)
    tb.attach(engine, log_handler=handler, event_name=PeriodEvents.ITERS_100_COMPLETED)

    engine.run(batch_generator)
    play_proc.kill()
    play_proc.join()


from budgetportal import models
from import_export import resources
from import_export.fields import Field
from import_export.instance_loaders import ModelInstanceLoader
from import_export.widgets import ForeignKeyWidget
from .irm_preprocessor import preprocess
from tablib import Databook
import logging

logger = logging.getLogger(__name__)

BASE_HEADERS = [
    "Project ID",
    "Project No",
    "Project Name",
    "Province",
    "Department",
    "Sector",
    "Local Municipality",
    "District Municipality",
    "Latitude",
    "Longitude",
    "Project Status",
    "Project Start Date",
    "Estimated Construction Start Date",
    "Estimated Project Completion Date",
    "Contracted Construction End Date",
    "Estimated Construction End Date",
    "Professional Fees",
    "Construction Costs",
    "Variation Orders",
    "Total Project Cost",
    "Project Expenditure from Previous Financial Years (Professional Fees)",
    "Project Expenditure from Previous Financial Years (Construction Costs)",
    "Project Expenditure from Previous Financial Years (TOTAL)",
    "Main Budget Appropriation (Professional Fees)",
    "Adjustment Budget Appropriation (Professional Fees)",
    "Main Budget Appropriation (Construction Costs)",
    "Adjustment Budget Appropriation (Construction Costs)",
    "Main Budget Appropriation (TOTAL)",
    "Adjustment Budget Appropriation (TOTAL)",
    "Actual Expenditure Q1",
    "Actual Expenditure Q2",
    "Actual Expenditure Q3",
    "Actual Expenditure Q4",
    "Budget Programme",
    "Primary Funding Source",
    "Nature of Investment",
    "Funding Status",
]
EXTRA_IMPLEMENTOR_HEADER = "Other parties"
IMPLEMENTORS = ["Program Implementing Agent", "Principal Agent", "Main Contractor"]
IMPLEMENTOR_HEADERS = IMPLEMENTORS + [EXTRA_IMPLEMENTOR_HEADER]
OUTPUT_HEADERS = BASE_HEADERS + IMPLEMENTOR_HEADERS

STATUS_ORDERING = [
    "Project Initiation",
    "Pre - Feasibility",
    "Feasibility",
    "Tender",
    "Design",
    "Site Handed - Over to Contractor",
    "Construction 1% - 25%",
    "Construction 26% - 50%",
    "Construction 51% - 75%",
    "Construction 76% - 99%",
    "Practical Completion (100%)",
    "Final Completion",
    "On Hold",
    "Terminated",
    "Other - Compensation of Employees",
    "Other - Packaged Ongoing Project",
]
status_order = {status: index for index, status in enumerate(STATUS_ORDERING)}

class InfraProjectSnapshotLoader(ModelInstanceLoader):
    def get_instance(self, row):
        """
        Gets an infrastructure project instance by IRM_project_id.
        """
        project_id = self.resource.fields["IRM_project_id"].clean(row)
        irm_snapshot_id = self.resource.fields["irm_snapshot"].clean(row)

        try:
            return models.InfraProjectSnapshot.objects.get(
                project=project_id, irm_snapshot=irm_snapshot_id
            )
        except models.InfraProjectSnapshot.DoesNotExist:
            pass

        return None

class InfraProjectForeignKeyWidget(ForeignKeyWidget):
    def get_queryset(self, value, row):
        project_id_qs = self.model.objects.filter(
            IRM_project_id=row["Project ID"], sphere_slug=row["sphere_slug"]
        )
        return project_id_qs

class InfraProjectSnapshotResource(resources.ModelResource):
    IRM_project_id = Field(
        attribute="project",
        # we use two fields for this lookup but it doesn't seem to run when I remove this
        column_name="Project ID",
        widget=InfraProjectForeignKeyWidget(models.InfraProject, "IRM_project_id"),
    )
    irm_snapshot = Field(
        attribute="irm_snapshot",
        column_name="irm_snapshot",
        widget=ForeignKeyWidget(models.IRMSnapshot),
    )
    sphere_slug = Field(column_name="sphere_slug",)
    project_number = Field(attribute="project_number", column_name="Project No")
    name = Field(attribute="name", column_name="Project Name")
    department = Field(attribute="department", column_name="Department")
    province = Field(attribute="province", column_name="Province")
    sector = Field(attribute="sector", column_name="Sector")
    local_municipality = Field(
        attribute="local_municipality", column_name="Local Municipality"
    )
    district_municipality = Field(
        attribute="district_municipality", column_name="District Municipality"
    )
    latitude = Field(attribute="latitude", column_name="Latitude")
    longitude = Field(attribute="longitude", column_name="Longitude")
    status = Field(attribute="status", column_name="Project Status")
    start_date = Field(attribute="start_date", column_name="Project Start Date")
    estimated_construction_start_date = Field(
        attribute="estimated_construction_start_date",
        column_name="Estimated Construction Start Date",
    )
    estimated_completion_date = Field(
        attribute="estimated_completion_date",
        column_name="Estimated Project Completion Date",
    )
    contracted_construction_end_date = Field(
        attribute="contracted_construction_end_date",
        column_name="Contracted Construction End Date",
    )
    estimated_constructtion_end_date = Field(
        attribute="estimated_construction_end_date",
        column_name="Estimated Construction End Date",
    )
    total_professional_fees = Field(
        attribute="total_professional_fees", column_name="Professional Fees"
    )
    total_construction_costs = Field(
        attribute="total_construction_costs", column_name="Construction Costs"
    )
    variation_orders = Field(
        attribute="variation_orders", column_name="Variation Orders"
    )
    estimated_total_project_cost = Field(
        attribute="estimated_total_project_cost", column_name="Total Project Cost"
    )
    expenditure_from_previous_years_professional_fees = Field(
        attribute="expenditure_from_previous_years_professional_fees",
        column_name="Project Expenditure from Previous Financial Years (Professional Fees)",
    )
    expenditure_from_previous_years_construction_costs = Field(
        attribute="expenditure_from_previous_years_construction_costs",
        column_name="Project Expenditure from Previous Financial Years (Construction Costs)",
    )
    expenditure_from_previous_years_total = Field(
        attribute="expenditure_from_previous_years_total",
        column_name="Project Expenditure from Previous Financial Years (TOTAL)",
    )
    project_expenditure_total = Field(
        attribute="project_expenditure_total", column_name="Project Expenditure (TOTAL)"
    )
    main_appropriation_professional_fees = Field(
        attribute="main_appropriation_professional_fees",
        column_name="Main Budget Appropriation (Professional Fees)",
    )
    adjusted_appropriation_professional_fees = Field(
        attribute="adjusted_appropriation_professional_fees",
        column_name="Adjustment Budget Appropriation (Professional Fees)",
    )
    main_appropriation_construction_costs = Field(
        attribute="main_appropriation_construction_costs",
        column_name="Main Budget Appropriation (Construction Costs)",
    )
    adjusted_appropriation_construction_costs = Field(
        attribute="adjusted_appropriation_construction_costs",
        column_name="Adjustment Budget Appropriation (Construction Costs)",
    )
    main_appropriation_total = Field(
        attribute="main_appropriation_total",
        column_name="Main Budget Appropriation (TOTAL)",
    )
    adjusted_appropriation_total = Field(
        attribute="adjusted_appropriation_total",
        column_name="Adjustment Budget Appropriation (TOTAL)",
    )
    actual_expenditure_q1 = Field(
        attribute="actual_expenditure_q1", column_name="Actual Expenditure Q1"
    )
    actual_expenditure_q2 = Field(
        attribute="actual_expenditure_q2", column_name="Actual Expenditure Q2"
    )
    actual_expenditure_q3 = Field(
        attribute="actual_expenditure_q3", column_name="Actual Expenditure Q3"
    )
    actual_expenditure_q4 = Field(
        attribute="actual_expenditure_q4", column_name="Actual Expenditure Q4"
    )
    budget_programme = Field(
        attribute="budget_programme", column_name="Budget Programme"
    )
    primary_funding_source = Field(
        attribute="primary_funding_source", column_name="Primary Funding Source"
    )
    nature_of_investment = Field(
        attribute="nature_of_investment", column_name="Nature of Investment"
    )
    funding_status = Field(attribute="funding_status", column_name="Funding Status")
    program_implementing_agent = Field(
        attribute="program_implementing_agent", column_name="Program Implementing Agent"
    )
    principle_agent = Field(attribute="principle_agent", column_name="Principal Agent")
    main_contractor = Field(attribute="main_contractor", column_name="Main Contractor")
    other_parties = Field(attribute="other_parties", column_name="Other parties")

    class Meta:
        model = models.InfraProjectSnapshot
        skip_unchanged = True
        report_skipped = False
        exclude = ("id",)
        import_id_fields = (
            "IRM_project_id",
            "sphere_slug",
        )
        instance_loader_class = InfraProjectSnapshotLoader

def import_snapshot(snapshot):
    file = snapshot.file.read()
    data_book = Databook().load(file, "xlsx")
    dataset = data_book.sheets()[0]
    preprocessed_dataset = preprocess(dataset)

    # Ensure projects exist
    for IRM_project_id in preprocessed_dataset["Project ID"]:
        if IRM_project_id:
            models.InfraProject.objects.get_or_create(
                IRM_project_id=IRM_project_id, sphere_slug=snapshot.sphere.slug
            )

    if len(preprocessed_dataset) > 0:
        preprocessed_dataset.append_col(
            [snapshot.id] * len(preprocessed_dataset), header="irm_snapshot"
        )
        preprocessed_dataset.append_col(
            [snapshot.sphere.slug] * len(preprocessed_dataset), header="sphere_slug"
        )
    resource = InfraProjectSnapshotResource()
    result = resource.import_data(preprocessed_dataset)
    return result


from django.contrib import admin

from book_shelf.models import (
    Author,
    Book,
    Genre,
    Language,
    Publisher,
    BookFormat
)

@admin.register(Book)
class BookAdmin(admin.ModelAdmin):
    list_display = ('title', 'get_author', 'publish_date', 'lend_type', 'book_format')

    class Media:
        css = {
            'all': ('resize-widget.css',),
        }

    def get_author(self, obj):
        return obj.author.full_name

    get_author.short_description = 'Author'
    get_author.admin_order_field = 'title'

    def lend_type(self, obj):
        return obj.lend_period.name

    lend_type.short_description = 'Borrow type'
    lend_type.admin_order_field = 'lend_period__name'

@admin.register(Genre)
class GenreAdmin(admin.ModelAdmin):
    class Media:
        css = {
            'all': ('resize-widget.css',),
        }

@admin.register(BookFormat)
class BookFormatAdmin(admin.ModelAdmin):
    class Media:
        css = {
            'all': ('resize-widget.css',),
        }

@admin.register(Language)
class LanguageAdmin(admin.ModelAdmin):
    class Media:
        css = {
            'all': ('resize-widget.css',),
        }

@admin.register(Author)
class AuthorAdmin(admin.ModelAdmin):
    list_display = ('get_author', 'get_books')

    def get_author(self, obj):
        return obj.full_name

    get_author.short_description = 'Author'
    get_author.admin_order_field = 'name'

    def get_books(self, obj):
        last_three = obj.book_set.all()[:3]
        return list(last_three)

    get_books.short_description = 'Books'

    css = {
        'all': ('resize-widget.css',),
    }

@admin.register(Publisher)
class PublisherAdmin(admin.ModelAdmin):
    class Media:
        css = {
            'all': ('resize-widget.css',),
        }


from argparse import ArgumentParser
import numpy as np
from sys import exit
import scipy
import scipy.stats
import itertools
from math import sqrt
from scipy.optimize import minimize
import json
import pickle
import os
import psycopg2
import pandas.io.sql as sqlio
import datetime

get_local_path = lambda s: os.path.join(os.path.dirname(os.path.realpath(__file__)), s)

def connect():
    connection = None
    try:
        # connection = psycopg2.connect( host = "127.0.0.1", database="WeBuildAi_development")
        connection = psycopg2.connect(database="WeBuildAi_development")
        print("Connection Successful")
    except Exception:
        print("Connection ERROR")
        exit(0)

    return connection

def get_possible_feature_values(connection):
    cursor = connection.cursor()
    query = "SELECT * from categorical_data_options"
    df = sqlio.read_sql_query(query, connection)

    possible_values = {}

    data_range_id_values = df['data_range_id'].values
    option_values = df['option_value'].values

    for i in range(len(option_values)):
        try:
            possible_values[int(data_range_id_values[i])].append(option_values[i])
        except KeyError as e:
            possible_values[int(data_range_id_values[i])] = [option_values[i]]

    return possible_values

def convert_categorical_features(feature):
    value = feature['feat_value']
    possible_values = feature['possible_values']

    if (("High" in possible_values) and ("Med" in possible_values) and ("Low" in possible_values)):
        if value == "High":
            mod_value = 1.0
        elif value == "Med":
            mod_value = 0.5
        elif value == "Low":
            mod_value = 0.0

        return mod_value

    elif (("Yes" in possible_values) and ("No" in possible_values)):
        if value == "Yes":
            mod_value = 1.0
        elif value == "No":
            mod_value = 0.0

        return mod_value

    else:
        arr = [0.0] * len(possible_values)
        arr[possible_values.index(value)] = 1.0

    return arr

def scale(feature, is_scale):
    '''
	:param feature: JSON object for the feature
	:param is_scale: Whether to scale OR not.
	:return: scaled OR unscaled value of the feature
	'''
    if (feature['feat_type'] == 'categorical'):
        mod_feature = convert_categorical_features(feature)
        return mod_feature
    else:
        value = float(feature['feat_value'])
        if (is_scale == False):
            mod_value = float(value)
        elif ((is_scale == True) and (feature['feat_type'] == 'continuous')):
            mod_value = float(value)
            mod_value = (value - float(feature['feat_min'])) / (float(feature['feat_max']) - float(feature['feat_min']))
        else:
            print("Not Implemented Yet- Scale")
            exit(0)

    return mod_value

def get_feature_info(connection):
    query = '''
    select a.id,name,description, category, b.is_categorical, b.lower_bound, b.upper_bound FROM features a, data_ranges b WHERE a.id=b.id;
    '''

    df = sqlio.read_sql_query(query, connection)

    feature_ids = df['id'].values
    feature_names = df['name'].values
    feature_desc = df['description'].values
    feature_categ = df['category'].values
    feature_type = df['is_categorical'].values
    feature_lb = df['lower_bound'].values
    feature_ub = df['upper_bound'].values

    feature_info = {}
    for i in range(len(feature_ids)):
        feature_info[feature_ids[i]] = [feature_names[i], feature_desc[i], feature_categ[i], feature_type[i], feature_lb[i], feature_ub[i]]

    return feature_info

def get_candidates(connection, pid, category, feature_info, possible_values, is_scale):
    cursor = connection.cursor()
    query = """
    SELECT id, participant_id, features FROM individual_scenarios WHERE participant_id=%s AND category='%s';
    """%(str(pid), str(category))

    all_samples = []
    candidate_ids = []
    df = sqlio.read_sql_query(query, connection)

    imp_features = set()

    scenarios = df['features'].values
    scenario_ids = df['id'].values

    for i in range(len(scenario_ids)):
        candidate_ids.append(scenario_ids[i])
        scenario = scenarios[i]
        feature_arr = []
        for f_key in scenario.keys():
            feature_obj = {}
            feature_id = int(f_key)
            imp_features.add(feature_id)
            feature_value = scenario[f_key]
            feature_name = feature_info[feature_id][0]
            feature_category = feature_info[feature_id][2]
            feature_type = feature_info[feature_id][3]
            if(feature_type == True):
                feature_type = "categorical"
            elif(feature_type == False):
                feature_type = "continuous"

            feature_min = feature_info[feature_id][4]
            feature_max = feature_info[feature_id][5]

            feature_obj['feat_id'] = feature_id
            feature_obj['feat_name'] = feature_name
            feature_obj['feat_category'] = feature_category
            feature_obj['feat_value'] = feature_value
            feature_obj['feat_type'] = feature_type
            if(feature_type == "categorical"):
                f_poss_values = possible_values[feature_id]
            else:
                f_poss_values = []
            feature_obj['possible_values'] = f_poss_values
            feature_obj['feat_min'] = feature_min
            feature_obj['feat_max'] = feature_max

            feature_arr.append(feature_obj)

        feature_arr = sorted(feature_arr, key=lambda elem: elem["feat_id"])
        num_features = len(feature_arr)

        array_f = []  # Actual Values
        for f1 in feature_arr:
            modified_features = scale(f1, is_scale)

            if (type(modified_features) is list):
                for mf in modified_features:
                    array_f.append(mf)
            else:
                array_f.append(modified_features)

        all_samples.append(array_f)

    return all_samples, imp_features, candidate_ids

def score_model(args):
    connection = connect()
    pid = args.pid
    fid = args.fid
    pairwise_type = args.type
    data_type = args.d
    normalize = args.n
    k = args.k
    loss_fun = args.l
    num_iters = args.num
    size_type = args.sz
    test_frac = args.tf

    #Load Model
    model = pickle.load(
        open(get_local_path('./RESULT/betas/Participant_' + str(pid) + '_' + str(pairwise_type) + '_BETA_Round' + str(fid) + '.pkl'),
             'rb'))
    #Score using this model
    possible_values = get_possible_feature_values(connection)
    feature_info = get_feature_info(connection)
    candidates, imp_features, candidate_ids = get_candidates(connection, pid, pairwise_type, feature_info, possible_values, True)
    #candidates, imp_features = get_candidates(pid, fid, connection)

    lambda_reg = 1
    d = len(imp_features)

    feat_trans = set([])
    feat_trans_dup = list(itertools.product(range(d + 1), repeat=k))
    for t in feat_trans_dup:
        feat_trans.add(tuple(sorted(t)))

    feat_trans = list(feat_trans)
    feat_trans.remove(tuple([d] * k))

    N = 1
    d_ext = len(feat_trans)
    learnt_beta = np.zeros((N, d_ext))

    score_arr = []
    for i in range(len(candidates)):
        altA = candidates[i]
        altA = list(altA)
        altA.append(1)
        k_altA = []
        for t in feat_trans:
            this_prod = 1
            for index in t:
                this_prod *= altA[index]
            k_altA.append(this_prod)

        k_altA = np.array(k_altA)
        score = np.dot(model, k_altA)
        score_arr.append(score)

    return score_arr, candidate_ids

def insert_score_arr(connection, candidate_ids, score_arr, participant_id, round):
    cursor = connection.cursor()

    # Get ranklist id i.e.
    # Select id from ranklists where participant_id = 18 and round=1;
    # Hopefully, get the most recent ranklist
    # Insert into that ranklist id based ranklist_element

    query = """
    SELECT id from ranklists WHERE participant_id=%s AND round=%s
    ORDER BY created_at DESC
    """%(str(participant_id), str(round))

    df = sqlio.read_sql_query(query, connection)
    ranklist_id = df['id'].values[0]

    ranks = np.argsort(-np.array(score_arr))
    insert_query = """
        INSERT INTO ranklist_element(ranklist_id, individual_scenario_id, model_rank, human_rank, created_at, updated_at)
        VALUES(%s, %s, %s, %s, %s, %s)
    """
    sql_args = []
    for i in range(len(candidate_ids)):
        sql_args.append([ranklist_id, candidate_ids[i], ranks[i]+1.0, 0, datetime.datetime.now(), datetime.datetime.now()])

    print(sql_args)
    cursor.executemany(insert_query, sql_args)
    cursor.close()
    connection.commit()

if __name__ == '__main__':
    parser = ArgumentParser(description="Inputs.")
    parser.add_argument('-pid', type=str, default='None', help='participant_id')
    parser.add_argument('-fid', type=str, default=0, help='feedback round number')
    parser.add_argument('-type', type=str, default='driver', help='Running model for social/individual preferences')
    parser.add_argument('-d', type=str, default='D', help='data type (str)')
    parser.add_argument('-n', type=int, default=1, help='normalize features (int)')
    parser.add_argument('-k', type=int, default=1, help='degree of polynomial (int)')
    parser.add_argument('-l', type=str, default='normal', help='loss function(str')
    parser.add_argument('-num', type=int, default=100, help='value of num iters>0 (int)')
    parser.add_argument('-sz', type=str, default='cardinalsizes', help='size type (str)')
    parser.add_argument('-tf', type=float, default=0.5, help='test frac (float)')
    args = parser.parse_args()

    score_arr, candidate_ids = score_model(args)
    connection = connect()
    insert_score_arr(connection, candidate_ids, score_arr, args.pid, args.fid)


import os
import matplotlib.pyplot as plt
import numpy as np
import cv2

Data_DIR = "./PetImages/"
Categories = []

for filename in os.listdir(Data_DIR):
    Categories.append(filename)
    category_path = os.path.join(Data_DIR, filename)
    for img_name in os.listdir(category_path):
        path = os.path.join(category_path, img_name)
        print(path)
        img = cv2.imread(path)
        cv2.imshow("data", img)
        cv2.waitKey(0)


'''
x1,y1=eval(input("请输入两个点坐标"))
print(x1,y1)
'''

import turtle
x1,y1=eval(input("请输入x1,y1"))
x2,y2=eval(input("请输入x1,y1"))
print(x1,y1)
print(x2,y2)

turtle.penup()
turtle.goto(x1,y1)
turtle.write(str(x1)+","+str(y1))
turtle.pendown()
turtle.goto(x2,y2)
turtle.write(str(x2)+","+str(y2))
turtle.goto((x1+x2)/2,(y1+y2)/2)

length=((x1-x2)**2 +(y1-y2)**2)**0.5
turtle.write("长度="+str(length))
turtle.done()


import json
import os

import shakedown

PACKAGE_NAME = 'cassandra'
DEFAULT_TASK_COUNT = 3

WRITE_DATA_JOB_FILENAME = 'write-data.json.template'
VERIFY_DATA_JOB_FILENAME = 'verify-data.json.template'
DELETE_DATA_JOB_FILENAME = 'delete-data.json.template'
VERIFY_DELETION_JOB_FILENAME = 'verify-deletion.json.template'

def get_job_dict(
        job_filename,
        node_address = os.getenv('CASSANDRA_NODE_ADDRESS', 'node-0-server.cassandra.autoip.dcos.thisdcos.directory'),
        node_port = os.getenv('CASSANDRA_NODE_PORT', '9042')):
    jobs_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'jobs')
    template_filename = os.path.join(jobs_dir, job_filename)
    with open(template_filename, 'r') as f:
        job_contents = f.read()
    job_contents = job_contents.replace('{{NODE_ADDRESS}}', node_address)
    job_contents = job_contents.replace('{{NODE_PORT}}', node_port)
    return json.loads(job_contents)

WRITE_DATA_JOB = get_job_dict(WRITE_DATA_JOB_FILENAME)
VERIFY_DATA_JOB = get_job_dict(VERIFY_DATA_JOB_FILENAME)
DELETE_DATA_JOB = get_job_dict(DELETE_DATA_JOB_FILENAME)
VERIFY_DELETION_JOB = get_job_dict(VERIFY_DELETION_JOB_FILENAME)


#! /usr/bin/python
# Python 2 diff of two lists.

# Author: Pete Jansz

from os import chdir
import collections, re, sys, string, os.path, io
from optparse import OptionParser

parser = OptionParser()

def parse_cli_args():
    parser.add_option('--f1', action='store', help='Input file 1', type=str, dest='f1', metavar='FILE')
    parser.add_option('--f2', action='store', help='Input file 2', type=str, dest='f2', metavar='FILE')
    parser.add_option('--difftype', action='store', help='1: f1-f2, 2: f2-f1, 3:xor(f1, f2)', type=int, dest='difftype')
    parser.add_option('--delim', action='store', help='Delimiter, default=\\t', type=str, dest='delim', default='\t')
    parser.add_option('--key', action='store', help='Primary key and to sort key: 0..tokens-1. Default=0', type=int, dest='key', default=0)
    return parser.parse_args()

def load_list( filename ):
    alist = []
    fin = open(filename, 'r')

    for line in fin.readlines():
        alist.append(line.strip().strip('+.'))

    fin.close()

    return alist

def load_dict( filename, options ):
    dictx = {}
    fin = open(filename, 'r')

    for line in fin.readlines():
        tokens = line.strip().strip('+.').split(options.delim)
        key = tokens[options.key].strip()
        dictx[key] = line.strip().strip('+.')

    fin.close()

    ordered_dict = collections.OrderedDict(sorted(dictx.items()))

    return ordered_dict

def diff_lists(list1, list2, choice):
    if choice == 1:
        return list(set(list1) - set(list2))
    elif choice == 2:
        return list(set(list2) - set(list1))
    elif choice == 3:
        return list(set(list1).symmetric_difference(set(list2)))

def mk_diff_key(key, left):
    diff_key = ''
    if left:
        diff_key = '< [' + key + ']'
    else:
        diff_key = '> [' + key + ']'
    return diff_key

def append_dict(d1, d2):
    for key in d2:
        d1[key] = d2[key]
    return d1

def diff_dicts(dict1, dict2, options):
    delta_dict = collections.OrderedDict()

    for key in dict1.keys():
        if key not in dict2:
            delta_dict[mk_diff_key(key, True)] = dict1[key]
        else:
            adict = diff_record(dict1[key], dict2[key], options)
            delta_dict = append_dict(delta_dict, adict)

    for key in dict2.keys():
        if key not in dict1:
            delta_dict[mk_diff_key(key, False)] = dict2[key]
        else:
            adict = diff_record(dict1[key], dict2[key], options)
            delta_dict = append_dict(delta_dict, adict)

    return delta_dict

def diff_record(s1, s2, options):
    """
        Tokenize s1, s2 and return a dictionary, identifying first tokens which differ
    """
    diff = {}
    s1_tokens = s1.split(options.delim)
    s2_tokens = s2.split(options.delim)

    if len(s1_tokens) == len(s2_tokens):
        if diff_lists(s1_tokens, s2_tokens, 3):
            for i in range(0, len(s1_tokens)-1):
                token1 = s1_tokens[i]
                token2 = s2_tokens[i]
                if token1 != token2:
                    key1 = mk_diff_key( token1, True )
                    diff[key1] = s1
                    key2 = mk_diff_key( token2, False )
                    diff[key2] = s2
                    break
    elif len(s1_tokens) > len(s2_tokens):
        for record in diff_lists(s1_tokens, s2_tokens, 1):
            key = mk_diff_key(record[options.key], True)
            diff[key] = record
    elif len(s1_tokens) < len(s2_tokens):
        for record in diff_lists(s1_tokens, s2_tokens, 2):
            key = mk_diff_key(record[options.key], False)
            diff[key] = record

    return diff

def main():
    exit_value = 1
    options, args = parse_cli_args()

    if not options.f1 or not options.f2 or not options.difftype or options.difftype > 3:
        parser.print_help()
        exit(exit_value)

    for f in options.f1, options.f2:
        if not os.path.exists( f ):
            print >>sys.stderr, 'File not found: ' + f
            exit(exit_value)

    # for item in diff_lists(load_list(options.f1), load_list(options.f2), options.difftype):
    #     print(item)
    dict1 = load_dict(options.f1, options)
    dict2 = load_dict(options.f2, options)
    diff_dict = diff_dicts(dict1, dict2, options)
    for key in diff_dict.keys():
        print(key + ' ' + diff_dict[key])

    exit_value = 0
    exit(exit_value)

if __name__ == "__main__":
    main()


from multiprocessing.queues import Queue
import sys
import subprocess as sp
import multiprocessing as mp
import os

ffmpeg = 'C:\\Program Files\\ffmpeg\\bin\\ffmpeg.exe'
nCPU = len(sys.argv[1:]) <= 4 and len(sys.argv[1:]) or 4

def ff(q):
    os.system('start cmd.exe')
    while True:
        try:
            file = q.get()
            # Раскладываем путь, имя и расширение файла по переменным
            fpath, fname = os.path.split(file)
            fname, fext = os.path.splitext(fname)

            # Назначаем выходной файл
            target = fpath + '\\ffmpeg\\' + fname + '.mpg'

            sp.call([ffmpeg, '-y', '-i', file, '-f', 'vob', '-bufsize', '240M', '-maxrate', '120M', '-q:v', '1', '-flags',
            '+ilme', '-top', '1', '-r', '25', '-ac', '2', '-sn', '-dn', target])
            q.task_done()
        except Queue.empty:
            return

if __name__ == "__main__":
    mp.freeze_support()
    # Создаем поддапку для скукоженных файлов
    try:
        os.mkdir(os.path.split(sys.argv[1])[0] + '\\ffmpeg')
    except OSError:
        print('Поддиректория уже есть')

    # Создаем очередь файлов
    q = mp.JoinableQueue()
    for item in sys.argv[1:]:
        q.put(item)

    # Стартуем воркеры
    for i in range(nCPU):
        p = mp.Process(target=ff, args=(q,))
        p.daemon = True
        p.start()

    q.join()
exit

"""
    nCPU=4
    l=len(sys.argv[1:])
    if l<=nCPU:
        nCPU=l
    else:
        for i in xrange(nCPU):
            a=i*l//nCPU+1
            b=(i+1)*l//nCPU
"""

"""
#Вызов медиаинфо для определения параметров видео
def finfo(f):
    duration, width, height = subprocess.check_output([mediainfo, param, f]).decode(errors="ignore").split(',')
    if int(height)>=720:
        if float(width)/int(height)>1.5:
            TVformat = 5
        else:
            TVformat = 1
    elif float(width)/int(height)>1.5:
        TVformat = 2
    else:
        TVformat = 1
    return TVformat, duration

#Функции для составления плейлиста
def cbi_head(f, TVf):
    f.write('<Cinegy><BatchIngestList><TargetFolder>{A7846D6C-2BAA-4B24-8955-298247D56B52}/{E4CF8CB2-CBBA-4B0E-A8CD-A22EC195EDFA}</TargetFolder><FromTape>0</FromTape><Silent>0</Silent><NoTapeDlg>0</NoTapeDlg><TV_Format>'+str(TVf)+'</TV_Format><CapturerID>Video File Ingestor : </CapturerID><ListName>%N</ListName>')
    return 1
def cbi_item(f, TVf, t, d):
    f.write('<CinegyItem><Name>%N</Name><Channel></Channel><Tagged>1</Tagged><TimeStart>0</TimeStart><TimeDuration>'+str(d)+'0000'+'</TimeDuration><IgnoreMediaIdDiff>1</IgnoreMediaIdDiff><TV_Format>'+str(TVf)+'</TV_Format><ProgramCode>%A</ProgramCode><Source>'+str(t)+'</Source><MediaID>1|0|-1|'+str(TVf)+'|'+str(t)+'|0||</MediaID></CinegyItem>')
    return 1
def cbi_tail(f):
    f.write('</BatchIngestList></Cinegy>')
    return 1

#Накидываем шапку плейлиста
CBI = open(target+'\\BatchList.cbi', 'w')
cbi_TVf, dur = finfo(sys.argv[1])
cbi_head(CBI, cbi_TVf)

#Добавляем итем в плейлист
cbi_TVf, dur = finfo(file)
cbi_item(CBI, cbi_TVf, target, dur)

#Дописываем и закрываем плелист
cbi_tail(CBI)
CBI.close()
"""


from xai.brain.wordbase.nouns._vortex import _VORTEX

#calss header
class _VORTICES(_VORTEX, ):
	def __init__(self,):
		_VORTEX.__init__(self)
		self.name = "VORTICES"
		self.specie = 'nouns'
		self.basic = "vortex"
		self.jsondata = {}


import numpy as np
import matplotlib.pyplot as pl

Demension = 7
Time = 3

def prox(W,l):
	(T,D) = W.shape
	TMP = np.zeros((T,D))
	for d in range(D):
		s = np.sqrt(np.sum(np.power(W[:,d],2)))
		if s == 0:
			W[:,d] = max([0,s-l])*W[:,d]
		else:
			W[:,d] = max([0,s-l])*W[:,d]/s
	return W

def grad(W,x_vec,y_vec,l2_reg):
	return W.dot(x_vec).dot(x_vec.T) \
	- y_vec.dot(x_vec.T) \
	+ l2_reg * W

def fobos(W,x_vec,y_vec,step_size,l2_reg,l2_1_reg):
	# grad step
	W = W - step_size * grad(W,x_vec,y_vec,l2_reg)
	# proximal operator step
	W = prox(W,l2_1_reg)
	return W

def exam(step,l2,l2_1):
	W = np.ones((Time,Demension))

	s = 0
	for line in open('sample.txt'):
		ll = line.split(',')
		data = [float(x) for x in ll]
		x_vec = np.matrix(data[0:7]).T
		y_vec = np.matrix(data[7:10]).T
		W = fobos(W,x_vec,y_vec,step,l2,l2_1)

	error=0
	itr=0

	for line in open('test.txt'):
		ll = line.split(',')
		data = [float(x) for x in ll]
		x_vec = np.matrix(data[0:7]).T
		y_vec = np.matrix(data[7:10]).T

		error = error + np.sum(np.power(y_vec-W.dot(x_vec),2))
		itr = itr + 1
	print(step,l2,l2_1,np.sqrt(error/itr/3)) # rmse
	print(W)

if __name__ == '__main__':

	exam(0.1,0.03,0.15)


from struct import unpack
import numpy as np
import cv2

class Bitmap:
    def __init__(self, filePath):
        self.fileHeader = self.BITMAPFILEHEADER(filePath)
        self.infoHeader = self.BITMAPINFOHEADER(filePath)

        self.height = self.infoHeader.biHeight
        self.width = self.infoHeader.biWidth
        bfOffBits = self.fileHeader.bfOffBits

        row = self.height
        column = self.width
        if (self.width * 3 % 4) != 0:
            self.placeholder = 4 - (self.width * 3 % 4)
        else:
            self.placeholder = 0
        self.img = np.ndarray((row, column), dtype=tuple)

        with open(filePath, 'rb') as bitmap:
            temp = bitmap.read(bfOffBits)
            for x in range(row):
                for y in range(column):
                    b = unpack('<B', bitmap.read(1))[0]
                    g = unpack('<B', bitmap.read(1))[0]
                    r = unpack('<B', bitmap.read(1))[0]
                    self.img[row - 1 - x][y] = (b, g, r)
                if (self.placeholder != 0):
                    temp = bitmap.read(self.placeholder)
        self.B, self.G, self.R = self.departBGR(self.img, row, column)

    def departBGR(self, img, row, column):
        B = np.zeros_like(img, dtype=np.uint8)
        G = np.zeros_like(img, dtype=np.uint8)
        R = np.zeros_like(img, dtype=np.uint8)
        for x in range(row):
            for y in range(column):
                BGR = img[x][y]
                B[x][y] = BGR[0]
                G[x][y] = BGR[1]
                R[x][y] = BGR[2]
        return B, G, R

    def getPixel(self):
        print('以左上角为原点，获得某像素点的RGB值:(R,G,B)')
        print('x为0至%d的整数，y为0至%d的整数' % (self.width - 1, self.height - 1))
        while True:
            x = int(input('请输入x:'))
            if x < 0 or x >= self.width:
                print('非法的x')
                continue
            y = int(input('请输入y:'))
            if y < 0 or y >= self.height:
                print('非法的y')
                continue
            print('像素点(%d,%d)的RGB值为(%d,%d,%d)' %
                  (x, y, self.R[x][y], self.G[x][y], self.B[x][y]))
            break

    def writeImg(self):
        b = self.B
        g = self.G
        r = self.R
        merged = cv2.merge([b, g, r])
        cv2.imwrite("output/fav.bmp", merged)

    def showImg(self):
        b = self.B
        g = self.G
        r = self.R
        merged = cv2.merge([b, g, r])
        cv2.imshow('any key to continue', merged)
        cv2.waitKey()
        cv2.destroyAllWindows()

    def printHeaderInfo(self):
        print('BITMAPFILEHEADER')
        print('bfType:', self.fileHeader.bfType)
        print('bfSize:', self.fileHeader.bfSize)
        print('bfReserved1:', self.fileHeader.bfReserved1)
        print('bfReserved2:', self.fileHeader.bfReserved2)
        print('bfOffBits:', self.fileHeader.bfOffBits)
        print('\nBITMAPINFOHEADER')
        print('biSize:', self.infoHeader.biSize)
        print('biWidth:', self.infoHeader.biWidth)
        print('biHeight:', self.infoHeader.biHeight)
        print('biPlanes:', self.infoHeader.biPlanes)
        print('biBitcount:', self.infoHeader.biBitcount)
        print('biCompression:', self.infoHeader.biCompression)
        print('biSizeImage:', self.infoHeader.biSizeImage)
        print('biXPelsPerMeter:', self.infoHeader.biXPelsPerMeter)
        print('biYPelsPerMeter:', self.infoHeader.biYPelsPerMeter)
        print('biClrUsed:', self.infoHeader.biClrUsed)
        print('biClrImportant:', self.infoHeader.biClrImportant)

    class BITMAPFILEHEADER:
        def __init__(self, filePath):
            with open(filePath, 'rb') as bitmap:
                #         bfType:2字节，说明文件类型，一般为19778，其转化为十六进制为0x4d42，对应的字符串为BM
                #         bfSize:4字节，文件大小，以字节为单位
                #         bfReserved1:2字节，保留，为0
                #         bfReserved1:2字节，保留，为0
                #         bfOffBits:4字节，从文件开始处到像素数据的偏移，也就是这两个结构体大小之和
                self.bfType = unpack('<h', bitmap.read(2))[0]
                self.bfSize = unpack('<i', bitmap.read(4))[0]
                self.bfReserved1 = unpack('<h', bitmap.read(2))[0]
                self.bfReserved2 = unpack('<h', bitmap.read(2))[0]
                self.bfOffBits = unpack('<i', bitmap.read(4))[0]

    class BITMAPINFOHEADER:
        def __init__(self, filePath):
            with open(filePath, 'rb') as bitmap:
                bitmap.read(14)
                #         bisize:4字节，此信息头大小
                #         biWidth:4字节，图像的宽
                #         biHeight:4字节，图像的高，正数代表位图为倒向，复数代表位图为正向，通常为正数
                #         biPlanes:2字节，图像的帧数，一般为1
                #         biBitcount:2字节，一像素所占的位数，一般为24
                #         biCompression:4字节，说明图像数据压缩类型，一般为0(不压缩)
                #         biSizeImage:4字节，像素数据所占大小，说明图像的大小，以字节为单位。当压缩类型为0时，总设置为0
                #         biXPelsPerMeter:4字节，水平分辨率，用像素/米表示，有符号整数
                #         biYPelsPerMeter:4字节，水平分辨率，用像素/米表示，有符号整数
                #         biClrUsed:4字节，说明位图实际使用的彩色表中的颜色索引数，若设为0则说明使用所有调色板项
                #         biClrImportant:4字节，说明对图像显示有重要影响的颜色索引的数目。若为0，表示都重要
                self.biSize = unpack('<i', bitmap.read(4))[0]
                self.biWidth = unpack('<i', bitmap.read(4))[0]
                self.biHeight = unpack('<i', bitmap.read(4))[0]
                self.biPlanes = unpack('<h', bitmap.read(2))[0]
                self.biBitcount = unpack('<h', bitmap.read(2))[0]
                self.biCompression = unpack('<i', bitmap.read(4))[0]
                self.biSizeImage = unpack('<i', bitmap.read(4))[0]
                self.biXPelsPerMeter = unpack('<i', bitmap.read(4))[0]
                self.biYPelsPerMeter = unpack('<i', bitmap.read(4))[0]
                self.biClrUsed = unpack('<i', bitmap.read(4))[0]
                self.biClrImportant = unpack('<i', bitmap.read(4))[0]

# 读bmp图像
filePath = 'bmp/fav.bmp'
bitmap = Bitmap(filePath)
# 打印文件头的信息头的信息
bitmap.printHeaderInfo()
# 展示该图像
bitmap.showImg()
# 将读到的bmp图像重新写入到output文件夹中
bitmap.writeImg()
# 获取某点的RGB值
bitmap.getPixel()

filePath = 'bmp/fav.bmp'

def slicing(filePath):
    img = cv2.imread(filePath)
    (y, x, alpha) = img.shape

    while True:
        chooseX = input('请输入选区宽度，不输入为全图%d像素:'%(x))
        if chooseX != '' and (int(chooseX) > x or int(chooseX) <= 0):
            print('输入非法！')
            continue
        chooseY = input('请输入选取高度，不输入为全图%d像素:'%(y))
        if chooseY != '' and (int(chooseY) > y or int(chooseY) <= 0):
            print('输入非法！')
            continue
        chooseM = int(input('请输入图块宽度:'))
        if chooseM > x or chooseM <= 0:
            print('输入非法！')
            continue
        chooseN = int(input('请输入图块高度:'))
        if chooseN > y or chooseN <= 0:
            print('输入非法！')
            continue
        break
    if chooseX != '':
        x = int(chooseX)
    if chooseY != '':
        y = int(chooseY)

    m = chooseM
    n = chooseN

    atomX = m
    atomY = n
    slicingY = int(y / atomY)
    slicingX = int(x / atomX)
    #     shuffleY = slicingY
    #     shuffleX = slicingX
    #     if y % atomY != 0:
    #         slicingY += 1
    #     if x % atomX != 0:
    #         slicingX += 1
    nowX = 0
    nowY = 0
    nextX = min(nowX + atomX, x)
    nextY = min(nowY + atomY, y)
    matrix = np.zeros((slicingY, slicingX), dtype=list)
    for i in range(slicingY):
        for j in range(slicingX):
            matrix[i][j] = img[nowY:nextY, nowX:nextX].copy()
            nowX = nowX + atomX
            nextX = min(nowX + atomX, x)
        nowY = nowY + atomY
        nextY = min(nextY + atomY, y)
        nowX = 0
        nextX = min(nowX + atomX, x)

#     cv2.imshow('slicing',matrix[6][9])
#     cv2.moveWindow('slicing',400,400)
#     cv2.waitKey()
#     cv2.destroyAllWindows()

#     shuffleMatrix = np.random.permutation(matrix[0:shuffleY, 0:shuffleX])
    preShuffleMatrix = matrix[0:slicingY, 0:slicingX].copy()
    shuffleMatrix = preShuffleMatrix.copy()
    sort = np.random.permutation(np.array(range(slicingY * slicingX)))
    sort = sort.reshape((slicingY, slicingX))
    for i in range(slicingY):
        for j in range(slicingX):
            index = sort[i][j]
            shuffleMatrix[i][j] = preShuffleMatrix[int(
                index / slicingX)][index % slicingX]

    shuffle = img.copy()
    for i in range(slicingY * atomY):
        for j in range(slicingX * atomX):
            for k in range(alpha):
                shuffle[i][j][k] = shuffleMatrix[int(i / atomY)][int(
                    j / atomX)][i % atomY][j % atomX][k]
    cv2.imwrite('output/shuffle.bmp', shuffle)

    cv2.imshow('any key to continue',shuffle)
    cv2.waitKey()
    cv2.destroyAllWindows()

slicing(filePath)


from dapiclient.client import DAPIClient
import cbor2
from thumbor_dash.error_handlers.sentry import ErrorHandler
from thumbor_dash.error_handlers import *

client = DAPIClient()

def getDocuments(handler, data):
    error_handler = ErrorHandler(handler.context.config)

    try:
        identity = client.getIdentity(data['owner_id'])
    except Exception as e:
        error_handler.handle_error(handler.context, handler, UnknownUserError)
        return

    else:
        docs = client.getDocuments(
            data['contract_id'],
            data['document_type'],
            data['where'],
            limit=2, # Only one document
            )

        documents = []

        for  doc in docs:
           documents.append(cbor2.loads(doc))

        return documents[0] # Return the only document in the list


import numpy as np
import pylab as pl
from coil_class import oneD
import Materials as mat

Io = 68e3  # [A]
Vo = 0.67*1e3  # [V]
Lf = 2*17.3/18  # [H]
L = 17.3-Lf  # [H]
Ro = Vo/Io
Rbp = 1e3  # [ohm]  # bypass resistance

time,dt = np.linspace(0,300,50000,retstep=True)
I = np.zeros(len(time))
If = np.zeros(len(time))
Ibp = np.zeros(len(time))
V = np.zeros(len(time))
Rf = 0.01*time  # [fault coil resistance]
E = 0

coil = oneD()
coil.temp(t=time)

coil.Tcond

If_old,I_old  = Io,Io

for i,t in enumerate(time):
    If[i] = (I_old*Rbp+Lf/dt*If_old)/(Rf[i]+Rbp+Lf/dt)  # fault coil
    Ibp[i] = I_old-If_old  # bypass current
    I[i] = (L*I_old/dt-Ibp[i] *Rbp)/(Ro+L/dt)

    R = Ro+(1/Rf[i]+1/Rbp)**-1
    V[i] = I[i]*R
    E+=V[i]*I[i]*dt
    I_old = I[i]
    If_old = If[i]

print('E',E*1e-9)

pl.figure(figsize=(9,6))
pl.plot(time,I*1e-3, 'k-',label='total')
pl.plot(time,If*1e-3, 'r--',label='fault')
pl.plot(time,Ibp*1e-3, 'b-',label='bypass')
pl.legend()
pl.xlabel('Time, t [s]')
pl.ylabel('Current, I [KA]')
pl.grid()
pl.savefig('../Figs/current_discharge.png', dpi=300)

pl.figure(figsize=(9,6))
pl.plot(time,V)
pl.xlabel('Time, t [s]')
pl.ylabel('Voltage, V [V]')
pl.grid()
pl.savefig('../Figs/voltage_discharge.png', dpi=300)

pl.figure(figsize=(9,6))
pl.plot(time,V*I*1e-6, 'r-')
pl.xlabel('Time, t [s]')
pl.ylabel('Power, P [MW]')
pl.grid()
pl.savefig('../Figs/power_discharge.png', dpi=300)


# -*- coding: utf-8 -*-
from django.db.models.fields.files import ImageFieldFile
from django.db. models import ImageField
from stored_settings.models import Settings
from stored_settings.settings import UPLOAD_TO_DIRECTORY

def stored_settings(request):
    context = {}
    for setting in Settings.objects.all():
        if setting.type == Settings.Type.BOOLEAN:
            setting.value = bool(setting.value) and 1 or 0
        elif setting.type == Settings.Type.IMAGE:
            setting.value = ImageFieldFile(instance=setting, field=ImageField(upload_to=UPLOAD_TO_DIRECTORY),
                                           name=setting.value)

        context[setting.key] = setting.value

    return {'stored_settings': context}


#Andres Mateo Calderón Ortega

def distance(a, b):
    r = abs(b - a)
    return r

def jolly(a):
    diff = []
    num = [int(c) for c in range(1, len(a))]
    cont = len(a) - 1
    cont2 = len(a) - 2
    for i in range(len(a) + 4):
        if cont2 < 0:
            break
        else:
            m = distance(a[cont], a[cont2])
            if m > len(a) - 1 or 0:
                return False
            elif m in diff:
                return False
            else:
                diff.append(m)
                cont = cont2
                cont2 -= 1
    diff.sort()
    if diff == num:
        return True
    else:
        return False

def main():
    a = [[1, 4, 2, 3], [1, 4, 2, -1, 6], [11, 7, 4, 2, 1, 6], [3, 5, 2, 1], [7, 2, 5, 4, -2, 1, 3]]
    for i in a:
        print(jolly(i))

main()


"""Streaming pickle implementation for efficiently serializing and
de-serializing an iterable (e.g., list)

Created on 2010-06-19 by Philip Guo

http://code.google.com/p/streaming-pickle/

Modified by Brian Thorne 2013 to add base64 encoding to support
python3 bytearray and the like.
"""
import base64
from cPickle import dumps, loads
import unittest
import tempfile

def s_dump(iterable_to_pickle, file_obj):
    """dump contents of an iterable iterable_to_pickle to file_obj, a file
    opened in write mode"""
    for elt in iterable_to_pickle:
        s_dump_elt(elt, file_obj)

def s_dump_elt(elt_to_pickle, file_obj):
    """dumps one element to file_obj, a file opened in write mode"""
    pickled_elt = dumps(elt_to_pickle,-1)
    encoded = base64.b64encode(pickled_elt)
    file_obj.write(encoded)

    # record separator is a blank line
    # (since pickled_elt as base64 encoded cannot contain its own newlines)
    file_obj.write(b'\n\n')

def s_load(file_obj):
    """load contents from file_obj, returning a generator that yields one
    element at a time"""
    cur_elt = []
    for line in file_obj:

        if line == b'\n':
            encoded_elt = b''.join(cur_elt)
            try:
                pickled_elt = base64.b64decode(encoded_elt)
                elt = loads(pickled_elt)
            except EOFError:
                print("EOF found while unpickling data")
                print(pickled_elt)
                raise StopIteration
            cur_elt = []
            yield elt
        else:
            cur_elt.append(line)

class TestStreamingPickle(unittest.TestCase):
    def setUp(self):
        pass

    def testSimpleList(self):
        # data = [1, 2, 3, 4, None,  b'test', '\n', '\x00', 3, b'\n\n\n\n', 5, 7, 9, 11, "hello", bytearray([2, 4, 4])]
        data = [1,[1,2,3,4],[8,9,29]]
        with tempfile.TemporaryFile() as f:
            s_dump(data, f)
            # reset the temporary file
            f.seek(0)
            i = 0
            for i, element in enumerate(s_load(f)):
                self.assertEqual(data[i], element)
                # print(i, element)
            self.assertEqual(i, len(data)-1)

if __name__ == "__main__":
    unittest.main()


from json import loads
from os import remove
from os.path import exists
from subprocess import Popen, PIPE
from time import sleep
from os.path import abspath, dirname

from PIL import Image

FFMPEG = 'C:/ffmpeg/bin/'
_ABSOLUTE = dirname(abspath(__file__))
_PIPES = {'stdin': PIPE, 'stdout': PIPE, 'stderr': PIPE}

class Video:
    __slots__ = ['bytes', 'info', 'err', 'de_color_model', 'en_color_model', 'temp']

    def __init__(self, video_bytes, temp, de_color_model=('gray', 1), en_color_model='yuv420p'):
        p = Popen([FFMPEG + 'ffprobe',
                   '-print_format', 'json',
                   '-show_streams',
                   '-i', '-'], **_PIPES)

        info = [i.decode('utf-8') for i in p.communicate(video_bytes)]
        self.info = loads(info[0])['streams'][0]
        self.err = info[1]

        self.temp = temp
        self.bytes = video_bytes
        self.de_color_model = de_color_model
        self.en_color_model = en_color_model

    @property
    def frames(self, fill=(128, 112)):
        frame_dump = _ABSOLUTE + '/temp/' + self.temp

        with open(frame_dump, 'wb+') as temp:
            temp.write(self.bytes)

        p = Popen([FFMPEG + 'ffmpeg',
                   '-f', 'image2pipe',
                   '-pix_fmt', self.de_color_model[0],
                   '-vcodec', 'rawvideo',
                   '-vf', r"scale='if(gt(iw,ih),-1,{0})':'if(gt(iw,ih),{1},-1)',crop={0}:{1}".format(*fill),
                   '-',
                   '-i', frame_dump], **_PIPES)

        data, err = p.communicate()
        self.err = err.decode('utf-8')

        remove(frame_dump)

        fs = fill[0] * fill[1] * self.de_color_model[1]
        return [Image.frombytes('L', fill, data[fs * i: fs * (i + 1)]) for i in range(len(data) // fs)]

    @frames.setter
    def frames(self, frames):
        dim = frames[0].width, frames[0].height

        frame_dump = _ABSOLUTE + '/temp/' + self.temp
        composite = _ABSOLUTE + '/temp/' + self.temp + '.mp4'

        with open(frame_dump, 'wb+') as temp:
            for f in frames:
                temp.write(f.tobytes())

        p = Popen([FFMPEG + 'ffmpeg',
                   '-f', 'rawvideo',
                   '-r', self.info['r_frame_rate'],
                   '-vcodec', 'rawvideo',
                   '-s', '{}x{}'.format(*dim),
                   '-pix_fmt', 'rgb24',
                   '-i', frame_dump,
                   '-an',
                   '-vcodec', 'libx264',
                   '-pix_fmt', self.en_color_model,
                   '-y', composite], **_PIPES)

        self.err = p.communicate()[1].decode('utf-8')

        for i in range(100):
            if exists(composite):
                with open(composite, 'rb') as temp:
                    self.__init__(temp.read(), self.temp, self.de_color_model, self.en_color_model)

                remove(composite)
                break

            sleep(.01)

        remove(frame_dump)