import numpy as np
import matplotlib.pyplot as plt
import multiprocessing as mp
import os
from time import time
import random
import math
import glob
from tqdm import tqdm
import pickle
import gc
from heiner.dataloader import DataLoader as HeinerDataloader
from myutils import printerror
from constants import *

# conclusion from profiling below:
# - inplace standardization: reduces the 0.8s per batch that the original version consumed!
# - threading (workers=1, multiprocessing=False) superior: 3.6s (2.7s without input standardization)
# - singleproc (workers=0) a bit slower: 4.4s
# - multiprocessing (workers=1, multiprocessing=True) inadequate because of slow serialization through
#                                                     queue across processes: 12.8s
# experiment2:
# inplace standardization vs original
# - original after 70 batches: 2.0s generator, 0.82s train_predict, 0.92s metrics
# - inplace after 70 batches: 1.18s generator, 0.52s train_predict, 0.96s metrics <--- first try
# - inplace after 70 batches: 0.9s generator, 0.59 train_predict, 0.86 metrics <-- second try
# - no input std after 70 batches: 0.5s generator, 0.54 train_predict, 0.95 metrics
#
# experiment 1:
# runtime profiling exploration (evaluated on sabik with batch size 128, sceneinstancebufsize 2000):
# - direct runtimes (via running this file):
#   - total time: ~2-3 sec
#   - instantiating scene instance buffers (i.e., loading file): 1...1.6 sec
#   - assignment of blocks to the batch array: 0.2 sec
#   - standardization: 0.8 sec
#   - all other parts are negligible
# - effective runtimes (via running training.py => generator_extension.py)
#   - singleprocessing (0 workers) / multiprocessing False
#     - total: ~4.4s (here/below: avg after 100 batches)
#     - generator: 2.6s (cf. above)
#     - train_predict: 0.7s
#     - metrics: 0.9s
#   - multithreading (1 worker) / multiprocessing False (same condition as others) [with standardization]
#       - total: ~3.6  (after ~200 batches)
#       - generator: 1.4s
#       - train_predict: 1.25s
#       - metrics: 0.95s
#   - multithreading (1 worker) / multiprocessing False (same condition as others) [without standardization]
#       - total: ~2.7s  (after ~1000 batches)
#       - generator: 0.8s
#       - train_predict: 0.6s
#       - metrics: 0.9s
#   - multiprocessing (1 worker) / multiprocessing True (note that the cpu was not freely available to me but similar to above/singleproc experiment)
#       - total: ~ 12.8s (here/below: avg after 60 batches)
#       - generator: ~10.0s [higher than singleproc!]
#       - train_predict: 1.25s [strange: higher than singleproc??]
#       - metrics 1.1s [strange: higher than singleproc??]

# buffer of a single scene instance (is instantiated e.g. 2000 times in a BatchLoader object in train mode)
class SceneInstanceBuffer:

    def __init__(self, filename, batchloader, mode, params):
        # #self.filename_and_or_sceneinstance_id = filename_and_or_sceneinstance_id
        self.mode = mode
        self.params = params
        self.filename = filename
        self.batchloader = batchloader

        # scene instance id
        self.scene_instance_id = self.batchloader.scene_instance_ids_dict[filename]
        # length
        self.scene_instance_length = self.batchloader.length_dict[filename]
        # positions and overlaps
        self.positions, self.overlaps = self.batchloader.block_positions_and_overlaps_dict[self.filename]

        # load data
        with np.load(filename) as data:
            # features
            self.x = data['x']
            # remark: input standardization will be done in buffer for the final batch (to be compatible with Heiner's code)

            # labels
            self.y = data['y'] if params['instantlabels'] else data['y_block']

        # saving iterator as attribute
        self.iter = self.block_iterator()

    def __len__(self):
        return len(self.positions)

    def block_iterator(self):

        empty = False
        blockid = 0
        while (not empty):

            # get current position and overlap
            position = self.positions[blockid]
            overlap = self.overlaps[blockid]

            # collect batchlength long features and labels from frame index position
            x_block = self.x[0, position:position+self.params['batchlength'], :] # ignore dummy batch dim
            y_block = self.y[0, position:position+self.params['batchlength'], :] # ignore dummy batch dim

            # adding scene instance id as second label id
            # remark: for compatibility with Heiner's accuracy utils we need to provide the scene instance id for each
            # frame although in our case all frames have the same scene instance id
            sid_block = self.scene_instance_id * np.ones_like(y_block, dtype=DTYPE_DATA)
            y_concat_sid_block = np.zeros((*y_block.shape, 2), dtype=DTYPE_DATA)
            y_concat_sid_block[:, :, 0] = y_block
            y_concat_sid_block[ :, :, 1] = sid_block

            # set mask values (overlapping frames should not be counted twice for loss and accuracy metrics)
            # remark: both the labels and the scene id need to be masked according to heiner's accuracy utils
            if overlap > 0: # except first overlap (would though be respected by following slicing as well)
                y_concat_sid_block[:overlap, :, :] = self.params['mask_value']

            yield x_block, y_concat_sid_block

            # last blockid processed:
            if blockid+1 == len(self):
                empty = True
            # next blockid:
            else:
                blockid += 1

# batchloader that internally uses a set of many (e.g. 2000) scene instance buffers to fetch blocks from
# one scene instance has between 3,000 and 20,000 frames, avg < 4,000 frames
# => avg host mem required per buffered scene instance: (160+13) * 4,000 * 4 Byte < 2.8 MB
# => a value of params['sceneinstancebufsize']=1000 implies up to 2.8 GB host mem occupancy
class BatchLoader(HeinerDataloader):
    def __init__(self, params, mode, fold_nbs, scene_nbs, batchsize, seed=1):
        label_mode = 'instant' if params['instantlabels'] else 'blockbased'
        # initializing super constructor with values from above or with their defaults (copied from super init)
        super().__init__(mode=mode, label_mode=label_mode,
                       fold_nbs=fold_nbs, scene_nbs=scene_nbs, batchsize=batchsize, timesteps=params['batchlength'], epochs=params['maxepochs'],
                       buffer=10, features=DIM_FEATURES, classes=DIM_LABELS, path_pattern=DATA_ROOT+'/',
                       seed=seed, seed_by_epoch=True, priority_queue=True, use_every_timestep=False, mask_val=params['mask_value'],
                       val_stateful=False, k_scenes_to_subsample=-1,
                       input_standardization=not params['noinputstandardization'])

        self.params = params

        self.free_memory_batches = 10 # run garbage collector after 10 batches

        # we require scene instance buffers to be larger than the batchsize (assumed in refill logic)
        assert self.params['sceneinstancebufsize'] > self.params['batchsize']

        # calculate or load filename->scene instance id, filename->scene instance length, filename->positions,overlaps dicts
        self.scene_instance_ids_dict_()
        self.length_dict_()
        self.block_positions_and_overlaps_dict = None # the above are initialized in HeinerDataloader
        self._block_positions_and_overlaps_dict()

        # get number of batches per epoch
        self._calculate_batchnumber()

        print('batchloader {} created ({} batches of size {} [per epoch] and length {} / historylength {}) using {} labels'.
              format(mode, self.batches_per_epoch, params['batchsize'], params['batchlength'], self.params['historylength'], label_mode))

        # set batch array
        self.batch_x = np.zeros((self.params['batchsize'], self.params['batchlength'], DIM_FEATURES), dtype=DTYPE_DATA)
        self.batch_y = np.zeros((self.params['batchsize'], self.params['batchlength'], DIM_LABELS, 2), dtype=DTYPE_DATA)

        # initialize state of the batch loader, will set self.epoch = 0
        self.init_epoch(first=True)

    def _calculate_batchnumber(self):
        blocks = 0
        assert len(self.filenames_all) == len(self.filenames)
        for filename in self.filenames_all:
            positions, overlaps = self.block_positions_and_overlaps_dict[filename]
            blocks += len(positions)

        self.batches_per_epoch = int(math.ceil(blocks/self.params['batchsize']))

    def _block_positions_and_overlaps_dict(self):
        if self.block_positions_and_overlaps_dict is None:
            pickle_path = os.path.join(self.pickle_path, 'block_positions_and_overlaps_dict.pickle')

            # update stored dict if current parametrization is not yet contained:
            if not os.path.exists(pickle_path):
                self._create_block_positions_and_overlaps_dict(pickle_path=pickle_path)

            # read stored dict(batchlength,historylength) of dicts(filename)
            with open(pickle_path, 'rb') as handle:
                all_block_positions_and_overlaps_dict = pickle.load(handle)

            dict_key = (self.params['batchlength'], self.params['historylength'])

            # update stored dict if current parametrization is not yet contained:
            if dict_key not in all_block_positions_and_overlaps_dict:
                self._create_block_positions_and_overlaps_dict(pickle_path=pickle_path)
                with open(pickle_path, 'rb') as handle:
                    all_block_positions_and_overlaps_dict = pickle.load(handle)

            # now we have ensured that the current parametrization exists in all_* dict
            self.block_positions_and_overlaps_dict = all_block_positions_and_overlaps_dict[dict_key]

        return self.block_positions_and_overlaps_dict

    def _create_block_positions_and_overlaps_dict(self, pickle_path):

        # calculate positions (startind frame indices) and overlaps (number of frames) of consecutive blocks
        # of a scene instance that is represented here only by its length
        # remark: the overlap with the previous block can vary though:
        #         first block (no overlap at all), intermediate blocks (overlap historysize-1) and
        #         the last block (overlap s.t. the end of the scene instance is exactly approached => quite large overlap possible)
        def calculate_block_positions_and_overlaps(scene_instance_length, batch_length, history_length):

            # assume that at least one full block (with batchlength) exists in the the scene instance
            assert batch_length <= scene_instance_length

            # first block
            block_idx = 0
            positions = [0]
            overlaps = [0]

            # there are more blocks, at least a second block:
            if batch_length < scene_instance_length:
                last_block = False
            # first block is the only block:
            else:
                last_block = True

            # until we have collected all blocks
            while (not last_block):
                # next block
                block_idx += 1

                # position candidate
                position = positions[block_idx - 1] + batch_length

                # for intermediate block overlapsuch that the first history_length-1 elements are shared with previous block
                # and the history_length's element is the first new frame (the overlapping part will be masked)
                overlap = history_length - 1

                # last block only:
                if position - overlap + batch_length >= scene_instance_length:
                    last_block = True
                    overlap = position + batch_length - scene_instance_length + 1
                    # ensure overlap is nonnegative and smaller than batch_length
                    assert overlap >= 0 and overlap < batch_length

                # update position to include overlap
                position -= overlap

                # ensure that beginning and end of block are within 0...scene_instance_length-1
                assert position >= 0 and position <= scene_instance_length - 1

                if last_block:
                    # ensure that last block really is last block
                    assert position == scene_instance_length - batch_length - 1

                positions.append(position)
                overlaps.append(overlap)

            # the length of both returned lists is the number of (batchlength long) blocks
            return positions, overlaps

        print('creating block positions and overlaps dict ')

        # initialize the dicitionary for filename->positions,overlap
        new_dict = {}

        # we need the scene instance lengths for each filename
        self.length_dict_()

        all_existing_files = glob.glob(self.pickle_path_pattern)
        all_existing_files = sorted(all_existing_files)
        for filename in tqdm(all_existing_files):
            positions, overlaps = calculate_block_positions_and_overlaps(self.length_dict[filename],
                                                                         self.params['batchlength'],
                                                                         self.params['historylength'])
            new_dict[filename] = (positions, overlaps)

        # load overall dictionary from file or, if file not existing, initialize it
        try:
            with open(pickle_path, 'rb') as handle:
                all_block_positions_and_overlaps_dict = pickle.load(handle)
        except IOError: #
            all_block_positions_and_overlaps_dict = {}

        # the to be pickled overall dictionary: dict(batchlength,historylength)->dict(filename->positions,overlaps)
        all_block_positions_and_overlaps_dict[(self.params['batchlength'], self.params['historylength'])] = new_dict

        with open(pickle_path, 'wb') as handle:
            pickle.dump(all_block_positions_and_overlaps_dict, handle, protocol=pickle.HIGHEST_PROTOCOL)

    def fill_scene_instance_buffers(self):

        t_start_allfill = time()

        # if (len(self.scene_instance_buffers) == 0):
        #     complete_fill = True
        #     t_start_buffer_filling = time() # keep (only) this time measurement after profiling
        #     print('batchloader {} (epoch {}): filling empty buffer with {} scene instances...'.format(self.mode, self.epoch+1,
        #                                                                                self.params['sceneinstancebufsize']))
        # else:
        #     complete_fill = False

        runtime_filename = 0.
        runtime_instantiation = 0.
        runtime_appending = 0.
        runtime_lengthcalc = 0.
        # add more scene instance buffers until buffer full or all files consumed (e.g. in test mode for a single scene)
        while (len(self.scene_instance_buffers) < self.params['sceneinstancebufsize'] and len(self.filenames) > 0):

            t_start = time()
            # get next of the (in train mode shuffled) filenames
            filename = self.filenames.pop()
            runtime_filename += time()-t_start

            t_start = time()
            # create buffer and append it to buffers list
            scene_instance_buf = SceneInstanceBuffer(filename,  self, self.mode, self.params)
            runtime_instantiation += time()-t_start

            t_start = time()
            self.scene_instance_buffers.append(scene_instance_buf)
            if self.mode == 'train':
                self.scene_instance_buffers_last = np.append(self.scene_instance_buffers_last, 10000)
                self.scene_instance_buffers_remaining.append(len(scene_instance_buf))
            runtime_appending += time()-t_start

            t_start = time()
            # calculate the total number of blocks in alle scene instance buffers
            self.blocks_allbuffers += len(scene_instance_buf)
            runtime_lengthcalc += time()-t_start

        # if complete_fill:
        #     print('batchloader {} (epoch {}): ...filled buffer in {:.2f} sec'.format(self.mode, self.epoch+1,
        #                                                                   time()-t_start_buffer_filling))

        # runtime_fillbuffer = time()-t_start_allfill

        # print('batchloader: fill scene instance buffers took {:.2f} => filename {:.2f}, instantiation {:.2f}, appending {:.2f}, lengthcalc {:.2f}'
        #       .format(runtime_fillbuffer, runtime_filename, runtime_instantiation, runtime_appending, runtime_lengthcalc))

    def init_epoch(self, first=False):

        if first:
            self.epoch = 0
            self.scene_instance_buffers = []
            if self.mode == 'train':
                # data for biased sampling from scene instance buffer ids in training mode
                self.scene_instance_buffers_last = np.array([], dtype=np.int)
                self.scene_instance_buffers_remaining = []
        else:
            self.epoch += 1
            # ensure that the scene instance buffer list is empty
            assert not self.scene_instance_buffers

        # recover original list of all filenames
        self.filenames = self.filenames_all.copy()

        # set new seed [only applicable in training mode, valid and test scene instances are always processed in order]
        if self.mode == 'train':
            # set new seed (control self.seed for reproducible results)
            random.seed(self.seed * (self.epoch+1)) # to have new seed per epoch (also for validation set)

            # shuffle filenames (not required for valid/test sets)
            random.shuffle(self.filenames)

        # number of all blocks that can be fetched from all buffers
        self.blocks_allbuffers = 0

        self.batchid = -1 # will be increased by each __next__() call

        # the scene instance buffers should be empty, otherwise we should not init an epoch
        assert len(self.scene_instance_buffers) == 0

        # print('batchloader (mode {}) is prepared for epoch {}'.format(self.mode, self.epoch+1))

    def __iter__(self):
        return self

    def __next__(self):

        t_start_nextbatch = time()

        t_start = time()
        # all batches created => next epoch or stop iteration
        if self.batchid > 0 and len(self.scene_instance_buffers) == 0:
            assert self.batches_per_epoch == self.batchid + 1 # TODO: remove if multiproc
            # start next epoch:
            if self.epoch + 1 < self.params['maxepochs']:
                self.init_epoch() # also sets self.batchid = -1
            # but only until maxepochs is reached
            else:
                raise StopIteration

        # fills buffers and update self.blocks_allbuffers
        # (only effective if a scene instance buffer is empty or last filename consumed)
        # refilling is reuiqred not only during epoch init but also before each batch construction
        self.fill_scene_instance_buffers() # randomness enters here via shuffled filenames

        # create a batch (potentially smaller than params['batchsize'] if it is the last one
        self.batchid += 1
        blockid = 0

        if self.mode == 'train':
            self.scene_instance_buffers_last += 1

        # free memory (unreferenced scene instance buffers) after every few batches
        if self.batchid % self.free_memory_batches == 0:
            gc.collect()

        runtime_sample_index = 0.
        runtime_next_block = 0.
        runtime_postproc_block = 0.
        runtime_postproc_block_assignment = 0.
        runtime_postproc_block_listops = 0.
        runtime_remove_block = 0.

        runtime_start = time()-t_start

        # add blocks to our batch until we have filled the batch or there are no blocks in the buffer left (last batch)
        while (blockid < self.params['batchsize'] and len(self.scene_instance_buffers) > 0):

            t_start = time()
            # training mode: get random scene instance buffer:
            if self.mode == 'train':
                    probs = self._scene_instance_buffer_probs()
                    sibuf_id = np.random.choice(len(self.scene_instance_buffers), size=1, replace=False, p=probs)[0]
                    # old version (without respecting batch correlation, without biasing towards longer sequences)
                    # sibuf_id = random.randint(0, len(self.scene_instance_buffers)-1)

            # validation/test mode: simply take the first scene instance:
            else:
                sibuf_id = 0

            runtime_sample_index += time()-t_start

            # extract next block from chosen scene instance (if existing)
            t_start = time()
            scene_instance_buffer = self.scene_instance_buffers[sibuf_id]
            buffer_output = next(scene_instance_buffer.iter, None)
            runtime_next_block += time()-t_start

            t_start = time()
            remove_buffer = False
            if buffer_output is not None:
                t_start_inner = time()
                self.batch_x[blockid, :, :], self.batch_y[blockid, :, :, :] = buffer_output
                runtime_postproc_block_assignment += time()-t_start_inner

                t_start_inner = time()
                # we fetched a valid block => update data for biased sampling
                if self.mode == 'train':
                    self.scene_instance_buffers_last[sibuf_id] = 0
                    self.scene_instance_buffers_remaining[sibuf_id] -= 1
                    if self.scene_instance_buffers_remaining[sibuf_id] == 0:
                        remove_buffer = True
                    else:
                        remove_buffer = False
                blockid += 1
                runtime_postproc_block_listops = time()-t_start_inner
            else:
                remove_buffer = True

            runtime_postproc_block += time()-t_start

            t_start = time()
            # remove scene instance that is completely used
            if remove_buffer:
                self.scene_instance_buffers.pop(sibuf_id)
                if self.mode == 'train':
                    self.scene_instance_buffers_last = np.delete(self.scene_instance_buffers_last, sibuf_id)
                    self.scene_instance_buffers_remaining.pop(sibuf_id)

            runtime_remove_block += time()-t_start

        # handling a case (relevant e.g. for blocklength 2000) that has a single (empty) remaining scene instance
        if blockid==0: # and remove_buffer
            raise StopIteration

        t_start = time()
        effective_batchsize = blockid # after the while loop blockid corresponds to the no of blocks taken

        # update number of remaining blocks
        self.blocks_allbuffers -= effective_batchsize

        # determine effective batch (size batchsize or for last batch smaller)
        if effective_batchsize < self.params['batchsize']:
            effective_batch_x = self.batch_x[:effective_batchsize, :, :]
            effective_batch_y = self.batch_y[:effective_batchsize, :, :, :]
        else:
            effective_batch_x = self.batch_x
            effective_batch_y = self.batch_y

        runtime_finish = time()-t_start

        runtime_nextbatch_total = time()-t_start_nextbatch
        # print('batchloader: total time to get the batch was {:.2f} => start {:.2f}, sample {:.2f}, next block {:.2f}, postproc block {:.2f} (assignment {:.2f}, listops {:.2f}), remove block {:.2f}, finish {:.2f}'
        #       .format(runtime_nextbatch_total, runtime_start, runtime_sample_index, runtime_next_block, runtime_postproc_block, runtime_postproc_block_assignment, runtime_postproc_block_listops, runtime_remove_block, runtime_finish))

        # we need to return a copied version of the batch in order not to overwrite it when creating the next batch
        return np.copy(self._input_standardization_if_wanted(effective_batch_x)), np.copy(effective_batch_y)

    # calculate probabilities that weight each scene instance buffer with its sequence lengt and also decrease weights of
    # scene instances that were sampled in the previous batch, i.e., decrease ensure that the last batches do not consist
    # of several batches of the last remaining long sequences, and also generally correlation between any two directly
    # successive batches are removed
    # optional improvement: decrease probability also if a block was taken two batches before (e.g. weight by inverse last seen)
    def _scene_instance_buffer_probs(self):
        assert len(self.scene_instance_buffers_remaining) == len(self.scene_instance_buffers_last) == len(self.scene_instance_buffers)
        remaining_squared = np.array(self.scene_instance_buffers_remaining)**2
        previous = self.scene_instance_buffers_last <= 1
        if previous.any():
            # remove probability for previously taken scene instance
            remaining_squared_reduced_previous = remaining_squared.copy().astype(np.float)
            remaining_squared_reduced_previous[previous] *= 0.001
        else:
            remaining_squared_reduced_previous = remaining_squared
        probs = remaining_squared_reduced_previous / remaining_squared_reduced_previous.sum()
        return probs

# testing:
if __name__ == '__main__':

    # ### train
    # mode = 'train'
    # folds = [1, 2, 3, 4, 5, 6] # final model
    # folds = [1, 2, 4, 5, 6] # lvl1
    # folds = [1, 2, 3, 5, 6] # lvl2
    # folds = [1, 3, 4, 5, 6] # lvl3
    # scenes = list(range(1, 80 + 1))  # -1 # all scenes (80 for train)

    # ### test
    mode = 'test'
    folds = [7, 8] # test folds
    scenes = list(range(1, 168 + 1)) # all 168 test scenes

    # ### val/toy
    # mode = 'val'
    # folds = [3]
    # folds = [4]
    # folds = [2]
    # scenes = [1]
    # scenes = list(range(1, 80 + 1))  # -1 # all scenes (80 for train)

    inputstd = True
    # inputstd = False
    params = {'sceneinstancebufsize': 200, #2000,
              'historylength': 1025,
              'batchsize': 128,
              'batchlength': 2000, #2500,
              'instantlabels': False,
              'maxepochs': 1,
              'noinputstandardization': not inputstd,
              'mask_value': -1}

    print('batchloader ({} mode, inputstd={}, folds {}, scenes {}) before initialization.....'.format(mode, inputstd,
                                                                                folds, scenes))

    t_start_allbatches = time()
    batchloader = BatchLoader(params=params, mode=mode, fold_nbs=folds,
                              scene_nbs=scenes, batchsize=params['batchsize'],
                              seed=1)  # seed for testing

    # for checking input standardization
    no_batches = batchloader.batches_per_epoch * params['maxepochs']
    mean_vec_batches = np.zeros((160, no_batches))
    std_vec_batches = np.zeros_like(mean_vec_batches)
    no_unmasked_features = np.zeros(no_batches, dtype=np.int)

    print('iterating once over the data set:')
    t_start_allbatches = time()
    batchcount = 0
    t_start_batch = time()

    for batch_x, batch_y in batchloader:
        batchcount += 1
        print('received batch (size {}) {}/{} in epoch {}/{} in {:.2f} sec'.format(batch_x.shape[0], batchloader.batchid % batchloader.batches_per_epoch + 1,
                                                                                batchloader.batches_per_epoch, batchloader.epoch + 1, params['maxepochs'],
                                                                                time() - t_start_batch))
        batch_x_unmasked = batch_x[batch_y[:, :, 0, 1] != params['mask_value']]
        no_unmasked_features[batchcount-1] = len(batch_x_unmasked)
        print('batch has {} unmasked of {} total feature vectors'.
              format(no_unmasked_features[batchcount-1],
                     batch_x.shape[0]*batch_x.shape[1]))
        mean_vec_batches[:, batchcount-1] = np.mean(batch_x_unmasked, axis=0)
        std_vec_batches[:, batchcount-1] = np.std(batch_x_unmasked, axis=0)
        t_start_batch = time()
    assert batchcount == batchloader.batches_per_epoch * params['maxepochs']
    print('...done receiving {} batches of {} epochs in {:.2f} seconds'.format(batchcount, params['maxepochs'],
                                                                               time() - t_start_allbatches))

    print('calculating mean and std of whole data set to validate input standardization:')
    # # assumption of the commented out code: size and length of all batches is equal (pot. smaller last batch is negligible)
    # mean_vec_total = np.mean(mean_vec_batches, axis=1)
    # var_vec_total = 1./batchcount * np.sum((std_vec_batches**2 + (mean_vec_batches-mean_vec_total[:, np.newaxis])**2), axis=1)
    # corrected  code (compared to commented out above): taking into account masking of features
    mean_vec_total = 1./np.sum(no_unmasked_features) * np.sum(no_unmasked_features[np.newaxis, :] * mean_vec_batches,
                                                              axis=1)
    var_vec_total = 1. / np.sum(no_unmasked_features) * np.sum(no_unmasked_features[np.newaxis, :] *
                                                               (std_vec_batches ** 2 +
                                                                (mean_vec_batches - mean_vec_total[:, np.newaxis]) ** 2),
                                                               axis=1)
    std_vec_total = np.sqrt(var_vec_total)
    print('total mean vector (should be close to component-wise zero): \n{}'.format(mean_vec_total))
    print('total std vector (should be close to component-wise unity): \n{}'.format(std_vec_total))


from typing import Type, Any, Callable, Union, List, Optional
import tensorflow as tf
import tensorflow.keras.layers
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, ReLU, BatchNormalization, Dropout, AveragePooling2D
from utils import drop_path
from operations import *
from utils import SeparableConv2d

def conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> Conv2D:
    """3x3 convolution with padding"""
    if dilation == 1:
        padding = "same"
    else:
        padding = "valid"
    return Conv2D(out_planes, kernel_size=(3,3), strides=(stride, stride), padding=padding, groups=groups, use_bias=False, dilation_rate=dilation)
    # return nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
    #                  padding=dilation, groups=groups, bias=False, dilation=dilation)

def sep_conv3x3(in_planes: int, out_planes: int, stride: int = 1, groups: int = 1, dilation: int = 1) -> SeparableConv2d:
    """3x3 convolution with padding"""
    if dilation == 1:
        padding = "same"
    else:
        padding = "valid"
    return SeparableConv2d(in_planes, out_planes, kernel_size=3, stride=stride,
                     padding=padding, bias=False, dilation=dilation)

def conv1x1(in_planes: int, out_planes: int, stride: int = 1) -> Conv2D:
    """1x1 convolution"""
    return Conv2D(out_planes, kernel_size=(1,1), strides=(stride, stride), use_bias=False)
    # return nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, bias=False)

def _bn_function_factory(norm, relu, conv):
    def bn_function(*inputs):
        #TODO
        concated_features = tf.concat(inputs, 1)
        # concated_features = torch.cat(inputs, 1)
        bottleneck_output = conv(relu(norm(concated_features)))
        return bottleneck_output

    return bn_function

class BasicBlock(tf.Module):
    expansion: int = 1

    def __init__(
        self,
        inplanes: int,
        planes: int,
        stride: int = 1,
        downsample: Optional[tf.Module] = None,
        groups: int = 1,
        base_width: int = 64,
        dilation: int = 1,
        norm_layer: Optional[Callable[..., tf.Module]] = None,
        separable: bool = True,
    ) -> None:
        super(BasicBlock, self).__init__()
        if norm_layer is None:
            norm_layer = BatchNormalization
        #if groups != 1 or base_width != 64:
        if groups != 1: # or base_width != 64:
            raise ValueError('BasicBlock only supports groups=1 and base_width=64')
        if dilation > 1:
            raise NotImplementedError("Dilation > 1 not supported in BasicBlock")
        # Both self.conv1 and self.downsample layers downsample the input when stride != 1
        if separable:
            self.conv1 = sep_conv3x3(inplanes, planes, stride)  #conv3x3(inplanes, planes, stride)
            self.conv2 = sep_conv3x3(planes, planes) #conv3x3(planes, planes)
        else:
            self.conv1 = conv3x3(inplanes, planes, stride)
            self.conv2 = conv3x3(planes, planes)

        self.bn1 = norm_layer()
        self.relu = ReLU()
        self.bn2 = norm_layer()
        self.downsample = downsample
        self.stride = stride

    def __call__(self, x: tf.Variable) -> tf.Variable:
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class Bottleneck(tf.Module):
    # Bottleneck in torchvision places the stride for downsampling at 3x3 convolution(self.conv2)
    # while original implementation places the stride at the first 1x1 convolution(self.conv1)
    # according to "Deep residual learning for image recognition"https://arxiv.org/abs/1512.03385.
    # This variant is also known as ResNet V1.5 and improves accuracy according to
    # https://ngc.nvidia.com/catalog/model-scripts/nvidia:resnet_50_v1_5_for_pytorch.

    def __init__(
        self,
        inplanes: int,
        planes: int,
        stride: int = 1,
        downsample: Optional[tf.Module] = None,
        groups: int = 1,
        base_width: int = 64,
        dilation: int = 1,
        norm_layer: Optional[Callable[..., tf.Module]] = None,
        separable: bool = True,
        expansion: int = 4,
    ) -> None:
        super(Bottleneck, self).__init__()
        if norm_layer is None:
            norm_layer = BatchNormalization
        width = int(planes * (base_width / 64.)) * groups
        # Both self.conv2 and self.downsample layers downsample the input when stride != 1
        self.conv1 = conv1x1(inplanes, width)
        self.bn1 = norm_layer()
        # self.bn1 = norm_layer(width)
        self.conv2 = conv3x3(width, width, stride, groups, dilation)
        self.bn2 = norm_layer()
        # self.bn2 = norm_layer(width)
        self.conv3 = conv1x1(width, planes * expansion)
        self.bn3 = norm_layer()
        # self.bn3 = norm_layer(planes * expansion)
        self.relu = ReLU()
        self.downsample = downsample
        self.stride = stride

    def __call__(self, x: tf.Variable) -> tf.Variable:
        identity = x

        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        out = self.conv3(out)
        out = self.bn3(out)

        if self.downsample is not None:
            identity = self.downsample(x)

        out += identity
        out = self.relu(out)

        return out

class _DenseLayer(tf.Module):
    def __init__(self, num_input_features, growth_rate, bn_size, drop_rate, efficient=False):
        super(_DenseLayer, self).__init__()
        self.add_module('norm1', BatchNormalization(num_input_features)),
        self.add_module('relu1', ReLU()),
        self.add_module('conv1', Conv2D(bn_size * growth_rate,
                        kernel_size=(1,1), strides=(1,1), use_bias=False)),
        self.add_module('norm2', BatchNormalization(bn_size * growth_rate)),
        self.add_module('relu2', ReLU()),
        self.add_module('conv2', Conv2D(growth_rate,
                        kernel_size=(3,3), strides=(1,1), padding="same", use_bias=False)),
        self.drop_rate = drop_rate
        self.efficient = efficient

    def __call__(self, *prev_features):
        bn_function = _bn_function_factory(self.norm1, self.relu1, self.conv1)
        if self.efficient and any(prev_feature.requires_grad for prev_feature in prev_features):
            bottleneck_output = tf.keras.Model.save_weights.checkpoint(bn_function, *prev_features)
        else:
            bottleneck_output = bn_function(*prev_features)
        new_features = self.conv2(self.relu2(self.norm2(bottleneck_output)))
        if self.drop_rate > 0:
            new_features = Dropout(new_features, p=self.drop_rate, training=self.training)
        return new_features

class BasicDenseLayer(tf.Module):
    def __init__(self, planes, bn_size):
        super(BasicDenseLayer, self).__init__()
        self.add_module('norm1', BatchNormalization(planes)),
        self.add_module('relu1', ReLU()),
        self.add_module('conv1', Conv2D(bn_size * planes,
                        kernel_size=(1,1), strides=(1,1), use_bias=False)),
        self.add_module('norm2', BatchNormalization(bn_size * planes)),
        self.add_module('relu2', ReLU()),
        self.add_module('conv2', Conv2D(planes,
                        kernel_size=(3,3), strides=(1,1), padding="same", use_bias=False)),

    def __call__(self, *prev_features):
        bn_function = _bn_function_factory(self.norm1, self.relu1, self.conv1)
        bottleneck_output = bn_function(*prev_features)
        new_features = self.conv2(self.relu2(self.norm2(bottleneck_output)))
        return new_features

class Transition(Sequential):
    def __init__(self, num_input_features, num_output_features):
        super(Transition, self).__init__()
        self.add_module('norm', BatchNormalization(num_input_features))
        self.add_module('relu', ReLU())
        self.add_module('conv', Conv2D(num_output_features,
                                          kernel_size=(1,1), strides=(1,1), use_bias=False))
        self.add_module('pool', AveragePooling2D(pool_size=(2,2), strides=2))

class DenseBlock(tf.Module):
    def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate, efficient=False):
        super(DenseBlock, self).__init__()
        for i in range(num_layers):
            layer = _DenseLayer(
                num_input_features + i * growth_rate,
                growth_rate=growth_rate,
                bn_size=bn_size,
                drop_rate=drop_rate,
                efficient=efficient,
            )
            self.add_module('denselayer%d' % (i + 1), layer)

    def __call__(self, init_features):
        features = [init_features]
        for name, layer in self.named_children():
            new_features = layer(*features)
            features.append(new_features)
        return tf.concat(features, 1)

class Cell(tf.Module):

  def __init__(self, genotype, C_prev_prev, C_prev, C, reduction, reduction_prev):
    super(Cell, self).__init__()
    print(C_prev_prev, C_prev, C)

    if reduction_prev:
      self.preprocess0 = FactorizedReduce(C_prev_prev, C)
    else:
      self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1, 1, 0)
    self.preprocess1 = ReLUConvBN(C_prev, C, 1, 1, 0)

    if reduction:
      op_names, indices = zip(*genotype.reduce)
      concat = genotype.reduce_concat
    else:
      op_names, indices = zip(*genotype.normal)
      concat = genotype.normal_concat
    self._compile(C, op_names, indices, concat, reduction)

  def _compile(self, C, op_names, indices, concat, reduction):
    assert len(op_names) == len(indices)
    self._steps = len(op_names) // 2
    self._concat = concat
    self.multiplier = len(concat)
    self._ops = tf.Module()
    # self._ops = nn.ModuleList()
    for name, index in zip(op_names, indices):
      stride = 2 if reduction and index < 2 else 1
      op = OPS[name](C, stride, True)
      self._ops += [op]
    self._indices = indices

  def __call__(self, s0, s1, drop_prob):
    s0 = self.preprocess0(s0)
    s1 = self.preprocess1(s1)

    states = [s0, s1]
    for i in range(self._steps):
      h1 = states[self._indices[2*i]]
      h2 = states[self._indices[2*i+1]]
      op1 = self._ops[2*i]
      op2 = self._ops[2*i+1]
      h1 = op1(h1)
      h2 = op2(h2)
      if self.training and drop_prob > 0.:
        if not isinstance(op1, Identity):
          h1 = drop_path(h1, drop_prob)
        if not isinstance(op2, Identity):
          h2 = drop_path(h2, drop_prob)
      s = h1 + h2
      states += [s]
    return tf.concat([states[i] for i in self._concat], dim=1)


import PySimpleGUI as sg
sg.theme('Dark Blue 3')
#to define a function which is used to create number in tens ,ones
#for e.g. cov(23,4) will give 234
def calculate(d):
    start_num=""
    end_num=""
    result=0
    for i in range (len(d)+1):
        if d[i] in ('+','-','*','/'):
            constant=d[i]
            break
        else:
            continue
    for j in range(len(d)):
        if(j<i):
            start_num=start_num+d[j]
        elif(j>i):
            end_num=end_num+d[j]
        else:
            i=i
    while(constant in ('+','-','*','/')):
        if(constant=='+'):
            result=float(start_num)+float(end_num)
            break
        elif(constant=='-'):
            result=float(start_num)-float(end_num)
            break
        elif(constant=='*'):
            result=float(start_num)*float(end_num)
            break
        elif(constant=='/'):
            result=float(start_num)/ float(end_num)
            break
        else:
            break
    return result

# please make your creations colorful
#layout = [[sg.Text("@"*45)], [sg.Text('LIC Lifeinsurance')], [sg.Input("Enter your name here:-")], [sg.Input()],[sg.FileBrowse()], [sg.OK(), sg.Cancel()]]
layout1 = [[sg.Text("_"*120)], [sg.Text('        Calculator       ')],
           [sg.Text("|         "),sg.Button("1",size=(5,1)),sg.Button("2",size=(5,1)),sg.Button("3",size=(5,1)),sg.Button("+ Add"),sg.Text(" "*100+"|")],
           [sg.Text("|         "),sg.Button("4",size=(5,1)),sg.Button("5",size=(5,1)),sg.Button("6",size=(5,1)),sg.Button("-  Sub"),sg.Text(" "*100+"|")],
           [sg.Text("|         "),sg.Button("7",size=(5,1)),sg.Button("8",size=(5,1)),sg.Button("9",size=(5,1)),sg.Button("*  Mul"),sg.Text(" "*100+"|")],
           [sg.Text("|            "),sg.Text("        "),sg.Button("0",size=(5,1)),sg.Text("           "),sg.Button("/ Div",size=(5,1)),sg.Text(" "*100+"|")],
           [sg.Text('',size=(105,1),key="cal",background_color="Black")],
           [sg.Button("=",size=(5,1)),sg.OK(), sg.Cancel()]]
window = sg.Window('Deepak Calculator', layout1)
d='' #used for stroring the command which user give
s=0  #used to store the starting number
e=0  #used to store the ending number
times=0 #no. of times loop used
flag=0 #used for number or string
while True:
    event, values = window.Read()
    if(event == '0' or event == '1' or event == '2' or event == '3' or event == '4' or event == '5'
       or event == '6' or event == '7' or event == '8' or event == '9'):
        print("Got it",event)
        d=d+event
        window["cal"].Update(d)
    elif(event=='+ Add'):
        print("Got it",event)
        d=d+"+"
        window["cal"].Update(d)
        #e+=d[0]
    elif(event=='- Sub'):
        print("Got it",event)
        d=d+"-"
        window["cal"].Update(d)
        #e-=d[0]
    elif(event=='* Mul'):
        print("Got it",event)
        d=d+"*"
        window["cal"].Update(d)
        #e+=d[0]
    elif(event=='/ Div'):
        print("Got it",event)
        d=d+"/"
        window["cal"].Update(d)
        #e+=d[0]
    elif (event == "="):
        print("Got it",event)
        result=calculate(d)
        d=str(result)
        print(result)
        window["cal"].Update(result)
    elif (event == "OK" or event == "Cancel"):
        print(event)
        window.Close()
    else:
        print("Got it",event)
        window.Close()
        break
#print("Why stopped me you $##@")
'''if(values[0]==''):
    print("Enter your NAME")
if(values[1]==''):
    print("Enter your AGE")
if(event=="OK"):
    print("Give me input $#%%")'''

'''print(event,values)
print(d)'''


from lib.database_handler import DatabaseDrivenObject
from PIL import Image
from werkzeug.utils import secure_filename
from flask import request
from lib.util.hash import sha256
from lib.settings import Settings
from typing import Dict, Tuple
from datetime import datetime
import lib.util.exceptions as APIExceptions
import os
import ast
from lib.user import User

class Lot(DatabaseDrivenObject):
    @staticmethod
    def get_settings():
        return Settings.get_enter_settings()['lot']

    @staticmethod
    def serialize_lot(lot_data: Tuple):
        lot = Lot(lot_data[0])
        user = User(lot_data[3])

        res = {
            'id': lot_data[0],
            'date': lot_data[1],
            'name': lot_data[2],
            'user': lot_data[3],
            'user_display_name': user.get_display_name(),
            'user_avatar': user.get_avatar_link(),
            'amount': lot_data[4],
            'currency': lot_data[5],
            'term': lot_data[6],
            'return_way': lot_data[7],
            'security': lot_data[8],
            'percentage': lot_data[9],
            'form': lot_data[10],
            'security_checked': ast.literal_eval(lot_data[11]),
            'guarantee_percentage': lot_data[12],
            'confirmed': ast.literal_eval(lot_data[13]),
            'commentary': lot_data[15],
            'photos': lot.get_photos(),
            'taken': lot.check_taken()
        }

        if lot.removed_by_moderator():
            res['remove_reason'] = lot.get_remove_reason()

        res['verification_requested'] = lot.security_verification_requested()
        res['club_guarantee_requested'] = lot.club_guarantee_requested()

        return res

    def __init__(self, lot_id: int):
        super().__init__()
        self.lot_id = lot_id

        if not self.exists(self.lot_id):
            raise APIExceptions.LotNotExistsError(self.lot_id)

    def club_guarantee_requested(self):
        self.cursor.execute(
            f"SELECT * FROM LotGuaranteeRequests WHERE `id` = ?",
            (self.lot_id,)
        )

        return self.cursor.fetchone() is not None

    def security_verification_requested(self):
        self.cursor.execute(
            f"SELECT * FROM LotSecurityVerificationRequests WHERE `id` = ?",
            (self.lot_id,)
        )

        return self.cursor.fetchone() is not None

    def get_remove_reason(self):
        self.cursor.execute(
            f"SELECT `reason` FROM LotVerificationDeclines WHERE `id` = ?",
            (self.lot_id, )
        )

        return self.cursor.fetchone()[0]

    def check_taken(self):
        self.cursor.execute(
            f"SELECT * FROM ConfirmedSubscriptions WHERE `lot` = ?",
            (self.lot_id, )
        )
        return self.cursor.fetchall() != []

    def get_photos_list(self):
        self.cursor.execute(
            f"SELECT `photos` FROM Lots WHERE `id` = ?",
            (self.lot_id, )
        )

        return eval(self.cursor.fetchone()[0])

    def get_photos(self):
        return {
            'lot_id': self.lot_id,
            'photos': [f'{request.host_url}image/lot/{photo}.jpg' for photo in self.get_photos_list()]
        }

    def exists(self, lot_id):
        self.cursor.execute(
            f"SELECT * FROM Lots WHERE `id` = ?",
            (lot_id, )
        )

        return self.cursor.fetchone() is not None

    def removed_by_moderator(self):
        self.cursor.execute(
            f"SELECT * FROM LotVerificationDeclines WHERE `id` = ?",
            (self.lot_id, )
        )

        return self.cursor.fetchall() != []

    def remove_moderator_delete_reason(self):
        self.cursor.execute(
            f"DELETE FROM LotVerificationDeclines WHERE `id` = ?",
            (self.lot_id, )
        )

        self.conn.commit()

    def approve(self):
        if self.removed_by_moderator():
            self.remove_moderator_delete_reason()

        self.cursor.execute(
            f"UPDATE Lots SET `confirmed` = 'True' WHERE `id` = ?",
            (self.lot_id, )
        )
        self.conn.commit()
        self.logger.info(f'New lot with id `{self.lot_id}` was approved')

    def unapprove(self):
        self.cursor.execute(
            f"UPDATE Lots SET `confirmed` = 'False' WHERE `id` = ?",
            (self.lot_id, )
        )
        self.conn.commit()

    def set_security_checked(self, checked: bool):
        self.cursor.execute(
            f"UPDATE Lots SET `security_checked` = ? WHERE `id` = ?",
            (str(checked), self.lot_id)
        )
        self.conn.commit()
        self.logger.debug(f'`Security checked` flag on lot with id `{self.lot_id}` is set to `{checked}`')

    def get_lot_data(self):
        self.cursor.execute(
            f"SELECT * FROM Lots WHERE `id` = ?",
            (self.lot_id, )
        )
        lot = self.cursor.fetchone()

        return self.serialize_lot(lot)

    def delete_lot(self):
        self.cursor.execute(
            f"UPDATE Lots SET `deleted` = 'True' WHERE `id` = ?",
            (self.lot_id, )
        )
        self.conn.commit()

    def add_moderator_delete_reason(self, moderator, reason):
        if self.removed_by_moderator():
            self.remove_moderator_delete_reason()

        self.cursor.execute(
            f"INSERT INTO LotVerificationDeclines VALUES(?,?,?)",
            (self.lot_id, reason, moderator.email)
        )

        self.conn.commit()

    def delete_lot_by_moderator(self, moderator, reason):
        self.add_moderator_delete_reason(moderator, reason)
        self.delete_lot()

    def restore_lot(self):
        self.cursor.execute(
            f"UPDATE Lots SET `deleted` = 'False' WHERE `id` = ?",
            (self.lot_id, )
        )
        self.conn.commit()

    def update_data(self, field, value):
        self.unapprove()

        self.cursor.execute(
            f"UPDATE Lots SET `{field}` = ? WHERE `id` = ?",
            (value, self.lot_id)
        )
        self.conn.commit()

    def get_lot_creator(self):
        self.cursor.execute(
            f"SELECT `user` FROM Lots WHERE `id` = ?",
            (self.lot_id, )
        )

        return self.cursor.fetchone()[0]

    def can_user_edit(self, user):
        return self.get_lot_creator() == user.email

    def add_photo(self, image):
        self.unapprove()

        self.create_directory_if_not_exists('data/images/temp')
        self.create_directory_if_not_exists('data/images/lots')

        temporary_file_location = f'data/images/temp/{secure_filename(image.filename)}'
        image.save(temporary_file_location)

        im = Image.open(temporary_file_location)
        photo_hash = sha256(str(datetime.now()))
        file_location = f'data/images/lots/{photo_hash}.jpg'
        im = im.convert("RGB")
        im.save(file_location)

        os.remove(temporary_file_location)

        self.cursor.execute(
            f"SELECT `photos` FROM Lots WHERE `id` = ?",
            (self.lot_id, )
        )
        photos = eval(self.cursor.fetchone()[0])
        photos.append(photo_hash)

        stringified_photos = str(photos).replace('\'', '"')

        self.cursor.execute(
            f"UPDATE Lots SET `photos` = ? WHERE `id` = ?",
            (stringified_photos, self.lot_id)
        )
        self.conn.commit()

        return 'Photo is successfuly added.'

    def remove_photo(self, photo_id):
        self.cursor.execute(
            f"SELECT `photos` FROM Lots WHERE `id` = ?",
            (self.lot_id, )
        )

        photos: list = eval(self.cursor.fetchone()[0])
        photos.pop(photo_id)
        stringified_photos = str(photos).replace('\'', '"')

        self.cursor.execute(
            f"UPDATE Lots SET `photos` = ? WHERE `id` = ?",
            (stringified_photos, self.lot_id)
        )
        self.conn.commit()

        return 'Photo is successfuly removed.'

    def is_archived(self) -> bool:
        self.cursor.execute(
            f"SELECT `deleted` FROM Lots WHERE `id` = ?",
            (self.lot_id, )
        )

        return self.cursor.fetchone()[0] == 'True'

    def delete_entirely(self):
        if not self.is_archived():
            raise APIExceptions.LotDeletionError()

        photos_list = self.get_photos_list()

        for photo in photos_list:
            os.remove(f'data/images/lots/{photo}.jpg')

        self.delete_data()

    def delete_data(self):
        self.cursor.execute(
            f"DELETE FROM LotVerificationDeclines WHERE `id` = ?",
            (self.lot_id, )
        )

        self.cursor.execute(
            f"DELETE FROM Lots WHERE `id` = ?",
            (self.lot_id, )
        )

        self.conn.commit()

    def set_guarantee_value(self, value):
        self.cursor.execute(
            f"UPDATE Lots SET `guarantee_percentage` = ? WHERE `id` = ?",
            (value, self.lot_id)
        )

        self.conn.commit()

    def set_lot_guarantee_requested(self, requested: bool = True):
        if requested:
            self.cursor.execute(
                f"INSERT OR IGNORE INTO LotGuaranteeRequests VALUES(?)",
                (self.lot_id, )
            )
        else:
            self.cursor.execute(
                f"DELETE FROM LotGuaranteeRequests WHERE `id` = ?",
                (self.lot_id, )
            )

        self.conn.commit()

    def set_lot_security_verification_requested(self, requested: bool = True):
        if requested:
            self.cursor.execute(
                f"INSERT OR IGNORE INTO LotSecurityVerificationRequests VALUES(?)",
                (self.lot_id, )
            )
        else:
            self.cursor.execute(
                f"DELETE FROM LotSecurityVerificationRequests WHERE `id` = ?",
                (self.lot_id, )
            )

        self.conn.commit()

    def request_for_guarantee(self):
        self.set_lot_guarantee_requested()

    def request_for_security_verification(self):
        self.set_lot_security_verification_requested()

    def remove_request_for_guarantee(self):
        self.set_lot_guarantee_requested(requested=False)

    def remove_request_for_security_verification(self):
        self.set_lot_security_verification_requested(requested=False)

class LotListGatherer(DatabaseDrivenObject):
    @staticmethod
    def __check_filter(lot_filter: Dict) -> Dict:
        from app import RestAPI

        settings = Settings.get_lot_filter_settings()
        lot_settings = Settings.get_enter_settings()['lot']

        result_filter = {}

        available_types = settings['available_types']
        available_types['order_by'] = list(lot_settings.keys()) + ['user', 'date', 'guarantee_percentage']
        available_types['order_type'] = ['ASC', 'DESC']

        for value in available_types:
            result_filter[value] = str(lot_filter[value]) if value in lot_filter else None
            if result_filter[value] is not None:
                RestAPI.validate_field(value, result_filter[value], available_types[value])

        result_filter['limit'] = result_filter['limit'] or settings['maximum_length']

        # limit check
        if int(result_filter['limit']) > settings['maximum_length']:
            raise APIExceptions.LotFiltrationError(f"Could not load {lot_filter['limit']} lots. Maximum is {settings['maximum_length']}")

        show_only = {
            key: value for key, value in filter(lambda i: isinstance(i[1], list), lot_settings.items())
        }

        if 'show_only' in lot_filter:
            if not isinstance(lot_filter['show_only'], dict):
                raise APIExceptions.LotFiltrationError(f"show_only field in lot filtration should be a Map[str, List[str]], but got `{lot_filter['show_only']}`")

            result_filter['show_only'] = {}

            for key, value in lot_filter['show_only'].items():
                result_filter['show_only'][key] = value

                if key not in show_only:
                    raise APIExceptions.LotFiltrationError(f"{key} is not a valid group. Available groups are: {', '.join(show_only.keys())}")

                if not isinstance(value, list) or any([not isinstance(v, str) for v in value]):
                    raise APIExceptions.LotFiltrationError(f"show_only field in lot filtration should be a Map[str, List[str]], but got `{lot_filter['show_only']}`")

                for v in value:
                    if v not in show_only[key]:
                        raise APIExceptions.LotFiltrationError(f"{v} is not available for {key} group. Available values are: {', '.join(show_only[key])}")

            if result_filter['show_only'] == {}:
                result_filter['show_only'] = None
        else:
            result_filter['show_only'] = None

        return result_filter

    @staticmethod
    def _format_sql_lot_filter_string(lot_filter, where_is_already_used: bool = False) -> str:
        res = ""
        if lot_filter['show_only'] is not None:
            res += (' WHERE ' if not where_is_already_used else ' AND ') + ' AND '.join(
                [' OR '.join([f"`{type}` = '{value}'" for value in lot_filter['show_only'][type]]) for type in lot_filter['show_only']]
            )
        if lot_filter['order_by'] is not None:
            res += f" ORDER BY `{lot_filter['order_by']}` {lot_filter['order_type'] if lot_filter['order_type'] is not None else ''}"
        if lot_filter['limit'] is not None:
            res += f" LIMIT {lot_filter['limit']}"
        if lot_filter['offset'] is not None:
            res += f" OFFSET {lot_filter['offset']}"
        return res

    def __init__(self, lot_filter = {}):
        super().__init__()
        self.lot_filter = self.__check_filter(lot_filter)

    def get_requested_for_guarantee(self):
        self.cursor.execute(
            "SELECT * FROM LotsWithGuaranteeRequested" + self._format_sql_lot_filter_string(self.lot_filter)
        )

        return [Lot.serialize_lot(lot) for lot in self.cursor.fetchall()]

    def get_requested_for_security_verification(self):
        self.cursor.execute(
            "SELECT * FROM LotsWithSecurityVerificationRequested" + self._format_sql_lot_filter_string(self.lot_filter)
        )

        return [Lot.serialize_lot(lot) for lot in self.cursor.fetchall()]

    def get_all_approved_lots(self):
        self.cursor.execute(
            "SELECT * FROM LiveLots" + self._format_sql_lot_filter_string(self.lot_filter)
        )

        return [Lot.serialize_lot(lot) for lot in self.cursor.fetchall()]

    def get_all_unapproved_lots(self):
        self.cursor.execute(
            f"SELECT * FROM LiveUnacceptedLots" + self._format_sql_lot_filter_string(self.lot_filter)
        )

        return [Lot.serialize_lot(lot) for lot in self.cursor.fetchall()]

    def get_archive(self):
        self.cursor.execute(
            "SELECT * FROM ArchiveLatestLots" + self._format_sql_lot_filter_string(self.lot_filter)
        )

        return [Lot.serialize_lot(lot) for lot in self.cursor.fetchall()]

    def get_archived_history(self, lot_id):
        self.cursor.execute(
            "SELECT * FROM LotsArchive" + self._format_sql_lot_filter_string(self.lot_filter)
        )

        table = self.cursor.fetchall()
        res_list = []

        for row in table:
            row = list(row)
            row[0], row[17] = row[17], row[0]
            res = Lot.serialize_lot(row)
            res['record_id'] = row[17]
            res['approve_date'] = row[18]
            res_list.append(res)

        return res_list

class UsersLotListGatherer(LotListGatherer):
    def __init__(self, user, lot_filter={}):
        super().__init__(lot_filter=lot_filter)
        self.user = user

    def get_favorites(self):
        self.cursor.execute(
            f"SELECT `favorite_lots` FROM UsersLots WHERE `email` = ?" +
            self._format_sql_lot_filter_string(self.lot_filter, where_is_already_used=True),
            (self.user, )
        )

        res: list = eval(self.cursor.fetchone()[0])

        return [Lot(lot_id).get_lot_data() for lot_id in reversed(res)]

    def get_personal(self):
        self.cursor.execute(
            "SELECT * FROM Lots "
            "WHERE `user` = ? AND `deleted` = 'False' "
            "AND `id` NOT IN ConfirmedLots "
            "AND `id` NOT IN FinishedLots" +
            self._format_sql_lot_filter_string(self.lot_filter, where_is_already_used=True),
            (self.user.email, )
        )

        return [Lot.serialize_lot(lot) for lot in self.cursor.fetchall()]

    def get_personal_confirmed(self):
        self.cursor.execute(
            "SELECT * FROM Lots WHERE `user` = ? AND `deleted` = 'False' "
            "AND `id` IN ConfirmedLots " +
            self._format_sql_lot_filter_string(self.lot_filter, where_is_already_used=True),
            (self.user.email, )
        )

        return [Lot.serialize_lot(lot) for lot in self.cursor.fetchall()]

    def get_personal_finished(self):
        self.cursor.execute(
            f"SELECT * FROM Lots WHERE `user` = ? AND `deleted` = 'False' "
            "AND `id` IN FinishedLots " +
            self._format_sql_lot_filter_string(self.lot_filter, where_is_already_used=True),
            (self.user.email, )
        )

        return [Lot.serialize_lot(lot) for lot in self.cursor.fetchall()]

    def get_personal_deleted(self):
        self.cursor.execute(
            f"SELECT * FROM Lots WHERE `user` = ? and `deleted` = 'True'" +
            self._format_sql_lot_filter_string(self.lot_filter, where_is_already_used=True),
            (self.user.email, )
        )

        return [Lot.serialize_lot(lot) for lot in self.cursor.fetchall()]


#!/usr/bin/env python
# coding: utf-8

# In[7]:

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.init as init
import pandas as pd
import numpy as np
from torch.utils.data import DataLoader, Dataset
import os, pickle
import argparse
from torch.utils.tensorboard import SummaryWriter
from sklearn.metrics import classification_report
import logging

# In[14]:

class BreathData_cycle(Dataset):
    def __init__(self, data_path, mode="train", val_ratio=0.2, test_ratio=0.2):
        with open(data_path, "rb") as file:
            data_dict = pickle.load(file)
        self.mode = mode
        all_data, all_labels = [],[]
        self.people = [i for i in data_dict.keys()]
        self.label_mapping = {self.people[i]:i for i in range(len(self.people))}
        label2person = {self.label_mapping[person] for person in self.label_mapping.keys()}
        self.save_mapping = {"person2label":self.label_mapping, "label2person":label2person}
        for person, this_data in data_dict.items():
            this_cycle = len(this_data)
            for each in this_data:
                print(each.shape)
            all_data += this_data
            all_labels += [self.label_mapping[person]] * this_cycle
        all_labels = np.asarray(all_labels)

        num_data = len(all_labels)
        train_ratio = 1 - val_ratio - test_ratio
        np.random.seed(0)
        random_idx = np.random.permutation(num_data)
        train_end_idx, val_end_idx = round(num_data * train_ratio), round(num_data * (train_ratio+val_ratio))
        train_idx, val_idx = random_idx[:train_end_idx], random_idx[train_end_idx:val_end_idx]
        if test_ratio != 0.0:
            test_idx = random_idx[val_end_idx:]
        if self.mode=="train":
            self.data = [all_data[i] for i in train_idx]
            self.labels = all_labels[train_idx]
        elif self.mode=="val":
            self.data = [all_data[i] for i in val_idx]
            self.labels = all_labels[val_idx]
        elif self.mode=="test":
            self.data = [all_data[i] for i in test_idx]
            self.labels = all_labels[test_idx]

        print(self.mode)
        #for each in self.data:
        #    print(each.shape)

    def __len__(self):
        return len(self.labels)

    def __getitem__(self,item):
        return self.data[item], self.labels[item]

class BreathData_cycle2(Dataset):
    def __init__(self, data_path, mode="train", val_ratio=0.2, test_ratio=0.2):
        with open(data_path, "rb") as file:
            data_dict = pickle.load(file)
        self.mode = mode
        train_data, train_labels, val_data, val_labels, test_data, test_labels = [],[],[],[],[],[]
        self.people = [i for i in data_dict.keys()]
        self.label_mapping = {self.people[i]:i for i in range(len(self.people))}
        label2person = {self.label_mapping[person]:person for person in self.label_mapping.keys()}
        self.save_mapping = {"person2label":self.label_mapping, "label2person":label2person}
        self.idx_record, self.idx_start = {"train":{}, "val":{}, "test":{}}, {"train":{}, "val":{}, "test":{}}
        idx_start_train, idx_start_val, idx_start_test = 0,0,0
        for itid, (person, this_data) in enumerate(data_dict.items()):

            this_cycle = len(this_data)
            cut_idx_train = int(np.round(this_cycle * (1 - val_ratio - test_ratio)))
            cut_idx_val = int(np.round(this_cycle * (1 - test_ratio)))
            num_val, num_test = cut_idx_val - cut_idx_train, this_cycle - cut_idx_val
            np.random.seed(itid*2)
            rand_idx = np.random.permutation(this_cycle)
            train_idx, val_idx, test_idx = rand_idx[:cut_idx_train], rand_idx[cut_idx_train:cut_idx_val], rand_idx[cut_idx_val:]
            self.idx_record["train"][person] = train_idx
            self.idx_record["val"][person] = val_idx
            self.idx_record["test"][person] = test_idx

            train_data += [this_data[i] for i in train_idx]
            train_labels += [self.label_mapping[person]] * cut_idx_train
            val_data += [this_data[i] for i in val_idx]
            val_labels += [self.label_mapping[person]] * (cut_idx_val - cut_idx_train)
            test_data += [this_data[i] for i in test_idx]
            test_labels += [self.label_mapping[person]] * (this_cycle - cut_idx_val)

            self.idx_start["train"][person] = (idx_start_train, idx_start_train+cut_idx_train)
            self.idx_start["val"][person] = (idx_start_val, idx_start_val+num_val)
            self.idx_start["test"][person] = (idx_start_test, idx_start_test+num_test)

        if self.mode=="train":
            self.data = train_data
            self.labels = train_labels
        elif self.mode=="val":
            self.data = val_data
            self.labels = val_labels
        elif self.mode=="test":
            self.data = test_data
            self.labels = test_labels

        print(self.mode)

        #for each in self.data:
        #    print(each.shape)

    def __len__(self):
        return len(self.labels)

    def __getitem__(self,item):
        return self.data[item], self.labels[item]

class RNN_model(nn.Module):
    def __init__(self, num_classes):
        super(RNN_model, self).__init__()
        self.project_layer = nn.Linear(39, 128)
        self.rnn_layer = nn.LSTM(128,128, batch_first=True)
        self.classifier = nn.Linear(128, num_classes)

    def forward(self, x):
        rnn_outputs, (hn, cn) = self.rnn_layer(self.project_layer(x))
        logits = self.classifier(rnn_outputs[:,-1])
        return logits

# In[4]:
def initialize_weights(model):
    if isinstance(model, nn.LSTM):
        for name, param in model.named_parameters():
            if 'weight_ih' in name:
                torch.nn.init.xavier_uniform_(param.data)
            elif 'weight_hh' in name:
                torch.nn.init.orthogonal_(param.data)
            elif 'bias' in name:
                param.data.fill_(0)

def run_epoch(data_loader, epoch, model, criterion, device, use_cuda=True, mode = 'train',
    optimizer=None, writer=None, idx_record=None):
    if mode == "train":
        model.train()
    elif mode in ['val', 'test']:
        model.eval()
    else:
        print("Wrong mode input!")
        return

    model.to(device)
    FloatTensor = torch.cuda.FloatTensor if use_cuda else torch.FloatTensor
    LongTensor = torch.cuda.LongTensor if use_cuda else torch.LongTensor
    ByteTensor = torch.cuda.ByteTensor if use_cuda else torch.ByteTensor
    avg_loss, avg_acc = 0.0, 0.0
    batches = 0
    output_all, label_all = [],[]
    for batch_idx, sample in enumerate(data_loader):
        data, labels = sample[0], sample[1]
        label_all.append(labels.numpy())
        data = data.type(dtype=FloatTensor).to(device)
        labels = labels.type(dtype=LongTensor).to(device)
        #print(data.size())
        #print(labels.size())
        output = model(data)
        output_class = torch.argmax(output, dim=1)
        output_all.append(output_class.cpu().numpy())
        loss = criterion(output, labels)
        if mode == "train":
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        avg_loss += loss.item()
        avg_acc += (output_class == labels).sum().item()
        batches += labels.size()[0]
    avg_loss /= batches
    avg_acc /= batches
    print("all data num:{}".format(batches))
    output_all, label_all = np.concatenate(output_all), np.concatenate(label_all)

    if writer is not None:
        writer.add_scalar(f"Loss/{mode}", avg_loss, epoch)
        writer.add_scalar(f"Acc/{mode}", avg_acc, epoch)

    print('%s Epoch: %d  , Loss: %.4f,  Accuracy: %.2f'%( mode, epoch, avg_loss, avg_acc))

    return avg_acc, output_all, label_all

def get_fpr(output_all, label_all, label2person):
    person_all = np.unique(label_all)
    fpr_each = {}
    mis_cls_all, total = 0,0
    for person in person_all:
        corr_output = output_all[label_all!=person]
        num_other = len(corr_output)
        mis_cls = (corr_output==person).sum()
        mis_cls_all += mis_cls
        total += num_other
        fpr_each[label2person[person]] = round(mis_cls / num_other)

    fpr = mis_cls_all / total

    return fpr, fpr_each

# In[17]:

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='train for user identity classification')
    parser.add_argument('--base_path', type=str, default="/home/qiaomu/codes/CSE518")
    parser.add_argument('--ckpt_path', type=str, default="/data/add_disk0/qiaomu/ckpts/CSE518")
    parser.add_argument('--optim', type=str, default="SGD")
    parser.add_argument('--epochs', type=int, default=35)
    parser.add_argument('--batch_size', type=int, default=1)
    parser.add_argument('--mode', type=str, default="train")
    parser.add_argument('--lr', type=float, default=0.001)
    parser.add_argument('--weight_decay', type=float, default=5e-5)
    parser.add_argument('--ckpt_name', type=str, default="best_model.pt")

    args = parser.parse_args()
    data_path = os.path.join(args.base_path, "extracted_features_new.pickle")
    train_dataset = BreathData_cycle2(data_path, mode="train")
    val_dataset = BreathData_cycle2(data_path, mode="val")
    save_mapping = train_dataset.save_mapping
    label2person = save_mapping["label2person"]
    target_names = [label2person[key] for key in label2person.keys()]
    num_people = len(train_dataset.people)
    print(f"num_people:{num_people}")
    #print("train dataset indices: {}".format(train_dataset.idx_record))
    #print("val dataset indices: {}".format(val_dataset.idx_record))

    writer = SummaryWriter('./logs')
    train_loader = DataLoader(train_dataset, batch_size= args.batch_size, num_workers=4, shuffle=True)
    val_loader = DataLoader(val_dataset, batch_size= args.batch_size, num_workers=4, shuffle=False)
    model = RNN_model(num_people)
    model.apply(initialize_weights)
    criterion = nn.CrossEntropyLoss()
    use_cuda = torch.cuda.is_available()
    print('use cuda: %s'%(use_cuda))
    if use_cuda:
        device = torch.device('cuda:0')
    else:
        device = torch.device('cpu')
    if args.optim=='SGD':
        optimizer = optim.SGD(model.parameters(),lr=args.lr, weight_decay=args.weight_decay)
    elif args.optim=="Adam":
        optimizer = optim.Adam(model.parameters(),lr=args.lr, weight_decay=args.weight_decay)
    elif args.optim=="RMSprop":
        optimizer = optim.RMSprop(model.parameters(),lr=args.lr, weight_decay=args.weight_decay)

    if not os.path.exists(args.ckpt_path):
        os.makedirs(args.ckpt_path)

    if args.mode=="train":
        for epoch in range(args.epochs):
            acc_train, output_train, label_train = run_epoch(train_loader, epoch, model, criterion, device, use_cuda, mode='train', optimizer=optimizer, writer=writer)
            acc_val, output_val, label_val = run_epoch(val_loader, epoch, model, criterion, device, use_cuda, mode="val", optimizer=optimizer, writer=None)
            fpr_val, fpr_val_each = get_fpr(output_val, label_val, label2person)
            print("Val fpr:{}".format(fpr_val))
            if acc_val > 0.90:
                save_path = os.path.join(args.ckpt_path, f"LSTM_{epoch}_{args.optim}_lr{args.lr}.pt")
                torch.save(model.state_dict(), save_path)
        writer.flush()
        writer.close()

    elif args.mode=="test":
        log_file = os.path.join(args.base_path, "logs", "test_log.log")
        logger = logging.getLogger("logger")
        logger.setLevel(logging.DEBUG)
        fh = logging.FileHandler(log_file)
        fh.setLevel(logging.DEBUG)
        sh = logging.StreamHandler()
        sh.setLevel(logging.INFO)
        logger.addHandler(sh)
        logger.addHandler(fh)

        model.load_state_dict(torch.load(os.path.join(args.ckpt_path, args.ckpt_name)))
        test_dataset = BreathData_cycle2(data_path, mode="test")
        test_loader = DataLoader(test_dataset, batch_size= args.batch_size, num_workers=8, shuffle=False)
        idx_record = test_dataset.idx_record["test"]
        acc_test, output_test, label_test = run_epoch(test_loader, 0, model, criterion, device, use_cuda, mode="test",
            optimizer=optimizer, writer=None, idx_record=idx_record)
        fpr_test, fpr_test_each = get_fpr(output_test, label_test, label2person)

        logger.info("Test TPR: {}, Test FPR:{}".format(acc_test, fpr_test))
        logger.info("Test FPR each: {}".format(fpr_test_each))
        logger.info(classification_report(label_test, output_test, target_names=target_names))

        val_dataset = BreathData_cycle2(data_path, mode="val")
        val_loader = DataLoader(val_dataset, batch_size= args.batch_size, num_workers=8, shuffle=False)
        acc_val, output_val, label_val = run_epoch(val_loader, 0, model, criterion, device, use_cuda, mode="val", optimizer=optimizer, writer=None)
        fpr_val, fpr_val_each = get_fpr(output_test, label_test, label2person)
        logger.info("Val TPR:{}, Val FPR:{}".format(acc_val, fpr_val))
        logger.info("Val FPR each: {}".format(fpr_val_each))
        logger.info(classification_report(label_val, output_val, target_names=target_names))

        with open("label_person_mapping_new.pickle","wb") as file:
            pickle.dump(save_mapping, file, protocol=pickle.HIGHEST_PROTOCOL)
        print(save_mapping)
        print(val_dataset.save_mapping)

# In[ ]:


from concurrent.futures import ThreadPoolExecutor
from instaloader import Instaloader, ProfileNotExistsException, Profile

L = Instaloader()

L.login("USER", "PASS")

def process_profile(L, PROFILE):
    try:
        posts = Profile.from_username(L.context, PROFILE).get_posts()
        try:
            most_recent_post = next(posts)

            post_url = f"https://www.instagram.com/p/{most_recent_post.shortcode}"

            with open('post_urls.txt', 'a') as output_file:
                print("Printing to file...")
                output_file.write(f"1172 | {post_url} | 2 | comment | comment \n")
                print("Done")
        except StopIteration:
            return
    except ProfileNotExistsException:
        return

with open('users.txt') as file:
    with ThreadPoolExecutor() as executor:
        for i in range(2000):#5 = 2 line of urls
            PROFILE = file.readline().strip()
            executor.submit(process_profile, L, PROFILE)


# Initial setup following http://docs.chainer.org/en/stable/tutorial/basic.html
import numpy as np
import chainer
import six
from chainer import cuda, Function, gradient_check, report, training, utils, Variable, computational_graph
from chainer import datasets, iterators, optimizers, serializers
from chainer import Link, Chain, ChainList
import chainer.functions as F
import chainer.links as L
from chainer.training import extensions

import matplotlib.pyplot as plt
import argparse

parser = argparse.ArgumentParser(description='Chainer example: MNIST')
parser.add_argument('--initmodel', '-m', default='',
                    help='Initialize the model from given file')
parser.add_argument('--batchsize', '-b', type=int, default=100,
                    help='Number of images in each mini-batch')
parser.add_argument('--epoch', '-e', type=int, default=20,
                    help='Number of sweeps over the dataset to train')
parser.add_argument('--gpu', '-g', type=int, default=-1,
                    help='GPU ID (negative value indicates CPU)')
parser.add_argument('--out', '-o', default='result/2',
                    help='Directory to output the result')
parser.add_argument('--resume', '-r', default='',
                    help='Resume the training from snapshot')
parser.add_argument('--n_units', '-u', type=int, default=50,
                    help='Number of units')

train, test = chainer.datasets.get_mnist()
args = parser.parse_args(['-g', '0'])

batchsize = args.batchsize
n_epoch = args.epoch
N = len(train)  # training data size
N_test = len(test)  # test data size

class MyChain(Chain):
    def __init__(self, n_units, n_out):
        super(MyChain, self).__init__()
        with self.init_scope():
            self.l1 = L.Linear(None, n_units)
            self.l2 = L.Linear(None, n_units)
            self.l3 = L.Linear(None, n_out)

    def __call__(self, x):
        h1 = F.relu(self.l1(x))
        h2 = F.relu(self.l2(h1))
        return self.l3(h2)

class SoftmaxCLassifier(Chain):
    def __init__(self, predictor):
        super(SoftmaxCLassifier, self).__init__(
            predictor=predictor
        )

    def __call__(self, x, t):
        # 在这里定义了损失函数的计算的话，就可以将此模型设置为优化器来训练
        y = self.predictor(x)
        self.loss = F.softmax_cross_entropy(y, t)
        self.accuracy = F.accuracy(y, t)
        return self.loss

# init
n_units = args.n_units
model = MyChain(n_units, 10)
classifier_model = SoftmaxCLassifier(model)
# Setup an optimizer
optimizer = chainer.optimizers.Adam()
optimizer.setup(classifier_model)
# Pass the loss function (Classifier defines it) and its arguments
# optimizer.update(classifier_model, x, t)

# 如果不是使用gpu就把变量gpu设置为-1
gpu = 0
if gpu >= 0:
    chainer.cuda.get_device(gpu).use()  # Make a specified GPU current
    classifier_model.to_gpu()  # Copy the model to the GPU
#     cuda.cupy相当于numpy的gpu版本
xp = np if gpu < 0 else cuda.cupy

if args.initmodel:
    print('Load model from', args.initmodel)
    serializers.load_npz(args.initmodel, classifier_model)
if args.resume:
    print('Load optimizer state from', args.resume)
    serializers.load_npz(args.resume, optimizer)
import os
if not os.path.exists(args.out):
    os.makedirs(args.out)
import time
for epoch in range(1, n_epoch + 1):
    print('epoch', epoch)

    # training
    perm = np.random.permutation(N)
    sum_accuracy = 0
    sum_loss = 0
    start = time.time()
    for i in range(0, N, batchsize):
        x = chainer.Variable(xp.asarray(train[perm[i:i + batchsize]][0]))
        t = chainer.Variable(xp.asarray(train[perm[i:i + batchsize]][1]))

        # Pass the loss function (Classifier defines it) and its arguments
        optimizer.update(classifier_model, x, t)

        if epoch == 1 and i == 0:
            with open('{}/graph.dot'.format(args.out), 'w') as o:
                g = computational_graph.build_computational_graph(
                    (classifier_model.loss,))
                o.write(g.dump())
            print('graph generated')

        sum_loss += float(classifier_model.loss.data) * len(t.data)
        sum_accuracy += float(classifier_model.accuracy.data) * len(t.data)
    end = time.time()
    elapsed_time = end - start
    throughput = N / elapsed_time
    print('train mean loss={}, accuracy={}, throughput={} images/sec'.format(
        sum_loss / N, sum_accuracy / N, throughput))

    # evaluation
    sum_accuracy = 0
    sum_loss = 0
    for i in six.moves.range(0, N_test, batchsize):
        index = np.asarray(list(range(i, i + batchsize)))
        x = chainer.Variable(xp.asarray(test[index][0]))
        t = chainer.Variable(xp.asarray(test[index][1]))
        with chainer.no_backprop_mode():
            # When back propagation is not necessary,
            # we can omit constructing graph path for better performance.
            # `no_backprop_mode()` is introduced from chainer v2,
            # while `volatile` flag was used in chainer v1.
            loss = classifier_model(x, t)
        sum_loss += float(loss.data) * len(t.data)
        sum_accuracy += float(classifier_model.accuracy.data) * len(t.data)

    print('test mean loss={}, accuracy={}'.format(
        sum_loss / N_test, sum_accuracy / N_test))


import re
from copy import copy
from tqdm import tqdm

with open('input.txt', 'r') as f:
    data = f.read()

test = """Valve AA has flow rate=0; tunnels lead to valves DD, II, BB
Valve BB has flow rate=13; tunnels lead to valves CC, AA
Valve CC has flow rate=2; tunnels lead to valves DD, BB
Valve DD has flow rate=20; tunnels lead to valves CC, AA, EE
Valve EE has flow rate=3; tunnels lead to valves FF, DD
Valve FF has flow rate=0; tunnels lead to valves EE, GG
Valve GG has flow rate=0; tunnels lead to valves FF, HH
Valve HH has flow rate=22; tunnel leads to valve GG
Valve II has flow rate=0; tunnels lead to valves AA, JJ
Valve JJ has flow rate=21; tunnel leads to valve II"""

class Node:
    def __init__(self, name, flow_rate):
        self.name = name
        self.flow_rate = flow_rate
        self.tunnels = []
        self.distances = None

    def add_tunnel(self, tunnel):
        self.tunnels.append(tunnel)

    def __repr__(self):
        return f'{self.name}'

cache = set()

def get_part_one(data):
    lines = data.split('\n')

    nodes = {}
    for line in lines:
        aa = re.findall(
            r"Valve (\w\w) has flow rate=(\d+); tunnel[s]? lead[s]? to valve[s]? (.*)", line)
        nodes[aa[0][0]] = {'flow_rate': int(
            aa[0][1]), 'tunnels': aa[0][2].split(', ')}

    nodes_objs = {}

    for node in nodes:
        nodes_objs[node] = Node(node, nodes[node]['flow_rate'])

    for node in nodes:
        for tunnel in nodes[node]['tunnels']:
            nodes_objs[node].add_tunnel(tunnel)

    for node in nodes:
        distances = get_distance(nodes_objs[node], nodes_objs)
        nodes_objs[node].distances = distances

    current_node = nodes_objs['AA']

    visit_mask = {k: False for k in nodes_objs}

    return search(current_node, 30, nodes_objs, 0, visit_mask)

def search(current_node, time, nodes, score, visit_mask):

    if (current_node, time, tuple(visit_mask.values())) in cache:
        return score

    visit_mask = copy(visit_mask)

    if time <= 0:
        return score

    scores = []

    score += current_node.flow_rate*time
    visit_mask[current_node.name] = True
    scores.append(score)

    for k in current_node.distances:
        if visit_mask[k] == False:
            scores.append(
                search(nodes[k], time-(current_node.distances[k]+1), nodes, score, visit_mask))

    cache.add((current_node, time, tuple(visit_mask.values())))
    return max(scores)

def get_distance(current_node, nodes):
    distance = 1
    distances = {}

    nn = [current_node.name]
    while len(distances) < len(nodes):
        tmp = []
        for n in nn:
            tmp = tmp + nodes[n].tunnels

        nn = list(set(tmp))
        for n in nn:
            if n not in distances:
                distances[n] = distance
        distance += 1
    distances = {k: v for k, v in distances.items(
    ) if nodes[k].flow_rate != 0 and k != current_node.name}
    return distances

assert get_part_one(test) == 1651
print(f'Part 1: {get_part_one(data)}')

# part 2

def int_to_visit_mask(n, nodes, bit_wise_not=False):
    mask = {}
    for i, node in enumerate(nodes):
        a = bool(n & (1 << i))
        if bit_wise_not:
            a = not a
        mask[node] = a
    return mask

def get_part_two(data):
    lines = data.split('\n')

    nodes = {}
    for line in lines:
        aa = re.findall(
            r"Valve (\w\w) has flow rate=(\d+); tunnel[s]? lead[s]? to valve[s]? (.*)", line)
        nodes[aa[0][0]] = {'flow_rate': int(
            aa[0][1]), 'tunnels': aa[0][2].split(', ')}

    nodes_objs = {}

    for node in nodes:
        nodes_objs[node] = Node(node, nodes[node]['flow_rate'])

    for node in nodes:
        for tunnel in nodes[node]['tunnels']:
            nodes_objs[node].add_tunnel(tunnel)

    for node in nodes:
        distances = get_distance(nodes_objs[node], nodes_objs)
        nodes_objs[node].distances = distances

    current_node = nodes_objs['AA']

    tbd = [k for k in nodes_objs if nodes_objs[k].flow_rate != 0 or k == 'AA']

    max_ = 0

    for i in tqdm(range(2**(len(tbd)))):
        mask = int_to_visit_mask(i, tbd)
        not_mask = int_to_visit_mask(i, tbd, bit_wise_not=True)
        l = search(current_node, 26, nodes_objs, 0, mask)
        r = search(current_node, 26, nodes_objs, 0, not_mask)
        max_ = max(max_, l + r)

    return max_

assert get_part_two(test) == 1707
print(f'Part 2: {get_part_two(data)}')


from pathlib import Path

from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

DB_URL= "sqlite:///"+str(Path(__file__).parent / 'guitar_listings.db')

column_name_translation = dict(
    modell="model",
    bauweise="shape",
    farbe="color",
    griffbrett="wood_fretboard",
    inkl_gigbag="gigbag_included",
    koffer="case_included",
    tonabnehmer="has_pickup",
    verkaufsrang="sales_rank",
    hersteller_id="manufacturer_id",
    erhaeltlich_seit="available_since",
    buende="frets",
    holz_decke="wood_top",
    holz_boden_zargen="wood_back_sides",
    artikelnummer="item_number"
)

# Create engine singleton and associated sessionmaker
engine = create_engine(DB_URL)
Session = sessionmaker()
Session.configure(bind=engine)


# Python 3.7.4
# OpenCV-Python 4.5.3.56

import cv2
import tkinter as tk
from utils import rescale_frame, make_1080p

cap = cv2.VideoCapture(0)
make_1080p(cap)

while True:
    ret, frame = cap.read()
    grayFrame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Upscale / Downscale method 1
    root = tk.Tk()
    screen_width = root.winfo_screenwidth()
    screen_height = root.winfo_screenheight()
    root.destroy()
    WIDTH = screen_width / 2
    HEIGHT = screen_height / 2
    frame = cv2.resize(frame, (int(WIDTH), int(HEIGHT)))

    # Upscale / Downscale method 2
    grayFrame = rescale_frame(grayFrame, 30)

    cv2.imshow("Normal Frame", frame)
    cv2.imshow('Grayscale Frame', grayFrame)

    # waitKey() basically determines how long cv waits for press key 'a', set to 1
    # ord('a') closes the browsers on 'a' keypress
    if cv2.waitKey(1) & 0xFF == ord('a'):
        break

cap.release()
cv2.destroyAllWindows()


from typing import List

import setuptools

from carsimg.constants import VERSION

def _requires_from_file(filename: str) -> List[str]:
    return open(filename).read().splitlines()

setuptools.setup(
    name="carsimg",
    version=VERSION,
    install_requires=_requires_from_file('requirements.txt'),
    author="Hiroaki Takahashi",
    author_email="aphiloboe@gmail.com",
    url="https://github.com/MiLL4U/carsimg",
    description="GUI application for controll CARS imaging system",
    packages=setuptools.find_packages(),
    package_data={"carsimg": ["ui/image/*"]},
    classifiers=[
        "Programming Language :: Python :: 3",
        'Operating System :: Microsoft :: Windows',
    ],
    python_requires='>=3.10',
)


#!/usr/bin/env python
# coding: utf-8

import numpy as np
import pandas as pd
import lightgbm as lgb
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import mean_absolute_error
from sklearn.manifold import LocallyLinearEmbedding

import pickle
import plotly.graph_objs as go
import os
import streamlit as st
import datetime
pd.set_option('plotting.backend', 'plotly')
import warnings

warnings.filterwarnings('ignore')

test_length = 7
model_path_C3 = 'pages/liquidC3_lgb.bin'
model_path_C2 = 'pages/liquidC2_lgb.bin'
model_path_C1 = 'pages/liquidC1_lgb.bin'

if __name__ == '__main__':
    df_gas_liquid = pd.read_csv('dataset/LiquidProduction.csv')
    df_gas_liquid['Date'] = pd.to_datetime(df_gas_liquid['Date'])
    df_gas_liquid['elapsed'] = (df_gas_liquid['Date'] - df_gas_liquid['Date'][0]).apply(lambda x: x.days) + 1

    lag_days = [i for i in range(1, 5)]
    df_gas_liquid = df_gas_liquid.assign(**{
        'lag_{}_C1'.format(l): df_gas_liquid['LiquidProductionC1'].transform(lambda x: x.shift(l)) for l in
        lag_days})
    df_gas_liquid = df_gas_liquid.assign(**{
        'lag_{}_C2'.format(l): df_gas_liquid['LiquidProductionC2'].transform(lambda x: x.shift(l)) for l in
        lag_days})
    df_gas_liquid = df_gas_liquid.assign(**{
        'lag_{}_C3'.format(l): df_gas_liquid['LiquidProductionC3'].transform(lambda x: x.shift(l)) for l in
        lag_days})

    featuresC1 = ['elapsed', 'GasProductionC1']  # + ['lag_{}_C1'.format(i) for i in lag_days]
    featuresC2 = ['elapsed', 'GasProductionC2']  # + ['lag_{}_C2'.format(i) for i in lag_days]
    featuresC3 = ['elapsed', 'GasProductionC3']  # + ['lag_{}_C3'.format(i) for i in lag_days]

    lgb_params = {
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'metric': 'mae',
        'num_leaves': 30,
        'min_data_in_leaf': 4,  # 叶子结点的最小数据数，避免过拟合
        'max_depth': 6,  # 单颗树的最大深度
        'subsample': 0.8,
        'colsample_bytree': 0.8,
        'learning_rate': 0.05,
        'max_bin': 1000,
        'n_estimators': 1000,
        'verbose': -1,
    }

    X_train_C1 = df_gas_liquid[featuresC1].iloc[:-test_length, :]
    y_train_C1 = df_gas_liquid['LiquidProductionC1'][:-test_length]
    X_test_C1 = df_gas_liquid[featuresC1].iloc[-test_length:, :]
    y_test_C1 = df_gas_liquid['LiquidProductionC1'][-test_length:]
    train_data_C1 = lgb.Dataset(X_train_C1, y_train_C1)
    if not os.path.exists(model_path_C1):
        regressor_C1 = lgb.train(lgb_params, train_data_C1, verbose_eval=100)
        pickle.dump(regressor_C1, open(model_path_C1, 'wb'))
    else:
        regressor_C1 = pickle.loads(open(model_path_C1, 'rb').read())
    y_pred_C1 = regressor_C1.predict(X_test_C1)
    mae = mean_absolute_error(y_test_C1, y_pred_C1)
    print('C1 MAE: {}'.format(mae))

    X_train_C2 = df_gas_liquid[featuresC2].iloc[:-test_length, :]
    y_train_C2 = df_gas_liquid['LiquidProductionC2'][:-test_length]
    X_test_C2 = df_gas_liquid[featuresC2].iloc[-test_length:, :]
    y_test_C2 = df_gas_liquid['LiquidProductionC2'][-test_length:]
    train_data_C2 = lgb.Dataset(X_train_C2, y_train_C2)
    if not os.path.exists(model_path_C2):
        regressor_C2 = lgb.train(lgb_params, train_data_C2, verbose_eval=100)
        pickle.dump(regressor_C2, open(model_path_C2, 'wb'))
    else:
        regressor_C2 = pickle.loads(open(model_path_C2, 'rb').read())
    y_pred_C2 = regressor_C2.predict(X_test_C2)
    mae = mean_absolute_error(y_test_C2, y_pred_C2)
    print('C2 MAE: {}'.format(mae))

    X_train_C3 = df_gas_liquid[featuresC3].iloc[:-test_length, :]
    y_train_C3 = df_gas_liquid['LiquidProductionC3'][:-test_length]
    X_test_C3 = df_gas_liquid[featuresC3].iloc[-test_length:, :]
    y_test_C3 = df_gas_liquid['LiquidProductionC3'][-test_length:]
    train_data_C3 = lgb.Dataset(X_train_C3, y_train_C3)
    if not os.path.exists(model_path_C3):
        regressor_C3 = lgb.train(lgb_params, train_data_C3, verbose_eval=100)
        pickle.dump(regressor_C3, open(model_path_C3, 'wb'))
    else:
        regressor_C3 = pickle.loads(open(model_path_C3, 'rb').read())
    y_pred_C3 = regressor_C3.predict(X_test_C3)
    mae = mean_absolute_error(y_test_C3, y_pred_C3)
    print('C3 MAE: {}'.format(mae))

    regressor = {'C1': regressor_C1, 'C2': regressor_C2, 'C3': regressor_C3}
    st.header("产液量预测")
    station = st.sidebar.selectbox(label='请选择集气站', options=['C1', 'C2', 'C3'], index=2)
    date_input = st.sidebar.date_input(label='请输入当前日期', value=pd.to_datetime('2021-12-31'))
    elasped = (pd.to_datetime(date_input) - df_gas_liquid['Date'][0]).days + 1
    gas_production = st.sidebar.number_input('请输入当日预计产气量', value=395.75)
    # lag_1 = st.sidebar.number_input(str(pd.to_datetime(date_input) - datetime.timedelta(days=1))[:10] + ' ' + station + ' 产液量:', value=142.78)
    # lag_2 = st.sidebar.number_input(str(pd.to_datetime(date_input) - datetime.timedelta(days=2))[:10] + ' ' + station + ' 产液量:', value=105.94)
    # lag_3 = st.sidebar.number_input(str(pd.to_datetime(date_input) - datetime.timedelta(days=3))[:10] + ' ' + station + ' 产液量:', value=128.68)
    # lag_4 = st.sidebar.number_input(str(pd.to_datetime(date_input) - datetime.timedelta(days=4))[:10] + ' ' + station + ' 产液量:', value=99.55)
    with st.form("my_form"):
        submitted = st.form_submit_button("submit")
        if submitted:
            # forecast = regressor[station].predict([[elasped, gas_production, lag_1, lag_2, lag_3, lag_4]])
            forecast = regressor[station].predict([[elasped, gas_production]])

            st.write('*'+str(pd.to_datetime(date_input))[:10] + ' ' + station + ' 预测产液量:   {}*'.format(forecast[0]))

    st.subheader('测试结果展示')
    fig = go.Figure()
    fig.add_trace(go.Scatter(
        x=df_gas_liquid['Date'][-test_length:],
        y=y_test_C1,
        name='Actual'
    ))

    fig.add_trace(go.Scatter(
        x=df_gas_liquid['Date'][-test_length:],
        y=y_pred_C1,
        name='Forecast'
    ))
    fig.update_layout(title_text='C1 预测产液量和真实产液量')
    st.plotly_chart(fig)

    fig = go.Figure()
    fig.add_trace(go.Scatter(
        x=df_gas_liquid['Date'][-test_length:],
        y=y_test_C2,
        name='Actual'
    ))

    fig.add_trace(go.Scatter(
        x=df_gas_liquid['Date'][-test_length:],
        y=y_pred_C2,
        name='Forecast'
    ))
    fig.update_layout(title_text='C2 预测产液量和真实产液量')
    st.plotly_chart(fig)

    fig = go.Figure()
    fig.add_trace(go.Scatter(
        x=df_gas_liquid['Date'][-test_length:],
        y=y_test_C3,
        name='Actual'
    ))

    fig.add_trace(go.Scatter(
        x=df_gas_liquid['Date'][-test_length:],
        y=y_pred_C3,
        name='Forecast'
    ))
    fig.update_layout(title_text='C3 预测产液量和真实产液量')
    st.plotly_chart(fig)


import note_events as ne

import json

class OngoingNote:
    # onset and duration in number of samples
    def __init__(self, onset: int, duration: int) -> None:
        self.onset: int = onset
        self.duration: int = duration

def parse_noteevents(digistring_results: str) -> ne.NoteEvents:
    note_events = ne.NoteEvents()

    # Read and parse the JSON file
    json_events = None
    try:
        with open(digistring_results) as file:
            json_events = json.load(file)
    except json.JSONDecodeError:
        print(f"'{digistring_results}' is not a valid JSON file")

    # JSON -> NoteEvents
    sample_rate = json_events["Sample rate (Hz)"]
    ongoing_notes = {}
    for event in json_events["note events"]:
        # No notes in frame
        if event["midi_number"] == None:
            # Flush all ongoing notes
            for on_midi, on_on in ongoing_notes.items():
                note_events.add_event(on_midi, on_on.onset / sample_rate, (on_on.onset + on_on.duration) / sample_rate)
            ongoing_notes = {}

        # If note was in previous frame...
        elif event["midi_number"] in ongoing_notes:
            matching_ongoing = ongoing_notes[event["midi_number"]]
            # ...and offset perfectly aligns with onset
            if matching_ongoing.onset + matching_ongoing.duration == event["note_start (samples)"]:
                # Extent the note event
                matching_ongoing.duration += event["note_duration (samples)"]
            elif matching_ongoing.onset + matching_ongoing.duration > event["note_start (samples)"]:
                # If the new note starts before the old one ends, invalid note overlap
                print("Warning: Overlapping note events; merging them into one...")
                print(f"Merged note {event['midi_number']} at [{matching_ongoing.onset / sample_rate}, {(matching_ongoing.onset + matching_ongoing.duration) / sample_rate}]"
                        + f" and [{event['note_start (s)']}, {event['note_start (s)'] + event['note_duration (s)']}]", end="")
                matching_ongoing.duration += event["note_duration (samples)"] - ((matching_ongoing.onset + matching_ongoing.duration) - event["note_start (samples)"])
                print(f" merged to [{matching_ongoing.onset / sample_rate}, {matching_ongoing.duration / sample_rate}]")
            else:
                # Space between notes, so flush old and start new
                note_events.add_event(event["midi_number"], matching_ongoing.onset / sample_rate, (matching_ongoing.onset + matching_ongoing.duration) / sample_rate)
                ongoing_notes[event["midi_number"]] = OngoingNote(event["note_start (samples)"], event["note_duration (samples)"])

        # Note is not in ongoing_notes
        else:
            # So add it
            ongoing_notes[event["midi_number"]] = OngoingNote(event["note_start (samples)"], event["note_duration (samples)"])

    # Flush all notes that were still ongoing at the end
    for on_midi, on_on in ongoing_notes.items():
        note_events.add_event(on_midi, on_on.onset / sample_rate, (on_on.onset + on_on.duration) / sample_rate)

    return note_events