import os
import argparse
import torch
from torchvision import datasets, transforms
from torch.utils.data.sampler import WeightedRandomSampler
from sklearn.model_selection import train_test_split
from collections import Counter
import numpy as np

# Transform and to normalize the data [0.0, 1.0]
transform_train = transforms.Compose([
    transforms.RandomCrop(32, padding=4),
    transforms.RandomHorizontalFlip(),
    transforms.ToTensor()
])
transform_test = transforms.Compose([transforms.ToTensor()])

# Cifar10 classes
cifar10_classes = ('airplane 0', 'automobile 1', 'bird 2', 'cat 3',
           'deer 4', 'dog 5', 'frog 6', 'horse 7', 'ship 8', 'truck 9')

def to_numpy(tensor):
    '''To convert the torch tensor into numpy
    '''
    return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()

def index_to_label(index_dict):
    '''To crete a new dict by replacing the keys (class-indexes) with class-labels
    '''
    new_dict = {}
    for key in range(len(index_dict)):
        new_dict[cifar10_classes[key]] = index_dict[key]
    return new_dict

def prepareTrainset(args, X_train, y_train):
    '''
    *** Usase:
    # prepare the training set where we limit no of samples in some classes
    # Rest of the classes will take all available samples

    *** parameters:
    args:
        args.classes_to_limit: Classes where we need less samples, [2,4,9]-> ['bird', 'deer', 'truck']
        args.data_limit_in_classes: samples limit, 2400

    X_train: Original training data(images)
    y_train: Original training targets(classes)

    *** Return
    training set with desired no of samples in each class
    '''
    X_train = np.rollaxis(X_train, 3, 1)
    X_train = (X_train/255.0)
    X_train = X_train.astype(np.float32)
    train_idx = []
    for i in range(10):
        indexes = [idx for idx in range(len(y_train)) if y_train[idx] == i]
        if i in args.classes_to_limit:
            indexes = indexes[:args.data_limit_in_classes]
            train_idx.extend(indexes)
        else:
            train_idx.extend(indexes)

    trainset = [(X_train[i], y_train[i]) for i in train_idx]

    if args.verbose:
        y_train = [y_train[id] for id in train_idx]
        print(f'\nTraining dataset: \n{len(y_train)}\n{index_to_label(dict(Counter(y_train)))}')

    return trainset

def prepareValset(args, X_val, y_val):
    '''Prepare validation set with 1,000 samples where each class has 100 samples
    '''
    X_val = np.rollaxis(X_val, 3, 1)
    X_val = (X_val/255.0)
    X_val = X_val.astype(np.float32)
    valset = [(X_val[i], y_val[i]) for i in range(len(X_val))]

    # Verbose
    if args.verbose:
        print(f'\nValidation dataset: \n{len(y_val)}\n{index_to_label(dict(Counter(y_val)))}')

    return valset

def train_data_sampler(args, y_train):
    ''' Sampling strategy for the training batches
    Weighted over sampling: Building a multinomial distribution over the set of observations
    where each observation behaves as its own class with a controlled probability of being drawn
    '''
    train_idx = []
    for i in range(10):
        indexes = [idx for idx in range(len(y_train)) if y_train[idx] == i]
        if i in args.classes_to_limit:
            indexes = indexes[:args.data_limit_in_classes]
            train_idx.extend(indexes)
        else:
            train_idx.extend(indexes)

    train_targets = [y_train[i] for i in train_idx]
    class_sample_count = np.unique(train_targets, return_counts=True)[1]
    weight = 1. / class_sample_count
    samples_weight = weight[train_targets]
    samples_weight = torch.from_numpy(samples_weight)

    sampler = WeightedRandomSampler(samples_weight, num_samples=len(samples_weight), replacement=False)
    return sampler

def loadCIFAR10(args):
    ''' Preparing the traning, val, test data loaders
        # Training set  : `args.classes_to_limit` classes will have `args.data_limit_in_classes` samples, other classes will have 4900 samples
        # Validation set: 1,000 samples     (making sure that 100 images are in each class)
        # Test set      : 10,000            (By default 1000 images are in each class)
    '''
    if args.verbose:
        print('\n***** CIFAR-10 DATASET')

    # path to save CIFAR10 data
    path = f'{os.path.dirname(os.path.dirname(__file__))}/data'

    # Download and load the CIFAR10 dataset
    train_val_set = datasets.CIFAR10(path, download = True, train = True, transform = transform_train)
    testset = datasets.CIFAR10(path, download = True, train = False, transform = transform_test)

    # Divide the CIFAR10 training samples into training and validation set
    # Training set      : 49,000 samples
    # Validation set    : 1,000 samples     (making sure that 100 images are in each class)
    X_train, X_val, y_train, y_val = train_test_split(train_val_set.data, train_val_set.targets, test_size=0.02, train_size=0.98, stratify=train_val_set.targets, shuffle=True, random_state=42)
    trainset = prepareTrainset(args, X_train, y_train)
    valset = prepareValset(args, X_val, y_val)

    # Train, Val, Test Dataset Loaders
    if args.data_sampling == None:
        trainLoader = torch.utils.data.DataLoader(trainset, batch_size = args.batch_size, shuffle=True)
    elif args.data_sampling == 'weightedOverSampling':
        # Weighted Oversampler for trainLoader
        train_sampler = train_data_sampler(args, y_train)
        trainLoader = torch.utils.data.DataLoader(trainset, batch_size = args.batch_size, sampler=train_sampler)
    valLoader = torch.utils.data.DataLoader(valset, batch_size = args.batch_size, shuffle = True)
    testLoader = torch.utils.data.DataLoader(testset, batch_size = args.batch_size, shuffle = True)

    if args.verbose:
        print(f'\nTest dataset: \n{len(testset.targets)}\n{index_to_label(dict(Counter(testset.targets)))}')

    return trainLoader, valLoader, testLoader

def loadCIFAR10_testset(batch_size = 100):
    # path to save CIFAR10 data
    path = f'{os.path.dirname(os.path.dirname(__file__))}/data'

    # Download and load the testset
    testset = datasets.CIFAR10(path, download = True, train = False, transform = transform_test)
    testLoader = torch.utils.data.DataLoader(testset, batch_size = batch_size, shuffle = True)
    return testLoader

# To check and visualize dataset independently
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Dataset_loader')
    parser.add_argument('--batch_size', default=64, type=int)
    parser.add_argument('--classes_to_limit', default=[2, 4, 9], choices=[i for i in range(10)])
    parser.add_argument('--data_limit_in_classes', default=2450, type=int)
    parser.add_argument('--verbose', default=True, type=bool)
    parser.add_argument('--visualize_a_batch', default=True, type=bool)
    parser.add_argument('--data_sampling', default='weightedOverSampling', type=str,
                        choices=['weightedOverSampling', None],
                        help='Data sampling to tackle imbalanced dataset')
    args = parser.parse_args()

    # Cifar10-dataset data loaders
    trainLoader, valLoader, testLoader = loadCIFAR10(args)

    if args.visualize_a_batch:
        print('\n***** Visualize a batch')
        dataiter = iter(trainLoader)
        images, labels = dataiter.next()
        print(images.shape, labels.shape)
        print(f'Pixel Values are B/W: [{torch.min(images).item()}, {torch.max(images).item()}]')

        print('\n***** Visualize some batches to see class distributions after applying weighted data over sampling')
        class_distribution = []
        for i, data in enumerate(trainLoader):
            _, labels = data
            class_distribution.append(np.unique(labels, return_counts=True)[1])
            print(class_distribution[i])
            if i > 9:
                break

        print('\n**** class-wise average distribution in batches after applying weighted data over sampling')
        class_distribution = np.array(class_distribution)
        class_distribution_avg = np.average(class_distribution, axis=0)
        print(f'{np.round(class_distribution_avg, decimals=2)}\n')


from queue import Queue
from adjacencyset import *

def sort_topology(graph):
    queue = Queue()
    in_degree_map = {}

    for v in range(graph.numVertices):
        in_degree_map[v] = graph.get_indegree(v)
        if in_degree_map[v] == 0:
            queue.put(v)

    sorted = []
    while not queue.empty():
        v = queue.get()
        sorted.append(v)

        for _v in graph.get_adjacent_vertices(v):
            in_degree_map[_v] = in_degree_map[_v] - 1
            if in_degree_map[_v] == 0:
                queue.put(_v)

    if len(sorted) != graph.numVertices:
        raise ValueError("Graph is a cyclic")
    print(sorted)

a = AdjacencyGraphSet(9, True)
a.add_edge(0,1)
a.add_edge(1,2)
a.add_edge(2,7)
a.add_edge(2,4)
a.add_edge(2,3)
a.add_edge(1,5)
a.add_edge(5,6)
a.add_edge(3,6)
a.add_edge(3,4)
a.add_edge(6,8)

sort_topology(a)


import cv2
import numpy as np
from scipy.spatial import cKDTree
from sklearn.decomposition import PCA

def sample_to_heatmap(points, x_adjust=0, y_adjust=0, z_threshold=None, nearest_k=3):
    # If a threshold is provided, keep only points with a z-coordinate above this threshold
    if z_threshold is not None:
        points = points[points[:, 2] > z_threshold]
        print('cropped')

    # Compute PCA
    pca = PCA(n_components=3)
    pca.fit(points)

    # The normal of the plane is the smallest principal component
    normal = pca.components_[-1]

    # The point on the plane can be the centroid of the point cloud
    centroid = np.mean(points, axis=0)

    # Now we can print the plane equation
    # The plane equation is of the form ax + by + cz + d = 0
    a, b, c = normal
    d = -centroid.dot(normal)
    print(f"The equation of the plane is {a:.5f}x + {b:.5f}y + {c:.5f}z + {d:.5f} = 0")

    # Get x, y, z coordinates
    x_coords = points[:, 0]
    y_coords = points[:, 1]
    z_coords = points[:, 2]

    # Calculate minimum and maximum values in x and y directions
    x_min, x_max = np.min(x_coords), np.max(x_coords)
    y_min, y_max = np.min(y_coords), np.max(y_coords)

    x_mid = (x_min + x_max) / 2 + x_adjust
    y_mid = (y_min + y_max) / 2 + y_adjust
    # The range of x and y values for the mesh grid
    x_range = np.linspace(x_mid - 15, x_mid + 15, 514)
    y_range = np.linspace(y_mid - 15, y_mid + 15, 514)

    x, y = np.meshgrid(x_range, y_range)

    # Compute corresponding z values for the plane
    z = (-a * x - b * y - d) / c

    tree = cKDTree(points)

    distances = []
    for point in np.vstack([x.flatten(), y.flatten(), z.flatten()]).T:
        # Find the three nearest points in the point cloud
        dists, idxs = tree.query(point, k=nearest_k)
        nearest_points = points[idxs]

        # For each nearest point, compute the distance to the point along the normal direction
        ds = []
        for nearest_point in nearest_points:
            displacement = nearest_point - point  # vector from point to nearest_point
            distance = np.dot(displacement, normal)  # project displacement onto normal
            ds.append(distance)
        distances.append(sum(ds) / len(ds))

    # 这里是用最小值纠正（normalisation）矩阵
    distances_array = (np.array(distances) - np.min(distances)) / 0.5 * 255
    distances_reshape = distances_array.reshape((514, 514))[1:513, 1:513].astype(int)

    return distances_reshape  # 这个就是image 直接save就行

# Create a heatmap using seaborn
def plot_heatmap(heatmap, save_path=None):
    cv2.imwrite(save_path, heatmap)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import numpy as np
import torch

class EMA_FM():
    def __init__(self, decay=0.9, first_decay=0.0, channel_size=512, f_map_size=196, is_use = False):
        self.decay = decay
        self.first_decay = first_decay
        self.is_use = is_use
        self.shadow = {}
        self.epsional = 1e-5
        if is_use:
            self._register(channel_size=channel_size, f_map_size= f_map_size)

    def _register(self, channel_size=512, f_map_size=196):
        Init_FM = torch.zeros((f_map_size, channel_size),dtype=torch.float)
        self.shadow['FM'] = Init_FM.cuda().clone()
        self.is_first = True

    def update(self, input):
        B, C, _ = input.size()
        if not(self.is_use):
            return torch.ones((C,C), dtype=torch.float)
        decay = self.first_decay if self.is_first else self.decay
        ####### FEATURE SIMILARITY MATRIX EMA ########
        # Mu = torch.mean(input,dim=0)
        self.shadow['FM'] = (1.0 - decay) * torch.mean(input,dim=0) + decay * self.shadow['FM']
        self.is_first = False
        return self.shadow['FM']

class Cluster_loss():
    def __init__(self):
        pass

    def update(self, correlation, loss_mask_num, loss_mask_den, labels):
        batch, channel, _ = correlation.shape
        c, _, _ = loss_mask_num.shape
        if labels is not None:
            label_mask = (1 - labels).view(batch, 1, 1)
            ## smg_loss if only available for positive sample
            correlation = correlation * label_mask
        correlation = (correlation / batch).view(1, batch, channel, channel).repeat(c, 1, 1, 1)

        new_Num = torch.sum(correlation * loss_mask_num.view(c, 1, channel, channel).repeat(1, batch, 1, 1),
                            dim=(1, 2, 3))
        new_Den = torch.sum(correlation * (loss_mask_den).view(c, 1, channel, channel).repeat(1, batch, 1, 1),
                            dim=(1, 2, 3))
        ret_loss = -torch.sum(new_Num / (new_Den + 1e-5))
        return ret_loss

class Multiclass_loss():
    def __init__(self, class_num=None):
        self.class_num = class_num

    def get_label_mask(self, label):
        label = label.cpu().numpy()
        sz = label.shape[0]
        label_mask_num = []
        label_mask_den = []
        for i in range(self.class_num):
            idx = np.where(label == i)[0]
            cur_mask_num = np.zeros((sz, sz))
            cur_mask_den = np.zeros((sz, sz))
            for j in idx:
                cur_mask_num[j][idx] = 1
                cur_mask_den[j][:] = 1
            label_mask_num.append(np.expand_dims(cur_mask_num, 0))
            label_mask_den.append(np.expand_dims(cur_mask_den, 0))
        label_mask_num = np.concatenate(label_mask_num, axis=0)
        label_mask_den = np.concatenate(label_mask_den, axis=0)
        return torch.from_numpy(label_mask_num).float().cuda(), torch.from_numpy(label_mask_den).float().cuda()

    def update(self, fmap, loss_mask_num, label):
        B, C, _, _ = fmap.shape
        center, _, _ = loss_mask_num.shape
        fmap = fmap.view(1, B, C, -1).repeat(center, 1, 1, 1)
        mean_activate = torch.mean(torch.matmul(loss_mask_num.view(center, 1, C, C).repeat(1, B, 1, 1), fmap),
                                   dim=(2, 3))
        # cosine
        mean_activate = torch.div(mean_activate, torch.norm(mean_activate, p=2, dim=0, keepdim=True) + 1e-5)
        inner_dot = torch.matmul(mean_activate.permute(1, 0), mean_activate).view(-1, B, B).repeat(self.class_num, 1, 1)
        label_mask, label_mask_intra = self.get_label_mask(label)

        new_Num = torch.mean(inner_dot * label_mask, dim=(1, 2))
        new_Den = torch.mean(inner_dot * label_mask_intra, dim=(1, 2))
        ret_loss = -torch.sum(new_Num / (new_Den + 1e-5))
        return ret_loss

def Cal_Center(fmap, gt):
    f_1map = fmap.detach().cpu().numpy()
    matrix = gt.detach().cpu().numpy()
    B, C, H, W = f_1map.shape
    cluster = []
    visited = np.zeros(C)
    for i in range(matrix.shape[0]):
        tmp = []
        if(visited[i]==0):
            for j in range(matrix.shape[1]):
                if(matrix[i][j]==1 ):
                    tmp.append(j)
                    visited[j]=1;
            cluster.append(tmp)
    center = []
    for i in range(len(cluster)):
        cur_clustet_fmap = f_1map[:,cluster[i],...]
        cluster_center = np.mean(cur_clustet_fmap,axis=1)
        center.append(cluster_center)
    center = np.transpose(np.array(center),[1,0,2,3])
    center = torch.from_numpy(center).float()
    return center


from itertools import chain

y, x = [int(i) for i in input().split()]

matrix = [[] for _ in range(y)]

for i in range(y):
    s = input()
    for u in s:
        if u == ".":
            matrix[i].append(True)
        elif u == "#":
            matrix[i].append(False)

def step(y, x, matrix):
    matrix[y][x] = True
    tmp = [[y, x+1],[y, x-1],[y-1, x],[y+1, x]]
    neighbors = []
    for i in tmp:
        try:
            if matrix[i[0]][i[1]] == False:
                neighbors.append(i)
        except:
            pass
        if len(neighbors) == 0:
            return
        else:
            for i in neighbors:
                step(i[0], i[1], matrix)

count = 0

while False in chain.from_iterable(matrix):
    coord = []
    for i in matrix:
        if False in i:
            coord = [matrix.index(i), i.index(False)]
    step(coord[0], coord[1], matrix)
    count += 1

print(count)


from glob import glob
from os import makedirs
from shutil import copy2

from tqdm import tqdm

SINGER = 'mixed'
RELEASE_DIR = 'release/mixed_---'
PATH_QUESTION = 'conf/jp_qst001_nnsvs_simple_4-4_mix.hed'
NAME_EXPERIMENT = 'simple_4-4_mix'

def copy_question(path_question, release_dir):
    """
    hedファイル(question)をコピー
    """
    makedirs(f'{release_dir}/conf', exist_ok=True)
    print('copying question')
    copy2(path_question, f'{release_dir}/{path_question}')

def copy_scaler(singer, release_dir):
    """
    dumpフォルダにあるファイルをコピー
    """
    makedirs(f'{release_dir}/dump/{singer}/norm', exist_ok=True)
    list_path_scaler = glob(f'dump/{singer}/norm/*_scaler.joblib')

    print('copying scaler')
    for path_scaler in tqdm(list_path_scaler):
        copy2(path_scaler, f'{release_dir}/{path_scaler}')

def copy_model(singer, name_exp, release_dir):
    """
    name_exp: 試験のID
    """
    name_exp = singer + '_' + name_exp
    makedirs(f'{release_dir}/exp/{name_exp}/acoustic', exist_ok=True)
    makedirs(f'{release_dir}/exp/{name_exp}/duration', exist_ok=True)
    makedirs(f'{release_dir}/exp/{name_exp}/timelag', exist_ok=True)
    list_path_model = glob(f'exp/{name_exp}/*/best_loss.pth')
    list_path_model += glob(f'exp/{name_exp}/*/latest.pth')
    list_path_model += glob(f'exp/{name_exp}/*/model.yaml')

    print('copying model')
    for path_model in tqdm(list_path_model):
        copy2(path_model, f'{release_dir}/{path_model}')

def main():
    """
    各種ファイルをコピーする
    """
    copy_question(PATH_QUESTION, RELEASE_DIR)
    copy_scaler(SINGER, RELEASE_DIR)
    copy_model(SINGER, NAME_EXPERIMENT, RELEASE_DIR)

if __name__ == '__main__':
    main()


#For example, given [(30, 75), (0, 50), (60, 150)], you should return 2.
classes = [(900, 910), (940, 1200), (950, 1120),(1100, 1130), (1500, 1900), (1800, 2000)]
started = []
for x,i in classes:
    started+= [(x,'started'),(i,'ended')]
started = sorted(started)
needed = 0
ongoing = 0
print(started)
for i in range(len(started)):
    if started[i][1] == 'started':
        ongoing +=1
        #print(started[i][0])
        if ongoing > needed:
            needed = ongoing
    else:
        ongoing -=1
print (needed)


#!/usr/bin/env python
from netCDF4 import Dataset
import copyNCVariable as copync
import sys, os
import random
import pdb
import numpy as np
import datetime as dt

#
#
#

def usage():
    print("Usage")
    print("    "+sys.argv[0]+" [filename] [dim name]")
    exit(1)

def change_time_units(var):
    """Change the time unit from epoch time to hours since 1800"""
    century18 = dt.datetime(1800,1,1,0)
    #for i,j in enumerate(var[:]):
    #    date = dt.datetime.utcfromtimestamp(j)
    #    seconds = (date - century18).total_seconds()
    #    hours = int( seconds / 60 / 60 )
    #    var[i] = hours
    def change_unit(date):
        date = dt.datetime.utcfromtimestamp(date)
        seconds = (date - century18).total_seconds()
        hours = int( seconds / 60 / 60 )
        return hours

    vfunc = np.vectorize(change_unit)
    new_data = vfunc(var[:])
    var[:] = new_data
    setattr(var, 'standard_name', "time")
    setattr(var, 'long_name', "time")
    setattr(var, "units","hours since 1800-01-01 00:00:00.0")
    setattr(var, "calendar", "proleptic_gregorian")
    return var

def add_utc_date(nc, time_var):
    """ Adds human readable date variable.
    Assumes date is in seconds since epoch.
    time_var is netCDF.Variable object.
    """
    # Create Variable
    utc = nc.createVariable('utc_time', int, ('time'))
    setattr(utc, 'standard_name', "time")
    setattr(utc, 'long_name', "UTC date yyyy-mm-dd hh:00:00 as yyyymmddhh")
    setattr(utc, "units","Gregorian_year month day hour")

    toUTC = lambda d: int(dt.datetime.fromtimestamp(d).strftime('%Y%m%d%H'))
    vfunc = np.vectorize(toUTC)
    utc_data = vfunc(time_var[:])
    utc[:] = utc_data

def find_variables_with_dimension(nc, dim_name):
    selected_vars = []
    for var_name in nc.variables:
        var = nc.variables[var_name]
        if dim_name in var.dimensions:
            selected_vars.append(var)
    return selected_vars

def find_variables_without_dimension(nc, dim_name):
    selected_vars = []
    for var_name in nc.variables:
        var = nc.variables[var_name]
        if dim_name not in var.dimensions:
            selected_vars.append(var)
    return selected_vars

def check_if_reduce_needed(vars_to_modify):
    """Return True if variable has missing start and end"""
    for var in vars_to_modify:
        if len(var.dimensions) > 2 and var[0,0,:].mask.all() and \
                var[-1,1,:,:].mask.all():
            return True
    return False

def add_time_bounds(nc, varname):
    """
    Adds a time bounds variable to variable.
    Assumes time dimension is called 'time'
    """
    THREE_HOURS = 60*60*3 # in seconds
    bnds_name = 'time_bnds'
    bounds_dim = 'nv'

    # Create bounds dimension
    nc.createDimension(bounds_dim, 2)

    # Get variable matching varname

    time_var = nc.variables['time']
    time_var.setncattr('bounds', bnds_name)
    time_data = time_var[:]
    time_length = len(time_data)

    # reshape time data
    bounds_data = np.dstack((time_data,time_data)).reshape(time_length,2)
    for i in bounds_data:
        i[0] = i[0] - (THREE_HOURS)
    bounds_var = nc.createVariable(bnds_name, time_var.dtype, ('time', bounds_dim), fill_value=9999)
    bounds_var[:] = bounds_data

def add_cell_methods(nc):
    methods = {
            'avg' : 'mean',
            'accum' : 'sum',
            'min' : 'minimum',
            'max' : 'maximum'
            }
    step_str = 'GRIB_stepType'
    for i in nc.variables:
        var = nc.variables[i]
        if step_str in var.ncattrs() and 'instant' not in var.getncattr(step_str):
            if 'cell_methods' in var.ncattrs():
                cur_str = var.getncattr('cell_methods')
                var.setncattr('cell_methods', cur_str + " time: " + methods[var.getncattr(step_str)])
            else:
                pass
                #var.setncattr('cell_methods', "time: " + methods[var.getncattr(step_str)])

def change_coordinates(nc):
    for i in nc.variables:
        var = nc.variables[i]
        if 'coordinates' in var.ncattrs():
            coord_str = var.getncattr('coordinates')
            coord_str = coord_str.replace('valid_time', '')
            coord_str = coord_str.replace('step', '')
            if 'time' not in coord_str:
                coord_str += " time"
            coord_str = ' '.join(coord_str.split())
            var.setncattr('coordinates', coord_str)

def remove_dimension(nc, dim_name, outfile=None):

    vars_to_modify = find_variables_with_dimension(nc, dim_name)
    vars_to_copy = find_variables_without_dimension(nc, dim_name)
    reduce_needed = check_if_reduce_needed(vars_to_modify)
    if outfile is None:
        outfile = 'tmp' + str(random.randint(1,10000)) + '.nc'
    tmp_nc = Dataset(outfile, 'w')
    # First copy global attrs
    copync.copy_global_attrs(nc, tmp_nc)
    # Then copy dimensions minus unwanted
    copync.copy_dimensions(nc, tmp_nc, ignore=['time',dim_name])
    if 'step' in nc.dimensions:
        if reduce_needed:
            tmp_nc.createDimension('time', (nc.dimensions['time'].size * nc.dimensions['step'].size) - 2)
        else:
            tmp_nc.createDimension('time', nc.dimensions['time'].size * nc.dimensions['step'].size )

    else:
        tmp_nc.createDimension('time', nc.dimensions['time'].size)
    if len(vars_to_modify) == 0: # not in dimensions, but need to get rid of step vars
        err_str = "'" + dim_name + "' is not in any of the variables."
        #raise Exception(err_str)
        time_var = None
        valid_var = None
        for var in vars_to_copy:
            if var.name != 'time' and var.name != 'step' and var.name != 'valid_time':
                copync.copy_variable(nc, tmp_nc, var.name)
            elif var.name == 'time':
                time_var = var
            elif var.name == 'valid_time':
                valid_var = var
        new_var = tmp_nc.createVariable('time', valid_var.dtype, ('time',))
        copync.copy_var_attrs(valid_var, new_var)
        new_var[:] = valid_var[:]
        return (outfile, tmp_nc)
    # Next, copy unchanged vars
    time_var = None
    for var in vars_to_copy:
        if var.name != 'time':
            copync.copy_variable(nc, tmp_nc, var.name)
        else:
            time_var = var

    for var in vars_to_modify:
        # If described by only unwanted dimension, then remove variable.
        if len(var.dimensions) == 1:
            # Remove variable
            pass
        else:
            # find dim index
            dims = var.dimensions
            dims_list = list(dims)
            shape = var.shape
            shape_list = list(shape)
            idx = dims.index(dim_name)

            if idx == 0:
                print('Need to implement')
                print('Exiting.')
                exit(1)

            size = shape_list.pop(idx)
            dims_list.pop(idx)
            dims = tuple(dims_list)
            shape_list[idx-1] = shape_list[idx-1]*size

            new_data = var[:].reshape(*shape_list)
            if reduce_needed:
                if len(dims) == 1:
                    new_data = new_data[1:-1]
                elif len(dims) > 1:
                    new_data = new_data[1:-1,:,:]
            varname = var.name
            if varname == 'valid_time':
                varname = 'time'

            new_var = tmp_nc.createVariable(varname, var.dtype, dims)
            copync.copy_var_attrs(var, new_var)
            new_var[:] = new_data
            step_str = 'GRIB_stepType'
            if step_str in new_var.ncattrs() and new_var.getncattr(step_str) is not 'instant':
                add_time_bounds(tmp_nc, new_var.name)

    return (outfile, tmp_nc)

def change_fill_value(nc, fill_value):
    """Changes fill value for all variables in file"""
    outfile = 'tmp' + str(random.randint(1,100000)) + '.nc'
    out_nc = copync.copy_dimensions(nc, outfile)
    copync.copy_variables(nc, out_nc, new_fill_value=fill_value)
    out_nc.close()
    return outfile

if __name__ == '__main__':
    if len(sys.argv) <= 2:
        usage()
    outfile = None
    nc_file = sys.argv[1]
    dim_name = sys.argv[2]
    nc = Dataset(nc_file, 'r+')
    if dim_name != "none":
        outfile,nc = remove_dimension(nc, dim_name)
    add_cell_methods(nc)
    change_coordinates(nc)
    add_utc_date(nc, nc.variables['time'])
    change_time_units(nc.variables['time'])
    if 'time_bnds' in nc.variables:
        change_time_units(nc.variables['time_bnds'])
    second_outfile = change_fill_value(nc, 9999)
    nc.close()
    if outfile is not None:
        os.remove(outfile)
    os.rename(second_outfile, nc_file)


import os
import sys
import yt_dlp
from tqdm import tqdm

def download_videos(links_file):
    if not os.path.exists(links_file):
        print("Error: The links file '{}' does not exist.".format(links_file))
        return

    ydl_opts = {
        'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',
        'outtmpl': '%(autonumber)s - %(title)s.%(ext)s',
        'merge_output_format': 'mp4',
    }

    with open(links_file, 'r') as f:
        video_links = f.read().splitlines()

    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
        for link in video_links:
            try:
                info_dict = ydl.extract_info(link, download=True)
                print("\n{}: Download completed!".format(info_dict['title']))
            except Exception as e:
                print("\nError downloading video: {}".format(e))
                continue

if __name__ == "__main__":
    links_file = "links.txt"
    download_videos(links_file)


import os
import pickle
import uvicorn
from fastapi import FastAPI

FAKE_HASH_TABLE_DB = './database/FakeHashTable.pickle'

class FakeHashTable:
    def __init__(self, bit_limitation=10):
        self.limitation = 2 ** bit_limitation
        self.hashtable = dict()
        self.id_list = set()
        self.history = list()
        self.avail_id = list(range(self.limitation))

    def hash(self, value, replacement=None):
        """
        :param value: value to hash
        :param replacement: if replacement = 'oldest' this instance will replace the object by the oldest record. If
        replacement is the value that existed in hashtable, it will remove old record and replace by new value.
        :return:
        """
        # For user replace ID by a new value
        if replacement is not None:
            if replacement == 'oldest' and self.history.__len__() > 2:
                old_id = self.hashtable[self.history[0]]
                del self.hashtable[self.history[0]]
                self.history = self.history[1:]
                self.history.append(value)
                self.hashtable[value] = old_id

            if replacement in self.hashtable:
                old_id = self.hashtable[replacement]
                self.history.remove(old_id)
                self.history.append(value)
                del self.hashtable[replacement]
                self.hashtable[value] = old_id
                return old_id
            return None

        if value in list(self.hashtable.keys()):
            return self.hashtable[value]

        # If larger than 10 bit, return None
        if self.hashtable.items().__len__() > self.limitation:
            return None

        # Add new ID
        new_id = self.avail_id.pop(0)
        self.history.append(value)
        self.id_list.add(new_id)
        self.hashtable[value] = new_id
        return new_id

    def remove(self, value):
        if value not in self.hashtable:
            return False
        old_id = self.hashtable[value]
        del self.hashtable[value]
        self.id_list.remove(value)
        self.avail_id.append(old_id)
        self.history.remove(value)

def backup_htb_object(in_htb):
    with open(FAKE_HASH_TABLE_DB, 'wb') as ff:
        pickle.dump(in_htb, ff)

def load_htb_object():
    with open(FAKE_HASH_TABLE_DB, 'rb') as ff:
        data = pickle.load(ff)
    return data

app = FastAPI()

if os.path.exists(FAKE_HASH_TABLE_DB):
    htb = load_htb_object()
else:
    htb = FakeHashTable()

@app.post("/")
async def get_machine_id(value):
    iid = htb.hash(value)
    backup_htb_object(htb)
    return {"id": iid}

if __name__ == '__main__':
    uvicorn.run(
        "hash_service:app",
        host='0.0.0.0',
        port=8000,
        reload=True,
        debug=True,
        workers=3
    )


n = int(input())
arr = list(map(int,input().split()))
left = 0
right = n-1
wube = 0
henock = 0
flag = True

while left<=right:
    if flag:
        if arr[left]>arr[right]:
            wube += arr[left]
            left += 1
        else:
            wube += arr[right]
            right -= 1
        flag = False
    elif not flag:
        if arr[left]>arr[right]:
            henock += arr[left]
            left += 1
        else:
            henock += arr[right]
            right -= 1
        flag = True
print(wube,henock)


from torchvision import models
import torch
import torch.nn as nn
from PIL import ImageGrab
import cv2
import torch.nn.functional as F
# import albumentations as A
# from albumentations.pytorch import ToTensorV2
from torchvision import transforms
import numpy as np
from PIL import Image
from input_keys import PressKey, ReleaseKey
import time

#import torch.nn.functional as F

#output = torch.randn(10, 5)  # example output tensor
#softmax_result = F.softmax(output, dim=1)

labels = {0: 'a', 1: 'w', 2: 'd'}
#labels = {0: 'a', 1: 'w', 2: 'd', 3: 's'}

def ingame_predic():
    test_transform = transforms.Compose(
        [
            # A.SmallestMaxSize(max_size=160),
            transforms.Resize((640, 480)),
            # A.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)),
            transforms.ToTensor()
        ]
    )

    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    #net = models.mobilenet_v3_large(pretrained=True, weights=models.mobilenet_v3_large(pretrained=True).weights.IMAGENET1K_V1)
    net = models.mobilenet_v3_large(pretrained=True)
    net.classifier[3] = nn.Linear(in_features=1280,out_features=3)

    #net = models.efficientnet_b4(pretrained=True)
    #net.classifier[1] = nn.Linear(in_features=1792,out_features=4)
    #net.load_state_dict(torch.load('./mbmodel.pt', map_location=device))

    net.load_state_dict(torch.load('./mbv3model.pt', map_location=device))
    net.to(device)
    net.eval()

    while(True):
        with torch.no_grad():
            screen = ImageGrab.grab(bbox=(0, 40, 1024, 768)) # 1024, 768 화면을 받아서 Numpy Array로 전환
            # screen = cv2.imread('./test_image2.jpg') # test image
            # input_image = Image.fromarray(screen)
            input_image = test_transform(screen).unsqueeze(0).to(device)

            output = net(input_image)
            softmax_result = F.softmax(output)
            top_prob, top_label = torch.topk(softmax_result, 1)
            prob = round(top_prob.item() * 100, 2)
            label = labels.get(int(top_label))
            # print(f'prob: {prob}, label: {label}')

            W = 0x11
            A = 0x1E
            S = 0x1F
            D = 0x20

            if (60 < prob) and (label == 'a'):
                PressKey(A)
                time.sleep(0.5)
                ReleaseKey(A)

            elif (60 < prob) and (label == 'w'):
                PressKey(W)
                time.sleep(0.5)
                ReleaseKey(W)

            elif (60 < prob) and (label == 'd'):
                PressKey(D)
                time.sleep(0.5)
                ReleaseKey(D)

            elif (60 < prob) and (label == 's'):
                PressKey(S)
                time.sleep(0.5)
                ReleaseKey(S)

            else:
                time.sleep(0.5)
        print(prob,label)
        #return prob, label

if __name__ == '__main__':
    predic_prob, predic_label = ingame_predic()
    print(predic_prob, predic_label)


import datetime

dogdict = {
    "american staffordshire terrier": True,
    "pitbull terrier": True,
    "bullterrier": True,
    "bullmastiff": True,
    "staffordshire bullterrier": True,
    "cane corso": True,
    "dogo argentino": True,
    "bordeaux dogge": True,
    "fila brasileiro": True,
    "mastin espanol": True,
    "französische bulldogge": False,
    "labrador": False,
    "chihuahua": False,
    "australian shepherd": False,
    "rottweiler": False,
    "border collie": False,
    "golden retriever": False,
    "rhodesian ridgeback": False,
    "mops": False,
    "berner sennenhund": False
}

citydict = {
    75015: 108,
    76359: 96,
    69254: 78,
    76275: 96,
    76287: 90,
    76337: 108,
    76307: 66,
    76327: 72,
    75045: 108,
    76356: 96,
    76297: 84,
    76344: 84,
    76351: 72,
    76707: 72,
    76676: 60,
    76689: 69,
    76646: 96,
    75053: 48,
    75038: 90,
    76703: 87,
    76698: 61,
    76707: 48,
    68753: 96,
    76661: 96,
    76709: 72,
    76669: 90,
    76684: 75,
    75059: 72
}

def serverHandler(clientString):
    dogInformations = splitStringFromClient(clientString, True)
    personalInformation = splitStringFromClient(clientString, False)
    if(dogInformations[3].lower() == "nein"):
        regionTax = getRegionTax(personalInformation)
        dogRaceSeperation = getDogRaceSeperation(dogInformations)
        dogTax = getDogTax(dogRaceSeperation, regionTax)

    else:
        dogTax = 0
    return dogTax

def splitStringFromClient(clientString, state):
    try:
        seperations = clientString.split(',')
        if(state):
            dogInformation = seperations[1].split(';')
            return dogInformation
        else:
            personalInformation = seperations[0].split(';')
            return personalInformation
    except:
        print("Error in SplitStringFormClient")

def getRegionTax(personalInformations):
    regionTax = citydict[int(personalInformations[6])]
    return regionTax
def getDogTax(dogRaceSeperation,regionTax):
    dogTax = None
    if(dogRaceSeperation == True):
       dogTax = int(regionTax) * 5
    else:
        dogTax = int(regionTax)
    if(dogTax != None):
        return dogTax
    else:
        return "Error in DogTax"

def getDogRaceSeperation(dogInformations):
    dogRaceSeperation = dogdict[dogInformations[2]]
    return dogRaceSeperation

def checkDate(value):
    try:
        day, month, year = map(int, value.split('.'))
        geburtstag_obj = datetime.date(year, month, day)
        try:
            date = value.strip()
            datetime.datetime.strptime(date, "%d.%m.%Y")
        except ValueError:
            return "Ungueltiges Datumsformat. Bitte geben Sie den Geburtstag im Format TT.MM.JJJJ ein.", False
        if geburtstag_obj >= datetime.date.today():
            return "Ungueltiges Datumsformat. Der Geburtstag muss in der Vergangenheit liegen.", False
        if month == 2 and day > 28:
            leap_year = (year % 4 == 0 and year % 100 != 0) or (year % 400 == 0)
            if (day == 29 and not leap_year) or (day > 29):
                return "Ungueltiges Datumsformat. Bitte geben Sie ein gueltiges Geburtsdatum ein.", False
        return "", True
    except:
        return "Bei der Datumsüberprüfung lief etwas schief. Bitte wenden Sie sich an einen Admin", False

def checkInput(value,nessesary, nameCheck):
    if(nessesary):
        if(value == None or value == ''):
            return False
        else:
            if("Geburtstag" in nameCheck):
                result = checkDate(value)
                return result
            elif(nameCheck == "Hunderasse"):
                if(value in dogdict):
                    return True
                else:
                    return False
            elif(nameCheck == "Ermaessigung"):
                if(value.lower() == "ja" or value.lower() == "nein"):
                    return True
                else:
                    return False
            elif(nameCheck == "PLZ"):
                if(value.isnumeric and int(value.strip()) in citydict):
                    return True
                else:
                    return False
            elif(nameCheck == "Hausnummer"):
                if(value.isnumeric()):
                    return True
                else:
                    return False
            else:
                return True
    else:
        if(value != None):
            return True
        else:
            return False


import time
import json
import board
import busio
import adafruit_ads1x15.ads1015 as ADS
from adafruit_ads1x15.analog_in import AnalogIn

max_val = None
min_val = None
# Create the I2C bus
i2c = busio.I2C(board.SCL, board.SDA)
# Create the ADC object using the I2C bus
ads = ADS.ADS1015(i2c)
# Create single-ended input on channel 0
chan = AnalogIn(ads, ADS.P2)
baseline_check = input("Is Light Sensor Covered? (enter 'y' to proceed): ")

if baseline_check == 'y':
    max_val = chan.value
print("------{:>5}\t{:>5}".format("raw", "v"))

for x in range(0, 10):
    if chan.value > max_val:
        max_val = chan.value
    print("CHAN 2: "+"{:>5}\t{:>5.3f}".format(chan.value, chan.voltage))
    time.sleep(0.5)

print('\n')

water_check = input("Does the Light Sensor receive the maximum light? (enter 'y' to proceed): ")
if water_check == 'y':
    min_val = chan.value

print("------{:>5}\t{:>5}".format("raw", "v"))
for x in range(0, 10):
    if chan.value < min_val:
        min_val = chan.value
    print("CHAN 2: "+"{:>5}\t{:>5.3f}".format(chan.value, chan.voltage))
    time.sleep(0.5)

config_data = dict()
config_data["min"] = min_val
config_data["max"] = max_val
with open('light_config.json', 'w') as outfile:
    json.dump(config_data, outfile)
print('\n')
print(config_data)
time.sleep(0.5)


import pytest

@pytest.fixture(name="fixer")
def fixer_fixture(two_to_three_test_case):
    return two_to_three_test_case("methodattrs")

attrs = ["func", "self", "class"]

def test(fixer):
    for attr in attrs:
        b = "a.im_%s" % attr
        if attr == "class":
            a = "a.__self__.__class__"
        else:
            a = "a.__%s__" % attr
        fixer.check(b, a)

        b = "self.foo.im_%s.foo_bar" % attr
        if attr == "class":
            a = "self.foo.__self__.__class__.foo_bar"
        else:
            a = "self.foo.__%s__.foo_bar" % attr
        fixer.check(b, a)

def test_unchanged(fixer):
    for attr in attrs:
        s = "foo(im_%s + 5)" % attr
        fixer.unchanged(s)

        s = "f(foo.__%s__)" % attr
        fixer.unchanged(s)

        s = "f(foo.__%s__.foo)" % attr
        fixer.unchanged(s)


from os.path import join as osjoin

paper_root_dir = 'paper'
stdj_dir = osjoin(paper_root_dir, 'stdj')
ctu_dir = osjoin(paper_root_dir, 'ctu')

stats_about_file_dir = osjoin('..', 'stats_about_files')
root_dir = osjoin('..', 'corpus')

# store all documents
all_doc_dir = osjoin(root_dir, 'all_doc')

# store source documents
src_dir = osjoin(root_dir, 'src')
# store raw suspicious documents
raw_susp_dir = osjoin(root_dir, 'raw_susp')
# store suspicious documents after making plagiarism docs
susp_dir = osjoin(root_dir, 'susp')

# store plagiarism paragraph location in suspicious docs
susp_stats_dir = osjoin(root_dir, 'susp_stats')
predict_susp_stats_dir = osjoin(root_dir, 'predicted_susp_stats')

# store embedding of documents, in pickle format
embeddings_dir = 'embeddings'
src_embeddings_dir = osjoin(embeddings_dir, 'src')
susp_embeddings_dir = osjoin(embeddings_dir, 'susp')
susp_embeddings_for_classification = osjoin(
    embeddings_dir, 'susp_for_classification'
)

train_classifier_dir = 'train_classifier'
csv_dir = osjoin(train_classifier_dir, 'csv')
parquet_train_classifier_dir = osjoin(train_classifier_dir, 'parquet')
train_classifier_log_file = osjoin('log', 'train_classification_model.log')

find_plagiarism_paragraph_dir = 'find_plagiarism_paragarph'
plg_dataframe_dir = osjoin(find_plagiarism_paragraph_dir, 'plg_dataframe')
predited_stats_dir = osjoin(find_plagiarism_paragraph_dir, 'predicted_stats')
find_plg_log_file = osjoin('log', 'find_plagiarism_paragraph.log')

production_susp_dir = osjoin(root_dir, 'production_susp')
production_susp_stats_dir = osjoin(root_dir, 'production_susp_stats')


import bs4
import requests

url = 'https://id.carousell.com/carousell_id'

contents = requests.get(url)

response = bs4.BeautifulSoup(contents.text, 'html.parser')

data = response.find('div', attrs={'class': 'D_apq D_eZ M_gF D_fb M_gH'})

datas = data.findAll('div', attrs={'class': 'D_jg', 'class': 'D_qq', 'class': 'D_qv'})
    # print(datas)
for obj in datas:
    judul = obj.find('p', attrs={'class': "D_bN M_aT D_aP M_aC D_bO M_aU D_bR M_aX D_bT M_aZ D_bW M_bc "
                                          "D_bZ M_bg D_bK"}).text
    image = obj.find('img', attrs={'class': 'D_bl', 'class': 'D_bi', 'class': 'D_tO'})['src']

    with open('images/' + judul + '.jpg', 'wb') as f:
        img = requests.get(image)
        f.write(img.content)


import numpy as np
import json
import matplotlib.pyplot as plt
import os

# Joints such as ear, eyes are not necassary
wanted_joints = list(range(5,18)) + [19]

def normalize_halpe26(poses, img):
    hip_idx = 13 #19 before removal of unneeded joints
    for i, det in enumerate(poses):
        nrows = det['box'][3]
        ncols = det['box'][2]

        # Define the centroid of pelvis (hip) as center of image
        center_x = det['keypoints'][hip_idx, 0]
        center_y = det['keypoints'][hip_idx, 1]

        keypts = poses[i]['keypoints'].copy()[:,:-1]
        keypts_norm = keypts.copy()
        keypts_norm[:, 0] = (center_x - keypts[:, 0]) / ncols
        keypts_norm[:, 1] = (center_y - keypts[:, 1]) / nrows

        poses[i]['keypoints'][:, :-1] = keypts_norm
    return poses

if __name__ == "__main__":
    print(wanted_joints, len(wanted_joints))


from fastapi import APIRouter

from api.docker.models import DockerStatsModel
from api.docker.retrieval import get_container_stats, ping_docker

router = APIRouter(
    prefix="/docker",
    tags=["Docker"],
)

@router.get("/", tags=["Ping"])
def get_docker_health():
    status = ping_docker()
    return {"status": "ok" if status else "error"}

@router.get("/stats/", response_model=DockerStatsModel)
def get_docker_stats():
    return get_container_stats()


# Title: IS 6713 Homework #1
# Author: Kalea Sebesta
# Date: Oct. 20, 2017
# Due Date: Oct 31, 2017

'''
Program Discription: Purpose is to maintian company info
for a small business. (first and last name, position, department,
and salary of each employee) The program continuously read input
from the user, output data, or quit the program depending on the
option the user selects. Menu items: the input of the company can
only be entered once, can input department, add employee, change
employee name, promote employee, output list of all current empl-
oyee and their info, output whether an employee exists (by name)
if not can add employee, or quit the program.
'''

# ---------------------------------------------------------
# FUNCTION DEFINITIONS
# ---------------------------------------------------------
# Function Name: Menu function
# Inputs: None
# Output: Returns the option the user input (str)
# Purpose: Displays menu and prompts for user input
# ---------------------------------------------------------
def Menu():
    # Prompts the user for input
    option = input('Choose one of the Following Options:\n'
                   'a. Input Company Name\n'
                   'b. Input Department Name\n'
                   'c. Add Employee\n'
                   'd. Change Employee Name\n'
                   'e. Promote Employee to New Position\n'
                   'f. Give Employee Raise\n'
                   'g. Current Employees and Info\n'
                   'h. Check Employee Existence\n'
                   'i. End Program\n')
    return option
# ----------------------------------------------------------
# Name: addEmployee function
# Input: Dictionary
# Output: Dictionary
# Purpose: Takes in a global dictionary and prompts the user
#          for an employees information and then adds the emp
#          to the dictionary if the emp doesnt already exists
# -----------------------------------------------------------
def addEmployee(dictionary):
    try:
        lname = str(input('Enter last name:'))
        fname= str(input('Enter first name:'))
        pos = str(input('Enter position: '))
        dept = str(input('Enter department: '))
        sal = int(input('Enter annual salary: '))

    except:
        # put a try or except to inform the user to enter the
        # salary as numeric value
        print('Please try again, the salary must include only numeric values')
        sal = int(input('Enter annual salary: '))

    # takes values inputted by user and saves them in the employee
    # dictionary and returns it. it also creates a new key called
    # full name that combines first and last names
    dictionary[dept] = {'lname': lname,
                         'fname': fname,
                         'fullName': fname + " " + lname,
                         'Position': pos,
                         'Department': dept,
                         'Salary': sal}
    return dictionary
# ------------------------------------------------------------
# Name: display_emp_counts
# Input: Two Lists and an index
# Output: Void (print statement
# Purpose: Takes the department list that contains the depart-
#          ment names and the list that contains the dictionaries
#          of the employees and the index that contains the
#          desired department in the department list. It then loops
#          through the employee list comparing the department in
#          the employee dictionary to the desired department, if
#          they are the same then the counter is increased (the
#          counter is counting how many employees are in that
#          department. Then the department name and number of emp
#          is printed out.
# -------------------------------------------------------------
def display_emp_counts(dept_lst, dept_emp_lst, index):
    # initializing variables
    count = 0
    size = len(dept_emp_lst)

    # printing the department
    print('The Below Employee List for Department:', dept_lst[index])

    for i in range(0, size, 1):
        # HELP HERE MY LOOP IS REPEATING LAST PERSON AND NOT PUTTING
        # IT IN THE CORRECT DEPARTMENT IF THE DEPARTMENT HAS ALREADY
        # BEEN ENTERED PREVIOUSLY
        # printing each employees info on one line prettily who is in
        # desired dept
        if dept_emp_lst[i]['Department'] == dept_lst[index]:
            print(dept_emp_lst[i]['fullName'],
                  dept_emp_lst[i]['Department'],
                  dept_emp_lst[i]['Position'],
                  "$"+'{:,}'.format(dept_emp_lst[i]['Salary']))
            count = count + 1

    # printing the department and total employee count
    print('The', dept_lst[index], 'department has', count, 'employee(s)\n')
# ----------------------------------------------------------------
# MAIN PROGRAM
# ----------------------------------------------------------------
# initialize variables
counter = 0
dept = tuple()
optionList = ['a','b','c','d','e','f','g','h','i']
count = 0
dept_emp_lst = []
global_emp = dict()

# Display Menu (start while loop)
while True:
    choice = Menu()

    # ----------------------------------------------------------

    # a. Input Company Name
    # if option a. is selected more than once an error message is
    # displayed and prompts the user to choose another option

    if choice == 'a' or choice == 'A' and counter == 0:
        comName = str(input('Enter the name of the company: '))
        counter = counter + 1

    elif choice == 'a' or choice == 'A' and counter > 0:
        # entering same company name results in error message
        comName = str(input('Enter the name of the company: '))
        print('Please choose another option (b-i) ', comName, 'already exists')
    # ----------------------------------------------------------

    # b. Input Department Name
    # prompts the user for a department name and checks to see
    # if it exists, if it doesnt it adds it to the dept tuple
    # if it does exists an error message is displayed

    elif choice == 'b' or choice == 'B' and counter > 0:
        deptName = input('Enter department name: ')
        if deptName not in dept:
            dept += (deptName,)
        else:
            print('Department Already Exists')
    # ----------------------------------------------------------

    # c. Add Employee
    # HOW CAN I CHECK AND IF EMPLOYEE ALREADY EXISTS DELETE DUPLICATES
    # add employee to the global dictionary and checks to see if
    # department is in department list, if it isn't it adds it to the
    # department tuple

    elif choice == 'c' or choice == 'C' and counter > 0:
            addEmployee(global_emp)
            for key in global_emp:
                if global_emp[key]["Department"] not in dept:
                    print("department not found")
                    print("adding to dept tuple")
                    dept += (global_emp[key]["Department"],)
            dept_emp_lst.append(global_emp[key])
    # ----------------------------------------------------------

    # d. Change Employee's Name
    # prompts user for the employee's name to change, if it is
    # in the system, it prompts the employee for the new name, if
    # it is not in the system it informs the user that the employee
    # isn't in the system

    elif choice == 'd' or choice == 'D' and counter > 0:
        # prompt user for original name
        oName = input('Enter original name:')

        if oName in global_emp[key]['fullName']:
            # prompt user for new name
            nName = input('Enter new name: ')

            # loop through employees in dept and find the original name
            # replace old name with new name in the empInfo
            for oName in global_emp[key]['fullName']:
                global_emp[key]['fullName'] = nName
        else:
            print('That employee does not exist')
    # ----------------------------------------------------------

    # e. Promote Employee to New Position
    # prompts the user for the employee that will be given a new
    # position, if the employee is in the system it prompts the
    # user for the new position title and replaces the old position
    # with the new position, if employee is not found in the system
    # the user is informed

    elif choice == 'e' or choice == 'E' and counter > 0:
        # prompt user for employee
        name = input('Enter name of employee that is changing positions: ')

        if name in global_emp[key]['fullName']:
            # prompt user for new position
            newPos = input('Enter the title of the new position: ')

            # change position in dictionary
            for name in global_emp[key]['fullName']:
                global_emp[key]['Position'] = newPos
        else:
            print('That employee does not exist')
    # ----------------------------------------------------------

    # f. Give Employee Raise
    # prompts the user for the employee that will be getting the
    # raise, if the employee is in the system it then prompts the
    # user for the percentage raise and calculates the new salary
    # with the new raise and replaces the old salary with the new
    # salary in the employee's dictionary

    elif choice == 'f' or choice == 'F' and counter > 0:
        # prompt user for employee
        name = input('Enter name of employee that is getting a raise: ')

        # checks to verify name is in the dictionary, if the employee is
        # in the dictionary then the user is prompt for the percentage raise
        # then calculate the raise and applies it to salary. if employee
        # is not in the system the user is notified
        if name in global_emp[key]['fullName']:
            percR = float(input('Enter the %raise: '))
            newSal = round((1 + (percR / 100)) * global_emp[key]['Salary'], 2)
            global_emp[key]['Salary'] = newSal
        else:
            print('That employee does not exist')
    # ---------------------------------------------------------

    # g. Current Employees and Info
    # for each department the employees info are printed out and
    # the total count for each department is also displayed
# ERRORS HERE WHEN A EMPLOYEE IT IT SALES MARKETING IT (THE LAST IT
# IS PRINTED AS A DUPLICATE OF THE MARKETING EMPLOYEE) LOGIC OF MY
# LOOP IS INCORRECT

    elif choice == 'g' or choice == 'G' and counter > 0:
        size = len(dept_emp_lst)
        for index in range(0, size, 1):
            display_emp_counts(dept, dept_emp_lst, index)
    # ---------------------------------------------------------

    # h. Does Employee Exist?
    # prompts the user for the name of the employee and checks it
    # against the names in the global employee dictionary, if emp-
    # loyee is already in the system the user is notified, if it
    # is not, it prompts the user to add the new employee

    elif choice == 'h' or choice == 'H' and counter > 0:
        # prompt user for original name
        name = input('Enter name of employee: ')
        # if yes display info
        if name in global_emp[key]['fullName']:
            print(name, 'is already an employee in the system')

        # if not then prompt to add to employee
        else:
            print('Employee not found, please add to system')
            addEmployee(global_emp)
            for key in global_emp:
                if global_emp[key]["Department"] not in dept:
                    print("department not found")
                    print("adding to dept tuple")
                    dept += (global_emp[key]["Department"],)
            dept_emp_lst.append(global_emp[key])
    # ---------------------------------------------------------

    # if the user inputs invalid option
    # if the user tries to input any information before entering
    # company name the user is notified and loops back to the menu

    elif (choice != 'a' or choice != 'A') and counter == 0:
        print('Company Name Must be Entered First')

    # if user enters a string instead of a value from the option
    # list the user is notified and loops back to the menu
    elif choice not in optionList:
        print('Not a valid option, please choose again')
    # --------------------------------------------------------

    # i. End Program
    # by ending the program an output file is written and the
    # title of the file is displayed to the user

    elif choice == 'i' or choice == 'I':
        # create output file
        # open and write
        fout = open('sebesta_HW1_output.txt', 'w')

        # prints company name and total number of employees
        fout.write('Company Name: %s\nNumber of Employees: %d\n\n' %
                   (comName, len(dept_emp_lst)))

        # prints out each employee and info in department
        # prints out alphabetically by department and last name
        # packages and imports
        from operator import itemgetter
        import operator
        newlst = sorted(dept_emp_lst,
                        key=operator.itemgetter('Department','lname'),
                        reverse=False)

        # calculates the number of employees for each department and
        # writes it to the file
        for i in dept:
            fout.write('The %s department has %d employees\n' %
                       (i, sum(x.get('Department') == i for x in newlst)))
        fout.write('\n')

        # LOGIC ERROR OCCURING HERE WHEN, IT IT Sales Marketing IT, IS ENTERED
        fout.write('\n'.join(d['fullName'] + ", " +
                             d['Department'] + ", " +
                             d['Position'] + ", $" +
                             '{:,}'.format(d['Salary'])for d in newlst))

        # print output file name
        print("The output file is: 'sebesta_HW1_output.txt' ")

        # close file
        fout.close()
        break
    # ---------------------------------------------------------


import requests
import optparse

from progressbar import *

CHUNK_SIZE = 1024

widgets = ['Downloading : ', Percentage(),
           ' ', Bar(marker='#',left='[',right=']'),
           ' ', ETA(), ' ', FileTransferSpeed()]

def download_file(url, file_name=None):
    response = requests.head(url)
    total_size = int(response.headers.get('content-length'))
    pbar = ProgressBar(widgets=widgets, maxval=total_size)
    pbar.start()

    going_size = 0

    if not file_name:
        file_name = url.split('/')[-1]
    elif os.path.isfile(file_name):
        file_name += 'new_' + file_name

    r = requests.get(url, stream=True)
    with open(file_name, 'wb') as f:
        for chunk in r.iter_content(chunk_size=CHUNK_SIZE):
            going_size += CHUNK_SIZE
            pbar.update(going_size)
            if chunk:
                f.write(chunk)
                f.flush()
    pbar.finish()
    return local_filename

parser = optparse.OptionParser()
parser.add_option('-u', default=False, dest='url')
parser.add_option('-n', default=False, dest='name')

options, remainder = parser.parse_args()
file_ = download_file(options.url, options.name)


from os import path
import warnings
import copy
import cv2
from PIL import Image
from infy_field_extractor.internal.constants import Constants

class ExtractorHelper():
    """Helper class for data extraction"""
    @staticmethod
    def extract_with_text_coordinates(
            image, bboxes_text, get_text_provider, file_data_list, additional_info,
            fieldboxes, logger, temp_folderpath, field="checkbox"):
        """
        Method to help extract fields using image, imagepath, bounding boxes coordinates of the text and
        bounding boxes coordinates of the field
        """
        _, width = image.shape
        # filter fieldboxes from bboxes_text
        bboxes_text = ExtractorHelper.filter_fieldboxes_from_ocr_words(
            fieldboxes, bboxes_text)

        # getting phrases
        # bboxes_text = ocr_parser_object.get_tokens_from_ocr(
        #     token_type_value=3, ocr_word_list=bboxes_text)
        additional_info['word_bbox_list'] = bboxes_text
        bboxes_text = get_text_provider.get_tokens(
            3, image, [], file_data_list, additional_info, temp_folderpath)

        # dividing a series of horizontal fieldboxes as each bounding box
        Y_SCALE = 6
        H_SCALE = 3
        c = fieldboxes[0]
        # appends the first line in bboxes_line_list
        bboxes_line_list = [[0, c[Constants.BB_Y]-c[Constants.BB_H]//Y_SCALE,
                             width, c[Constants.BB_H]+c[Constants.BB_H]//H_SCALE]]
        # list to add a new line
        temp_list = []
        # to track the count of the number of lines in bboxes_line_list
        count = 0
        # to track if any new word found which is not present in any bboxes_line_list
        flag = False
        for f in fieldboxes:
            for i in bboxes_line_list:
                count += 1
                # if words already there in the bboxes_list_line then set flag
                # as True and moves to the next line
                if(i[Constants.BB_Y] <= f[Constants.BB_Y] <= i[Constants.BB_Y]+i[Constants.BB_H]):
                    flag = True
                elif(flag is False and count == len(bboxes_line_list)):
                    temp_list.append(
                        [0, f[Constants.BB_Y]-f[Constants.BB_H]//Y_SCALE, width,
                            f[Constants.BB_H]+f[Constants.BB_H]//H_SCALE])
            bboxes_line_list = bboxes_line_list + temp_list
            temp_list = []
            flag = False
            count = 0

        # getting the final result
        # for each line divided calls the __get_status_for_each_line method
        result = {}
        done_fields_dList = []
        count = 0
        for bbox_line in bboxes_line_list:
            count += 1
            logger.info(
                "Extracting checkboxes from line "+str(count)+":")
            r, done_fieldsList = ExtractorHelper.get_status_for_each_line(
                bbox_line, bboxes_text, fieldboxes, image, logger, field)
            done_fields_dList = done_fields_dList+done_fieldsList
            result.update(r)
        return result, done_fields_dList

    @staticmethod
    def get_status_for_each_line(bbox_line, bboxes_text, fieldboxes, image, logger, field):
        """
        It returns a dictionary with text as key and the field's status or bbox as value
        """

        # stores the x,y,width and height of the bbox of the line
        _ = bbox_line[Constants.BB_X]
        y_l = bbox_line[Constants.BB_Y]
        _ = bbox_line[Constants.BB_W]
        h_l = bbox_line[Constants.BB_H]
        # filter fieldboxes present in bbox_line
        fieldboxes_line = []
        for c in fieldboxes:
            if(y_l <= c[Constants.BB_Y] <= y_l+h_l):
                fieldboxes_line.append(c)

        # filter texts present in bbox_line
        texts_line = []
        for t in bboxes_text:
            # gets all the text even if a small region of the text is in the line, therefore
            # matches both the y-coordinate and y-coordinate+height of the text
            # lying inside the line's bbox
            if((y_l <= t.get("bbox")[Constants.BB_Y] <= (y_l+h_l)) or
                    (y_l <= (t.get("bbox")[Constants.BB_Y]+t.get("bbox")[Constants.BB_H]) <= (y_l+h_l))):
                texts_line.append(t)

        # check if the fieldboxes are at the right or left side of the texts
        # initializing isfieldRight as True, assuming that the last bbox in the line is of checkbox
        isfieldRight = True
        last_field = fieldboxes_line[len(fieldboxes_line)-1]
        y_c = last_field[Constants.BB_Y]
        h_c = last_field[Constants.BB_H]
        for t in texts_line:
            x_t = t.get("bbox")[Constants.BB_X]
            y_t = t.get("bbox")[Constants.BB_Y]
            # if the last bbox in the line is a phrase then fieldboxes are on the left side
            if((y_c-(h_c//2) <= y_t <= y_c + h_c) and x_t > last_field[Constants.BB_X]):
                isfieldRight = False
        logger.info(
            "Fieldboxes on the right side of value:"+str(isfieldRight))

        result = {}

        # get the final result
        # the variable adds dictionary with key as the text used for radiobutton and value as its bbox
        done_fields_dList = []
        for f in fieldboxes_line:
            # declare closest variable to consider the key for the fielbox which is closest to it
            closest = texts_line[0]
            # if key are to the right of fields, the closest text to the right
            #  of the field is key for that field
            if(isfieldRight is False):
                for t in texts_line:
                    x_t = t.get("bbox")[Constants.BB_X]
                    t_dist = x_t - (f[Constants.BB_X]+f[Constants.BB_W])
                    close_dist = closest.get(
                        "bbox")[Constants.BB_X] - (f[Constants.BB_X]+f[Constants.BB_W])
                    if(close_dist < 0):
                        closest = t
                    if(close_dist > 0 and t_dist > 0 and t_dist < close_dist):
                        closest = t
            # if key are to the left of fields, the closest text to the left of the field
            #  is key for that field
            else:
                for t in texts_line:
                    x_t = t.get("bbox")[Constants.BB_X]
                    w_t = t.get("bbox")[Constants.BB_W]
                    t_dist = f[Constants.BB_X] - x_t - w_t
                    close_dist = f[Constants.BB_X] - (
                        closest.get("bbox")[Constants.BB_X]+closest.get("bbox")[Constants.BB_W])
                    if(close_dist < 0):
                        closest = t
                    if(close_dist > 0 and t_dist > 0 and t_dist < close_dist):
                        closest = t
            text = closest.get("text")
            done_fields_dList.append(closest)
            # if two phrases arranged vertically is meant for that field, it looks for the texts
            # which has almost the same y-coordinate
            X_SCALE = 2
            Y_SCALE = 2
            for t in texts_line:
                x_t = t.get("bbox")[Constants.BB_X]
                y_t = t.get("bbox")[Constants.BB_Y]
                w_t = t.get("bbox")[Constants.BB_W]
                h_t = t.get("bbox")[Constants.BB_H]
                x_ct = closest.get("bbox")[Constants.BB_X]
                y_ct = closest.get("bbox")[Constants.BB_Y]
                # compares the closest text's y-coordinates with the current text
                # which should be more than
                # heigth of the phrase and the x- coordinate should be almost equal
                if((x_t-w_t//X_SCALE) <= x_ct <= (x_t+w_t//X_SCALE) and (Y_SCALE*abs(y_t - y_ct) > h_t)):
                    done_fields_dList.append(t)
                    if(y_ct < y_t):
                        text = closest.get("text") + " " + t.get("text")
                    else:
                        text = t.get("text") + " " + closest.get("text")
                    break
            # if the field is a checkbox then calls the method to see if checkbox checked or not
            if(field == "checkbox"):
                isCheck = ExtractorHelper.check_if_true(
                    image, f, field)
                result[text] = isCheck
            # if the fiels is radio then returns the text as key and radiobutton bbox as value
            elif(field == "radio"):
                result[text] = f

        return result, done_fields_dList

    @staticmethod
    def check_if_true(
            image, field_bbox, field, field_coordinate=[], debug_mode_check=False,
            temp_folderpath=None, img_name=None):
        """
        checks the status of the checkbox/radio using contour detection method
        """
        # to get the image of only field
        x, y, w, h = field_bbox[Constants.BB_X], field_bbox[Constants.BB_Y], \
            field_bbox[Constants.BB_W], field_bbox[Constants.BB_H]
        img = image[y:y+h, x:x+w]
        _, threshold = cv2.threshold(
            img, 170, 255, cv2.THRESH_BINARY_INV)
        contours, _ = cv2.findContours(
            threshold, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        if(debug_mode_check is True):
            img = cv2.drawContours(img, contours, -1, (180, 105, 255), 3)
            cv2.imwrite(f'{temp_folderpath}\\{img_name}_contours.png', img)
        if(field == "radio"):
            isCheck = False
            x, y, r = field_coordinate[0] - \
                x, field_coordinate[1] - y, field_coordinate[2]
            for i in range(0, len(contours)):
                cnt = contours[i]
                for c in cnt:
                    if(int(x-r/3) < c[0][0] < int(x+r/3) and int(y - r/3) < c[0][1] < int(y+r/3)):
                        isCheck = True
            return isCheck
        elif(field == "checkbox"):
            MY_CONS_1 = 6
            cv2.drawContours(img, contours, -1, (100, 255, 150), 5)
            # x,y,w,h of the outer most boundary of the checkbox
            x, y = 0, 0
            # to count the number of squares
            count = 0
            # to count junk contours
            count_false_cnt = 0
            # checked_area = 0
            for i in range(0, len(contours)):
                cnt = contours[i]
                epsilon = 0.04*cv2.arcLength(cnt, True)
                approx = cv2.approxPolyDP(cnt, epsilon, True)
                x1, _, w1, h1 = cv2.boundingRect(cnt)
                # counts the if the contours has four edges and if the x-coordinate lies in the range of
                # x-coordinate of the outermost boundary of the checkbox
                if (len(approx) == Constants.RECT_EDGES and x-(w//MY_CONS_1) <= x1 <= x+(w//MY_CONS_1)):
                    count += 1
                elif w1*h1 < 0.05*w*h:
                    count_false_cnt += 1
                # else:
                #     checked_area += w1*h1
                # if there is another contour other than the margins of the checkboxes, then true
            if(len(contours)-count - count_false_cnt > 0):
                return True
            else:
                return False

    @staticmethod
    def filter_fieldboxes_from_ocr_words(fieldboxes, bboxes_text):
        filter_list = []
        # filter fieldboxes from bboxes_text
        for t in bboxes_text:
            for c in fieldboxes:
                x_t = t.get("bbox")[Constants.BB_X]
                y_t = t.get("bbox")[Constants.BB_Y]
                w_t = t.get("bbox")[Constants.BB_W]
                h_t = t.get("bbox")[Constants.BB_H]
                # checks if the fieldbox x-coordinate is in the range of bboxes_text x -coordinate
                # and x-coordinate+width or vice-versa
                # also checks for the fieldbox y-coordinate is in the range of bboxes _text
                if((x_t <= c[Constants.BB_X] <= (x_t+w_t) and
                    y_t <= c[Constants.BB_Y] <= (y_t+h_t)) or
                    (c[Constants.BB_X] <= x_t <= (c[Constants.BB_X]+c[Constants.BB_W]) and
                     c[Constants.BB_Y] <= y_t <= (c[Constants.BB_Y]+c[Constants.BB_H]))):
                    filter_list.append(t)
                    continue
                # checks if the fieldbox y-coordinate is in the range of bboxes_text y-coordinate
                # and x-coordinate+width or vice-versa
                # also checks for the fieldbox x-coordinate is in the range of bboxes _text
                if((x_t <= c[Constants.BB_X] <= (x_t+w_t) and
                    c[Constants.BB_Y] <= y_t <= (c[Constants.BB_Y]+c[Constants.BB_H])) or
                    (c[Constants.BB_X] <= x_t <= (c[Constants.BB_X]+c[Constants.BB_W]) and
                     y_t <= c[Constants.BB_Y] <= (y_t+h_t))):
                    filter_list.append(t)
                    continue
        bboxes_text = [x for x in bboxes_text if x not in filter_list]
        return bboxes_text

    @staticmethod
    def check_image_dpi(imagepath, logger):
        im = Image.open(imagepath)
        try:
            dpi = im.info['dpi']
            if(dpi[0] < Constants.TESSERACT_MIN_DPI and dpi[1] < Constants.TESSERACT_MIN_DPI):
                warning = "The result might be not accurate due to low dpi"
                warnings.warn(warning)
                logger.warning(warning)
        except Exception:
            warning = ("Dpi of the image cannot be extracted: "
                       "The result might be not accurate if the dpi is less than 300")
            warnings.warn(warning)
            logger.warning(warning)

    @staticmethod
    def read_image(image_path, logger, temp_folderpath, coordinates=[]):
        if(path.exists(image_path) is False):
            logger.error("property imagepath not found")
            raise Exception("property imagepath not found")
        img_name = path.splitext(path.split(image_path)[1])[0]
        image = cv2.imread(image_path)
        try:
            img = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
        except Exception:
            img = image
        if(coordinates != []):
            if type(coordinates[0]) == dict:
                coordinates = coordinates[0]["bbox"]
            x_img, y_img, width, height = coordinates[Constants.BB_X], coordinates[
                Constants.BB_Y], coordinates[Constants.BB_W], coordinates[Constants.BB_H]
            image = img[y_img:y_img+height, x_img:x_img+width]
            imagepath = temp_folderpath + "//" + img_name + '_crop.jpg'
            PILimage = Image.fromarray(image)
            PILimage.save(imagepath, dpi=(300, 300))
            # cv2.imwrite(path.join(imagepath), image)
        else:
            image = img
            imagepath = image_path
        return (image, imagepath, img_name)

    @staticmethod
    def get_closest_fieldbox(fieldboxes, field_pos, phrase_bbox):
        closest = fieldboxes[0]
        if(field_pos == "right"):
            for c in fieldboxes:
                c_dist = c[Constants.BB_X] - \
                    (phrase_bbox[Constants.BB_X] + phrase_bbox[Constants.BB_Y])
                close_dist = closest[Constants.BB_X] - \
                    (phrase_bbox[Constants.BB_X] + phrase_bbox[Constants.BB_Y])
                closest = ExtractorHelper.closest_fieldbox_if_left_right(
                    phrase_bbox, c, close_dist, c_dist, closest)
        elif(field_pos == "left"):
            for c in fieldboxes:
                c_dist = phrase_bbox[Constants.BB_X] - \
                    (c[Constants.BB_X] + c[Constants.BB_W])
                close_dist = phrase_bbox[Constants.BB_X] - \
                    (closest[Constants.BB_X]+closest[Constants.BB_W])
                closest = ExtractorHelper.closest_fieldbox_if_left_right(
                    phrase_bbox, c, close_dist, c_dist, closest)
        elif(field_pos == "bottom"):
            for c in fieldboxes:
                c_dist = c[Constants.BB_Y] - \
                    (phrase_bbox[Constants.BB_Y] + phrase_bbox[Constants.BB_H])
                close_dist = closest[Constants.BB_Y] - \
                    (phrase_bbox[Constants.BB_Y] + phrase_bbox[Constants.BB_H])
                closest = ExtractorHelper.closest_fieldbox_if_top_bottom(
                    phrase_bbox, c, close_dist, c_dist, closest)
        elif(field_pos == "top"):
            for c in fieldboxes:
                c_dist = phrase_bbox[Constants.BB_Y] - \
                    (c[Constants.BB_Y] + c[Constants.BB_H])
                close_dist = phrase_bbox[Constants.BB_Y] - \
                    (closest[Constants.BB_Y]+closest[Constants.BB_H])
                closest = ExtractorHelper.closest_fieldbox_if_top_bottom(
                    phrase_bbox, c, close_dist, c_dist, closest)
        else:
            dist_list = []
            for f in fieldboxes:
                dist_dict = {}
                dist_dict["fieldbox"] = f
                dist_dict["x_dist"] = abs(
                    f[Constants.BB_X] - phrase_bbox[Constants.BB_X])
                dist_dict["y_dist"] = abs(
                    f[Constants.BB_Y] - phrase_bbox[Constants.BB_Y])
                dist_list.append(dist_dict)
            dist_list.sort(key=lambda x: (x["x_dist"], x["y_dist"]))
            return dist_list[0]["fieldbox"]
        return closest

    @staticmethod
    def closest_fieldbox_if_top_bottom(
            phrase_bbox, fieldbox, closest_fieldbox_dist, fieldbox_dist, closest_fieldbox):
        close_dist = closest_fieldbox_dist
        c = fieldbox
        c_dist = fieldbox_dist
        closest = closest_fieldbox
        if(close_dist < 0):
            if(phrase_bbox[Constants.BB_X] >= c[Constants.BB_X] and
                    phrase_bbox[Constants.BB_X]+phrase_bbox[Constants.BB_W] <= c[Constants.BB_X]+c[Constants.BB_W]):
                closest = c
            elif(phrase_bbox[Constants.BB_X] <= c[Constants.BB_X] <= phrase_bbox[Constants.BB_X]+phrase_bbox[Constants.BB_W]):
                closest = c
            elif(phrase_bbox[Constants.BB_X] <= c[Constants.BB_X]+c[Constants.BB_W] <= phrase_bbox[Constants.BB_X]+phrase_bbox[Constants.BB_W]):
                closest = c
        elif(close_dist > 0 and c_dist > 0 and c_dist <= close_dist):
            if(phrase_bbox[Constants.BB_X] >= c[Constants.BB_X] and
                    phrase_bbox[Constants.BB_X]+phrase_bbox[Constants.BB_W] <= c[Constants.BB_X]+c[Constants.BB_W]):
                closest = c
            elif(phrase_bbox[Constants.BB_X] <= c[Constants.BB_X] <= phrase_bbox[Constants.BB_X]+phrase_bbox[Constants.BB_W]):
                closest = c
            elif(phrase_bbox[Constants.BB_X] <= c[Constants.BB_X]+c[Constants.BB_W] <= phrase_bbox[Constants.BB_X]+phrase_bbox[Constants.BB_W]):
                closest = c
        return closest

    @staticmethod
    def closest_fieldbox_if_left_right(phrase_bbox, fieldbox, closest_fieldbox_dist, fieldbox_dist, closest_fieldbox):
        close_dist = closest_fieldbox_dist
        c = fieldbox
        c_dist = fieldbox_dist
        closest = closest_fieldbox
        if(close_dist < 0):
            if(phrase_bbox[Constants.BB_Y] >= c[Constants.BB_Y] and
                    phrase_bbox[Constants.BB_Y]+phrase_bbox[Constants.BB_H] <= c[Constants.BB_Y]+c[Constants.BB_H]):
                closest = c
            elif(phrase_bbox[Constants.BB_Y] <= c[Constants.BB_Y] <= phrase_bbox[Constants.BB_Y]+phrase_bbox[Constants.BB_H]):
                closest = c
            elif(phrase_bbox[Constants.BB_Y] <= c[Constants.BB_Y]+c[Constants.BB_W] <= phrase_bbox[Constants.BB_Y]+phrase_bbox[Constants.BB_H]):
                closest = c
        elif(close_dist > 0 and c_dist > 0 and c_dist <= close_dist):
            if(phrase_bbox[Constants.BB_Y] >= c[Constants.BB_Y] and
                    phrase_bbox[Constants.BB_Y]+phrase_bbox[Constants.BB_H] <= c[Constants.BB_Y]+c[Constants.BB_H]):
                closest = c
            elif(phrase_bbox[Constants.BB_Y] <= c[Constants.BB_Y] <= phrase_bbox[Constants.BB_Y]+phrase_bbox[Constants.BB_H]):
                closest = c
            elif(phrase_bbox[Constants.BB_Y] <= c[Constants.BB_Y]+c[Constants.BB_W] <= phrase_bbox[Constants.BB_Y]+phrase_bbox[Constants.BB_H]):
                closest = c
        return closest

    @staticmethod
    def get_box_region(
            image, img_name, debug_mode_check, temp_folderpath,
            MIN_BOX_HEIGHT, MIN_BOX_WIDTH, MAX_BOX_HEIGHT=None, MAX_BOX_WIDTH=None):
        # get regions
        img = image.copy()
        (_, img_bin) = cv2.threshold(img, 128, 255,
                                     cv2.THRESH_BINARY | cv2.THRESH_OTSU)
        img_bin = 255-img_bin

        # Defining a kernel length
        kernel_length = img.shape[1]//200

        # A verticle kernel of (1 X kernel_length), which will detect
        # all the verticle lines from the image.
        verticle_kernel = cv2.getStructuringElement(
            cv2.MORPH_RECT, (1, kernel_length))
        # A horizontal kernel of (kernel_length X 1), which will help
        # to detect all the horizontal line from the image.
        hori_kernel = cv2.getStructuringElement(
            cv2.MORPH_RECT, (kernel_length, 1))
        # A kernel of (3 X 3) ones.
        # Morphological operation to detect verticle lines from an image
        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))
        img_temp1 = cv2.erode(img_bin, verticle_kernel, iterations=3)
        verticle_lines_img = cv2.dilate(
            img_temp1, verticle_kernel, iterations=3)
        # Morphological operation to detect horizontal lines from an image
        img_temp2 = cv2.erode(img_bin, hori_kernel, iterations=3)
        horizontal_lines_img = cv2.dilate(img_temp2, hori_kernel, iterations=3)
        # Weighting parameters, this will decide the quantity of an image
        # to be added to make a new image.

        alpha = 0.5
        beta = 1.0 - alpha
        # This function helps to add two image with specific weight
        # parameter to get a third image as summation of two image.
        img_final_bin = cv2.addWeighted(
            verticle_lines_img, alpha, horizontal_lines_img, beta, 0.0)
        img_final_bin = cv2.erode(~img_final_bin, kernel, iterations=2)
        (_, img_final_bin) = cv2.threshold(img_final_bin, 128,
                                           255, cv2.THRESH_BINARY | cv2.THRESH_OTSU)
        if(debug_mode_check is True):
            cv2.imwrite(temp_folderpath+"//"+img_name +
                        "verticle_line.png", verticle_lines_img)
            cv2.imwrite(temp_folderpath+"//"+img_name +
                        "horizontal_line.png", horizontal_lines_img)
            cv2.imwrite(temp_folderpath+"//"+img_name +
                        "img_final_bin.png", img_final_bin)
        # Find contours for image, which will detect all the boxes
        contours, _ = cv2.findContours(
            img_final_bin, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        idx = 0
        bboxes_region = []
        for c in contours:
            # Returns the location and width,height for every contour
            x, y, w, h = cv2.boundingRect(c)
            if(MAX_BOX_HEIGHT is None and MAX_BOX_WIDTH is None):
                if (w >= MIN_BOX_WIDTH and h >= MIN_BOX_HEIGHT):
                    idx += 1
                    bboxes_region.append([x, y, w, h])
                    if(debug_mode_check is True):
                        new_img = img[y:y+h, x:x+w]
                        cv2.imwrite(temp_folderpath+"//"+img_name+str(x) +
                                    '_'+str(y) + '.png', new_img)
            else:
                if (MAX_BOX_WIDTH >= w >= MIN_BOX_WIDTH and MAX_BOX_HEIGHT >= h >= MIN_BOX_HEIGHT):
                    idx += 1
                    bboxes_region.append([x, y, w, h])
                    if(debug_mode_check is True):
                        new_img = img[y:y+h, x:x+w]
                        cv2.imwrite(temp_folderpath+"//"+img_name+str(x) +
                                    '_'+str(y) + '.png', new_img)
        return bboxes_region

    @staticmethod
    def get_updated_within_box(within_bbox, scaling_factor):
        if(len(within_bbox) > 0):
            for i in [0, 2]:
                within_bbox[i] = round(
                    within_bbox[i] * scaling_factor.get('hor', 1))
            for i in [1, 3]:
                within_bbox[i] = round(
                    within_bbox[i] * scaling_factor.get('ver', 1))
        return within_bbox

    @staticmethod
    def get_updated_text_bbox(text_bboxes, scaling_factor):
        if(len(text_bboxes) > 0):
            for bbox in text_bboxes:
                for i in [0, 2]:
                    bbox['bbox'][i] = round(
                        bbox['bbox'][i] * scaling_factor.get('hor', 1))
                for i in [1, 3]:
                    bbox['bbox'][i] = round(
                        bbox['bbox'][i] * scaling_factor.get('ver', 1))
        return text_bboxes

    @staticmethod
    def get_invalid_keys(truth_dict, test_dict) -> list:
        """Compare two dictionary objects and return invalid keys by using one of them as reference

        Args:
            truth_dict (dict): The object containing all valid keys
            test_dict (dict): The object to evaluate for presence of invalid keys

        Returns:
            list: The list of invalid keys
        """

        def __get_all_keys_recursively(parent_key, dict_obj):
            all_keys = []
            for k, val in dict_obj.items():
                key = k if parent_key is None or len(
                    parent_key) == 0 else f"{parent_key}->{k}"
                if not key in all_keys:
                    all_keys.append(key)
                if isinstance(val, dict):
                    all_keys += __get_all_keys_recursively(key, val)
            return all_keys

        truth_keys = __get_all_keys_recursively(None, truth_dict)
        test_keys = __get_all_keys_recursively(None, test_dict)
        return list(set(test_keys)-set(truth_keys))

    @staticmethod
    def get_updated_config_dict(from_dict, default_dict):
        config_dict_temp = copy.deepcopy(default_dict)
        for key in from_dict:
            if isinstance(from_dict[key], dict):
                if config_dict_temp.get(key) is None:
                    config_dict_temp[key] = from_dict[key]
                else:
                    config_dict_temp[key] = ExtractorHelper.get_updated_config_dict(
                        from_dict[key], config_dict_temp[key])
            else:
                if config_dict_temp.get(key) is None:
                    config_dict_temp[key] = from_dict[key]
        return config_dict_temp


IDUN = True

RANDOM_SEED = 2021
TEST_SIZE = 0.15
VAL_SIZE = 0.10
TUNING_SIZE = 0.3

# Dataset sizes
LABELED_VALIDATION_ABSOLUTE_SIZE = 345
LABELED_TRAIN_SIZE = 1380
WEAK_NUMBER_OF_ARTICLES_PER_CLASS = 2760

# TF-IDF params
TITLE_MAX_FEATURES = 1000
CONTENT_MAX_FEATURES = 5000

# LR params
MAX_ITER = 4000
SOLVER = 'liblinear'
PENALTY = 'l1'
CV = 5

# LR Tuned params
WEAK_BEST_C = 5
SUPERVISED_BEST_C = 1

# BERT config
BERT_N_WORDS = 512

# Numerical cols
NUMERICAL_COLS = ['id',
                  'content_exclamation_count',
                  'title_exclamation_count',
                  'content_word_count',
                  'title_word_count',
                  'content_word_count_with_punctuation',
                  'title_word_count_with_punctuation',
                  'content_sentence_count',
                  'title_sentence_count',
                  'content_capital_word_count',
                  'title_capital_word_count',
                  'content_stop_word_count',
                  'title_stop_word_count',
                  'content_stop_word_ratio',
                  'title_stop_word_ratio',
                  'content_words_per_sentence',
                  'content_quote_marks_count',
                  'content_ttr_score',
                  'title_ttr_score',
                  'title_nouns_count',
                  'title_proper_nouns_count',
                  'content_avg_word_length',
                  'title_avg_word_length',
                  'content_avg_word_length_no_stop_words',
                  'title_avg_word_length_no_stop_words',
                  'content_url_count',
                  'content_verb_ratio',
                  'content_past_tense_verb_ratio',
                  'content_past_tense_verb_ratio_of_all_verbs',
                  'content_adjective_ratio',
                  'content_adverb_ratio',
                  'title_verb_ratio',
                  'title_past_tense_verb_ratio',
                  'title_past_tense_verb_ratio_of_all_verbs',
                  'title_adjective_ratio',
                  'title_adverb_ratio',
                  'content_capital_word_ratio',
                  'title_capital_word_ratio',
                  'content_personal_pronouns_count',
                  'content_personal_pronouns_ratio',
                  'content_quote_marks_ratio',
                  'title_nouns_ratio',
                  'title_proper_nouns_ratio',
                  'content_exclamation_ratio',
                  'title_exclamation_ratio',
                  'content_sentiment_word_sub',
                  'content_sentiment_word_pos',
                  'content_sentiment_word_neg',
                  'title_sentiment_word_sub',
                  'title_sentiment_word_pos',
                  'title_sentiment_word_neg',
                  'content_sentiment_sentence_sub',
                  'content_sentiment_sentence_pos',
                  'content_sentiment_sentence_neg',
                  'title_sentiment_sentence_sub',
                  'title_sentiment_sentence_pos',
                  'title_sentiment_sentence_neg',
                  'content_sentiment_text_sub',
                  'content_sentiment_text_pos',
                  'content_sentiment_text_neg',
                  'title_sentiment_text_sub',
                  'title_sentiment_text_pos',
                  'title_sentiment_text_neg',
                  'title_swn_pos_score',
                  'title_swn_neg_score',
                  'title_swn_obj_score',
                  'content_swn_pos_score',
                  'content_swn_neg_score',
                  'content_swn_obj_score'
                  ]

LEMMATIZED_COLS = ['id', 'content_lemmatized_lowercase_no_stopwords', 'title_lemmatized_lowercase_no_stopwords']
RAW_TEXT_COLS = ['id', 'content', 'title']

LABELED_LEMMATIZED_COLS = ['id', 'content_lemmatized_lowercase_no_stopwords', 'title_lemmatized_lowercase_no_stopwords', 'label']
LABELED_RAW_TEXT_COLS = ['id', 'content', 'title', 'label']

BERT_SAVE_COLS = ['text', 'label']


from sys import stdin
from heapq import heappop, heappush
readline = stdin.readline

heap = []
while True:
    line = readline()
    if line[1] == 'x':
        print(-heappop(heap))
    elif line[0] == 'i':
        heappush(heap, -int(line.split()[1]))
    else:
        break

'''
def insert(heap, key):
    heap.append(-1000000)
    heapIncreaseKey(heap, key)

def parent(i):
    return (i - 1) // 2

def heapIncreaseKey(heap, key):
    heap[len(heap) - 1] = key
    i = len(heap) - 1
    while i > 0 and heap[parent(i)] < heap[i]:
        heap[i], heap[parent(i)] = heap[parent(i)], heap[i]
        i = parent(i)

def heapExtractMax(heap):
    if len(heap) < 1:
        return None
    else:
        MAX = heap[0]
        heap[0] = heap.pop()
        maxHeapify(heap, 0)

        return MAX

def maxHeapify(heap, i):
    left = (i + 1) * 2 - 1
    right = (i + 1) * 2
    largest = left if left < len(heap) and heap[left] > heap[i] else i
    largest = right if right < len(heap) and heap[right] > heap[largest] else largest
    if largest != i:
        heap[i], heap[largest] = heap[largest], heap[i]
        maxHeapify(heap, largest)

heap = []
while True:
    line = sys.stdin.readline()
    if line[0] == 'i':
        insert(heap, int(line.split()[1]))
    elif line[0:2] == 'ex':
        print(str(heapExtractMax(heap)))
    else:
        break
'''


import requests
from lxml import etree

BASE_DOMIN = 'https://ygdy8.net'
URL = []
HEADERS = {
    'Referer': 'https://c.02kdid.com/b/1/1754/22432/960X90/960X90.html',
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.116 Safari/537.36'
}

def get_detail_url(url):
    response = requests.get(url, headers=HEADERS)
    text = response.content
    html = etree.HTML(text)
    detail_url = html.xpath('//table[@class="tbspan"]//a/@href')
    detail_url = map(lambda url: BASE_DOMIN + url, detail_url)
    return detail_url

def parse_detail_page(url):
    response = requests.get(url, headers=HEADERS)
    text = response.content
    html = etree.HTML(text)
    movie = {}
    movie['title'] = html.xpath('//div[@class="title_all"]//font[@color="#07519a"]/text()')
    zoom = html.xpath('//div[@id="Zoom"]')[0]
    imgs = html.xpath('//img/@src')
    movie['cover'] = imgs[0]
    infos = zoom.xpath('.//text()')
    def info_parse(text, rule):
        return text.replace(rule, '').strip()

    for index, info in enumerate(infos):
        if info.startswith('◎年　　代'):
            info = info_parse(info, '◎年　　代')
            movie['year'] = info
        elif info.startswith('◎产　　地'):
            info = info_parse(info, '◎产　　地')
            movie['plase'] = info
        elif info.startswith('◎类　　别'):
            info = info_parse(info, '◎类　　别')
            movie['catergory'] = info
        elif info.startswith('◎主　　演'):
            info = info_parse(info, '◎主　　演')
            actors = [info]
            for x in range(index+1, len(infos)):
                actor = infos[x].strip()
                if actor.startswith('◎'):
                    break
                actors.append(actor)
            movie['actors'] = actors
        elif info.startswith('◎简　　介'):
            info = info_parse(info, '◎简　　介')
            proflie = ''
            for x in range(index+1, len(infos)):
                proflie += infos[x].strip()
                if infos[x+1].startswith('【下载地址】'):
                    break
            movie['proflie'] = proflie
    downloadurl = html.xpath('//td[@bgcolor="#fdfddf"]/a/@href')[0]
    movie['downloadurl'] = downloadurl
    return movie

def spider():
    base_url = "https://ygdy8.net/html/gndy/dyzz/list_23_{}.html"
    for x in range(1, 2):
        url = base_url.format(x)
        detail_urls = get_detail_url(url)
        for detail_url in detail_urls:
            movie = parse_detail_page(detail_url)

if __name__ == '__main__':
    spider()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Apr 11 10:56:48 2020

@author: alexanderfalk
"""

import itertools
import time
import sys

class ThreeOPT:

    def __init__(self, computed_solution, time_limit=60):
        self.solution = computed_solution
        self.time_limit = time_limit

    def construct(self, time_left):
        return self.algorithm(time_left)

    def algorithm(self, start_time):
        for index, route in enumerate(self.solution.routes):
            segments = self.tour_segments(route)
            for i, j, k in segments:
                self.solution.routes[index] = self.improvement(route, i, j, k)

                t1 = time.time() # End time
                if t1 - start_time > self.time_limit:
                    sys.stdout.write("Time Expired\n")
                    return self.solution

        return self.solution

    def distance(self, i, j):
        return self.solution.instance.pre_distance(i, j)

    def tour_segments(self, route):
        indices = [index for index in range(len(route))]
        return list(itertools.combinations(indices, r = 3))

    def improvement(self, route, i, j, k):
        A, B, C, D, E, F = route[i-1], route[i], route[j-1], route[j], route[k-1], route[k % len(route)]
        dist0 = self.distance(A, B) + self.distance(C, D) + self.distance(E, F)
        dist1 = self.distance(A, C) + self.distance(B, D) + self.distance(E, F)
        dist2 = self.distance(A, B) + self.distance(C, E) + self.distance(D, F)
        dist3 = self.distance(A, D) + self.distance(E, B) + self.distance(C, F)
        dist4 = self.distance(F, B) + self.distance(C, D) + self.distance(E, A)

        if dist0 > dist1:
            route[i:j] = reversed(route[i:j])

        elif dist0 > dist2:
            route[j:k] = reversed(route[j:k])

        elif dist0 > dist4:
            route[i:k] = reversed(route[i:k])

        elif dist0 > dist3:
            route[i:k] = route[j:k] + route[i:j]

        return route


class Solution(object):
    def splitArray(self, nums):
        """
        :type nums: List[int]
        :rtype: bool
        """
        n = len(nums)
        target = 0
        total = sum(nums)
        for i in range(1, n-5):
            if i != 1 and nums[i-1]==0 and nums[i] == 0:
                continue
            target += nums[i - 1]
            j_start = i + 1
            left = total - nums[i] - target
            if self.j_helper(nums, target, j_start, left):
                return True
        return False

    def j_helper(self, nums, target, start, left):
        j_target = 0
        n = len(nums)
        for j in range(start+1, n-3):
            j_target += nums[j-1]
            k_left = left - nums[j] - j_target
            k_start = j + 1
            if j_target == target and self.k_helper(nums, target, k_start, k_left):
                return True
        return False

    def k_helper(self, nums, target, start, left):
        k_target = 0
        n = len(nums)
        for k in range(start + 1, n-1):
            k_target += nums[k-1]
            last_left = left - nums[k] - k_target
            if k_target == target and last_left == target:
                return True
        return False

leetcode/548. Split Array with Equal Sum.py


from sys import stdin

while(True):
    try:
        xa,ya,xb,yb,xc,yc,xd,yd = map(float, stdin.readline().split(","))
        ab = [xb-xa,yb-ya]
        bc = [xc-xb,yc-yb]
        cd = [xd-xc,yd-yc]
        da = [xa-xd,ya-yd]
        cr1 = ab[0]*bc[1] - ab[1]*bc[0]
        cr2 = bc[0]*cd[1] - bc[1]*cd[0]
        cr3 = cd[0]*da[1] - cd[1]*da[0]
        cr4 = da[0]*ab[1] - da[1]*ab[0]
        if (cr1<0 and cr2<0 and cr3<0 and cr4<0) or (cr1>0 and cr2>0 and cr3>0 and cr4>0):
            print("YES")
        else:
            print("NO")
    except:
        break