import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms
from torch.autograd import Variable
from progressbar import *

def getData(name='cifar10', train_bs=128, test_bs=1000):
    if name == 'svhn':
        train_loader = torch.utils.data.DataLoader(
    datasets.SVHN('../../data', split='extra', download=True,
                   transform=transforms.Compose([
                       transforms.ToTensor()
                   ])),
    batch_size=train_bs, shuffle=True)
        test_loader = torch.utils.data.DataLoader(
    datasets.SVHN('../../data', split='test', download=True,transform=transforms.Compose([
                       transforms.ToTensor()
                   ])),
    batch_size=test_bs, shuffle=False)

    if name == 'cifar10':
        transform_train = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

        transform_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

        trainset = datasets.CIFAR10(root='../../data', train=True, download=True, transform=transform_train)
        train_loader = torch.utils.data.DataLoader(trainset, batch_size=train_bs, shuffle=True)

        testset = datasets.CIFAR10(root='../../data', train=False, download=False, transform=transform_test)
        test_loader = torch.utils.data.DataLoader(testset, batch_size=test_bs, shuffle=False)

    if name == 'cifar100':

        transform_train = transforms.Compose([
        transforms.RandomCrop(32, padding=4),
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

        transform_test = transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),
    ])

        trainset = datasets.CIFAR100(root='../../data', train=True, download=True, transform=transform_train)
        train_loader = torch.utils.data.DataLoader(trainset, batch_size=train_bs, shuffle=True)

        testset = datasets.CIFAR100(root='../../data', train=False, download=False, transform=transform_test)
        test_loader = torch.utils.data.DataLoader(testset, batch_size=test_bs, shuffle=False)

    if name == 'tinyimagenet':
        normalize = transforms.Normalize(mean=[0.44785526394844055, 0.41693055629730225, 0.36942949891090393],
                                     std=[0.2928885519504547, 0.28230994939804077, 0.2889912724494934])
        train_dataset = datasets.ImageFolder(
        '../../data/tiny-imagenet-200/train',
        transforms.Compose([
            transforms.RandomCrop(64, padding=4),
            transforms.RandomHorizontalFlip(),
            transforms.ToTensor(),
            normalize,
        ]))

        train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=train_bs, shuffle=True, num_workers=4, pin_memory=False)

        test_dataset = datasets.ImageFolder(
        '../../data/tiny-imagenet-200/val',
        transforms.Compose([
            transforms.ToTensor(),
            normalize,
        ]))

        test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=test_bs, shuffle=False)

    return train_loader, test_loader

def train(epoch, model, train_loader,optimizer, criterion=nn.CrossEntropyLoss()):
    model.train()
    print('\nTraing, Epoch: %d' % epoch)
    train_loss = 0
    correct = 0
    total = 0
    for batch_idx, (input, target) in enumerate(train_loader):
        input, target = input.cuda(), target.cuda()
        optimizer.zero_grad()
        output = model(input)
        loss = criterion(output, target)
        loss.backward()
        optimizer.step()

        train_loss += loss.item()
        _, predicted = output.max(1)
        total += target.size(0)
        correct += predicted.eq(target).sum().item()

        progress_bar(batch_idx, len(train_loader), 'Loss: %.3f | Acc: %.3f%% (%d/%d)'
                     % (train_loss/(batch_idx+1),
                        100.*correct/total, correct, total))

def test(model, test_loader):
    print('\nTesting')
    model.eval()
    correct = 0
    total_num = 0
    for data, target in test_loader:
        data, target = data.cuda(), target.cuda()
        output = model(data)
        pred = output.data.max(1, keepdim=True)[1] # get the index of the max log-probability
        correct += pred.eq(target.data.view_as(pred)).cpu().sum().item()
        total_num += len(data)
    print('testing_correct: ', correct / total_num)
    return correct / total_num

def exp_lr_scheduler(epoch, optimizer, strategy=True, decay_eff=0.1, decayEpoch=[]):
    """Decay learning rate by a factor of lr_decay every lr_decay_epoch epochs"""
    print(strategy)

    if strategy=='normal':
        if epoch in decayEpoch:
            for param_group in optimizer.param_groups:
                param_group['lr'] *= decay_eff
    else:
        print('wrong strategy')
        raise ValueError('A very specific bad thing happened.')

    return optimizer

def fb_warmup(optimizer, epoch, baselr, large_ratio):
    for param_group in optimizer.param_groups:
        param_group['lr'] = epoch * (baselr * large_ratio-baselr) / 5. + baselr
    return optimizer

# useful
def group_add(params, update, lmbd=1):
    """
    params = params + update*alpha
    :param params: list of variable
    :param update: list of data
    :return:
    """
    for i,p in enumerate(params):
        params[i].add_(update[i]*lmbd+0.)
    return params

def group_product(xs, ys):
    """
    the inner product of two lists of variables xs,ys
    :param xs:
    :param ys:
    :return:
    """
    return sum([torch.sum(x*y) for (x, y) in zip(xs, ys)])

def get_p_g_m(opt, layers):
    i = 0
    paramlst = []
    grad = []
    mum = []

    for group in opt.param_groups:
        weight_decay = group['weight_decay']
        momentum = group['momentum']

        for p in group['params']:
            if p.grad is None:
                continue
            p.grad.data = p.grad.data
            d_p = p.grad.data
            if momentum != 0:
                param_state = opt.state[p]
                if 'momentum_buffer' not in param_state:
                    buf = param_state['momentum_buffer'] = torch.zeros_like(p.data)
                else:
                    buf = param_state['momentum_buffer']
            if i in layers:
                paramlst.append(p.data)
                grad.append(d_p+0.) # get grad
                mum.append(buf*momentum+0.)
            i += 1
    return paramlst, grad, mum

def manually_update(opt, grad):
    for group in opt.param_groups:
        weight_decay = group['weight_decay']
        momentum = group['momentum']
        dampening = group['dampening']
        nesterov = group['nesterov']

        for i,p in enumerate(group['params']):
            d_p = grad[i]
            if weight_decay != 0:
                d_p.add_(weight_decay, p.data)
            if momentum != 0:
                param_state = opt.state[p]
                if 'momentum_buffer' not in param_state:
                    buf = param_state['momentum_buffer'] = torch.zeros_like(p.data)
                    buf.mul_(momentum).add_(d_p)
                else:
                    buf = param_state['momentum_buffer']
                    buf.mul_(momentum).add_(1 - dampening, d_p)
                if nesterov:
                    d_p = d_p.add(momentum, buf)
                else:
                    d_p = buf
            p.data.add_(-group['lr'], d_p)

def change_lr_single(optimizer, best_lr):
    """change learning rate"""
    for param_group in optimizer.param_groups:
        param_group['lr'] = best_lr
    return optimizer

def bad_params(net):
    '''Init layer parameters.'''
    for m in net.modules():
        if isinstance(m, nn.Conv2d):
            nn.init.normal(m.weight, std=0.5)
            if m.bias is not None:
                nn.init.constant(m.bias, 0)
        elif isinstance(m, nn.Linear):
            nn.init.normal(m.weight, std=0.5)
            if m.bias is not None:
                nn.init.constant(m.bias, 0)


import os
from shutil import copyfile, copytree
import pandas as pd
import numpy as np
import math
import argparse

arg_parser = argparse.ArgumentParser(
description="Use a trained DeepSDF decoder to reconstruct a shape given SDF samples."
)
arg_parser.add_argument(
"--meshes_dir", "-d", dest="meshes_dir", required=True, help="The data source directory."
)

data_dir = '/home/nugrinovic/datasets/3DPeopleOcc'

subjects = ['all']
subjects_out = ['']

# get subjects from folders
root_path = data_dir

if not 'subjects' in locals():
    subjects = os.listdir(root_path)
    subjects.sort()
    subjects = [c for c in subjects if os.path.isdir(os.path.join(root_path, c))]
if 'all' in subjects:
    subjects = os.listdir(data_dir)
    subjects.sort()
    subjects = [c for c in subjects if os.path.isdir(os.path.join(root_path, c))]

# get actions from folders
source_metadata = []
target_metadata = []

im_by_cam = {}
train = []
val = []
test = []
data = []

epoch = 100
args = arg_parser.parse_args()
meshes_dir = args.meshes_dir
exp_dir = meshes_dir.split('reconstruction/')[0]
eval_dir = os.path.join(exp_dir, 'eval')

for subject in subjects:
    if subject in subjects_out:
        continue
    f = os.path.join(eval_dir, '%d_%s_eval.csv' % (epoch, subject))

    df = pd.read_csv(f)
    # eval_data = df.to_dict()
    eval_by_iou = df.sort_values(by=['iou'])
    # filter around inflection iou=0.77+-0.1
    set = 0.7900
    # delta = 0.02
    # this line added due to .lst data mal generation
    eval_by_iou = eval_by_iou[eval_by_iou['subject_id'].str.startswith(subject )]

    eval_by_iou = eval_by_iou[eval_by_iou['iou'] > set ]

    subs_by_iou = eval_by_iou['subject_id'].tolist()
    iou_sorted = eval_by_iou['iou'].tolist()
    instances = subs_by_iou
    instances.sort()

    subject_path = os.path.join(root_path, subject)
    actions = os.listdir(subject_path)
    actions.sort()
    actions = [c for c in actions if os.path.isdir(os.path.join(subject_path, c))]
    data = []

    print('start to generate list for every instance of subject: %s'%(subject))
    l = len(instances)
    train = ['/'.join(c.split('/')[1:]) for c in instances]
    train = [c for c in train if 'points.npz' in os.listdir(os.path.join(subject_path, c))]

    data.append(train)

    data =  [item for sublist in data for item in sublist]
    # chunks = [data[x:x + 100] for x in range(0, len(data), 100)]

    df_train = pd.DataFrame(data)
    dst = os.path.join(subject_path, 'data_above_07900_iou.lst')
    np.savetxt(dst, df_train.values, fmt='%s')

    # for i, data in enumerate(chunks):
    #     df_train = pd.DataFrame(data)
    #     dst = os.path.join(subject_path, 'data_all_%d.lst'%(i))
    #     np.savetxt(dst, df_train.values, fmt='%s')


import urllib.request
import json
import bs4
from tournamentClasses import *
import warnings

warnings.filterwarnings("ignore", category=UserWarning, module='bs4')

def get_html(yr):
    print('...')
    url = 'http://www.sports-reference.com/cbb/postseason/{}-ncaa.html'.format(yr)
    ls = [urllib.request.urlopen(url).read().decode('utf-8')]
    url = 'http://www.sports-reference.com/cbb/seasons/{}-polls.html'.format(yr)
    ls.append(urllib.request.urlopen(url).read().decode('utf-8'))
    print('done with {}'.format(yr))
    return ls

# html_dict = {yr:get_html(yr) for yr in range(1985,2017)}  # uncomment to rebuild html_dict
# json.dump(html_dict, open("html_dict_file.txt", "w"))

html_dict = json.load(open("html_dict_file.txt", "r"))
html_dict = {int(yr): html_dict[yr] for yr in html_dict.keys()}

def get_teams(yr):
    soup = bs4.BeautifulSoup(html_dict[yr][1])
    data = []
    table = soup.find(lambda tag: tag.name == 'table' and tag.has_key('id') and tag['id'] == 'ap')
    rows = table.findAll('tr')
    for row in rows:
        cols = row.findAll('td')
        cols = [ele.text.strip() for ele in cols]
        data.append(data.append([ele for ele in cols]))
    data = [ele for ele in data if isinstance(ele, list) and len(ele) >= 1 and ele[0] == 'Final']
    data = [[ele for i, ele in enumerate(team) if i not in [0, 3, 4]] for team in data]
    data = [[ele[:-2] if i == 1 else ele for i, ele in enumerate(team)] for team in data]

    # replace all names that change within the bracket:
    replacement_dict = {'North Carolina': 'UNC', 'Louisiana State': 'LSU', 'Nevada-Las Vegas': 'UNLV',
                        'Virginia Commonwealth': 'VCU', 'Texas-El Paso': 'UTEP', 'Pittsburgh': 'Pitt',
                        'Connecticut': 'UConn', 'East Tennessee State': 'ETSU', 'Southern Mississippi': 'Southern Miss',
                        'Southern California': 'SCU', 'Massachusetts': 'UMass', "Saint Joseph's": "St. Joseph's",
                        'Mississippi': 'Ole Miss', 'Brigham Young': 'BYU',
                        'Southern Methodist': 'SMU'}  # add replacements here`
    data = [[ele if ele not in replacement_dict.keys() else replacement_dict[ele] for ele in team] for team in data]

    '''rank_dict = {ele[1]: [int(ele[0]), ele[2]] for ele in data}
    return rank_dict'''
    return [Team(ele[1], ele[0], ele[2]) for ele in data]

def find_team_by_name(name, teams):
    for team in teams:
        if team.name == name:
            return team
    return None

region_dict = {}
for yr in range(1985, 2017):
    if yr != 2011 and (yr in range(2007, 2017)) or (yr in range(1998, 2004)):
        region_dict[yr] = ['east', 'west', 'south', 'midwest', 'national']
    elif yr in range(1985, 1998):
        region_dict[yr] = ['east', 'west', 'southeast', 'midwest', 'national']
region_dict[2004] = ['stlouis', 'atlanta', 'eastrutherford', 'phoenix', 'national']
region_dict[2005] = ['syracuse', 'albuquerque', 'austin', 'chicago', 'national']
region_dict[2006] = ['minneapolis', 'atlanta', 'oakland', 'washington', 'national']
region_dict[2011] = ['east', 'southeast', 'southwest', 'west', 'national']

def gen_year_dict(start, end):
    year_dict = {}
    url_tag_dict = {}
    for yr in range(start, end + 1):
        yr_url_tags = []
        teams = get_teams(yr)
        soup = bs4.BeautifulSoup(html_dict[yr][0])
        brackets = soup.find("div", {"id": "brackets"})

        conference_and_record_dict = json.load(open('conferences_doc_full.txt','r'))
        conference_and_record_dict = {int(key):conference_and_record_dict[key] for key in conference_and_record_dict.keys()}
        regions = region_dict[yr]
        games = []

        for region in regions:
            bracket = brackets.find("div", {"id": "{}".format(region)})
            bracket_results = bracket.find_all(['a', 'span'])
            url_tag_data = [str(item)[22:str(item)[22:].find('/') + 22] for item in bracket_results if
                            str(item)[1:7] == 'a href' and str(item)[22:str(item)[22:].find('/') + 22] != 's']
            results = [info.text for info in bracket_results if info.text[:2] != 'at']
            results = [results[6 * i:6 * i + 6] for i in range(16)]
            if region != "national":
                num_games = 15
            else:
                num_games = 3
            for game_i in range(num_games):
                if region != 'national':
                    if game_i in range(8):
                        tourn_round = "Round of 64"
                    elif game_i in range(8, 12):
                        tourn_round = "Round of 32"
                    elif game_i in range(12, 14):
                        tourn_round = "Sweet 16"
                    elif game_i == 14:
                        tourn_round = "Elite 8"
                else:
                    if game_i in range(2):
                        tourn_round = 'Final 4'
                    else:
                        tourn_round = 'Championship'
                if not find_team_by_name(results[game_i][1], teams):
                    team1 = Team(results[game_i][1], 'NR', '???', results[game_i][0], region)
                    teams.append(team1)
                else:
                    team1 = find_team_by_name(results[game_i][1], teams)
                    team1.seed = results[game_i][0]
                if region != 'national':
                    team1.region = region
                if not find_team_by_name(results[game_i][4], teams):
                    team2 = Team(results[game_i][4], 'NR', '???', results[game_i][3], region)
                    teams.append(team2)
                else:
                    team2 = find_team_by_name(results[game_i][4], teams)
                    team2.seed = results[game_i][3]
                    if region != 'national':
                        team2.region = region
                if tourn_round == 'Round of 64':
                    team1.url_tag = url_tag_data[game_i * 2]
                    team2.url_tag = url_tag_data[game_i * 2 + 1]
                    yr_url_tags.append(team1.url_tag)
                    yr_url_tags.append(team2.url_tag)
                    team1.conf = conference_and_record_dict[yr][team1.url_tag][0]
                    team1.record = conference_and_record_dict[yr][team1.url_tag][1]
                    team2.conf = conference_and_record_dict[yr][team2.url_tag][0]
                    team2.record = conference_and_record_dict[yr][team2.url_tag][1]
                games.append(Game(tourn_round, team1, results[game_i][2], team2, results[game_i][5]))
        year_dict[yr] = games
        url_tag_dict[yr] = yr_url_tags
        print(str(yr) + " completed")
    # uncomment next line to recreate the url_tag_dictionary file
    # json.dump(url_tag_dict,open('url_tags_by_yr.txt','w'))
    return year_dict

def write_csv(start, end, filename):
    with open(filename, 'w') as f:
        year_dict = gen_year_dict(start, end)
        for yr in range(start, end + 1):
            f.write(str(yr) + '\n')
            for game in year_dict[yr]:
                f.write('{}\n'.format(",".join(game.results()[0])))
                f.write('{}\n\n'.format(",".join(game.results()[1])))

#write_csv(1985, 2016, 'output.csv')
#gen_year_dict(1985, 2016)


import cv2, os
import numpy as np

from PIL import Image

recognizer = cv2.face.LBPHFaceRecognizer_create()
path='dataSet'

def getImagesAndLabels(path):
    imagePaths=[os.path.join(path,f) for f in os.listdir(path)]
    faces = []
    IDs = []
    for imagePath in imagePaths:
        faceImg = Image.open(imagePath).convert('L')
        faceNp = np.array(faceImg,'uint8')
        ID=int(os.path.split(imagePath)[-1].split('.')[1])
        faces.append(faceNp)
        IDs.append(ID)
        cv2.imshow("traning",faceNp)
        cv2.waitKey(10)
    return faces, IDs

faces, IDs = getImagesAndLabels(path)
#trainning
recognizer.train(faces, np.array(IDs))
if not os.path.exists('recognizer') :
    os.makedirs('recognizer')
recognizer.save('recognizer/trainningData.yml')
cv2.destroyAllWindows()


from __future__ import absolute_import, division, print_function

import tensorflow as tf
from tensorflow.keras import Model, layers
import numpy as np
import matplotlib.pyplot as plt
from timer import timer
from load_basic import DATA_FOLDER, EXPER_FOLDER, DATA_SIZE, NET_NAME
from read_data import ideal_data_generator, exper_data_generator, DATA_FEATURES, EXPER_FEATURES

learning_rate  = 1e-3
batch_size     = 1
epoch          = 1
training_steps = DATA_SIZE*epoch
display_step   = 1
print(training_steps)

num_features =  DATA_FEATURES # number of wave length separate.
print("data features: ", num_features)

#  num_hidden_1 = 512 # 1st layer num features.
#  num_hidden_2 = 128 # 2nd layer num features (the latent dim).
#  num_hidden_3 = 32
#  num_hidden_4 = 1

train_size = int(0.8*DATA_SIZE)
#  test_size = int(0.3*DATA_SIZE)

dataset = tf.data.Dataset.from_generator(
    ideal_data_generator,
    (tf.float32, tf.float32, tf.float32),
    (tf.TensorShape(None), tf.TensorShape(None), tf.TensorShape(None))
)

train_set = dataset.take(train_size)
test_set  = dataset.skip(train_size)

train_set = train_set.shuffle(train_size).repeat(epoch).batch(batch_size).prefetch(epoch)

class ConvNet(Model):
    # Set layers.
    def __init__(self):
        super().__init__()
        # Convolution Layer with 32 filters and a kernel size of 5.
        self.flatten = layers.Flatten()

        # Fully connected layer.
        self.fc2 = layers.Dense(2048)
        #  self.fc1 = layers.Dense(1313)
        self.fc0 = layers.Dense(512)
        # Apply Dropout (if is_training is False, dropout is not applied).
        self.dropout = layers.Dropout(rate=0.5)

        # Output layer, class prediction.
        self.out = layers.Dense(1)

    # Set forward pass.
    def call(self, x, is_training=False):
        x = tf.reshape(x, [-1, DATA_FEATURES, 1])
        #  x = self.conv1(x)
        #  x = self.conv2(x)
        #  x = self.conv3(x)
        #  x = self.conv4(x)
        #  x = self.conv5(x)
        #  x = self.conv6(x)
        #  x = self.conv7(x)
        #  x = self.conv8(x)
        #  x = self.conv9(x)
        #  x = self.conv10(x)
        #  x = self.conv11(x)
        #  x = self.conv12(x)
        #  x = self.conv13(x)
        #  x = self.conv14(x)
        #  x = self.conv15(x)
        x = self.flatten(x)
        x = self.fc2(x)
        #  x = self.fc1(x)
        x = self.fc0(x)
        x = self.dropout(x, training=is_training)
        x = self.out(x)
        #  if not is_training:
        #      # tf cross entropy expect logits without softmax, so only
        #      # apply softmax when not training.
        #      x = tf.nn.softmax(x)
        return x

# Build neural network model.
conv_net = ConvNet()

# Cross-Entropy Loss.
# Note that this will apply 'softmax' to the logits.

def lossfunc(pred, ori):
    return tf.reduce_mean(tf.pow(ori-pred, 2))
    #  cp = tf.math.abs(tf.subtract(pred, ori))
    #  return tf.reduce_mean(tf.cast(cp, tf.float32))

# Accuracy metric.
def accuracy(y_pred, y_true):
    # Predicted class is the index of highest score in prediction vector (i.e. argmax).
    correct_prediction = tf.math.abs(tf.subtract(y_pred, y_true))
    return 1-tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

# Stochastic gradient descent optimizer.
optimizer = tf.optimizers.Adam(learning_rate)

# Optimization process.
def run_optimization(x, y):
    # Wrap computation inside a GradientTape for automatic differentiation.
    with tf.GradientTape() as g:
        # Forward pass.
        pred = conv_net(x, is_training=True)
        # Compute loss.
        loss = lossfunc(pred, y)

    # Variables to update, i.e. trainable variables.
    trainable_variables = conv_net.trainable_variables

    # Compute gradients.
    gradients = g.gradient(loss, trainable_variables)

    # Update W and b following gradients.
    optimizer.apply_gradients(zip(gradients, trainable_variables))

if __name__=="__main__":
    LOSS = []
    ACC = []
    # Run training for the given number of steps.
    for step, (v, r, batch_x) in enumerate(train_set.take(training_steps), 1):
        # Run the optimization to update W and b values.
        run_optimization(batch_x, v)

        if step % display_step == 0:
            pred = conv_net(batch_x)
            loss = lossfunc(pred, v)
            LOSS.append(loss)
            #  acc = accuracy(pred, v)
            #  ACC.append(acc)
            #  print("step: %i, loss: %f, accuracy: %f" % (step, loss, acc))
            print("step: %i, loss: %f" % (step, loss))

    plt.plot(LOSS)
    plt.show()
    plt.plot(np.log(LOSS))
    plt.show()
    #  plt.plot(ACC)
    #  plt.show()
    # Test model on validation set.
    #  pred = conv_net(x_test)
    conv_net.save_weights(NET_NAME)
    #
    #  new_model = ConvNet()
    #  new_model.load_weights('testSave')
    #  pred = new_model(x_test)
    #  print("Test Accuracy2: %f" % accuracy(pred, y_test))

#
#  #  Visualize predictions.
#  import matplotlib.pyplot as plt
#
#
#  # Predict 5 images from validation set.
#  n_images = 5
#  test_images = x_test[:n_images]
#  predictions = conv_net(test_images)
#
#  # Display image and model prediction.
#  for i in range(n_images):
#      plt.imshow(np.reshape(test_images[i], [28, 28]), cmap='gray')
#      plt.show()
#      print("Model prediction: %i" % np.argmax(predictions.numpy()[i]))
#


from django.conf.urls import url
from . import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^search$', views.search, name='search'),
    url(r'^result$', views.result, name='result'),
    url(r'^entries$', views.show_entries, name='show'),
    url(r'^entries/new$', views.create_article, name='new'),
    url(r'^entries/(?P<daily_id>[0-9]+)$', views.detail, name='detail'),
    url(r'^entries/(?P<daily_id>[0-9]+)/edit$', views.edit, name='edit'),
    url(r'^entries/(?P<daily_id>[0-9]+)/delete$', views.delete, name='delete'),
    url(r'^users$', views.show_users, name='show_users'),
    url(r'^users/(?P<user_id>[0-9]+)$', views.show_user_page, name='show_user_page'),
]


from velociraptor.observations.objects import ObservationalData

import unyt
import numpy as np
import os
import sys

# Exec the master cosmology file passed as first argument
with open(sys.argv[1], "r") as handle:
    exec(handle.read())

# Cosmologies
h_obs = 0.7
h_sim = cosmology.h

input_filename = "../raw/Tremonti_2004_ascii.txt"
delimiter = "\t"

output_filename = "Tremonti2004_Data.hdf5"
output_directory = "../"

if not os.path.exists(output_directory):
    os.mkdir(output_directory)

# Read the data
raw = np.loadtxt(input_filename, delimiter=delimiter)
M_star = 10 ** raw[:, 0] * unyt.Solar_Mass * (h_sim / h_obs) ** -2
Z_median = raw[:, 3] * unyt.dimensionless  # 12 + log(O/H)
Z_lo = raw[:, 2] * unyt.dimensionless  # 12 + log(O/H)
Z_hi = raw[:, 4] * unyt.dimensionless  # 12 + log(O/H)

# Define the scatter as offset from the mean value
y_scatter = unyt.unyt_array((Z_median - Z_lo, Z_hi - Z_median))

# Meta-data
comment = (
    "Data obtained assuming a Kroupa IMF and h=0.7. "
    f"h-corrected for SWIFT using cosmology: {cosmology.name}. "
    "The metallicity is expressed as 12 + log10(O/H). "
    "In these units the solar metallicity is 8.69."
    "The error bars given the 16th and 84th percentile of the distribution."
)
citation = "Tremonti et al. (2004) (SDSS)"
bibcode = "2004ApJ...613..898T"
name = "Stellar mass - Gas phase metallicity relation"
plot_as = "line"
redshift = 0.1
h = h_sim

# Write everything
processed = ObservationalData()
processed.associate_x(
    M_star, scatter=None, comoving=True, description="Galaxy Stellar Mass"
)
processed.associate_y(
    Z_median, scatter=y_scatter, comoving=True, description="Gas phase metallicity"
)
processed.associate_citation(citation, bibcode)
processed.associate_name(name)
processed.associate_comment(comment)
processed.associate_redshift(redshift)
processed.associate_plot_as(plot_as)
processed.associate_cosmology(cosmology)

output_path = f"{output_directory}/{output_filename}"

if os.path.exists(output_path):
    os.remove(output_path)

processed.write(filename=output_path)


import os
import tkinter
from tkinter import *
from PyPDF2 import PdfFileReader #PDF
import comtypes.client #DOCX
import win32api # install pypiwin32, PyMuPDF, python-pptx, python-docx
from docx import Document
from docx.enum.text import WD_COLOR_INDEX
from pptx import Presentation
import xlsxwriter #EXCEL
from xlsxwriter.utility import xl_rowcol_to_cell
import xlrd
import fitz # PDF

def init(window):
    window.title("Best File Search")
    window.geometry("700x640")
    window.resizable(False, False)

    # Configure Widgets
    frame.place(x=300, y=78)  # set for fileList(Listbox) scrollbar
    frame2.place(x=300, y=412)
    scrollbar.configure(command=fileList.yview)
    fileList.configure(yscrollcommand=scrollbar.set)
    scrollbar2.configure(command=fileList.yview)
    openedFileList.configure(yscrollcommand=scrollbar.set)
    backgroundlabel.configure(image=backgroundimage)
    bClear.configure(command=buttonClear)
    bOpen.configure(command=buttonOpen)
    btnSearch.configure(command=executeSearchButton)
    bClearTwo.configure(command=buttonClearTwo)
    # Place Widgets
    # labelsearchtext.place(x=40, y=80)
    # labledesiredpath.place(x=40, y=230)
    desiredpath.place(x=130, y=250)
    searchText.place(x=41, y=125)
    # searchedFileName.place(x=300, y=30)
    # openedFileName.place(x=300, y=380)
    btnSearch.place(x=170, y=110, width=50, height=50)
    bClear.place(x=620, y=385, width=50, height=20)
    bClearTwo.place(x=565,y=50, width=50, height=20)
    bOpen.place(x=620, y=50, width=50, height=20)

def buttonClear():
    openedFileList.delete(0, END)

def buttonClearTwo():
    fileList.delete(0, END)

def buttonOpen():
    value = fileList.get(fileList.curselection()[0])
    #os.startfile(value)

    searchInput = searchText.get()

    ext = os.path.splitext(value)[-1]
    pathExceptExt = os.path.splitext(value)[0]

    if ext == ".pptx":
        powerpoint = comtypes.client.CreateObject("Powerpoint.Application")
        powerpoint.Visible = 1

        Filename = value
        deck = powerpoint.Presentations.Open(Filename)
        deck.SaveAs(pathExceptExt + ".pdf", 32)  # formatType = 32 for ppt to pdf

        value2 = pathExceptExt + ".pdf"
        doc = fitz.open(value2)
        pdf = PdfFileReader(value2)
        saveNumPages = pdf.getNumPages()

        for i in range(saveNumPages):
            page = doc[i]
            text_instances = page.searchFor(searchInput)

            for inst in text_instances:
                highlight = page.addHighlightAnnot(inst)
        doc.save(pathExceptExt + "_modified.pdf", garbage=4, deflate=True, clean=True)
        os.startfile(pathExceptExt + "_modified.pdf")

    elif ext == ".txt":
        with open(value, mode='r', encoding='utf-8') as fo:
            textthis = fo.read()

        with open(pathExceptExt + ".html", "w") as e:
            if searchInput in textthis:
                e.write("
" + textthis.replace(searchInput, '{}'.format(
                    searchInput)) + "

\n")
        os.startfile(pathExceptExt + ".html")
    elif ext == ".xlsx":

        wbk = xlsxwriter.Workbook(value)
        wks = wbk.add_worksheet()
        myPath = value

        cell_format = wbk.add_format()
        cell_format.set_bg_color('yellow')
        for sh in xlrd.open_workbook(myPath).sheets():
            for row in range(sh.nrows):
                for col in range(sh.ncols):
                    mycell = sh.cell(row, col)
                    if mycell.value == searchInput:
                        wks.write(xl_rowcol_to_cell(row, col), searchInput, cell_format)
                    else:
                        wks.write(xl_rowcol_to_cell(row, col), mycell.value)
        wbk.close()
        os.startfile(value)
    elif ext == ".docx":
        document = Document(value)

        for para in document.paragraphs:
            start = para.text.find(searchInput)
            if start > -1:
                pre = para.text[:start]
                post = para.text[start + len(searchInput):]
                para.text = pre
                para.add_run(searchInput)
                para.runs[1].font.highlight_color = WD_COLOR_INDEX.YELLOW
                para.add_run(post)
        document.save(pathExceptExt + "_modified.docx")
        os.startfile(pathExceptExt + "_modified.docx")

    elif ext == ".pdf":

        doc = fitz.open(value)
        for i in range(10):
            try:
                    page = doc[i]

                    text_instances = page.searchFor(searchInput)

                    for inst in text_instances:
                        highlight = page.addHighlightAnnot(inst)

            except IndexError:
                break
        doc.save(pathExceptExt + "_mod.pdf", garbage=4, deflate=True, clean=True)
        os.startfile(pathExceptExt + "_mod.pdf")

    openedFileList.insert(END, value)

# Determine whether there are specific strings in some files such as .txt, .docx, .pptx files
# then insert their path and name to the list
def searchDir(root_folder, searchInput):

    #exceptFolderList = [ "AppData", "WINDOWS", "Windows", "Program Files (x86)", "Program Files"]

    try:
        filenames = os.listdir(root_folder)

        for filename in filenames:
            full_filename = os.path.join(root_folder, filename)

            if os.path.isdir(full_filename):
                if filename == "AppData" or filename == "WINDOWS" or filename == "Windows" or filename == "Program Files (x86)" or filename == "Program Files":
                    pass
                else:
                    searchDir(full_filename, searchInput)

            else:
                ext = os.path.splitext(full_filename)[-1]
                #Search TXT files
                if ext == ".pdf":
                    try:
                        doc = fitz.open(full_filename)
                        page = doc[0]
                        text_instances = page.searchFor(searchInput)
                        if text_instances:
                            print(full_filename)
                            fileList.insert(END, full_filename)
                            break
                    except:
                        pass

                elif ext == ".txt":
                    try:
                        fo = open(full_filename, 'r', encoding='utf-8', errors='ignore')

                        if searchInput in fo.read():
                            print(full_filename)
                            fileList.insert(END, full_filename)
                            break
                    except:
                        pass

                elif ext == ".docx":
                    try:
                        print(full_filename)
                        document = Document(full_filename)

                        for para in document.paragraphs:
                            if searchInput in para.text:
                                fileList.insert(END, full_filename)
                                break
                    except:
                        pass
                elif ext == ".xlsx":
                    try:
                        print(full_filename)
                        wbk = xlsxwriter.Workbook(full_filename)
                        wks = wbk.add_worksheet()

                        myPath = full_filename
                        for sh in xlrd.open_workbook(myPath).sheets():
                            for row in range(sh.nrows):
                                for col in range(sh.ncols):
                                    myCell = sh.cell(row, col)
                                    if myCell.value == searchInput:
                                        fileList.insert(END, full_filename)
                                        break
                                break
                    except:
                        pass
                elif ext == ".pptx":
                    try:
                        print(full_filename)
                        prs = Presentation(full_filename)
                        for slide in prs.slides:
                            for shape in slide.shapes:
                                if hasattr(shape, "text"):
                                    if searchInput in shape.text:
                                        fileList.insert(END, full_filename)
                                    break
                            break
                    except:
                        pass

    except PermissionError:
        pass

# Search User's HDD. Send the information of HDD and filetypes.
def searchInfo():
    searchInput = searchText.get()
    HDD_List = []

    for drive in win32api.GetLogicalDriveStrings().split('\000')[:-1]:
        # insert the name of HDDs to the ListBox to display them for User to see.
        onlyHDDname = drive.split(":")
        desiredpath.delete(0, END)
        HDD_List.append(onlyHDDname[0])

        for i in HDD_List[0:]:
            desiredpath.insert(END, i + " ")

        # Specify the file_type.
        print("Finding files")
        searchDir(drive, searchInput)

# Execute Search Button
def executeSearchButton():
    searchInfo()

window = Tk()
C = Canvas(window, bg="blue", height=250)
backgroundimage = PhotoImage(file="image.png")
backgroundlabel = Label(window)
backgroundlabel.pack()

frame = Frame(window)
frame2 = Frame(window)

bOpen = tkinter.Button(window, text="open")
bClear = tkinter.Button(window, text="Clear")
bClearTwo = tkinter.Button(window, text="Clear")
btnSearch = tkinter.Button(window, text="search")
labelsearchtext = tkinter.Label(window, text="Enter text")
labledesiredpath = tkinter.Label(window, text="Searched HDD")
openedFileName = tkinter.Label(window, text="Opened Files")
searchedFileName = tkinter.Label(window, text="Searched Files")

fileList = Listbox(frame, width=50, height=17)
fileList.pack(side='left', fill='y')
scrollbar = Scrollbar(frame, orient=VERTICAL)
scrollbar.pack(side="right", fill="y")
scrollbar2 = Scrollbar(frame2, orient=VERTICAL)
scrollbar2.pack(side="right", fill="y")

openedFileList = Listbox(frame2, width=50)
openedFileList.pack(side='left', fill='y')
fileList.pack(side='left', fill='y')
desiredpath = tkinter.Listbox(window, width=10, height=5)
searchText = tkinter.Entry(window, width=16)

#fileList.bind('<<ListboxSelect>>', openFile)
#fileList.bind('<Double-Button>', buttonOpen)
openedFileList.bind('<Double-Button>', buttonOpen)
# initialise and run main loop
init(window)
mainloop()

Best-File-Search-master/Best File Search/Best File Search.py


# -*- coding: utf-8 -*-
from setuptools import find_packages, setup

with open("README.md") as f:
    readme = f.read()

with open("LICENSE") as f:
    license = f.read()

setup(
    name="maodevice",
    version="1.0",
    description="Python 3.7 package to control MAO devices",
    long_description=readme,
    author="Project MAO Contributors",
    author_email="tueda1207@gmail.com",
    url="https://github.com/mao-wfs/maodevice",
    license=license,
    packages=find_packages(exclude=("docs", "tests")),
    install_requires=["pyserial>=3.4"],
)


'''
## iam_user_attach_policy
What it does: Attaches a policy (passed in as a variable) to the user
Usage: AUTO: iam_user_attach_policy policy_arn=<policy_arn>
Limitations: none
Examples:
    AUTO: iam_user_attach_policy policy_arn=arn:aws:iam::aws:policy/AlexaForBusinessFullAccess
    AUTO: iam_user_attach_policy policy_arn=arn:aws:iam::621958466464:policy/sumo_collection
    AUTO: iam_user_attach_policy policy_arn=arn:aws:iam::$ACCOUNT_ID:policy/sumo_collection
'''

import boto3
from botocore.exceptions import ClientError

#Poll the account and check if the provided policy exists.
def check_for_policy(iam_client,policy_arn):
    found_policy = False
    try:
        #Check to see if the policy exists in the account currently
        get_policy_response = iam_client.get_policy(PolicyArn=policy_arn)

        if get_policy_response['ResponseMetadata']['HTTPStatusCode'] < 400:
            text_output =  "IAM policy exists in this account.\n"
            found_policy = True

    except ClientError as e:
        error = e.response['Error']['Code']
        if error == 'NoSuchEntity':
            text_output = "The policy %s does not exist. Please check and try again.\n" % policy_arn
        else:
            text_output = "Unexpected error: %s \n" % e

    return text_output,found_policy

#Create attach the policy to the group
def add_policy_to_user(iam_client,user,policy_arn):
    try:
        attach_policy_response = iam_client.attach_user_policy(
            UserName=user,
            PolicyArn=policy_arn
        )

        text_output = "Policy attached to user: \" %s \"\n" % user

    except ClientError as e:
            text_output = "Unexpected error: %s \n" % e

    return text_output

### Update user - core method
def run_action(boto_session,rule,entity,params):
    text_output = ""
    account_id = entity['accountNumber']
    user = entity['name']

    iam_client = boto_session.client('iam')

    # Get the policy_arn from the params
    usage = "Usage: AUTO: iam_user_attach_policy policy_arn=<policy_arn>\n Examples: \nAUTO: iam_user_attach_policy policy_arn=arn:aws:iam::aws:policy/AlexaForBusinessFullAccess\nAUTO: iam_user_attach_policy policy_arn=arn:aws:iam::621958466464:policy/sumo_collection\niam_user_attach_policy policy_arn=arn:aws:iam::$ACCOUNT_ID:policy/sumo_collection\n"
    if len(params) == 1:
        try:
            key_value = params[0].split("=")
            key = key_value[0]
            value = key_value[1]

            if key == "policy_arn":
                if "arn:aws:iam::aws:policy" in value:
                    policy_arn = value
                    text_output = text_output + "AWS Managed policy specified for attachment\nARN: %s \n" % value

                elif "$ACCOUNT_ID" in value:
                    # Look for "$ACCOUNT_ID" and replace it with the current account number
                    account_id = entity['accountNumber']
                    policy_arn = value.replace("$ACCOUNT_ID", account_id)
                    text_output = text_output + "Policy ARN that we're attaching to the user: %s \n" % policy_arn

                else:
                    policy_arn = value
                    text_output = text_output + "Policy ARN specified for attachment: %s \n" % value

            else:
                text_output = text_output + "Params don't match expected values. Exiting.\n" + usage
                return text_output

        except:
            text_output = text_output + "Params handling error. Please check params and try again.\n" + usage
            return text_output

    else:
        text_output = "Wrong amount of params inputs detected. Exiting.\n" + usage
        return text_output

    ## Check and add the policy to the user
    try:
        function_output, found_policy = check_for_policy(iam_client,policy_arn)
        text_output = text_output + function_output

        if found_policy:
            text_output = text_output + add_policy_to_user(iam_client,user,policy_arn)
        else:
            return text_output

    except ClientError as e:
        text_output = "Unexpected error: %s \n" % e

    return text_output


#!/usr/bin/python3
# coding=utf-8
import dandan
import time
import random
import confluent_kafka as confluent
import config

logger = config.getLogger()

topic = 'test-partitions'

conf = {
    'bootstrap.servers': config.hosts,
}
producer = confluent.Producer(conf)

count = 10000
partitions = 2
future = time.time() + 100000

while future > time.time():
    message = bytes("test message", encoding='utf8')
    logger.debug('send message %s', message)
    producer.produce(
        topic=config.topic,
        value=message,
        # key=bytes(str(var % partitions), encoding='utf-8')
    )
    producer.flush()
    time.sleep(random.random() * 0.01)


import numpy as np

from dnn.image import DilatedCNNExtractor
from data import fetch

import os, sys, random, glob

import IPython

from keras.preprocessing.image import ImageDataGenerator

import keras
from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Model
from keras.layers import Input, Dense, Dropout, Activation, Flatten, Reshape
from keras.layers import Conv2D, MaxPooling2D, GlobalMaxPooling2D, UpSampling2D

import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split

import random

input_img = Input((128, 128, 3))
x = Conv2D(16, (3, 3), padding='same', activation='relu') (input_img)
x = Conv2D(16, (3, 3), padding='same', activation='relu') (x)
x = Dropout(rate=0.1) (x)
x = MaxPooling2D((2, 2)) (x)

x = Conv2D(32, (3, 3), padding='same', activation='relu') (x)
x = Conv2D(32, (3, 3), padding='same', activation='relu') (x)
x = Dropout(rate=0.1) (x)
x = MaxPooling2D((2, 2)) (x)

x = Conv2D(64, (3, 3), padding='same', activation='relu') (x)
x = Conv2D(64, (3, 3), padding='same', activation='relu') (x)
x = Dropout(rate=0.1) (x)
x = MaxPooling2D((2, 2)) (x)

x = Conv2D(128, (3, 3), padding='same', activation='relu') (x)
x = Conv2D(128, (3, 3), padding='same', activation='relu') (x)
x = Dropout(rate=0.1) (x)
x = MaxPooling2D((2, 2)) (x)

x = Conv2D(64, (3, 3), padding='same', activation='relu') (x)
x = Conv2D(32, (3, 3), padding='same', activation='relu') (x)
x = Dropout(rate=0.1) (x)
x = MaxPooling2D((2, 2)) (x)

x = Conv2D(32, (3, 3), padding='same', activation='relu') (x)
x = Conv2D(16, (3, 3), padding='same', activation='relu') (x)

x = Flatten() (x)

x = Dense(128, activation='sigmoid')(x)
x = Dense(1)(x)

model = Model(input_img, x)
model.compile(loss='mean_squared_error', optimizer='adadelta', metrics=['mape'])

model.summary()

augment = ImageDataGenerator(
        featurewise_center=False,  # set input mean to 0 over the dataset
        samplewise_center=False,  # set each sample mean to 0
        featurewise_std_normalization=False,  # divide inputs by std of the dataset
        samplewise_std_normalization=False,  # divide each input by its std
        zca_whitening=False,  # apply ZCA whitening
        rotation_range=180,  # randomly rotate images in the range (degrees, 0 to 180)
        width_shift_range=0.15,  # randomly shift images horizontally (fraction of total width)
        height_shift_range=0.15,  # randomly shift images vertically (fraction of total height)
        horizontal_flip=True,  # randomly flip images
        vertical_flip=True)  # randomly flip images

def datagen():
    while True:
        for file_name in glob.glob("results/slides_cache*.npz"):
            slide_data = np.load(file_name)["data"].tolist()

            X = []
            Y = []

            for slide_info in slide_data:
                case = slide_info[1].get('case', None)
                if case is None: continue
                days_to_death = fetch.days_to_death(case)

                if days_to_death is None: continue
                if not days_to_death:
                    days_to_death = 3000
                days_to_death /= 3000.0
                days_to_death += 1.0

                X.append(slide_info[0])
                Y.append(days_to_death)

            X = np.concatenate(X)
            Y = np.array(Y)
            Y = np.repeat(Y, 24)

            print (X.shape, Y.shape)
            X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.1)

            aug = augment.flow(X, Y)

            for i, (x, y) in enumerate(aug):
                yield x, y
                if i == 120:
                    break

model.fit_generator(datagen(), samples_per_epoch=120, epochs=10)

IPython.embed()


import aiohttp
from bs4 import BeautifulSoup
#import pandas as pd
import requests
import asyncio

#MODIFY YOUR INFO/REQUIREMENTS#####

ourServer = "twisting-nether"
ourRaidsPerWeek = "2"
updateThresholdDays = 10
pagesToCheck = 10

###################################
#KEYS##############################
urlBase = "https://www.wowprogress.com"
key = "/character/eu/"
nextkey = "/gearscore/char_rating/next/"
transferKey = "Yes, ready to transfer"
timeKey = "day ago"
timeKey2 = "days ago"
###################################

print("\n\n Class & spec combinations available: \n ~~~~~~~~~~~~~~~~~~\n deathknight - dd, tank \n demon hunter - dd, tank \n druid - balance, restoration, feral dd, feral tank \n hunter - beastmastery, marksmanship, survival \n mage - arcane, fire, frost \n monk - healer, dd, tank \n paladin - holy, protection, retribution \n priest - healer, dd \n rogue - assassination, outlaw, subtlety \n shaman - elemental, enhancement, restoration \n warlock - affliction, demonology, destruction \n warrior - dd, protection \n ~~~~~~~~~~~~~~~~~~\n")

ourClass = input("Class: ")
ourSpecs = input("Specs: ")
minGearscore = input("Min Gearscore: ")
minKills = input("Min mythic kills this tier: ")

async def fetch(session, url):
        try:
            async with session.get(url) as response:
                text = await response.text()
                return text, url
        except Exception as e:
            print(str(e))

async def main():

    url = urlBase + "/gearscore/class." + ourClass + "?lfg=1&raids_week=" + ourRaidsPerWeek + "&lang=en"
    theList = []

    print("Searching for recruits now...")

    ##Check the class summary pages to gather urls for all characters under the search parameters
    for x in range(pagesToCheck):
        msg = "Checking page " + str(x) + "/" + str(pagesToCheck)
        print(msg, end='\r')

        r = requests.get(url)
        data = r.text
        soup = BeautifulSoup(data,'html.parser')

        for link in soup.find_all('a'):
            target = link.get('href')
            if nextkey in target:
                url = urlBase + target

        for tr in soup.find('table').find_all('tr'):
            for link in tr.find_all('a'):
                target = link.get('href')
                if nextkey in target:
                    url = urlBase + target
                if key in target:
                    for spanText in tr.find_all('span'):
                        if timeKey in spanText.get_text() or timeKey2 in spanText.get_text():
                            timeText = spanText.get_text()
                            theTime = timeText[:timeText.find(' ')]
                            if int(theTime) < updateThresholdDays:
                                theList.append(target)
                        else:
                            theList.append(target)

    msg = "Found " + str(len(theList)) + " potential recruits."
    print(msg)

    tasks = []
    finalList = []

    headers = {
        "user-agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"}
    ##Asynchronously get all the raw html data for each url
    async with aiohttp.ClientSession(headers=headers) as session:
        for item in theList:
            url = urlBase + item
            tasks.append(fetch(session, url))

        htmls = await asyncio.gather(*tasks)
        msg = "Successfully retrieved " + str(len(htmls)) + " URLs."
        print(msg)
        print("Processing...")
        index = 0
        for html in htmls:
            if html is not None:
                url = html[1]
                data = html[0]
                soup = BeautifulSoup(data,'html.parser')
                ##traverse html data for relevant match to search criteria (TODO Add more filters)
                needsTransfer = True
                if ourServer in item:
                    needsTransfer = False

                for strongText in soup.find_all('strong'):
                    if ourSpecs in strongText.get_text():
                        for gs in soup.find_all(class_='gearscore'):
                            if "Item Level" in gs.get_text():
                                gsText = gs.get_text()
                                gsF = float(gsText[12:])
                                if gsF >= float(minGearscore):
                                    bossKillTable = soup.find(id="tiers_details")
                                    if bossKillTable is not None:
                                        kills = 0
                                        for kill in bossKillTable.find_all(class_='progress_heroic'):
                                            if "Mythic" in kill.get_text():
                                                kills+=1
                                        if kills >= int(minKills):
                                            if needsTransfer:
                                                for spanText in soup.find_all('span'):
                                                    if transferKey in spanText.get_text():
                                                       finalList.append(url)
                                            else:
                                               finalList.append(url)
            else:
                continue
            index += 1
            msg = "Progress: " + str(round(len(htmls) / 100 * index, 2)) + "%"
            print(msg, end='\r')

    ##Write the relevent urls for hits to file; TODO Directly to google sheet?
    file_name = ourClass + "_" + ourSpecs + ".txt"
    with open(file_name, "w") as f:
      [f.write(f'{final_item}\n') for final_item in finalList]
    print(f"{str(len(finalList))} recruit(s) found and written to file.")

if __name__ == "__main__":
    asyncio.run(main())


"""
decision tree
"""
import math
from collections import Counter, defaultdict
import numpy as np
from typing import NamedTuple, Any
import matplotlib.pyplot as plt

# NamedTuple 을 상속받는 클래스 선언
class Candidate(NamedTuple):
    level: str
    lang: str
    tweet: bool
    phd: bool
    result: bool = None  # 기본값 쓸 수 있다! (클래스 선언 방식의 장점!)

def uncertainty(p):
    """
    엔트로피 : 불확실성의 정도 0 < p <= 1
    확률 p=0이면, 사건이 항상 발생하지 않는다라고 확신 -> 불확실성 0
    확률 p=1이면, 사건이 항상 발생한다고 확신 -> 불확실성 1
    확률 0 < p < 1이면, 사건이 발생할 수도, 발생하지 않을 수도 있다 -> 불확실성이 있다.
     """
    # math.log(p, base=2) -> log_2 가 밑수
    # 1보다 작은 로그는 음수에 -을 붙여서 양수로 만든다
    return -p * math.log(p, 2)

def entropy(class_probabilities):
    """
    주어진 확률들의 리스트에 대해서 엔트로피를 계산
    E = sum(i) [uncertainty(p_i)] = -p_1 * log(p_1) - p_2 * log(p_2) * ...
    E = 모든 경우에 대한 불확실성 = 각 사건에 대한 불확실성 + 각 사건에 대한 불확실성 + ...
    가장 불확실성이 큰 경우 : 50%, 50% -> 엔트로피 max (최댓값)
    """
    ent = 0
    for p in class_probabilities:
        if p != 0:
            ent += uncertainty(p)
            # 만약 p=0이면 log(p)를 계산할 때 Error 가 발생
        # 더하지 않는 건, 더하기 0과 같으므로, else 부분은 없어도 되~
    return ent

def binary_entropy(p):
    """사건이 두 가지인 경우, 사건이 일어날 확률 p, 사건이 일어나지 않을 확률 1-p
    두가지만 더하면 된다"""
    return uncertainty(p) + uncertainty(1-p)

def class_probabilities(labels):
    """"""
    total_count = len(labels)
    counts = Counter(labels)  # 갯수, {label1: x개, label2: y개.. } -> dict 와 비슷
    print(counts)
    probabilities = []
    for count in counts.values():
        p = count / total_count  # 각 레이블의 확률
        probabilities.append(p)
    return probabilities
#
# def class_probabilities(labels):
#     total_count = len(labels)
#     counts = Counter(labels)  # 갯수, {label1: x개, label2: y개.. } -> dict 와 비슷
#     print(counts)
#     pass

def partition_by(dataset, attr_name):
    """NamedTuple 들의 리스트로 이루어진 dataset 을 NamedTuple 의 특정 attribute 로 partitioning"""
    # list 를 value 로 갖는 dict 만듬
    # 전체적 흐름 : 지원자 1 row 가 senior 그룹에 들어감. (맞다면), 아니면 그룹 만들어 들어감
    partitions = defaultdict(list)
    for sample in dataset:
        # 지원자 1 row 추출
        # sample.attr_name 은 글자 그대로 인식함
        # sample 의 getattr 함수에 지정된 attr_name(ex_level) 으로 준다. defaultdict 의 key 로 준다.
        key = getattr(sample, attr_name)
        # dict 의 키로 사용하여 sample 을 value 에 저장(새로 생성)
        partitions[key].append(sample)
    return partitions

# defaultdict : dictionary 에 기본값을 정의하고, 키값이 없더라도 에러를 출력하지 않고 기본값을 출력한다.

def partition_entropy_by(dataset, by_partition, by_entropy):
    """
    attr_name 으로 분리된 각 파티션에서 label_name 의 엔트로피를 각각 계산하고, 파티션 내에서의 엔트로피 * 파티션의 비율들의 합을 리턴
    dataset : 데이터 전체
    by_partition : patitioning 을 나눌 컬럼명(특성)
    by_entropy : 합격 불합격 결과(엔트로피를 계산할 컬럼명)

    과정
    : attr_name 이 예를 들어 level 일 때, level 이 Junior 일 때, 합격할 확률의 엔트로피를 구한다.
    level 이 mid 일 때, 합격할 확률의 엔트로피를 구한다.
    level 이 Senior 일 때, 합격할 확률의 엔트로피를 구한다.
    """
    # partition 을 나눔
    partitions = partition_by(dataset, by_partition)
    # label(class) 별 확률을 계산하기 위해서 각 label 들의 리스트 만들기
    labels = []
    for partition in partitions.values():  # partition 갯수 만큼 반복
        values = []
        for sample in partition:  # 부분집합인 partition 안의 sample 들(candidate)의 갯수 - 원소 1개 만큼 반복
            values.append(getattr(sample, by_entropy))
        labels.append(values)  # [Junior 의 T, F, T, F], [Senior 의 T, F, T, F]..
    print(labels)  # [[t,f,t,f], [..], [..] ..] 위의 리스트를 모아 하나의 리스트로!
    # 각 파티션이 차지하는 비율을 계산하고, 각 파티션에서의 엔트로피에 그 비율을 곱해주기 위해서
    total_count = sum(len(label) for label in labels)
    # 함수 안에도 for 문 쓸 수 있음
    # labels 은 리스트를 3개 가지고 있음, 실제로는 5 + 5 + 5 = 15개이다.
    # 따라서 for 문 :  [[t,f,t,f], [..], [..] ..] 안의 [t,f,t,f] 에서의 label 의 갯수 계산
    ent = 0
    for label in labels:
        # 파티션이 가지고 있는 [t,f,t,f] 에서 확률 계산 (ex_P(t) = 1/2, P(f) = 1/2)
        cls_prob = class_probabilities(label)  # 파티션의 엔트로피 계산
        part_ent = entropy(cls_prob)
        ent += part_ent * len(label) / total_count  # 파티션 내에서의 엔트로피 * 파티션의 비율들의 합
    return ent

# 데이터 타입을 먼저 지정해준다
class Leaf(NamedTuple):  # NamedTuple 을 상속받는 클래스
    value: Any  # 어떤 타입이든 올 수 있다는 뜻, 합격 불합격 저장 용도로 사용

class Split(NamedTuple):
    # 무엇으로 가지를 치는지 attribute 이름이 있어야 해요
    attribute: str  # 트리에서 가지(branch)가 나눠지는 기준 : key 값은 무조건 문자열 ex_level
    subtree: dict  # 중간에 있는 tree (ex_senior 의 subtree)

# predict 함수
def predict(model, sample):
    """sample을 model(의사결정나무)에 적용했을 때, 예측 결과를 리턴"""
    # 기본원리 : leaf type 이 들어오면 우리가 예측을 멈춘다!
    if isinstance(model, Leaf):
        # model 이 최종 노드인 Leaf 타입이면, Leaf 가 가지고 있는 value (값) 을 리턴
        return model.value

    # model이 아닌 경우에는 가지를 따라 내려가야 하기 때문에
    # sample(ex_candidate) 이 attribute(ex_level) 로 가지고 있는 값을 찾아서, 해당 가지로 내려감
    subtree_key = getattr(sample, model.attribute)  # sample 의 attribute 의 값을 찾는다.
    # ex_model.attribute 가 level 이라면 sample 의 값인 ex_'Senior' 값이 나온다.
    print('subtree_key : ', subtree_key)

    # model 이 가지고 있는 subtree (ex_'Senior'라는 subtree 로 다시 간다)
    subtree = model.subtree[subtree_key]
    return predict(subtree, sample)  # 재귀함수 : (ex_ Senior)

def build_tree(dataset, by_splits, target):
    print('\n>> building tree.. ')
    print(f'dataset len : {len(dataset)} = {dataset}')
    print(f'by_split : {by_splits}, target : {target}')
    # dataset (data), by_splits (tree 나누는 기준), target(결과)

    # target 의 갯수를 셈 by Counter 객체 생성 Counter : {True: x, False: y}
    target_counts = Counter(getattr(sample, target)
                            for sample in dataset)  # 리스트로 만들어진 True, False 갯수를 센다. (counter 객체 dict 타입 리턴)
    print('target_counts : ', target_counts)  # dict 에서는 key:value 가 1개의 data, 그래서 dict 의 length 는 2

    # Counter 의 length 가 1개이면
    # leaf 가 될 수 있는 상태란? target 갯수, 즉 Counter 의 length 가 1개인 상태 - Leaf 생성하고 종료
    if len(target_counts) == 1:
        keys = list(target_counts.keys())
        # = [k for k in target_counts.keys()] -> [] 연산자 사용 불가.. why? keys() 는 generator 객체이므로,,
        # target_counts.keys() generator (반복문 안에서 리스트를 만들어주는 객체)이므로 리스트를 먼저 만들어 주어야 함 -> 리스트 함수 쓰면 됨
        result = keys[0]
        leaf = Leaf(result)
        print('leaf : ', leaf)
        return leaf

    # tree 의 depth 가 깊어져서 더이상 서브 트리를 나눌 기준이 없을 때 (ex_ 4가지 기준 모두 씀)
    # by_splits = []일때, 원소가 없으면 False
    # if len(by_splits) == 0:
    if not by_splits:
        return Leaf(list(target_counts.keys())[0])  # leaf node 를 무조건 리턴

    # Counter 의 length 가 1이 아니면, 파티션을 나눌 수 있음.
    # by_splits(가지 나누는 기준)의 각 변수로 파티션을 나눔.
    # 각 파티션별 엔트로피(level 로 나눴을 때 엔트로피.. )를 계산해서 가장 낮은 엔트로피 변수를 선택
    # best_splilter = min(by_splits, key=partition_entropy_by())
    # partition_entropy_by()를 key= 에 바로 못쓰는 이유
    # 파라미터 1개밖에 못주기 때문, by_splits 는 key 의 함수(dataset, ..)에 파라미터로 주는데, partition_entropy_by()는 파라미터 3개이다.
    # (ex_ sorted(list, key=lambda x: len(x)) # 리스트의 문자열 1개가 x가 되어서 글자 수 세고 글자수 기준으로 정렬 - 파라미터 1개만 된다)
    # 어디다 넣어야 할 지 알려주기 위해서 partition_entropy_by()를 호출할 수 있는 wrapper 함수(helper 함수)를 작성

    # wrapper 함수(helper 함수)
    # 내부함수(자신의 바깥쪽에 선언된 변수를 모두 가져다 쓸 수 있다 - ??? ctrl + 클릭으로 확인)
    def splitted_entropy(split_attr):
        print('split_attr = ', split_attr)
        # 파라미터 3개 중 1개를 딱 정해준다.(split_attr) / build_tree 의 파라미터 dataset, target 이 자동으로 파라미터로 들어간다 - ???)
        result = partition_entropy_by(dataset, split_attr, target)
        print('splitted entropy = ', result)
        return result

    best_splilter = min(by_splits, key=splitted_entropy)
    print('best_spliter : ', best_splilter)

    # 선택된 변수(entropy 최솟값을 주는 변수)로 Split 객체를 생성 - 첫번째 가지를 치자!
    partitions = partition_by(dataset, best_splilter)
    print('partitions : ', partitions)

    # partition 을 만든 이유 : 나눠준 candidate 들을 key 값(ex_level)의 value 들을 부분집합으로 만들어주기 위해서!
    # 각 부분집합들은 또다른 가지의 dataset 이 된다.
    # 선택한 변수를 제외한 나머지 변수들로 sub tree 를 만듬
    # branch 기준 리스트에서 선택된 변수 제거
    new_split = [x for x in by_splits
                 if x != best_splilter]  # best_spliter 와 다른 새로운 split 기준 리스트 생성
    print(f'제거 후 by_splits : {by_splits}, new_splits : {new_split}')

    # subtree 생성
    subtree = {k: build_tree(subset, new_split, target)  # 재귀함수로 가지 밑 다른 트리를 만듬 *by_splits 전의 변수 없어진 상태
               for k, subset in partitions.items()}
    # 해석 : {'Senior' - key ! : [Candidate(level='Senior', ..), ...- value 는 원소 5개짜리 부분집합

    # Split 객체를 생성하여 리턴
    return Split(best_splilter, subtree)

# leaf 하고 나서 다시 올라가는데 왜 올라가지? 다시 올라가서 재귀함수인가?
# 같은 depth 에서는 같은 기준으로 나누게 하기 위하여 new_split 을 써야 한다. (근데 다시 올라가는 거 모르겠음)

if __name__ == '__main__':
    candidates = [Candidate('Senior', 'Java', False, False, False),
                  Candidate('Senior', 'Java', False, True, False),
                  Candidate('Mid', 'Python', False, False, True),
                  Candidate('Junior', 'Python', False, False, True),
                  Candidate('Junior', 'R', True, False, True),
                  Candidate('Junior', 'R', True, True, False),
                  Candidate('Mid', 'R', True, True, True),
                  Candidate('Senior', 'Python', False, False, False),
                  Candidate('Senior', 'R', True, False, True),
                  Candidate('Junior', 'Python', True, False, True),
                  Candidate('Senior', 'Python', True, True, True),
                  Candidate('Mid', 'Python', False, True, True),
                  Candidate('Mid', 'Java', True, False, True),
                  Candidate('Junior', 'Python', False, True, False)]

# decision tree = 스무고개
# 첫 질문이 제일 중요하다! (확률적 - 엔트로피 개념 파악하기)
# 질문 순서에 따라 decision tree 갯수 엄청 많아져~
# 그런데 트리 크기가 짧은게 꼭 좋은 건 아니다 -> 다른 데이터에 예측력이 떨어질 수 있다.

    # 엔트로피 : 질문을 선택하는 기준
    # uncertainty 함수 그래프
    x_pts = np.linspace(0.0001, 0.9999, 100)  # 시작값, 끝값, 그사이를 나누는 값
    y_pts = [uncertainty(p) for p in x_pts]
    plt.plot(x_pts, y_pts)
    plt.xlim(0.0)  # 0 <= x 그 밑은 그리지 않는다.
    plt.ylim(0.0)  # 0 <= y 그 밑은 그리지 않는다.
    plt.title('-p * log(p)')
    plt.show()
    # 해석
    # 엔트로피는 불확실성! https://web.mit.edu/16.unified/www/FALL/thermodynamics/notes/node56.html
    # 확률 1 = 모두 발생하는 경우, 불확실성이 없다. 확실하게 정해져 있으므로
    # 확률 0 = 발생하지 않는 경우, 불확실성이 없다. 확실하게 정해져 있으므로

    # binary entropy graph
    x_pts = np.linspace(0.0001, 0.9999, 100)
    y_pts = [binary_entropy(p) for p in x_pts]
    plt.plot(x_pts, y_pts)
    plt.xlim(0)
    plt.axvline(0.5)
    plt.ylim(0)
    plt.show()
    # 0.5 일때 가장 불확실성이 커진다. 소금물100% + 맹물 100% = 50%, 50%으로 불확실성이 가장 커지는 방향으로 변화한다.

    # entropy 함수 테스트
    # 확률 1개, p=1
    rain_prob = [1]  # 내일 비올확률 1 (확률 1개)
    ent = entropy(rain_prob)
    print('entropy = ', ent)  # binary entropy 의 그래프의 최솟값 0

    # 확률 2개, p1=0.5, p2=0.5 (p1+p2=1)
    rain_prob = [0.5, 0.5]
    ent = entropy(rain_prob)
    print('entropy = ', ent)  # binary entropy 의 그래프의 최댓값 1

    # 확률 2개, p1=0.9, p2=0.1 (p1+p2=1) 비가 올 확률 90%
    rain_prob = [0.9, 0.1]
    ent = entropy(rain_prob)
    print('entropy = ', ent)  # entropy 0.47

# decision tree = 스무고개에서의 엔트로피 활용
# E(level), E(lang), E(mid) .. 계산 -> 엔트로피가 가장 낮은 질문을 첫번째로 삼는다.
# 2번째 질문 -> partitioning group 1 을 다시 partitioning 하여 엔트로피를 계산한다.

    # class_probabilities 함수 테스트
    level = ['junior', 'senior', 'mid', 'junior']
    cls_prob = class_probabilities(level)
    print(cls_prob)

    # 실전!
    # 전체 집합을 나누는 기준(Partitioning) 에 따라 엔트로피가 달라질 수 있다.
    # 함수 만듬
    partition_by_level = partition_by(candidates, 'level')
    print('partitioning by level: \n', partition_by_level)
    partition_by_tweets = partition_by(candidates, 'tweet')
    print('partitioning by tweets: \n', partition_by_tweets)

    # partition entropy by 함수 테스트
    # 전체 지원자들을 level 로 partition 을 나눠서 result 의 엔트로피를 계산하자~
    ent_level = partition_entropy_by(candidates, 'level', 'result')
    print(f'entropy partitioned by level : {ent_level}')
    # 결과 : [[False, False, True, True, True], [True, True, True, True], [True, True, False, True, False]]
    ent_lang = partition_entropy_by(candidates, 'lang', 'result')
    print(f'entropy partitioned by lang : {ent_lang}')
    ent_tweets = partition_entropy_by(candidates, 'tweet', 'result')
    print(f'entropy partitioned by tweets : {ent_tweets}')
    ent_phd = partition_entropy_by(candidates, 'phd', 'result')
    print(f'entropy partitioned by phd : {ent_phd}')

    """ 결과 해석 - 불확실성이 적은 질문을 먼저 던지는 것이 좋다. 즉 엔트로피가 가장 적은 것부터 트리를 그리자.
    [[False, False, False, True, True], [True, True, True, True], [True, True, False, True, False]] : 파티션
    Counter({False: 3, True: 2}) : 1번의 FALSE 3, 3 엔트로피 계산 * 비율 곱하여 더한다.
    Counter({True: 4})           : 2번의 TRUE 4
    Counter({True: 3, False: 2}) : 3번의
    entropy partitioned by level : 0.6935361388961919

    [[False, False, True], [True, True, False, True, True, True, False], [True, False, True, True]]
    Counter({False: 2, True: 1})
    Counter({True: 5, False: 2}) : 확실성 더 높다. 엔트로피 더 낮다.
    Counter({True: 3, False: 1})
    entropy partitioned by lang : 0.8601317128547441  불확실성이 더 많아 전체 엔트로피 더 커질 수 밖에 없다.

    [[False, False, True, True, False, True, False], [True, False, True, True, True, True, True]]
    Counter({False: 4, True: 3}) : 불확실성 크다
    Counter({True: 6, False: 1}) : 불확실성 상대적으로 적다
    entropy partitioned by tweets : 0.7884504573082896

    [[False, True, True, True, False, True, True, True], [False, False, True, True, True, False]]
    Counter({True: 6, False: 2})
    Counter({False: 3, True: 3})
    entropy partitioned by phd : 0.8921589282623617  --> 박사학위가 엔트로피가 제일 크다.
    """

    # test - 자료구조를 만들고 있다!
    hire_tree = Split(
        'level',                                          # branch 나누는 기준 : root node
        {
            'Senior': Split(                              # 첫번째 가지
                'tweet',
                {True: Leaf(True), False: Leaf(False)}),  # subtree
            'Mid': Leaf(True),  # 전부 합격인 leaf 노드
            'Junior': Split(
                 'phd',
                 {True: Leaf(False), False: Leaf(True)})
        }
    )

    # 두번째 그림도 만들어보기 (노트 필기)

    # predict test
    print(hire_tree)
    candidate_1 = Candidate('Senior', 'Java', False, False, False)
    result = predict(hire_tree, candidate_1)
    print(result)

    candidate_2 = Candidate('Mid', 'Python', False, False, True)
    result = predict(hire_tree, candidate_2)
    print(result)

    # built tree 함수 테스트
    tree = build_tree(candidates, ['level', 'lang', 'tweet', 'phd'], 'result')
    print(tree)  # 최소 엔트로피로 이루어진 tree 를 출력한다.

    # pip install graphviz


import cv2
import numpy as np

# input is numpy image array
def opticalflow(img1, img2):
    prvs = cv2.cvtColor(img1, cv2.COLOR_RGB2GRAY)
    next = cv2.cvtColor(img2, cv2.COLOR_RGB2GRAY)
    hsv = np.zeros_like(prvs)
    hsv[..., 1] = 255

    flow = cv2.calcOpticalFlowFarneback(prvs, next, None, 0.5, 3, 15, 3, 5, 1.2, 0)

    mag, ang = cv2.cartToPolar(flow[...,0], flow[...,1])
    hsv[...,0] = ang*180/np.pi/2
    hsv[...,2] = cv2.normalize(mag,None,0,255,cv2.NORM_MINMAX)
    rgb = cv2.cvtColor(hsv,cv2.COLOR_HSV2RGB)
    gray = cv2.cvtColor(rgb, cv2.COLOR_RGB2GRAY)

    mask = np.where(x > 0.5, 0, 1)
    gray[mask]

    return rgb, gray[mask]


from datetime import datetime
import pandas
import json

import luigi
import urllib

import os
from itertools import groupby
from SlackNotifier import SlackNotifier

from ParquetFileInsertion import IngestParquetFile
from LoggerFactory import LoggerFactory

import argparse
import logging
import traceback

class RunAllEolHostingIngestions(luigi.WrapperTask):
    data_folder = luigi.Parameter()
    eol_tables_config_path = luigi.Parameter()
    environments_to_ingest = luigi.ListParameter()
    ingest_to = luigi.Parameter()
    ingestion_date = luigi.DateParameter()
    logs_file_path = luigi.Parameter()

    # The logic of starting multiple tasks basing on our files/partitioning structure
    # Warning! Dependent workers (for IngestParquetFile) will run in a separate process. Unfortunatelly, that complicates troubleshooting and logging.
    # To debug code logic, change luigi.cfg [core] workers = 1. Don't forget to undo this change before commiting.
    # Note, that for multiprocess behaviour number of workers should be > 1.
    # To troubleshoot multiprocess issues use logging since debugging will be unable.
    def requires(self):
        # Logger for this task somehow breaking the multi-thread execution.
        # Therefore, no logger is used for this task.
        eol_tables_config = self.read_tables_configuration(self.eol_tables_config_path)

        configured_files_to_ingest = self.aggregate_files_configured_to_ingest(self.data_folder, eol_tables_config)
        ### For debugging a specific parquet file
        #configured_files_to_ingest = [f for f in configured_files_to_ingest if f.FileName=='33413888-000000-0000.parquet']

        for environment, files_to_ingest_for_environement in groupby(configured_files_to_ingest, lambda x: x.Environment):
            if (not environment in self.environments_to_ingest):
                continue
            for file_to_ingest in files_to_ingest_for_environement:
                # Skip not enabled tables and files for not today
                if(file_to_ingest.Date < self.ingestion_date or
                        not file_to_ingest.IsEnabled):
                    continue
                yield IngestParquetFile(
                    ingest_to = self.make_db_connection_string(),

                    ingestion_date = self.ingestion_date,
                    environment = file_to_ingest.Environment,

                    parquet_source = file_to_ingest.FileName,
                    parquet_source_location = file_to_ingest.FullPath,
                    backup_date = file_to_ingest.Date,
                    table = file_to_ingest.TargetTable,
                    column_names = file_to_ingest.TargetColumns,

                    logs_file_path = self.logs_file_path
                )

    def make_db_connection_string(self):
        connection_string = "Driver={SQL Server};Server=" + self.ingest_to + ";Database=CustomerIntelligence;UID=;PWD="
        connection_string = urllib.parse.quote_plus(connection_string)
        connection_string = "mssql+pyodbc:///?odbc_connect=%s" % connection_string
        return connection_string

    def read_tables_configuration(self, config_file_path):
        if config_file_path is None or not os.path.isfile(config_file_path):
            raise FileNotFoundError(config_file_path)
        tables = pandas.read_json(config_file_path)
        if len(tables) < 1:
            raise Exception("List of tables to ingest is empty at: {}".format(config_file_path))
        return json.loads(tables.to_json(orient='records'))

    def aggregate_files_configured_to_ingest(self, root_folder, eol_tables_config):
        local_files = self.list_all_files_of_eol_hosting_structure(root_folder)
        files_to_ingest = [f for f in local_files
                                if any(table['source'] == f.EntityName for table in eol_tables_config)]
        for file in files_to_ingest:
            config = ([c for c in eol_tables_config if c['source'] == file.EntityName])[0]
            file.ExtendWithIngestionConfig(config)
        return files_to_ingest

    def check_path(self, file_path):
        # path structure: data folder path\\environment=NL\\Tables\\2019\\12\\31
        return len(file_path.split('\\')[-5:][0].split('=')) > 1

    def list_all_files_of_eol_hosting_structure(self, folder_path):
        paths = []
        for path, subdirs, files in os.walk(folder_path):
            if not self.check_path(path):
                continue
            for name in files:
                paths.append(os.path.join(path, name))
        cig_files = [CigIngestionFile(p) for p in paths]
        return cig_files

class CigIngestionFile():
    def __init__(self, file_path):
        if file_path is None:
            raise FileNotFoundError(file_path)
        # path structure: data folder path\\environment=NL\\2019\\12\\31\\file.parquet
        splitted_path = file_path.split('\\')
        splitted_path = splitted_path[-6:]

        if len(splitted_path) < 1:
            raise FileNotFoundError(file_path)

        environment = splitted_path[0].split('=')[1]
        date_string = "{}-{}-{}".format(splitted_path[2], splitted_path[3], splitted_path[4])

        self.FullPath = file_path
        self.FileName = splitted_path[5]
        self.Environment = environment
        self.Date = datetime.strptime(date_string, "%Y-%m-%d").date()
        self.EntityName = splitted_path[1]

    def ExtendWithIngestionConfig(self, ingestion_config):
        self.TargetTable = ingestion_config['target_name']
        self.TargetColumns = ingestion_config['columns']
        self.IsEnabled = ingestion_config['is_enabled']

def read_ingestion_config(ingestion_config_filename):
    config = {}
    currentFolder = os.path.dirname(__file__)
    with open(os.path.join(currentFolder, ingestion_config_filename)) as file:
        config = json.load(file)
    return config

def send_execution_summary(execution_summary, environments, db_server_name, ingestion_date):
    icon = ":heavy_check_mark:" if execution_summary.worker.run_succeeded else ":fish_cake:"
    luigi_page = 'http://nlc1prodci02:8082/static/visualiser/index.html'

    header = "{}\n EOL Hosting ETL : Ingestion summary\n\nFor the following Environments: {}\nIngestion date (CIGCopyTime): {}"\
        .format(icon, ", ".join(environments), ingestion_date)
    body = "\nRun visualization page: {}\n\n{}\n".format(luigi_page, execution_summary.summary_text)
    footer = "The SQL Server DB: {}\n{}".format(db_server_name, "----------------------------------------------------------------")
    message = "{}\n{}\n{}".format(header, body, footer)
    SlackNotifier().notify_with_message(message)

# The entry point
# Before starting it luigi server must be started
# Via cmd : luigid
#      Or : python start_luigi_server.py
if __name__ == "__main__":
    try:
        ingestion_config_filename = 'ingestion_config.json'
        parser = argparse.ArgumentParser()
        parser.add_argument('--ingestion_config_filename', help='Specify filename of ingestion configuration (Default="ingestion_config.json")')
        args = parser.parse_args()
        if not args.ingestion_config_filename is None and len(args.ingestion_config_filename) > 0:
            ingestion_config_filename = args.ingestion_config_filename

        now = datetime.now()
        config = read_ingestion_config(ingestion_config_filename)
        data_folder = config['data_folder']+'\\'
        environments_to_ingest = config['environments']
        ingest_to = config['ingest_to']
        ingestion_date = datetime.strptime(config['ingestion_date'], '%Y-%m-%d') if len(config['ingestion_date']) > 0 else now
        currentFolder = os.path.dirname(__file__)
        tables_to_upload_config_file = os.path.join(currentFolder, config['tables_to_upload_config_file'])
        logs_file_path = "{}\luigi.{}.log".format(config['logs_folder'], now.strftime('%Y-%m'))

        logger = LoggerFactory.create_logger_for_multiprocess_use(logs_file_path, logging.INFO, 'main')
        logger.info("The ingestion is configured and started.")

        execution_summary = luigi.build(
            [RunAllEolHostingIngestions(
                eol_tables_config_path = tables_to_upload_config_file,
                data_folder = data_folder,
                environments_to_ingest = environments_to_ingest,
                ingest_to = ingest_to,
                ingestion_date = ingestion_date,
                logs_file_path = logs_file_path
                )],
            detailed_summary=True)

        logger.info(execution_summary.summary_text)

        if config['is_execution_summary_via_slack_enabled'] and (
            len(execution_summary.worker._scheduled_tasks) > 1 or
            not execution_summary.worker.run_succeeded):
                send_execution_summary(execution_summary, environments_to_ingest, ingest_to, ingestion_date)

    except Exception as e:
        message = ":rage4: \n EOL Hosting ETL : Unexpected failure \nError:\n{}\n{}"\
            .format(e, traceback.format_exc())
        SlackNotifier().notify_with_message(message)
        logger = LoggerFactory.create_logger_for_multiprocess_use(logs_file_path, logging.ERROR, 'main-crashed')
        logger.fatal(e, exc_info=True)


from flask import Flask, render_template, request, redirect
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.tree import DecisionTreeClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import GridSearchCV
from sklearn.svm import SVC
import time
import warnings
import os

app = Flask(__name__)

main_list = ['radius_mean', 'texture_mean', 'perimeter_mean', 'area_mean', 'smoothness_mean', 'compactness_mean',
 'concavity_mean', 'concave points_mean', 'symmetry_mean', 'fractal_dimension_mean', 'radius_se',
 'perimeter_se', 'texture_se', 'area_se', 'smoothness_se', 'compactness_se', 'concavity_se', 'concave points_se',
 'symmetry_se', 'fractal_dimension_se', 'radius_worst', 'texture_worst', 'perimeter_worst', 'area_worst', 'smoothness_worst',
 'compactness_worst', 'concavity_worst', 'concave points_worst', 'symmetry_worst', 'fractal_dimension_worst']

def getContext(list_of_parameters, test_case = None):

	data = pd.read_csv('data.csv', index_col=False)
	#print(data)
	data['diagnosis'] = data['diagnosis'].apply(lambda x: '1' if x == 'M' else '0')
	data = data.set_index('id')
	#del data['Unnamed: 32']
	data = data.drop(list_of_parameters, axis=1)
	#print(data)

	my_path = os.path.dirname(__file__)
	#print(my_path)
	my_file = 'static/plot1.png'
	i_s =0
	data.plot(kind='density', subplots=True, layout=(5,7), sharex=False, legend=False,fontsize=1)
	if os.path.isfile(os.path.join(my_path, my_file)):
		print("Yes")
		os.remove(os.path.join(my_path, my_file))
		i_s = 1
		my_file = 'static/plot13.png'
	plt.savefig(os.path.join(my_path, my_file))

	from matplotlib import cm as cm
	fig = plt.figure()
	ax1 = fig.add_subplot(111)
	cmap = cm.get_cmap('jet', 30)
	cax = ax1.imshow(data.corr(), interpolation="none", cmap=cmap)
	ax1.grid(True)
	plt.title('Breast Cancer Attributes Correlation')
	# Add colorbar, make sure to specify tick locations to match desired ticklabels
	fig.colorbar(cax, ticks=[.75,.8,.85,.90,.95,1])
	my_file = 'static/plot2.png'
	if os.path.isfile(os.path.join(my_path, my_file)):
		print("Yes")
		os.remove(os.path.join(my_path, my_file))
		my_file = 'static/plot24.png'
	plt.savefig(os.path.join(my_path, my_file))

	Y = data['diagnosis'].values
	X = data.drop(['diagnosis'], axis=1).values
	#print(X)
	X_train, X_test, Y_train, Y_test = train_test_split (X, Y, test_size = 0.20, random_state=11)

	# prepare the model
	with warnings.catch_warnings(): warnings.simplefilter("ignore")
	scaler = StandardScaler().fit(X_train)
	X_train_scaled = scaler.transform(X_train)
	model = SVC(C=2.0, kernel='rbf')
	start = time.time()
	model.fit(X_train_scaled, Y_train)
	end = time.time()
	#print( "Run Time: %f" % (end-start))

	# estimate accuracy on test dataset
	with warnings.catch_warnings(): warnings.simplefilter("ignore")

	X_test_scaled = scaler.transform(X_test)
	predictions = model.predict(X_test_scaled)
	if test_case is not None:
		test_case_scaled = scaler.transform(test_case)
		p = model.predict(test_case_scaled)
		#print("+++++++++++"+p)
		if p == '1':
			#print("Malignant")
			p = "Malignant"
		else:
			#print("Benign")
			p = "Benign"
	else:
		p=0

	#print("Accuracy score %f" % accuracy_score(Y_test, predictions))
	accu = round(accuracy_score(Y_test, predictions),4)*100
	x= classification_report(Y_test, predictions, output_dict=True)
	precision_1 = x['0']['precision']
	recall_1 = x['0']['recall']
	f1_1 = round(x['0']['f1-score'],2)
	sc_1 = x['0']['support']

	precision_2 = x['1']['precision']
	recall_2 = x['1']['recall']
	f1_2 = round(x['1']['f1-score'],2)
	sc_2 = x['1']['support']

	y= confusion_matrix(Y_test, predictions)
	v_11 = y[0][0]
	v_10 = y[0][1]
	v_01 = y[1][0]
	v_00 = y[1][1]

	return {
	'precision_1':round(precision_1,3),
	'precision_2':round(precision_2,3),
	'recall_1':round(recall_1,3),
	'recall_2':round(recall_2,3),
	'f1_1':f1_1,
	'f1_2':f1_2,
	'sc_1':sc_1,
	'sc_2':sc_2,
	'accuracy':accu,
	'v_11':v_11,
	'v_10':v_10,
	'v_01':v_01,
	'v_00':v_00,
	'i_s':i_s,
	'p':p,
	}

@app.route('/index.html')
@app.route('/')
def index():
    return render_template('index.html')

@app.route('/team.html')
def team():
    return render_template('team.html')

@app.route('/pulse.html')
def pulse():
    return render_template('pulse.html')

@app.route('/contact.html')
def contact():
    return render_template('contact.html')

@app.route('/files.html', methods=['POST'])
def files():
	z = request.form.getlist('options')
	y=[]
	for i in z:
		if request.form.get(i):
			y.append(float(request.form.get(i)))
	print(y)

	x = list(set(main_list) - set(z))
	#x.append('diagnosis')
	#print(x)
	print("length of z: ",len(z))
	print("length of y: ",len(y))
	if len(y) != 0:
		if len(y) == len(z):
			print("++++++++++1")
			context = getContext(x,[y])
		else:
			print("++++++++++2")
			return redirect('pulse.html')
	else:
		print("++++++++++3")
		context = getContext(x)
	return render_template('files.html', context=context)

@app.after_request
def add_header(response):
    # response.cache_control.no_store = True
    response.headers['Cache-Control'] = 'no-store, no-cache, must-revalidate, post-check=0, pre-check=0, max-age=0'
    response.headers['Pragma'] = 'no-cache'
    response.headers['Expires'] = '-1'
    return response

if __name__ == "__main__":
	app.static_folder = 'static'
	app.config["CACHE_TYPE"] = "null"
	app.run()


#!/usr/bin/python
# -*- coding: utf-8 -*-
import StringIO

import xlsxwriter
from django.utils.translation import ugettext

def WriteToExcel(licenses, user):
    output = StringIO.StringIO()
    workbook = xlsxwriter.Workbook(output)
    worksheet_s = workbook.add_worksheet(ugettext(licenses[0].doc_number.replace("/", ".")))

    # excel styles
    title = workbook.add_format({
        'bold': True,
        'font_size': 14,
        'align': 'center',
        'valign': 'vcenter'
    })
    header = workbook.add_format({
        'bg_color': '#F7F7F7',
        'color': 'black',
        'align': 'center',
        'valign': 'top',
        'border': 1
    })
    cell = workbook.add_format({
        'align': 'left',
        'valign': 'top',
        'text_wrap': True,
        'border': 1
    })
    cell_center = workbook.add_format({
        'align': 'center',
        'valign': 'top',
        'text_wrap': True,
        'border': 1
    })
    cell_no_border = workbook.add_format({
        'valign': 'top',
        'text_wrap': True,
        'border': 0
    })

    # write title
    title_text = u"{0}".format(ugettext(licenses[0].doc_number.replace("/", ".")))
    # merge cells
    worksheet_s.merge_range('B2:F2', title_text, title)

    # write header
    worksheet_s.write(4, 0, ugettext(u"№ п.п"), header)
    worksheet_s.write(4, 1, ugettext(u"ФИО получателя"), header)
    worksheet_s.write(4, 2, ugettext(u"Должность получателя"), header)
    worksheet_s.write(4, 3, ugettext(u"Номер электронного ключа eToken"), header)
    worksheet_s.write(4, 4, ugettext(u"Номер лицензии на право использования программного продукта КриптоАРМ"), header)
    worksheet_s.write(4, 5, ugettext(u"Номер лицензии на право использования программного продукта КриптоПро"), header)
    worksheet_s.write(4, 6, ugettext(u"Подпись"), header)
    # write data

    for idx, data in enumerate(licenses):
        row = 5 + idx
        worksheet_s.write_number(row, 0, idx + 1, cell_center)
        worksheet_s.write_string(row, 1, licenses[idx].name_of_recepient, cell)
        worksheet_s.write_string(row, 2, licenses[idx].position_of_recepient, cell)
        worksheet_s.write_string(row, 3, licenses[idx].eToken_number, cell)
        worksheet_s.write_string(row, 4, licenses[idx].ca_license[:5] + "-...-" + licenses[idx].ca_license[-5:], cell)
        worksheet_s.write_string(row, 5, licenses[idx].cp_license[:5] + "-...-" + licenses[idx].cp_license[-5:], cell)
        worksheet_s.write_string(row, 6, "", cell)

    worksheet_s.write(row + 2, 3, ugettext(u"Выдал"), cell_no_border)
    worksheet_s.write(row + 2, 4, ugettext(u""), cell_no_border)
    worksheet_s.write(row + 2, 5, ugettext(user.get_full_name()), cell_no_border)

    # close workbook
    workbook.close()
    xlsx_data = output.getvalue()
    return xlsx_data


#Encoding=UTF8

import tornado.web
import mongo_helper
import json
import logger

class InventoryHandler(tornado.web.RequestHandler):
    def get(self):
        name = self.get_argument('name', '')
        if not name:
            return

        inv = mongo_helper.get_inventory(name)
        inv['test'] = 'wangtt'
        self.write(json.dumps(inv))

    def put(self):
        name = self.get_argument('name', '')
        if not name:
            return

        count = self.get_argument('count', '100')
        count = int(count)

        mongo_helper.add_inventory(name, count)

    def post(self):
        name = self.get_argument('name', '')
        if not name:
            return

        count = self.get_argument('count', '100')
        count = int(count)

        mongo_helper.update_inventory(name, count)


# coding: utf-8
import re
import time
import asyncio
import aiohttp
import pyautogui

class DanmuServer():
	def __init__(self, roomid, csrf_token):
		self.url = "https://api.live.bilibili.com/ajax/msg"
		self.form = {"roomid":roomid,
			"csrf_token":csrf_token}
		self.allow_keyboard_dicts = {'q':'q', 'w':'w', 'e':'e', 'r':'r',
									'w':'w', 'a':'a', 's':'s', 'd':'d', 'f':'f',
									"space":" "}
		self.allow_mouse_dicts = {"mlc":"left", "mrc":"right", "mmc":"middle"}

	def danmu_execution(self, dm):
		'''
		处理鼠标/键盘
		'''
		print(dm)
		dm_cmouse = re.findall(r".*?(mlc|mrc|mmc).*?", dm)
		dm_mmouse = re.findall(r".*?mmove\s*(\d+)\s+(\d+).*?", dm)
		dm_instructions = re.findall(r".*?(\w)(\d?).*?", dm)
		if dm_cmouse:
			pyautogui.click(button=self.allow_mouse_dicts[dm_cmouse[0]])
			time.sleep(0.3)
			return
		elif dm_mmouse:
			now_x, now_y = pyautogui.position()
			x = int(dm_mmouse[0][0]) if dm_mmouse[0][0] else now_x
			y = int(dm_mmouse[0][1]) if dm_mmouse[0][1] else now_y
			pyautogui.moveRel(x, y, duration=0.25)
			return
		if "space" in dm:
			dm_instructions = [("space", "")]
		if dm_instructions:
			cmd = dm_instructions[0]
			if cmd[0] in self.allow_keyboard_dicts.keys():
				if cmd[1]:
					time_wait = int(cmd[1])
				else:
					time_wait = int(1)
				pyautogui.keyDown(self.allow_keyboard_dicts[cmd[0]])
				time.sleep(time_wait)
				pyautogui.keyUp(self.allow_keyboard_dicts[cmd[0]])

	async def getDanmu(self):
		'''
		使用异步 请求api
		'''
		async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False)) as session:
			while True:
				print("连接弹幕网...(゜-゜)つロ 干杯~")
				async with session.post(self.url, data=self.form) as resp:
					print("获取弹幕数据...(゜-゜)つロ 干杯~")
					danmuData = await resp.json()
					# dm = [ (_tmp['nickname'], _tmp['text'], _tmp['timeline']) for _tmp in danmuData['data']['room'][-1:-6:-1] ]
					dm_text = danmuData['data']['room'][-1]['text']
					# self.push_danmu(data_list)
					print("获取弹幕数据成功...(゜-゜)つロ 干杯~")
					print("解析弹幕数据;并运行...(゜-゜)つロ 干杯~")
					self.danmu_execution(dm_text)
					# 太快了。我要减减速度
					time.sleep(0.3)

roomid = 3758547
csrf_token = "088056*********584eda17"

loop = asyncio.get_event_loop()
danmuserver = DanmuServer(roomid, csrf_token)
loop.run_until_complete(danmuserver.getDanmu())
loop.close()


import torch
import torchvision
import torchvision.transforms as transforms
from PIL import Image
import numpy as np
import cv2
import numpy
import torch.utils.data as data
import os

def default_loader(path):
    return Image.open(path).convert('RGB')

mytransform = transforms.Compose([
    transforms.Grayscale(1),
    transforms.ToTensor(),
])

class myImageFolder(data.Dataset):
    def __init__(self, root, fig_cls, fig_t, transform = None, loader = default_loader):
        fh = open(fig_cls)
        imgs = []
        for fh_line in fh.readlines():
            fn = fh_line.split()[0].split('.')[0]
            if os.path.isfile(os.path.join(root, fn + '.jpg')):
                imgs.append((str(fn), int(fh_line.split()[1])))
            else:
                print('can not find the path : ' + os.path.abspath(os.path.join(root, fn)))
        self.root = root
        self.t_root = fig_t
        self.imgs = imgs
        self.transform = transform
        self.loader = loader

    def __getitem__(self, index):
        fn, label_cls = self.imgs[index]
        img = self.loader(os.path.join(self.root, fn + '.jpg'))
        label_t = torch.from_numpy(numpy.load(os.path.join(self.t_root, fn + '.npy')))
        if self.transform is not None:
            img = self.transform(img)
        return img, torch.tensor(label_cls).view(-1), label_t

    def __len__(self):
        return len(self.imgs)

train_dataset = myImageFolder(root = './data/data2/train_fig', fig_cls = './data/data2/train_out.txt', fig_t = './data/data2/train_t', transform= mytransform)
test_dataset = myImageFolder(root = './data/data2/test_set', fig_cls = './data/data2/output_test.txt', fig_t = './data/data2/test_t', transform= mytransform)

BATCH_SIZE = 2048
NUM_WORKERS = 8

train_loader = data.DataLoader(train_dataset, batch_size= BATCH_SIZE, shuffle = True, num_workers= NUM_WORKERS)
test_loader = data.DataLoader(test_dataset, batch_size= BATCH_SIZE, shuffle = True, num_workers= NUM_WORKERS)


""" Tests Reading of PypeIt Input files """
from pathlib import Path
import os
import shutil
import glob

from IPython import embed

from pypeit import inputfiles
from pypeit.scripts.setup import Setup

def test_read_fluxing_files():
    """ Test reading fluxing files """
    # Grab em
    fluxing_files = glob.glob(os.path.join(
        os.getenv('PYPEIT_DEV'), 'fluxing_files', '*.flux'))
    # Loop
    for ifile in fluxing_files:
        fluxFile = inputfiles.FluxFile.from_file(ifile)
        assert len(fluxFile.data['filename']) > 0

def test_read_coadd1d_files():
    """ Test reading coadd1d files """
    # Grab em
    coadd1d_files = glob.glob(os.path.join(
        os.getenv('PYPEIT_DEV'), 'coadd1d_files', '*.coadd1d'))
    # Loop
    for ifile in coadd1d_files:
        coadd1dFile = inputfiles.Coadd1DFile.from_file(ifile)
        assert len(coadd1dFile.data['filename']) > 0

def test_read_coadd2d_files():
    """ Test reading coadd2d files """
    # Grab em
    coadd2d_files = glob.glob(os.path.join(
        os.getenv('PYPEIT_DEV'), 'coadd2d_files', '*.coadd2d'))
    # Loop
    for ifile in coadd2d_files:
        coadd2dFile = inputfiles.Coadd2DFile.from_file(ifile)
        assert len(coadd2dFile.data['filename']) > 0

def test_read_flexure_files():
    """ Test reading flexure files """
    # Grab em
    flexure_files = glob.glob(os.path.join(
        os.getenv('PYPEIT_DEV'), 'flexure_files', '*.flex'))
    # Loop
    for ifile in flexure_files:
        flexureFile = inputfiles.FlexureFile.from_file(ifile)
        assert len(flexureFile.data['filename']) > 0

def test_read_pypeit_files():
    """ Test reading PypeIt files """
    # Grab em
    pypeit_files = glob.glob(os.path.join(
        os.getenv('PYPEIT_DEV'), 'pypeit_files', '*.pypeit'))
    # Loop
    for ifile in pypeit_files:
        pypeitFile = inputfiles.PypeItFile.from_file(ifile)

def test_get_spectrograph():
    pypeit_file = Path(os.getenv('PYPEIT_DEV')) / 'pypeit_files' \
                    / 'shane_kast_blue_452_3306_d57.pypeit'
    assert pypeit_file.exists(), 'Missing test pypeit file'

    pypeitFile = inputfiles.PypeItFile.from_file(pypeit_file)
    spec = pypeitFile.get_spectrograph()
    assert spec.name == 'shane_kast_blue', 'Wrong spectrograph'

def test_get_pypeitpar():

    spec = 'shane_kast_blue'
    setup = '600_4310_d55'

    # Define the path with the raw data
    data_root = Path(os.environ['PYPEIT_DEV']).resolve() / 'RAW_DATA' / spec / setup
    assert data_root.exists(), 'TEST ERROR: Raw data path does not exist'

    # Define the output directory and remove it if it already exist
    setup_path = Path().resolve() / f'{spec}_A'
    if setup_path.exists():
        shutil.rmtree(setup_path)

    args = ['-r', str(data_root), '-s', spec, '-c', 'all']
    pargs = Setup.parse_args(args)
    Setup.main(pargs)

    assert setup_path.exists(), 'No setup_files directory created'
    pypeit_file = setup_path / f'{spec}_A.pypeit'
    assert pypeit_file.exists(), 'PypeIt file not written'

    pypeitFile = inputfiles.PypeItFile.from_file(pypeit_file)
    spectrograph, par, file = pypeitFile.get_pypeitpar()

    assert spectrograph.name == spec, 'Wrong spectrograph'
    assert par['rdx']['spectrograph'] == spec, 'Wrong spectrgraph'
    assert par['rdx']['scidir'] == 'Science', 'Wrong default science directory'
    assert Path(file).name == 'b27.fits.gz', 'File name changed'

    # Clean-up
    shutil.rmtree(setup_path)