import csv
import multiprocessing
import sys
import warnings
import output_change

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from deap import base
from deap import creator
from deap import gp
from sklearn.cluster import KMeans
from sklearn.metrics import adjusted_rand_score
from sklearn.metrics import pairwise_distances
from sklearn.metrics import silhouette_score

import ea_simple_elitism
import multi_tree as mt
import vector_tree as vt
from ParallelToolbox import ParallelToolbox
from read_data import read_data
from selection import *

from get_features import FeatureMaker
import random
import copy
from scipy.spatial import distance
from clustering import cluster

POP_SIZE = 1024
NGEN = 1
CXPB = 0.8
MUTPB = 0.2
ELITISM = 10
MAX_HEIGHT = 8
SEED = 100
REP = mt  # individual representation {mt (multi-tree) or vt (vector-tree)}
N_TREES = 1
DATA_DIR = '.'  # "/home/schofifinn/PycharmProjects/SSResearch/data"
METRIC = 'intra'
MAXIMISE = METRIC != 'intra'

def make_feature_file(fm, input_data_name):
    feature_file = open(input_data_name + ".data", 'w')
    feature_file.write("classLast,{0},{1},space\n".format(fm.term_num(), fm.class_num()))
    for test_case_info in fm.case_info_list:
        str_vector = map(str, test_case_info.get_vector())
        feature_file.write("{0} {1}\n".format(" ".join(str_vector), test_case_info.get_classification()))
    feature_file.close()

def connectedness(cluster):
    print(cluster)

def evaluate(individual, toolbox, data, k, metric, maximise, distance_vector=None, labels_true=None):
    """
    Evaluates an individuals fitness. The fitness is the clustering performance on the data
    using the metric.

    :param individual: the individual to be evaluated
    :param toolbox: the evolutionary toolbox
    :param data: the data to be used to evaluate the individual
    :param k: the number of clusters for k-means
    :param metric: the metric to be used to evaluate clustering performance
    :param distance_vector: a pre-computed distance vector, required for silhouette-pre metric
    :param labels_true: the ground truth cluster labels, required for ari metric
    :return: the fitness of the individual
    """
    X = REP.process_data(individual, toolbox, data, MAX_HEIGHT)
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        # print("Kmenas SEED",SEED)
        kmeans = KMeans(n_clusters=k, random_state=SEED).fit(X)
    labels = kmeans.labels_
    # print(labels)
    # print("\n")
    # print("\n")
    # print("\n")
    # individuals that find single cluster are unfit
    nlabels = len(set(labels))
    if nlabels == 1:
        return [np.inf] if maximise else [-1]
    # uses precomputed distances of original data to avoid trending towards single dimension to
    # minimise silhouette.
    if metric == 'silhouette_pre':
        if distance_vector is None:
            raise Exception("Must provide distance vector for silhouette-pre metric.")
        return [silhouette_score(distance_vector, labels, metric='precomputed')]
    elif metric == 'silhouette':
        return [silhouette_score(X, labels, metric='euclidean')]
    elif metric == 'ari':
        if labels_true is None:
            raise Exception("Must provide ground truth labels for ARI")
        return [adjusted_rand_score(labels_true, labels)]
    elif metric == 'intra':
        sum = 0.
        for l in np.unique(labels):
            pts = data[labels == l]
            avg = pts.mean(axis=0)
            diff = np.linalg.norm(pts - avg)
            sum += diff
        return [sum]
    elif metric == 'connectedness':
        sum = 0.
        for l in np.unique(labels):
            idxs = labels == l
            dists = distance_vector[idxs][:, idxs]
            # not a single point in a cluster by itself
            if dists.shape[0] > 1:
                dists = np.divide(1., dists, out=np.zeros_like(dists), where=dists != 0)
                dists = np.minimum(dists, 10)
                np.fill_diagonal(dists, 0)
                sum += dists[dists > 0].sum() / dists.shape[0]  # .mean()
        return [sum / nlabels]
    else:
        raise Exception("invalid metric: {}".format(metric))

def write_ind_to_file(ind, run_num, results, datafile, data, toolbox):
    """
    Writes the attributes of an individual to file.

    :param run_num: the number of the current run
    :param ind: the individual
    :param results: a dictionary of results, titles to values
    """
    line_list = []
    # add constructed features to lines
    if REP is mt:
        for cf in [str(tree) for tree in ind]:
            line_list.append(cf + "\n")
    elif REP is not vt:
        raise Exception("Invalid representation")
    line_list.append("\n")
    fl = open("%d_ind.txt" % run_num, 'w')
    if REP is mt:
        fl.writelines(line_list)
    else:
        for i in ind:
            fl.write(str(i))
            fl.write("\n")
    fl.close()
    the_rep = 0
    if REP is vt:
        the_rep = 1
    output_change.change_output(str(run_num) + "_ind.txt", N_TREES, datafile + ".data", run_num, the_rep)

def init_toolbox(toolbox, pset):
    """
    Initialises the toolbox with operators.
    :param toolbox: the toolbox to initialise
    :param pset: primitive set for evolution
    """
    REP.init_toolbox(toolbox, pset, N_TREES)
    toolbox.register("select", selElitistAndTournament, tournsize=7, elitism=ELITISM)

def init_stats():
    """
    Initialises a MultiStatistics object to capture data.
    :return: the MultiStatistics object
    """
    fitness_stats = tools.Statistics(lambda ind: ind.fitness.values)
    stats = tools.MultiStatistics(fitness=fitness_stats)
    stats.register("avg", np.mean)
    stats.register("std", np.std)
    stats.register("min", np.min)
    stats.register("max", np.max)
    return stats

def final_evaluation(best, data, labels, num_classes, toolbox, print_output=True):
    """
    Performs a final performance evaluation on an individual.

    :param best: the individual to evaluate
    :param data: the dataset associated with the individual
    :param labels: the ground-truth labels of the dataset
    :param num_classes: the number of classes of the dataset
    :param toolbox: the evolutionary toolbox
    :param print_output: whether or not to print output
    :return: a dictionary of results, titles to values
    """
    return

rd = {}

def eval_wrapper(*args, **kwargs):
    return evaluate(*args, **kwargs, toolbox=rd['toolbox'], data=rd['data'], k=rd['num_classes'], metric=rd['metric'],
                    maximise=rd['maximise'], distance_vector=rd['distance_vector'])

# copies data over from parent process
def init_data(rundata):
    global rd
    rd = rundata

def maingptree(datafile, run_num):
    print("maing tree NGEN", NGEN)
    print("maing tree SEED", SEED)
    random.seed(SEED)
    all_data = read_data("%s/%s.data" % (DATA_DIR, datafile))
    rd['data'] = all_data["data"]
    rd['labels'] = all_data["labels"]
    rd['num_classes'] = len(set(rd['labels']))
    print("%d classes found." % rd['num_classes'])
    rd['distance_vector'] = pairwise_distances(rd['data'])
    rd['num_instances'] = rd['data'].shape[0]
    rd['num_features'] = rd['data'].shape[1]
    rd['metric'] = METRIC
    rd['maximise'] = MAXIMISE
    pset = gp.PrimitiveSet("MAIN", rd['num_features'], prefix="f")
    pset.context["array"] = np.array
    REP.init_primitives(pset)
    weights = (1.0,) if MAXIMISE else (-1.,)
    creator.create("FitnessMax", base.Fitness, weights=weights)
    # set up toolbox
    toolbox = ParallelToolbox()  # base.Toolbox()
    init_toolbox(toolbox, pset)
    toolbox.register("evaluate", eval_wrapper)
    rd['toolbox'] = toolbox
    pop = toolbox.population(n=POP_SIZE)
    hof = tools.HallOfFame(1)
    if REP is vt:
        hof = tools.HallOfFame(N_TREES)
    stats = init_stats()
    with multiprocessing.Pool(initializer=init_data, initargs=[rd]) as pool:
        toolbox.map = pool.map
        pop, logbook = ea_simple_elitism.eaSimple(pop, toolbox, CXPB, MUTPB, ELITISM, NGEN, stats,
                                                  halloffame=hof, verbose=True)
    if REP is not vt:
        best = hof[0]
    else:
        best = []
        for i in range(N_TREES):
            if i < len(hof):
                best.append(hof[i])
    res = final_evaluation(best, rd['data'], rd['labels'], rd['num_classes'], toolbox)
    write_ind_to_file(best, run_num, res, datafile, rd['data'], toolbox)
    return pop, stats, hof

if __name__ == "__main__":
    feature_maker = FeatureMaker()
    make_feature_file(feature_maker, "features")
    print("Getting feature information done")

    while True:
        vector_name = input(
            "Please input which type of representation you will use, vt(vetor tree) or mt(multi tree): ")
        if vector_name == "vt":
            REP = vt
            break
        elif vector_name == "mt":
            REP = mt
            break

    NGEN = int(input("Please input the length of generations: "))
    N_TREES = int(input("Please input the number of features: "))
    SEED = int(input("Please input seed number: "))
    maingptree("features", SEED)
    print("GP process done")

    k = int(input("Please input the number of clusters, k: "))
    cluster("result_" + str(SEED) + ".txt", k, method='random', num_iter=10)


#import libraries
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import scipy.io as sio

#read data
xy=sio.loadmat('./SimulatedDataWithTTF.mat')
x=xy['SimulateData']
#convert images to 1d array to feed simple RNN
image_stream=np.zeros([np.size(x,axis=0),np.size(x,axis=1),x[1,1].size])
for i in range(np.size(x,axis=0)):
    for j in range(np.size(x,axis=1)):
        image_stream[i,j,:]=np.reshape(x[i,j],(x[1,1].size,))
train_size=int(np.size(image_stream,axis=0)*0.8)
train_set_input=image_stream[:train_size,:,:]
test_set_input=image_stream[train_size:,:,:]

TTF=xy['TTF1']
TTF=np.squeeze(TTF)
TTF_train=np.zeros([np.size(train_set_input,axis=0),np.size(train_set_input,axis=1)])
TTF_test=np.zeros([np.size(test_set_input,axis=0),np.size(test_set_input,axis=1)])
for i in range(np.size(train_set_input,axis=0)):
    image_stream_len=int(TTF[i]*0.01)
    train_set_input[i,image_stream_len:,:]=0
    TTF_train[i,:image_stream_len]=np.fliplr([np.arange(TTF[i]-int(TTF[i]*0.01)*100,TTF[i],100)])

for i in range(np.size(test_set_input,axis=0)):
    image_stream_len=int(TTF[i+train_size]*0.01)
    test_set_input[i,image_stream_len:,:]=0
    TTF_test[i,:image_stream_len]=np.fliplr([np.arange(TTF[i+train_size]-int(TTF[i+train_size]*0.01)*100,TTF[i+train_size],100)])

TTF_train=TTF_train/8000.0
TTF_train[:,-4:]=0
TTF_test=TTF_test/8000.0
TTF_test[:,-4:]=0

timestep=np.size(train_set_input,axis=1)
input_dim=np.size(train_set_input,axis=2)
num_units=10
batch_size=8
initial_prediction_time=10

#mean absolute percentage error for performance evaluation
def mean_absolute_percentage_error(true_val,predicted_val):
    count=np.zeros([np.size(true_val,axis=2),1])
    percentage_error=np.zeros([np.size(true_val,axis=2),1])
    for i in range(np.size(true_val,axis=0)):
        for j in range(np.size(true_val,axis=1)):
            for k in range(initial_prediction_time,np.size(true_val,axis=2)-4):
                if(true_val[i,j,k]!=0.0):
                    percentage_error[k]+=(np.abs((true_val[i,j,k]-predicted_val[i,j,k])/true_val[i,j,k]))
                    count[k]+=1
    for i in range(np.size(true_val,axis=2)):
        if count[i]==0:
            percentage_error[i]=0
        else:
            percentage_error[i]=percentage_error[i]/count[i]
    return percentage_error

#Inputs & Outputs => Data
encoder_input=tf.placeholder(tf.float32,[batch_size,timestep,input_dim])#input placeholder
ground_truth=tf.placeholder(tf.float32,[batch_size,timestep])#true output placeholder
#Build encoder part
encoder_cell=tf.nn.rnn_cell.BasicLSTMCell(num_units,name='encoder',dtype=tf.float32)#LSTM cell
encoder_cell=tf.nn.rnn_cell.DropoutWrapper(encoder_cell,input_keep_prob=0.8,output_keep_prob=0.8)
initial_state = encoder_cell.zero_state(batch_size, dtype=tf.float32)
encoder_outputs,encoder_state=tf.nn.dynamic_rnn(encoder_cell,encoder_input,
                                                initial_state=initial_state)#Run dynamic RNN
#output vector (It is called as vector because outputs are created as vectors of size timestep)
W2=tf.Variable(np.random.rand(num_units,1),dtype=tf.float32)
b2=tf.Variable(np.random.rand(1,1),dtype=tf.float32)
predictions=[tf.nn.relu(tf.matmul(tf.squeeze(encoder_outputs[i,:,:]),W2)+b2)
                for i in range(encoder_outputs.get_shape().as_list()[0])]
predictions=tf.stack(predictions)

ground_truth=tf.expand_dims(ground_truth,axis=2)
x=np.arange(1,81)/np.sum(np.arange(1,81))
mask=tf.expand_dims(tf.constant(np.matlib.repmat(x,batch_size,1),dtype=tf.float32),axis=2)
error=tf.reduce_sum(((ground_truth[:,:,:]-predictions[:,:,:])**2)*mask)#error
train_step=tf.train.AdamOptimizer(0.0001).minimize(error)

with tf.Session() as sess:
    epoch_num=5000
    mse_track=[]
    maxe_track=[]
    mp_track=[]
    sess.run(tf.global_variables_initializer())#initialize variables
    data_length=int(len(TTF_train)/batch_size)
    for epoch in range(epoch_num):
        sum_error=0.0
        max_error=0.0
        count_sum=0.0
        print(epoch)
        for frame_id in range(data_length):
            batch_in=train_set_input[frame_id*batch_size:(frame_id+1)*batch_size,:,:]
            batch_out=TTF_train[frame_id*batch_size:(frame_id+1)*batch_size,:]
            [_error,_train_step,_predictions]=sess.run(
                    [error,train_step,predictions],
                    feed_dict={
                        encoder_input:batch_in,
                        ground_truth:np.expand_dims(batch_out,axis=2)
                     }
                )
            sum_error+=_error
            max_error=max(_error,max_error)
            count_sum+=1
        print("Train:",epoch_num,sum_error/count_sum,max_error)
        mse_track.append(sum_error/count_sum)
        maxe_track.append(max_error)
        #Test

        dummy_error=[]
        cum_batch_out=[]
        cum_predictions=[]
        for frame_id in range(int(np.size(TTF_test,axis=0)/batch_size)):
            batch_in=test_set_input[frame_id*batch_size:(frame_id+1)*batch_size,:,:]
            batch_out=TTF_test[frame_id*batch_size:(frame_id+1)*batch_size,:]
            [_error,_predictions]=sess.run(
                    [error,predictions],
                    feed_dict={
                        encoder_input:batch_in,
                        ground_truth:np.expand_dims(batch_out,axis=2)
                    }
                )
            cum_batch_out.append(batch_out)
            cum_predictions.append(_predictions)
        cum_predictions=np.squeeze(np.asarray(cum_predictions))
        cum_batch_out=np.asarray(cum_batch_out)
        mperror=mean_absolute_percentage_error(cum_batch_out,cum_predictions)
        mp_track.append(mperror)
        if epoch==epoch_num-1:
            np.savez('nowarmsigmoid.npz', original=cum_batch_out, predicted=cum_predictions,
                     mse=np.asarray(mse_track), maxe=np.asarray(maxe_track), mp=np.asarray(mp_track))


#!/usr/bin/env python

from mutagen.easyid3 import EasyID3
from mutagen.mp3 import MP3
from shutil import move
from glob import glob
import os

class PackSongs(object):
    """
    PackSongs pack songs into folders according to property choosen
    by the user making easy to manage songs and keep them into folders
    """

    def __init__(self, input_folder, tag=''):
        """ Initializing info to use for creating folders """
        self.tag = tag
        self.folder = input_folder
        self.change_cwd()
        self.list_mp3files()

    def find_info(self, song_name):
        """ Finding song info """
        info = dict()
        # Loading Song
        try:
            self.mp3file = MP3(song_name, ID3=EasyID3)
            # Storing all details of song into "info" dictionary
            try:
                info['title'] = str(self.mp3file['title'][0])
            except:
                info['title'] = ''
            try:
                info['artist'] = str(self.mp3file['artist'][0])
            except:
                info['artist'] = ''
            try:
                info['album'] = str(self.mp3file['album'][0])
            except:
                info['album'] = ''
            try:
                info['year'] = str(self.mp3file['date'][0])
            except:
                info['year'] = ''
            try:
                info['duration'] = int(self.mp3file.info.length * 1000)
            except:
                info['duration'] = ''
        except:
            info = {}

        self.song_info = info

    def change_cwd(self):
        """
        Changing current working directory to user input folder to access songs
        """
        os.chdir(self.folder)

    def remove_schars(self, fn, char):
        """ Removing special chars from folder name """
        fl = 0
        while fl != 1:
            try:
                fn.remove(char)
            except:
                fl = 1
        return fn

    def check_folder(self, folder_name):
        """ Checking if required folder exists otherwise create a new one """
        if not os.path.exists(folder_name):
            try:
                os.makedirs(folder_name)
            except:
                fn = list(folder_name)
                special_chars = [':', '/', '\\', '*', '?', '<', '>', '|']
                for i in special_chars:
                    fn = self.remove_schars(fn, i)
                folder_name = ''.join(fn)
                os.makedirs(folder_name)

    def list_mp3files(self, folder=''):
        """ Storing list of all .mp3 files """
        # All posibilities of writing mp3 considered ;)
        if folder == '':
            self.songs = glob('*.??3')
        else:
            return glob(folder+'/*.??3')

    def move_song(self, song, folder_name):
        """ Moving song to specific folder """
        try:
            move(song, folder_name)
        except:
            pass

    def generate_log(self):
        """
        Generate a log file stating the songs inside each newly created
        directory
        """
        # Listing all newly made folders
        new_folders = [i for i in os.listdir('.') if(os.path.isdir(i))]
        # Writing nfo data into log file
        fh = open('PackLog.txt', 'w')
        for folder in new_folders:
            text = 'Folder  :   ' + folder + '\n' + "-"*(12+len(folder)) + \
            '\n\n' + 'Track Listing' + '\n' + "-"*len('Track Listing') + '\n'
            try:
                songs = os.listdir(folder)
                for index,name in enumerate(songs):
                    text += str(index+1) + '. ' + name + '\n'
            except:
                continue
            fh.write(text+'\n\n\n')
        fh.close()

    def put_songs(self):
        """ Putting all songs in folders according to user choice """
        # Traversing all songs
        for song in self.songs:
            # Calling function to find song info
            self.find_info(song)
            info = self.song_info
            if info != {}:
                folder_name = info[self.tag]
                if folder_name in [None, '', 'Unknown']:
                    folder_name = 'Random'
                self.check_folder(folder_name)
                self.move_song(song, folder_name)


#******************************************#
#     File Name: OVERFLOW_ADC_DAC.py
#        Author: Takahiro Yamamoto
# Last Modified: 2023/06/06 22:02:47
#******************************************#

#######################################
### Modules
#######################################
import os
import numpy as np
from gwpy.timeseries import TimeSeries, TimeSeriesDict
from gwpy.segments import DataQualityFlag

#######################################
### Functions
#######################################
def _make_overflow_flag(sigs:TimeSeriesDict, name:str, round:bool=False) -> DataQualityFlag:
    '''
    Core function for making DataQualityFlag about ADC/DAC Overflows.

    name:str
        'OMC', 'ETMX', or 'ETMY'
    '''
    ### [NOTE] Definition of this flag.
    ### [HACK] This flag should be included in category 1.
    dqflag = DataQualityFlag(name='K1:{0}_OVF:1'.format(name),
                             label='Overflows on {0}'.format(name),
                             category=None,
                             description='Overflows on {0}'.format(name),
                             isgood=False)

    threshold = TimeSeries([0.0], unit='NONE', name='threshold:0.0')
    for sig in sigs.values():
        state = sig != threshold
        #dqflag |= state.to_dqflag(round=round)
        dqflag |= state.to_dqflag().round(contract=round) # Modified by Uchikata 2023/06/23

    return dqflag

#Added by Uchikata 2023/06/23 #
def _make_overflow_ok_flag(sigs:TimeSeriesDict, name:str, round:bool=False) -> DataQualityFlag:
    '''
    Core function for making DataQualityFlag about No ADC/DAC Overflows.

    name:str
        'OMC', 'ETMX', or 'ETMY'
    '''
    temp = DataQualityFlag(name='K1:{0}_OVF_OK:1'.format(name),
                           label='No overflows on {0}'.format(name),
                           category=None,
                           description='No overflows on {0}'.format(name),
                           isgood=True)
    threshold = TimeSeries([0.0], unit='NONE', name='threshold:0.0')
    sigN = 1
    for sig in sigs.values():
        state = sig == threshold
        #print(state)
        if sigN == 1:
            temp = state.to_dqflag().round(contract=round)
        else:
            temp &= state.to_dqflag().round(contract=round)
        sigN = sigN + 1

    ### [NOTE] Definition of this flag.
    dqflag = DataQualityFlag(name='K1:{0}_OVF_OK:1'.format(name),
                             label='No overflows on {0}'.format(name),
                             category=None,
                             description='No overflows on {0}'.format(name),
                             isgood=True)
    dqflag |= temp

    return dqflag

def make_overflow_flag(t0:float, t1:float, name:str, round:bool=False, host:str='k1nds1', port:int=8088) -> DataQualityFlag:
    '''
    User interface for making DataQualityFlag about ADC/DAC Overflows.

    name:str
        'OMC', 'ETMX', or 'ETMY'
    '''
    ### [NOTE] Channels checked in this flag.
    chans = {'OMC': ['K1:FEC-79_ADC_OVERFLOW_0_0',     ### DCPD_A
                     'K1:FEC-79_ADC_OVERFLOW_0_1'],    ### DCPD_B
             'ETMX': ['K1:FEC-103_DAC_OVERFLOW_1_0',   ### MN_V3
                      'K1:FEC-103_DAC_OVERFLOW_1_1',   ### MN_H1
                      'K1:FEC-103_DAC_OVERFLOW_1_2',   ### MN_H2
                      'K1:FEC-103_DAC_OVERFLOW_1_3',   ### MN_H3
                      'K1:FEC-103_DAC_OVERFLOW_1_4',   ### MN_V1
                      'K1:FEC-103_DAC_OVERFLOW_1_5',   ### MN_V2
                      'K1:FEC-103_DAC_OVERFLOW_1_6',   ### IM_V1
                      'K1:FEC-103_DAC_OVERFLOW_1_7',   ### IM_V2
                      'K1:FEC-103_DAC_OVERFLOW_1_8',   ### IM_V3
                      'K1:FEC-103_DAC_OVERFLOW_1_9',   ### IM_H1
                      'K1:FEC-103_DAC_OVERFLOW_1_10',  ### IM_H2
                      'K1:FEC-103_DAC_OVERFLOW_1_11',  ### IM_H3
                      'K1:FEC-103_DAC_OVERFLOW_1_12',  ### TM_H1
                      'K1:FEC-103_DAC_OVERFLOW_1_13',  ### TM_H2
                      'K1:FEC-103_DAC_OVERFLOW_1_14',  ### TM_H3
                      'K1:FEC-103_DAC_OVERFLOW_1_15'], ### TM_H4
             'ETMY': ['K1:FEC-108_DAC_OVERFLOW_1_0',   ### MN_V3
                      'K1:FEC-108_DAC_OVERFLOW_1_1',   ### MN_H1
                      'K1:FEC-108_DAC_OVERFLOW_1_2',   ### MN_H2
                      'K1:FEC-108_DAC_OVERFLOW_1_3',   ### MN_H3
                      'K1:FEC-108_DAC_OVERFLOW_1_4',   ### MN_V1
                      'K1:FEC-108_DAC_OVERFLOW_1_5',   ### MN_V2
                      'K1:FEC-108_DAC_OVERFLOW_1_6',   ### IM_V1
                      'K1:FEC-108_DAC_OVERFLOW_1_7',   ### IM_V2
                      'K1:FEC-108_DAC_OVERFLOW_1_8',   ### IM_V3
                      'K1:FEC-108_DAC_OVERFLOW_1_9',   ### IM_H1
                      'K1:FEC-108_DAC_OVERFLOW_1_10',  ### IM_H2
                      'K1:FEC-108_DAC_OVERFLOW_1_11',  ### IM_H3
                      'K1:FEC-108_DAC_OVERFLOW_1_12',  ### TM_H1
                      'K1:FEC-108_DAC_OVERFLOW_1_13',  ### TM_H2
                      'K1:FEC-108_DAC_OVERFLOW_1_14',  ### TM_H3
                      'K1:FEC-108_DAC_OVERFLOW_1_15'], ### TM_H4
    }[name]
    sigs = TimeSeriesDict.fetch(chans, t0, t1, host=host, port=port)
    return _make_overflow_flag(sigs, name, round=round)

def write_dqflag(dqflag:DataQualityFlag, name:str):
    '''
    '''
    xmlfile = '{0}.xml'.format(name)
    ### [NOTE] If file already exists, new dqflag is merged.
    if os.path.exists(xmlfile):
        old = DataQualityFlag.read(xmlfile)
        dqflag = old + dqflag
    dqflag.write(xmlfile, overwrite=True, format='ligolw')
    np.savetxt('{0}.txt'.format(name), dqflag.active, fmt = "%.4f")

#######################################
### sample code
#######################################
if __name__ == '__main__':
    import gpstime
    import argparse
    parser = argparse.ArgumentParser(
        description='make SegmentList of Overflows.',
        epilog='> python3 test.py --gps0 1369843000 --gps1 1369846000')
    parser.add_argument('--gps0', required=True, type=float, help='start gpstime')
    parser.add_argument('--gps1', required=True, type=float, help='end gpstime')
    parser.add_argument('--name', required=True, type=str, choices=['OMC', 'ETMX', 'ETMY'],  help='end gpstime')
    parser.add_argument('--round', action='store_true', help='round integer GPS')
    parser.add_argument('--output', action='store_true', help='round integer GPS')
    parser.add_argument('--only-nevent', action='store_true', help='round integer GPS')
    args = parser.parse_args()

    t0 = min(args.gps0, args.gps1)
    t1 = max(args.gps0, args.gps1)
    round = args.round

    x = make_overflow_flag(t0, t1, name=args.name, round=round)
    if args.only_nevent:
        print('{0}'.format(len(x.active)))
    else:
        print('       name: {0}'.format(x.name))
        print('      label: {0}'.format(x.label))
        print('   category: {0}'.format(x.category))
        print('description: {0}'.format(x.description))
        print('     isgood: {0}'.format(x.isgood))
        print('    segment: {0}'.format(x.known))
        print('     active: {0}'.format(len(x.active)))
        for seg in x.active:
            print('        {0}'.format(seg))

    if args.output:
        date = gpstime.parse(t0)
        write_dqflag(x, '{0:4d}-{1:02d}-{2:02d}'.format(date.year, date.month, date.day))

#######################################
### EOF
#######################################


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import pandas as pd
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt

from scipy.io import loadmat
from scipy.optimize import minimize

from sklearn.linear_model import LinearRegression, Ridge
from sklearn.preprocessing import PolynomialFeatures

pd.set_option('display.notebook_repr_html', False)
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', 150)
pd.set_option('display.max_seq_items', None)

import seaborn as sns
sns.set_context('notebook')
sns.set_style('white')

# In[2]:

data = loadmat('ex5data1.mat')
data.keys()

# In[3]:

y_train = data['y']
X_train = np.c_[np.ones_like(data['X']), data['X']]

yval = data['yval']
Xval = np.c_[np.ones_like(data['Xval']), data['Xval']]

print('X_train:', X_train.shape)
print('y_train:', y_train.shape)
print('Xval:', Xval.shape)
print('yval:', yval.shape)

# In[4]:

def linearRegCostFunction(theta, X, y, reg):
    m = y.size

    h = X.dot(theta)

    J = (1/(2*m))*np.sum(np.square(h-y)) + (reg/(2*m))*np.sum(np.square(theta[1:]))

    return(J)

# In[5]:

def lrgradientReg(theta, X, y, reg):
    m = y.size

    h = X.dot(theta.reshape(-1,1))

    grad = (1/m)*(X.T.dot(h-y))+ (reg/m)*np.r_[[[0]],theta[1:].reshape(-1,1)]

    return(grad.flatten())

# In[7]:

initial_theta = np.ones((X_train.shape[1],1))
cost = linearRegCostFunction(initial_theta, X_train, y_train, 0)
gradient = lrgradientReg(initial_theta, X_train, y_train, 0)
print(cost)
print(gradient)

# In[8]:

def trainLinearReg(X, y, reg):
    initial_theta = np.array([[15],[15]])

    res = minimize(linearRegCostFunction, initial_theta, args=(X,y,reg), method=None, jac=lrgradientReg,
                   options={'maxiter':5000})

    return(res)

# In[9]:

fit = trainLinearReg(X_train, y_train, 0)
fit

# In[10]:

regr = LinearRegression(fit_intercept=False)
regr.fit(X_train, y_train.ravel())
print(regr.coef_)
print(linearRegCostFunction(regr.coef_, X_train, y_train, 0))

# In[11]:

plt.plot(np.linspace(-50,40), (fit.x[0]+ (fit.x[1]*np.linspace(-50,40))), label='Scipy optimize')
plt.scatter(X_train[:,1], y_train, s=50, c='r', marker='x', linewidths=1)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.ylim(ymin=-5)
plt.xlim(xmin=-50)
plt.legend(loc=4);

# In[12]:

def learningCurve(X, y, Xval, yval, reg):
    m = y.size

    error_train = np.zeros((m, 1))
    error_val = np.zeros((m, 1))

    for i in np.arange(m):
        res = trainLinearReg(X[:i+1], y[:i+1], reg)
        error_train[i] = linearRegCostFunction(res.x, X[:i+1], y[:i+1], reg)
        error_val[i] = linearRegCostFunction(res.x, Xval, yval, reg)

    return(error_train, error_val)

# In[13]:

t_error, v_error = learningCurve(X_train, y_train, Xval, yval, 0)

# In[14]:

plt.plot(np.arange(1,13), t_error, label='Training error')
plt.plot(np.arange(1,13), v_error, label='Validation error')
plt.title('Learning curve for linear regression')
plt.xlabel('Number of training examples')
plt.ylabel('Error')
plt.legend();

# In[15]:

poly = PolynomialFeatures(degree=8)
X_train_poly = poly.fit_transform(X_train[:,1].reshape(-1,1))

regr2 = LinearRegression()
regr2.fit(X_train_poly, y_train)

regr3 = Ridge(alpha=20)
regr3.fit(X_train_poly, y_train)

plot_x = np.linspace(-60,45)
plot_y = regr2.intercept_+ np.sum(regr2.coef_*poly.fit_transform(plot_x.reshape(-1,1)), axis=1)
plot_y2 = regr3.intercept_ + np.sum(regr3.coef_*poly.fit_transform(plot_x.reshape(-1,1)), axis=1)

plt.plot(plot_x, plot_y, label='Scikit-learn LinearRegression')
plt.plot(plot_x, plot_y2, label='Scikit-learn Ridge (alpha={})'.format(regr3.alpha))
plt.scatter(X_train[:,1], y_train, s=50, c='r', marker='x', linewidths=1)
plt.xlabel('Change in water level (x)')
plt.ylabel('Water flowing out of the dam (y)')
plt.title('Polynomial regression degree 8')
plt.legend(loc=4);


"""
This is the file containing all of the endpoints for our flask app.
The endpoint called `endpoints` will return all available endpoints.
"""

from flask import Flask
from flask_restx import Resource, Api
import json
from werkzeug.exceptions import NotFound

app = Flask(__name__)
api = Api(app)

HOME = 'Home'
CHATROOM = 'Chatroom'
HELLO = 'hello'
AVAILABLE = 'Available endpoints:'
DATA_DIR = '../data'
HOW_TO_JSON = DATA_DIR + '/' + 'how_to.json'

def get_how_to():
    print("Going to open {HOW_TO_JSON}")
    try:
        with open(HOW_TO_JSON) as file:
            return json.loads(file.read())
    except FileNotFoundError:
        print('Failed to find {}'.format(HOW_TO_JSON))
        return None

@api.route('/<int:roomkey>')
class Chatroom(Resource):
    """
    Attempting to add a new route: Chatroom
    """
    def put(self, roomkey):
        """
        For now this will just return a line of text
        """
        return {CHATROOM: "Joined."}

@api.route('/')
class Home(Resource):
    """
    The landing page for the web app
    """
    def get(self):
        """
        Trivial check to make sure that the endpoint is working
        """
        return {HOME: 'endpoint is available'}

@api.route('/hello')
class HelloWorld(Resource):
    """
    The purpose of the HelloWorld class is to have a simple test to see if the
    app is working at all.
    """
    def get(self):
        """
        A trivial endpoint to see if the server is running.
        It just answers with "hello world."
        """
        return {HELLO: 'world'}

@api.route('/endpoints/list')
class Endpoints(Resource):
    """
    This class will serve as live, fetchable documentation of what endpoints
    are available in the system.
    """
    def get(self):
        """
        The `get()` method will return a list of available endpoints.
        """
        epts = sorted(rule.rule for rule in api.app.url_map.iter_rules())
        return {AVAILABLE: epts}

@api.route('/how_to')
class HowTo(Resource):
    """
    This class will serve as live, fetchable documentation of what endpoints
    are available in the system.
    """
    @api.response(200, 'Success')
    @api.response(404, 'Not Found')
    def get(self):
        """
        The `get()` method will return the main menu.
        """
        how_to = get_how_to()
        if how_to is None:
            raise (NotFound("How To not found."))
        return how_to


from subprocess import call
import argparse

parser = argparse.ArgumentParser(description="Create subsamples of datasets.")
parser.add_argument("dataset", help="name of dataset collection XML file")
args = parser.parse_args()

sample_script = "../../data/collections/sample.py"
size_ratios = [.1, .2, .3, .4, .5, .6, .7, .8, .9]
sample_ids = range(5)
for size_ratio in size_ratios:
    for sample_id in sample_ids:
        sample_name = args.dataset.split('.')[0] + "-" + str(int(size_ratio * 100)) + "-" + str(sample_id) + ".xml"
        command = "python " + sample_script + " " + args.dataset + " " + sample_name + " --size_ratio " + str(size_ratio)
        call(command, shell=True)


import os
import shutil
import gzip
import tarfile
import zipfile
from stat import *
from unicodedata import east_asian_width

def get_file_names(dir_path: str):
    """指定したディレクトリ直下のファイル名一覧を取得する関数.

    Args:
        param1 dir_path: ファイル名一覧を取得するディレクトリパス.

    Returns:
        ファイル名の一覧を格納したリスト.
    """
    file_names = list()
    if not dir_path[-1] in "/":
        dir_path += "/"
    for file in os.listdir(dir_path):
        if os.path.isfile(r"{0}{1}".format(dir_path, file)):
            file_names.append(file)

    return file_names

def get_dir_names(dir_path: str):
    """指定したディレクトリ直下のディレクトリ名一覧を取得する関数.

    Args:
        param1 dir_path: ディレクトリ名一覧を取得するディレクトリパス。

    Returns:
        ディレクトリ名の一覧を格納したリスト.
    """
    dir_names = list()
    if not dir_path[-1] in "/":
        dir_path += "/"
    for dir in os.listdir(r"{}".format(dir_path)):
        if os.path.isdir(r"{0}{1}".format(dir_path, dir)):
            dir_names.append(dir)

    return dir_names

def dir_exists(path: str):
    """wrapper of os.path.isdir().

    Args:
        param1 path: directory path.

    Returns:
        if directory exists, return True, or False.
    """
    return os.path.isdir(r"{}".format(path))

def make_dirs(path: str):
    """wrapper of os.makedirs().

    Args:
        param1 path: directory path.

    Returns:
        Not returns value.
    """
    os.makedirs(r"{}".format(path))

def rm_filedir(path: str):
    """wrappaer of os.remove()

    Args:
        path: target file to remove
    """
    os.remove(r'{}'.format(path))

def remove_dir(path: str):
    """wrapper of os.rmdir()

    Args:
        param1 path: directory path.

    Returns:
        Not Returns value.
    """
    os.rmdir(r"{}".format(path))

def remove_dirs(path: str):
    """wrapper of os.removedirs()

    Args:
        param1 path: directory path.

    Returns:
        Not Returns value.
    """
    os.removedirs(r"{}".format(path))

def f_remove_dirs(path: str, ignore_errors=False):
    """wrapper of shutil.rmtree()

    Args:
        param1 path: directory path.

    Returns:
        Not Returns value.
    """
    shutil.rmtree(r"{}".format(path), ignore_errors=ignore_errors)

def compress_gz(path: str):
    """existed file compress to gzip and remove one.

    Args:
        param1 path: target file to compress.
    """
    with open(r'{}'.format(path), 'rb') as f_read:
        with gzip.open(r'{}.gz'.format(path), 'wb') as f_out:
            shutil.copyfileobj(f_read, f_out)

def create_tararchive(path: str, mode=None):
    """create a tar archive from an existed file/dir.

    Args:
        param1 path: the target file/dir
        param2 mode: compression mode. default is gz.
            other mode ... bz2 xz

    Raises:
        TarError
    """
    if mode is None:
        mode = 'gz'
    prefix_mode = "w:"
    mode = prefix_mode + mode

    archive_name = path + '.tar.gz'
    try:
        archive = tarfile.open(r"{}".format(archive_name), mode=mode)
        archive.add(archive_name)
    except tarfile.TarError:
        raise
    finally:
        archive.close()

def zip_data(file_path: str, archive_name=None):
    """
    ファイル及びフォルダごとZIP化関数
    :param file_path: the target file to archive.
    :param archive_name: archive name saved(not path)
    :return:

    Raises:
        FileNotFoundError: raises if file does not exist.
    """
    # head... base directory.
    # tail... file name or directory name.
    head, tail = os.path.split(file_path)
    basedir_idx = len(head)
    if archive_name is None:
        zip_path = file_path + ".zip"
    else:
        zip_path = head + "/" + archive_name + ".zip"

    with zipfile.ZipFile(file=zip_path, mode='w') as zipobj:
        if os.path.isfile(file_path):
            fname = os.path.split(file_path)[1]
            zipobj.write(file_path, arcname=fname)
            print(">> archived...   {}".format(fname))
            return
        for dir, sub_dir, file_names in os.walk(file_path):
            dir_name = os.path.split(dir)[1]
            print(">> archived...   {}".format(dir))
            zipobj.write(dir, arcname=dir[basedir_idx:])
            for file_name in file_names:
                print(">> archived...   {}".format(os.path.join(dir, file_name)))
                zipobj.write(os.path.join(dir, file_name),
                             arcname=os.path.join(dir[basedir_idx:], file_name))

def change_write_permissions(path: str, target='u', recursive=False):
    """add "write" permission to specified targets.

        Args:
            param1 path: file path.
            param2 mode: target to permission.
            param3 recursive: recursive option. defualt is False.
                if this param is true, processing recursively.

        Raises:
            KeyError: raises when referenced unknwon key.
            FileNotFoundError
    """
    target_map = {
        'u': S_IWUSR,
        'ug': S_IWUSR | S_IWGRP,
        'uo': S_IWUSR | S_IWOTH,
        'g': S_IWGRP,
        'go': S_IWGRP | S_IWOTH,
        'o': S_IWOTH,
        'a': S_IWUSR | S_IWGRP | S_IWOTH,
    }

    if recursive:
        for dir, sub_dir, filenames in os.walk(path):
            os.chmod(path=dir, mode=target_map[target])
            for filename in filenames:
                os.chmod(path=os.path.join(dir, filename), mode=target_map[target])
    else:
        os.chmod(path=path, mode=target_map[target])

def change_read_permissions(path: str, target='u', recursive=False):
    """add "read" permission to specified targets.

        Args:
            param1 path: file path.
            param2 mode: target to permission.
            param3 recursive: recursive option. defualt is False.
                if this param is true, processing recursively.

        Raises:
            KeyError: raises when referenced unknwon key.
            FileNotFoundError
    """
    target_map = {
        'u': S_IRUSR,
        'ug': S_IRUSR | S_IRGRP,
        'uo': S_IRUSR | S_IROTH,
        'g': S_IRGRP,
        'go': S_IRGRP | S_IROTH,
        'o': S_IROTH,
        'a': S_IRUSR | S_IRGRP | S_IROTH,
    }

    if recursive:
        for dir, sub_dir, filenames in os.walk(path):
            os.chmod(path=dir, mode=target_map[target])
            for filename in filenames:
                os.chmod(path=os.path.join(dir, filename), mode=target_map[target])
    else:
        os.chmod(path=path, mode=target_map[target])

def change_exc_permissions(path: str, target='u', recursive=False):
    """add "execution" permission to specified targets.

        Args:
            param1 path: file path.
            param2 mode: target to permission.
            param3 recursive: recursive option. defualt is False.
                if this param is true, processing recursively.

        Raises:
            KeyError: raises when referenced unknwon key.
            FileNotFoundError
    """
    target_map = {
        'u': S_IXUSR,
        'ug': S_IXUSR | S_IXGRP,
        'uo': S_IXUSR | S_IXOTH,
        'g': S_IXGRP,
        'go': S_IXGRP | S_IXOTH,
        'o': S_IXOTH,
        'a': S_IXUSR | S_IXGRP | S_IXOTH,
    }

    if recursive:
        for dir, sub_dir, filenames in os.walk(path):
            os.chmod(path=dir, mode=target_map[target])
            for filename in filenames:
                os.chmod(path=os.path.join(dir, filename), mode=target_map[target])
    else:
        os.chmod(path=path, mode=target_map[target])

def join_path(head: str, tail: str):
    """os.path.join wrapper method."""
    return os.path.join(head, tail)

def get_strlength(line: str):
    """get length of string.

    Args:
        param1 line: string

    Returns:
        string length(int)
    """

    width_dict = {
        'F': 2,   # Fullwidth
        'H': 1,   # Halfwidth
        'W': 2,   # Wide
        'Na': 1,  # Narrow
        'A': 2,   # Ambiguous
        'N': 1    # Neutral
    }

    chars = [char for char in line]
    eaw_list = [east_asian_width(char) for char in chars]
    char_length = [width_dict[eaw] for eaw in eaw_list]
    return sum(char_length)

def change_permission(path: str, user_mode: str, grp_mode: str, oth_mode: str,
                      recursive=False):
    """
    """
    user_mode_map = {
        '0': '',
        '1': S_IXUSR,
        '2': S_IWUSR,
        '3': S_IWUSR | S_IXUSR,
        '4': S_IRUSR,
        '5': S_IRUSR | S_IXUSR,
        '6': S_IRUSR | S_IWUSR,
        '7': S_IRUSR | S_IWUSR | S_IXUSR
    }
    grp_mode_map = {
        '0': '',
        '1': S_IXGRP,
        '2': S_IWGRP,
        '3': S_IWGRP | S_IXGRP,
        '4': S_IRGRP,
        '5': S_IRGRP | S_IXGRP,
        '6': S_IRGRP | S_IWGRP,
        '7': S_IRGRP | S_IWGRP | S_IXGRP
    }
    oth_mode_map = {
        '0': '',
        '1': S_IXOTH,
        '2': S_IWOTH,
        '3': S_IWOTH | S_IXOTH,
        '4': S_IROTH,
        '5': S_IROTH | S_IXOTH,
        '6': S_IROTH | S_IWOTH,
        '7': S_IROTH | S_IWOTH | S_IXOTH
    }
    mode = 0

    mode_list = {'u_mode': user_mode_map[user_mode],
                 'g_mode': grp_mode_map[grp_mode],
                 'o_mode': oth_mode_map[oth_mode]}
    if user_mode == '0':
        mode_list.pop('u_mode')
    if grp_mode == '0':
        mode_list.pop('g_mode')
    if oth_mode == '0':
        mode_list.pop('o_mode')
    for key, value in mode_list.items():
        mode |= value

    if recursive:
        for dir, sub_dir, filenames in os.walk(path):
            os.chmod(path=dir, mode=mode)
            for filename in filenames:
                os.chmod(path=os.path.join(dir, filename), mode=mode)
    else:
        os.chmod(path=path, mode=mode)

class FileOpeException(Exception):
    pass


import sys

from bs4 import BeautifulSoup as bs4
from django.conf import settings
from django.utils.module_loading import import_string

from wagtail_toolbox.wordpress.models.config import StreamBlockSignatureBlocks

class WagtailBlockBuilder:
    def __init__(self, fallback_block_name=None, rich_text_block_name=None):
        if fallback_block_name is None and not hasattr(
            settings, "WPI_FALLBACK_BLOCK_NAME"
        ):
            self.fallback_block = (
                "wagtail_toolbox.wordpress.wagtail_builder_utils.raw_html_block_builder"
            )
        else:
            self.fallback_block = settings.WPI_FALLBACK_BLOCK_NAME

        if rich_text_block_name is None and not hasattr(
            settings, "WPI_RICHTEXT_BLOCK_NAME"
        ):
            self.rich_text_block = (
                "wagtail_toolbox.wordpress.wagtail_builder_utils.richtext_block_builder"
            )
        else:
            self.rich_text_block = settings.WPI_RICHTEXT_BLOCK_NAME

        self.stream_blocks = []
        self.stream_block_signatures = (
            StreamBlockSignatureBlocks.objects.all().values_list(
                "signature", "block_name", "block_kwargs"
            )
        )

    @staticmethod
    def make_tag_signature(element):
        """Make a signature for a BS4 tag and its children."""
        signature = f"{element.name}:"
        current = element.find()
        while current:
            signature += f"{current.name}:"
            current = current.find() if current.find() else None
        return signature

    def build(self, html):
        soup = bs4(html, "html.parser")
        block_stack = []

        for element in soup.findChildren(recursive=False):
            signature = self.make_tag_signature(element)

            try:
                stream_block_config = self.stream_block_signatures.get(
                    signature=signature
                )
            except StreamBlockSignatureBlocks.DoesNotExist:
                sys.stderr.write(f"Signature not found: {signature}\n")
                continue

            is_richtext = stream_block_config[1] == self.rich_text_block

            block_builder = import_string(stream_block_config[1])
            block_builder_kwargs = stream_block_config[2]
            block = block_builder(
                str(element), block_builder_kwargs=block_builder_kwargs
            )

            block_stack.append(block)  # add everything to the block stack

            # now fix up rich_text blocks as they should be one block
            # and not consecutive blocks
            if is_richtext and len(block_stack) > 1:
                # are the last 2 blocks on the stack a type="rich_text" block?
                # we know the last one is because of is_richtext
                if block_stack[-2]["type"] == "rich_text":
                    # pop the last block and get it's value
                    last_block_value = block_stack.pop()["value"]
                    # add the last_block_value to the last_block
                    block_stack[-1]["value"] += last_block_value

        return block_stack

make_tag_signature = WagtailBlockBuilder.make_tag_signature  # for convenience


from corsheaders.defaults import default_headers
from .base import *
from sentry_sdk.integrations.celery import CeleryIntegration

SECRET_KEY = env('SECRET_KEY')
SESSION_COOKIE_SECURE = True
CSRF_COOKIE_SECURE = True
ALLOWED_HOSTS = ['*']
CORS_ORIGIN_ALLOW_ALL = True
DJANGO_DEBUG = False
DJANGO_READ_DOT_ENV_FILE = False

CORS_ALLOW_HEADERS = list(default_headers) + [
    'content-disposition',
    'access-control-max-age'
]

# Email backend for Production
EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'

# Initialize Sentry
sentry_sdk.init(
    dsn=env.str('SENTRY_DSN'),
    integrations=[DjangoIntegration(), CeleryIntegration()],
    send_default_pii=True
)


import sqlite3 as sql
from typing import List

from pydojoctl.models.dojo import (
    Evento,
    EventoAtuacao,
    EventoConfiguracao,
    Participante,
)

def conectar(database='codedojo.db') -> sql.Connection:
    try:
        con = sql.connect(database)
        con.row_factory = sql.Row
        con.execute('PRAGMA foreign_keys=ON')
        return con
    except sql.DatabaseError as erro:
        print(erro)
        raise

def criar_base(connection=None):
    try:
        with (connection if connection else conectar()) as con:
            con.execute(
                """CREATE TABLE IF NOT EXISTS evento (
                  id INTEGER PRIMARY KEY AUTOINCREMENT,
                  nome text NOT NULL,
                  data text NOT NULL,
                  duracao_prevista integer,
                  duracao_real integer,
                  ativo integer
                )
            """
            )
    except sql.DatabaseError as erro:
        print(erro)
        raise

    # Participante
    try:
        with (connection if connection else conectar()) as con:
            con.execute(
                """CREATE TABLE IF NOT EXISTS participante (
                  id INTEGER PRIMARY KEY AUTOINCREMENT,
                  nome text NOT NULL,
                  email text NOT NULL,
                  presente integer
                )
            """
            )
    except sql.DatabaseError as erro:
        print(erro)
        raise

    # Configuração do evento
    try:
        with (connection if connection else conectar()) as con:
            con.execute(
                """CREATE TABLE IF NOT EXISTS evento_config (
                  id INTEGER PRIMARY KEY AUTOINCREMENT,
                  evento integer NOT NULL,
                  tempo_piloto integer NOT NULL,
                  tempo_audiencia integer NOT NULL,
                  FOREIGN KEY(evento) REFERENCES evento(id)
                )
            """
            )
    except sql.DatabaseError as erro:
        print(erro)
        raise

    # Atuação no evento
    try:
        with (connection if connection else conectar()) as con:
            con.execute(
                """CREATE TABLE IF NOT EXISTS evento_atuacao (
                  id INTEGER PRIMARY KEY AUTOINCREMENT,
                  evento integer NOT NULL,
                  participante integer NOT NULL,
                  ordem_sorteio integer,
                  FOREIGN KEY(evento) REFERENCES evento(id),
                  FOREIGN KEY(participante) REFERENCES participante(id)
                )
            """
            )
    except sql.DatabaseError as erro:
        print(erro)
        raise

def inserir_evento(evento: Evento, connection=None):
    try:
        with (connection if connection else conectar()) as con:
            return con.execute(
                """INSERT into evento
                (nome, data, duracao_prevista, duracao_real, ativo)
                VALUES
                (:nome, :data, :duracao_prevista, :duracao_real, :ativo)
            """,
                (
                    evento.nome,
                    evento.data,
                    evento.duracao_prevista,
                    evento.duracao_real,
                    evento.ativo,
                ),
            )
    except sql.DatabaseError as erro:
        print(erro)
        raise

def listar_eventos(connection=None) -> List[Evento]:
    eventos = []
    try:
        with (connection if connection else conectar()) as con:
            result = con.execute('SELECT * FROM evento')
            rows = result.fetchall()
            for row in rows:
                eventos.append(Evento(**row))
    except sql.DatabaseError as erro:
        print(erro)
        raise
    return eventos

def buscar_evento(evento_id, connection=None) -> Evento:
    try:
        with (connection if connection else conectar()) as con:
            result = con.execute(
                'SELECT * FROM evento where id = :evento_id',
                {'evento_id': evento_id},
            )
            row = result.fetchone()
            return Evento(**row)
    except sql.DatabaseError as erro:
        print(erro)
        raise

def inserir_participante(participante: Participante, connection=None):
    try:
        with (connection if connection else conectar()) as con:
            return con.execute(
                """INSERT into participante
                (nome, email, presente)
                VALUES
                (:nome, :email, :presente)
            """,
                (participante.nome, participante.email, participante.presente),
            )
    except sql.DatabaseError as erro:
        print(erro)
        raise

def inserir_configuracao(evento_config: EventoConfiguracao, connection=None):
    try:
        with (connection if connection else conectar()) as con:
            return con.execute(
                """INSERT into evento_config
                (evento, tempo_piloto, tempo_audiencia)
                VALUES
                (evento, tempo_piloto, tempo_audiencia)
            """,
                (
                    evento_config.evento.id,
                    evento_config.tempo_piloto,
                    evento_config.tempo_audiencia,
                ),
            )
    except sql.DatabaseError as erro:
        print(erro)
        raise

def inserir_atuacao(evento_atuacao: EventoAtuacao, connection=None):
    try:
        with (connection if connection else conectar()) as con:
            return con.execute(
                """INSERT into evento_atuacao
                (evento, participante, ordem_sorteio)
                VALUES
                (:evento, :participante, :ordem_sorteio)
            """,
                (
                    evento_atuacao.evento.id,
                    evento_atuacao.participante.id,
                    evento_atuacao.ordem_sorteio,
                ),
            )
    except sql.DatabaseError as erro:
        print(erro)
        raise


from django.shortcuts import render

# Create your views here.

def home(response):
    degree = 0
    if response.method == "POST":
        answers = response.POST.get('answers')
        result = response.POST.get('result')
        if answers == "django":
            degree += 50
        if result == "3":
            degree += 50
    return render(response, "main/home.html", {"degree": degree})


from django.urls import path
from rest_framework.urlpatterns import format_suffix_patterns
from .views import itemlist,item_detail,bidlist,item_bids

app_name = 'api'

urlpatterns = [
    path('itemlist/',itemlist),
    path('item_detail/<int:pk>/',item_detail),
    path('bidlist/',bidlist),
    path('item_bids/<int:pk>/',item_bids),
    # path('itemlist/',itemlist),

]

urlpatterns = format_suffix_patterns(urlpatterns)


import os
from datetime import datetime

import matplotlib.pyplot as plt
import numpy as np
import seaborn as sns
from matplotlib.pyplot import figure
from sklearn.manifold import TSNE
from sklearn.neighbors import KNeighborsClassifier

from src.experiments import Adult
from src.utils.normalizer import Normalizer

sns.set()
sns.set_context("paper")

def create_meshgrid(points, h=0.1):
    """
    Create the mesh grid to plot in.

    :param h: The mesh size
    :param points: The dataset

    :return: The mesh grid to plot in
    """
    x_min, x_max = points[:, 0].min() - 0.1, points[:, 0].max() + 0.1
    y_min, y_max = points[:, 1].min() - 0.1, points[:, 1].max() + 0.1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

    return xx, yy

def plot_decision_surface(running_instance, query_idx=None, y_pred=None, soft=True, title=""):
    """
    Plots the decision surface of the model together with the data points.

    :param running_instance: An object containing the information about the current experiment and arguments
    :param query_idx: The index of chosen least confident example
    :param y_pred: The predictions of the model
    :param soft: Whether to plot  kernel-like boundary
    :param title: The title of the plot

    """
    experiment = running_instance.experiment
    known_idx = running_instance.known_idx
    train_idx = running_instance.train_idx

    if experiment.X.shape[1] > 2:
        plot_decision_surface_tsne(running_instance, query_idx, y_pred, title)
    else:
        X_known, y_known = experiment.X[known_idx], experiment.y[known_idx]
        X_train, y_train = experiment.X[train_idx], experiment.y[train_idx]
        X_known_norm, X_train_norm = Normalizer(experiment.normalizer).normalize_known_train(X_known, X_train)
        X_known_train_norm = np.concatenate((X_known_norm, X_train_norm), axis=0)
        model = experiment.model

        figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
        # create a mesh to plot in
        h = 0.05  # step size in the mesh
        xx, yy = create_meshgrid(X_known_train_norm, h)

        if soft:
            if hasattr(model.sklearn_model, "decision_function"):
                Z = model.decision_function(np.c_[xx.ravel(), yy.ravel()])
            else:
                Z = model.predict(np.c_[xx.ravel(), yy.ravel()])
        else:
            Z = model.predict(np.c_[xx.ravel(), yy.ravel()])

        # Put the result into a color plot
        Z = Z.reshape(xx.shape)
        plt.contourf(xx, yy, Z, cmap=plt.cm.RdBu_r, alpha=0.8)

        if y_pred is not None:
            wrong_points = X_train_norm[np.where(y_pred != y_train)]
            plt.scatter(X_train_norm[:, 0], X_train_norm[:, 1], c=y_pred, cmap=plt.cm.RdBu_r, s=45)
            plt.scatter(wrong_points[:, 0], wrong_points[:, 1], s=120, cmap=plt.cm.RdBu_r, facecolors='none',
                        edgecolors='green', linewidths=2)
        else:
            plt.scatter(X_train_norm[:, 0], X_train_norm[:, 1], c=y_train, cmap=plt.cm.RdBu_r, s=45)
            plt.scatter(X_known_norm[:, 0], X_known_norm[:, 1], c=y_known, cmap=plt.cm.RdBu_r, s=45,
                    edgecolors="yellow", linewidths=2)

        if query_idx is not None:
            least_conf = experiment.X[query_idx]
            idx_array = np.where((X_train[:, 0] == least_conf[0]) & (X_train[:, 1] == least_conf[1]))[0]
            if len(idx_array):
                idx_in_train = idx_array[0]
                least_conf_norm = X_train_norm[idx_in_train]
                plt.scatter(least_conf_norm[0], least_conf_norm[1], marker='x', s=400, linewidths=5, color='yellow')

        plt.xlim(X_known_train_norm[:, 0].min() - 0.1, X_known_train_norm[:, 0].max() + 0.1)
        plt.ylim(X_known_train_norm[:, 1].min() - 0.1, X_known_train_norm[:, 1].max() + 0.1)

        save_plot(plt, experiment.path, title, title, False)

def plot_decision_surface_tsne(running_instance, query_idx, y_pred, title):
    exp = running_instance.experiment
    X_known, y_known, X_train, y_train, _, _ = running_instance.get_all_data()
    X_known_norm, X_train_norm = Normalizer(exp.normalizer).normalize_known_train(X_known, X_train)

    X_known_train_norm = np.concatenate((X_known_norm, X_train_norm), axis=0)

    X_initial = Normalizer(exp.normalizer).normalize(exp.X)
    X_embedded = get_tsne_embedding(X_initial)
    X_known_embedded = X_embedded[running_instance.known_idx]
    X_train_embedded = X_embedded[running_instance.train_idx]
    X_known_train_embedded = np.concatenate((X_known_embedded, X_train_embedded), axis=0)

    y_predicted = running_instance.predict(X_known_train_norm)

    h = 10
    x_min, x_max = X_known_train_embedded[:, 0].min() - 1, X_known_train_embedded[:, 0].max() + 1
    y_min, y_max = X_known_train_embedded[:, 1].min() - 1, X_known_train_embedded[:, 1].max() + 1
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, h), np.linspace(y_min, y_max, h))
    # approximate Voronoi tesselation on resolution x resolution grid using 1-NN
    background_model = KNeighborsClassifier(n_neighbors=1).fit(X_known_train_embedded, y_predicted)
    voronoiBackground = background_model.predict(np.c_[xx.ravel(), yy.ravel()])
    voronoiBackground = voronoiBackground.reshape((h, h))

    plt.figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
    plt.contourf(xx, yy, voronoiBackground, cmap=plt.cm.RdBu_r, alpha=0.8)

    if y_pred is not None:
        plt.scatter(X_train_embedded[:, 0], X_train_embedded[:, 1], c=y_pred, cmap=plt.cm.RdBu_r, s=45)
    else:
        plt.scatter(X_train_embedded[:, 0], X_train_embedded[:, 1], c=y_train, cmap=plt.cm.RdBu_r, s=45)
        plt.scatter(X_known_embedded[:, 0], X_known_embedded[:, 1], c=y_known, cmap=plt.cm.RdBu_r, s=45,
                    edgecolors="yellow", linewidths=2)

    if query_idx is not None:
        least_conf = running_instance.get_from_indexes(exp.X, query_idx)
        if isinstance(exp, Adult):
            X_train = X_train.to_numpy()
        idx_array = np.where((X_train[:, 0] == least_conf[0]) & (X_train[:, 1] == least_conf[1]))[0]
        if len(idx_array):
            idx_in_train = idx_array[0]
            least_conf_norm = running_instance.get_from_indexes(X_train_embedded, idx_in_train)
            plt.scatter(least_conf_norm[0], least_conf_norm[1], marker='x', s=400, linewidths=5, color='yellow')

    save_plot(plt, exp.path, title, title, False)

def plot_kmedoids(points, kmedoids_instance, other_points, path):
    # Plot the decision boundary
    plt.figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
    plt.xlim(points[:, 0].min() - 0.1, points[:, 0].max() + 0.1)
    plt.ylim(points[:, 1].min() - 0.1, points[:, 1].max() + 0.1)
    xx, yy = create_meshgrid(points)

    # Obtain labels for each point in mesh.
    Z = kmedoids_instance.predict((np.c_[xx.ravel(), yy.ravel()]))
    Z = Z.reshape(xx.shape)

    plt.imshow(Z, interpolation='nearest',
               extent=(xx.min(), xx.max(), yy.min(), yy.max()),
               cmap=plt.cm.Paired,
               aspect='auto', origin='lower')

    if other_points is not None:
        plt.scatter(other_points.iloc[:, 0], other_points.iloc[:, 1], c=other_points.predictions, marker='o', s=30)
    else:
        plt.scatter(points[:, 0], points[:, 1], c=points[:, 2], marker='o', s=30)

    # Plot the centroids
    centroids_idx = kmedoids_instance.get_medoids()
    centroids = points[centroids_idx]
    plt.scatter(centroids[:, 0], centroids[:, 1],
                marker='x', s=169, linewidths=3,
                color='red', zorder=10)

    save_plot(plt, path, "K-Medoids", "K-Medoids")

def plot_rules(clf, X, y, title, path, rules_f1):

    if X.shape[1] > 2:
        plot_rules_tsne(X, y, title, path)
    else:
        figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
        xx, yy = create_meshgrid(X, 0.005)
        if hasattr(clf, "decision_function"):
            Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
        else:
            Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]
        Z = Z.reshape(xx.shape)
        plt.contourf(xx, yy, Z, cmap=plt.cm.RdBu_r, alpha=0.8)
        plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.RdBu_r, s=45)

        save_plot(plt, path, " rules " + str(title), "F1 rules wrt svm: {}".format(rules_f1), False)

def plot_rules_tsne(X, y, title, path):

    X_embedded = get_tsne_embedding(X)
    h = 10
    x_min, x_max = X_embedded[:, 0].min() - 1, X_embedded[:, 0].max() + 1
    y_min, y_max = X_embedded[:, 1].min() - 1, X_embedded[:, 1].max() + 1
    xx, yy = np.meshgrid(np.linspace(x_min, x_max, h), np.linspace(y_min, y_max, h))
    # approximate Voronoi tesselation on resolution x resolution grid using 1-NN
    background_model = KNeighborsClassifier(n_neighbors=1).fit(X_embedded, y)
    voronoiBackground = background_model.predict(np.c_[xx.ravel(), yy.ravel()])
    voronoiBackground = voronoiBackground.reshape((h, h))

    plt.figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
    plt.contourf(xx, yy, voronoiBackground, cmap=plt.cm.RdBu_r, alpha=0.8)
    plt.scatter(X_embedded[:, 0], X_embedded[:, 1], c=y, cmap=plt.cm.RdBu_r, s=45)

    save_plot(plt, path, title, " rules", False)

def plot_initial_points(known_idx, test_idx, experiment):

    X_initial = Normalizer(experiment.normalizer).normalize(experiment.X)
    if experiment.X.shape[1] > 2:
        X_initial = get_tsne_embedding(X_initial)

    plot_points(X_initial, experiment.y, "Initial points", experiment.path)
    plot_points(X_initial[known_idx], experiment.y[known_idx], "Known points", experiment.path)
    plot_points(X_initial[test_idx], experiment.y[test_idx], "Test points", experiment.path)

def plot_points(X, y, title="", path=None):
    figure(num=None, figsize=(10, 8), facecolor='w', edgecolor='k')
    plt.scatter(X[:, 0], X[:, 1], c=y, marker='o', s=45, cmap=plt.cm.coolwarm)

    save_plot(plt, path, title, title)

def get_tsne_embedding(X):
    return TSNE(n_components=2, n_iter=300, random_state=0).fit_transform(X)

def save_plot(plt, path, img_name, plot_title=None, use_grid=True, use_date=True):
    img_name = "{}.pdf".format(img_name)
    plt.grid(use_grid)
    if plot_title is not None:
        plt.title(plot_title)
    if path:
        try:
            if use_date:
                img_name = "{}-{}.pdf".format(datetime.now().strftime('%Y-%m-%d_%H-%M-%S.%f'), img_name)
            plt.savefig(os.path.join(path, img_name), bbox_inches='tight', format='pdf')
        except ValueError:
            print("Something went wrong while saving image: ", img_name)
    else:
        plt.show()
    plt.close()


import os
import json
import logging

logger = logging.getLogger(__name__)

def test_post(nc):
    with open('./test/integration/data/employee/data.json') as oj:
        s = oj.read()
        employee = json.loads(s)
    logger.debug('employee = %s', employee)
    res = nc.employees.post(employee)
    logger.debug('res = %s', res)
    assert res['externalId'] == employee['externalId'], 'ID Number does not match'
    assert res['type'] == 'employee', 'Type does not match'


from sklearn.utils import shuffle
from tensorflow import keras
from keras.datasets import cifar10
import os
import time
import numpy as np
# Library for plot the output and save to file
import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt

import math
from pycuda import gpuarray
import pycuda.driver as cuda
from pycuda.compiler import SourceModule
import pycuda.autoinit

baseDir = os.path.dirname(os.path.abspath('__file__')) + '/'
classesName = ['plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck']
(xTrain, yTrain), (xTest, yTest) = cifar10.load_data()
xVal = xTrain[49000:, :].astype(np.float)
yVal = np.squeeze(yTrain[49000:, :])
xTrain = xTrain[:49000, :].astype(np.float)
yTrain = np.squeeze(yTrain[:49000, :])
yTest = np.squeeze(yTest)
xTest = xTest.astype(np.float)

# Normalize the data by subtract the mean image
meanImage = np.mean(xTrain, axis=0)
xTrain -= meanImage

xVal -= meanImage
xTest -= meanImage

# Reshape data from channel to rows
xTrain = np.reshape(xTrain, (xTrain.shape[0], -1))
xVal = np.reshape(xVal, (xVal.shape[0], -1))
xTest = np.reshape(xTest, (xTest.shape[0], -1))
# Add bias dimension columns
xTrain = np.hstack([xTrain, np.ones((xTrain.shape[0], 1))])
xVal = np.hstack([xVal, np.ones((xVal.shape[0], 1))])
xTest = np.hstack([xTest, np.ones((xTest.shape[0], 1))])

class Svm (object):
    """" Svm classifier """
    # (xTrain.shape[1], numClasses)
    def __init__ (self, inputDim, outputDim):
        self.W = None
        sigma =0.01
        self.W = sigma * np.random.randn(inputDim,outputDim)

    def sgd (self, xTrain, yTrain, learning_rate, reg_strength, max_epochs=200, batchSize=200, verbose=False):
        # train model with cuda
        Loss = []
        length_of_features = xTrain.shape[1]
        dim0 = 32
        block_dim = (10, dim0, 1)
        #grid_dim = (1, math.ceil(length_of_features/dim0), 1)
        grid_dim = (1, 1, 1)
        prg_sgd = SourceModule(open("../kernels/sgd_cifar_single_blk_normal_mult.cu", "r").read())
        func = prg_sgd.get_function("sgd")
        weights = self.W.astype(np.float32)
        time_gpu = 0
        s = time.time()
        dot_XW = np.zeros(10).astype(np.float32)
        DW = np.zeros((length_of_features,10)).astype(np.float32)
        loss = np.zeros(1)
        for i in range(max_epochs):
            num_train = np.random.choice(xTrain.shape[0], batchSize)
            xBatch = xTrain[num_train] # xBatch is a 2D array, each row is a data point,
            yBatch = yTrain[num_train]
            # Transfer variables onto device
            _start = cuda.Event()
            _end = cuda.Event()
            _start.record()
            d_x = gpuarray.to_gpu(xBatch.astype(np.float32))
            d_y = gpuarray.to_gpu(yBatch.astype(np.float32))
            d_weights = gpuarray.to_gpu(self.W.astype(np.float32))
            d_single_dw = gpuarray.to_gpu(DW)
            d_dot_XW = gpuarray.to_gpu(dot_XW)
            d_loss = gpuarray.to_gpu(loss.astype(np.float32))
            func(d_x, d_y, d_weights, d_single_dw,
                 np.float32(reg_strength),
                 np.float32(learning_rate),
                 np.int32(batchSize),
                 d_dot_XW,
                 d_loss,
                 block=block_dim,
                 grid=grid_dim)

            _end.record()
            _end.synchronize()
            self.W = d_weights.get()
            Loss.append(d_loss.get())
            time_gpu +=   _start.time_till(_end) # in ms
        return Loss

        #################

    def predict (self, x,):
        yPred = np.zeros(x.shape[0])
        s = x.dot(self.W)
        yPred = np.argmax(s, axis=1)
        return yPred

    def calAccuracy (self, x, y):
        acc = 0
        yPred = self.predict(x)
        acc = np.mean(y == yPred)*100
        return acc

#xTrain = xTrain[0:10000]
#yTrain = yTrain[0:10000]

numClasses = np.max(yTrain) + 1
classifier = Svm(xTrain.shape[1], numClasses)

# Training classifier

t = []
acc_tr = []
acc_te = []
loss_a = []

itr = 150
for i in range(itr):
    max_iter = (i+1) * 10
    startTime = time.time()
    Loss = classifier.sgd(xTrain, yTrain,
                          learning_rate=1e-7,
                          reg_strength=5e4,
                          max_epochs=max_iter,
                          batchSize=200, verbose=True)
    endTime = time.time()
    loss_a.append(Loss[0])
    print(loss_a)
    print ("max_epochs = ", max_iter, "\n")
    print ('Training time: {0}'.format(endTime - startTime))
    print ('Training acc:   {0}%'.format(classifier.calAccuracy(xTrain, yTrain)))
    #    print ('Validating acc: {0}%'.format(classifier.calAccuracy(xVal, yVal)))
    print ('Testing acc:    {0}%'.format(classifier.calAccuracy(xTest, yTest)))
    t.append(endTime - startTime)
    acc_tr.append(classifier.calAccuracy(xTrain, yTrain))
    acc_te.append(classifier.calAccuracy(xTest, yTest))

print(np.array(loss_a))

### print loss with max_epochs ####
x = np.linspace(0, itr, itr)
fig, ax = plt.subplots()
line1, = ax.plot(x, loss_a, label='LOSS')
ax.set_title('Loss-CF-naive')
ax.set_xlabel('max epochs (*10)')
ax.set_ylabel('Loss')
ax.set_yscale('log')
ax.grid(True)
ax.set_xlim(0, itr + 1)
ax.xaxis.set_ticks(np.arange(0,itr + 1, 1))
ax.legend()
plt.savefig("../CF-Naive-Loss.png")

### print time with max_epochs ####
x0 = np.linspace(0, itr, itr)
fig0, ax0 = plt.subplots()
line1, = ax0.plot(x0, t, label='LOSS')
ax0.set_title('Training-Time-CF-naive')
ax0.set_xlabel('max epochs (*10)')
ax0.set_ylabel('Time (s)')
#ax0.set_yscale('log')
ax0.grid(True)
ax0.set_xlim(0, itr + 1)
ax0.xaxis.set_ticks(np.arange(0,itr + 1, 1))
ax0.legend()
plt.savefig("../CF-Naive-Time.png")

### print accuracy on test and train with max_epochs ###

x1 = np.linspace(0, itr, itr)
fig1, ax1 = plt.subplots()
line1, = ax1.plot(x1, acc_tr, label='Training Accuracy')
line1, = ax1.plot(x1, acc_te, label='Testing Accuracy')
ax1.set_title('Accuracy-CF-Naive')
ax1.set_xlabel('max epochs (*10)')
ax1.set_ylabel('Accuracy (%)')

ax1.grid(True)
ax1.set_xlim(0, itr + 1)
ax1.xaxis.set_ticks(np.arange(0,itr + 1, 1))
ax1.legend()
plt.savefig("../CF-Naive-Accuracy.png")


### Distortion coefficients
import numpy as np

import cv2
from .common import make_gray, apply_pipeline

def get_corners(image, shape, nxy):
    gray = make_gray(image)
    if gray.shape != shape:
        return None

    ret, corners = cv2.findChessboardCorners(gray, nxy)
    if ret:
        return corners

    return None

def get_object_points(nx, ny, count):
    objpoint = np.zeros((nx*ny,3), np.float32)
    objpoint[:,:2] = np.mgrid[0:nx,0:ny].T.reshape(-1,2)
    return [objpoint] * count

### Transform matix

def apply_shape(arr, shape):
    arr[:,0] *= shape[1]
    arr[:,1] *= shape[0]

    return arr.astype(np.float32)

def get_transform_src(shape):
    x1, x2, x3, x4 = 0.1, 0.37, 0.65, 0.95
    y1, y2 = 0.7, 0.95
    return apply_shape(np.array([
        [x1, y2],
        [x2, y1],
        [x3, y1],
        [x4, y2],
    ]), shape)

def get_transform_dst(shape):
    return apply_shape(np.array([
        [0, 1],
        [0, 0],
        [1, 0],
        [1, 1]
    ]), shape)

def get_transform_matrices(shape):
    src = get_transform_src(shape)
    dst = get_transform_dst(shape)
    M = cv2.getPerspectiveTransform(src, dst)
    Minv = cv2.getPerspectiveTransform(dst, src)
    return M, Minv

### CameraCalibrator

class CameraCalibrator:
    def __init__(self, nx, ny, shape=(720, 1280)):
        self.image_points = []
        self.__nxy = (nx, ny)

        self.__M, self.__Minv  = get_transform_matrices(shape)

        self.__shape = shape
        self.__mtx = None
        self.__dist = None

    def calibrate(self, image):
        corners = get_corners(image, self.__shape, self.__nxy)
        if corners is not None:
            self.image_points.append(corners)
            return True
        return False

    def finish_calibrate(self):
        nx, ny = self.__nxy
        object_points = get_object_points(nx, ny, len(self.image_points))
        ret, mtx, dist, _, _ = cv2.calibrateCamera(
            object_points,
            self.image_points,
            self.__shape[::-1],
            None,
            None
        )
        assert(ret)
        self.__mtx = mtx
        self.__dist = dist

    def undistort(self, image):
        return cv2.undistort(image, self.__mtx, self.__dist, None, self.__mtx)

    def transform(self, image):
        shape = [image.shape[1], image.shape[0]]
        return cv2.warpPerspective(image, self.__M, shape)

    def un_transform(self, image):
        shape = [image.shape[1], image.shape[0]]
        return cv2.warpPerspective(image, self.__Minv, shape)

    def process_image(self, image):
        return list(apply_pipeline(image, [
            self.undistort,
            self.transform
        ]))


import cv2
import sqlite3
import numpy as np
from database_ops import *
import requests

faceDetect = cv2.CascadeClassifier("haarcascade_frontalface_alt2.xml")
recognizer = cv2.face.LBPHFaceRecognizer_create()
url = 'http://192.168.43.1:8080/shot.jpg'

recognizer.read('recognizer/training_data.yml')

while True:
    img_resp = requests.get(url)
    img_array = np.array(bytearray(img_resp.content),dtype = np.uint8)
    img = cv2.imdecode(img_array, -1)
    img = cv2.resize(img,(1280,720))
    gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    faces = faceDetect.detectMultiScale(gray,scaleFactor=1.5,minNeighbors=5)

    for(x,y,w,h) in faces:
        cv2.rectangle(img,(x,y),(x+w,y+h),(0,0,0),2)
        id,conf = recognizer.predict(gray[y:y+h, x:x+w])
        profile = 0
        if conf >= 45 and conf <= 85:
            profile = get_profile(id)
        if profile != 0:
            cv2.putText(img, profile[1], (x, y-40), 2, 0.8, (255, 255, 255), 1)
            cv2.putText(img, 'Age : ' + str(profile[2]), (x, y-20), 2, 0.8, (255, 255, 255), 1)
            cv2.putText(img, 'Gender : ' + str(profile[3]), (x, y), 2, 0.8, (255, 255, 255), 1)
        else:
            cv2.putText(img, 'Unknown', (x, y), 2, 0.8, (255, 255, 255), 1)

    cv2.imshow("Face_Detection_Window",img)
    if cv2.waitKey(1) == ord('q'):
        break

cv2.destroyAllWindows()


#!/usr/bin/env python3

''' Original code from: https://github.com/xysun/pychat
    Updated for use in OU ACM Chapter's networking talk
    This file contains code for running the client'''

import select
import socket
import sys
import utils

READ_BUFFER = 4096

def prompt():
    ''' Prompt for client input '''
    print('#> ', end=' ', flush=True)

def main():
    ''' Main function for the client script '''
    if len(sys.argv) < 2:
        print("Usage: Python3 client.py [hostname]", file=sys.stderr)
        sys.exit(1)
    else:
        server_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        server_connection.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        server_connection.connect((sys.argv[1], utils.PORT))

    print("Connected to server\n")
    msg_prefix = ''

    socket_list = [sys.stdin, server_connection]

    while True:
        read_sockets, _, _ = select.select(socket_list, [], [])
        # the _ means drop the value in python
        for sock in read_sockets:
            if sock is server_connection: # incoming message
                msg = sock.recv(READ_BUFFER)
                if not msg:
                    print("Server down!")
                    sys.exit(2)
                else:
                    if msg == utils.QUIT_STRING.encode():
                        sys.stdout.write('Bye\n')
                        sys.exit(2)
                    else:
                        sys.stdout.write(msg.decode())
                        if 'Please tell us your name' in msg.decode():
                            msg_prefix = 'name: ' # identifier for name
                        else:
                            msg_prefix = ''
                        prompt()

            else:
                msg = msg_prefix + sys.stdin.readline()
                server_connection.sendall(msg.encode())

if __name__ == '__main__':
    main()


from users.models import Customer, Executer
from task.models import Task
from rest_framework import serializers

class CustomerSerializer(serializers.ModelSerializer):
    class Meta:
        model = Customer
        fields = ('id', 'first_name', 'balance')

class ExecuterSerializer(serializers.ModelSerializer):
    class Meta:
        model = Executer
        fields = ('id', 'first_name', 'balance')

class TaskSerializer(serializers.ModelSerializer):
    # customer = CustomerSerializer(read_only=True)
    # executer = ExecuterSerializer(read_only=True)

    class Meta:
        model = Task
        fields = ('id', 'title', 'description', 'price', 'owner', 'created_by')


import os
import yaml
import sys

def get_bom_dir():
    return os.path.join(sys.argv[1], "bom")

def get_default_tkr_bom():
    bomDir = get_bom_dir()
    for bomFile in os.listdir(bomDir):
        if(bomFile.startswith("tkr-bom")):
            return bomFile
    return None

def main():
    bomFile = get_default_tkr_bom()
    bomDir = get_bom_dir()
    bomFilePath = os.path.join(bomDir, bomFile)
    with open(bomFilePath) as file:
        bom = yaml.safe_load(file)
    osInfo = bom["azure"][0]["osinfo"]

    # Create a file with no azure VM image info in BOM
    del bom["azure"]
    bom["release"]["version"] = "v0.0.0+no-azure-image"
    with open(os.path.join(bomDir, "bom-clustergen-no-azure-image-test.yaml"), 'w') as file:
        documents = yaml.dump(bom, file)

    # Create a file with marketplace azure VM image info in BOM
    bom["azure"] = [{
        "publisher": "bom-image-publisher",
        "offer":"bom-image-offer",
        "sku":"bom-image-sku",
        "version":"bom-image-version",
        "thirdPartyImage":True,
        "osinfo": osInfo
    }]
    bom["release"]["version"] = "v0.0.0+marketplace-image"
    with open(os.path.join(bomDir, "bom-clustergen-marketplace-image-test.yaml"), 'w') as file:
        documents = yaml.dump(bom, file)

    # Create a file with marketplace azure VM image(no thirdPartyImage flag) info in BOM
    bom["azure"] = [{
        "publisher": "bom-image-publisher",
        "offer":"bom-image-offer",
        "sku":"bom-image-sku",
        "version":"bom-image-version",
        "osinfo": osInfo
    }]
    bom["release"]["version"] = "v0.0.0+marketplace-image-no-thirdpartyimage"
    with open(os.path.join(bomDir, "bom-clustergen-marketplace-no-thirdparty-image-test.yaml"), 'w') as file:
        documents = yaml.dump(bom, file)

    # Create a file with shared gallery azure VM image info in BOM
    bom["azure"] = [{
        "resourceGroup": "bom-resource-group",
        "name":"bom-image-name",
        "subscriptionID":"bom-subscription-id",
        "version":"bom-image-version",
        "gallery":"bom-image-gallery",
        "osinfo": osInfo
    }]
    bom["release"]["version"] = "v0.0.0+shared-gallery-image"
    with open(os.path.join(bomDir, "bom-clustergen-shared-gallery-image-test.yaml"), 'w') as file:
        documents = yaml.dump(bom, file)

if __name__ == "__main__":
    main()


# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.

import argparse
import logging
import os
import sys
from collections import namedtuple

from wptrunner.wptmanifest.backends import base
from wptrunner.wptmanifest.node import KeyValueNode
from wptrunner.wptmanifest.serializer import serialize

here = os.path.dirname(__file__)
logger = logging.getLogger(__name__)

class Compiler(base.Compiler):
    def visit_KeyValueNode(self, node):
        key_name = node.data
        values = []
        for child in node.children:
            values.append(self.visit(child))

        self.output_node.set(key_name, values)

    def visit_ConditionalNode(self, node):
        assert len(node.children) == 2
        # For conditional nodes, just return the subtree
        return serialize(node.children[0]), self.visit(node.children[1])

    def visit_UnaryExpressionNode(self, node):
        raise NotImplementedError

    def visit_BinaryExpressionNode(self, node):
        raise NotImplementedError

    def visit_UnaryOperatorNode(self, node):
        raise NotImplementedError

    def visit_BinaryOperatorNode(self, node):
        raise NotImplementedError

class ExpectedManifest(base.ManifestItem):
    def __init__(self, node):
        """Object representing all the tests in a particular manifest"""
        base.ManifestItem.__init__(self, node)
        self.child_map = {}

    def append(self, child):
        """Add a test to the manifest"""
        base.ManifestItem.append(self, child)
        self.child_map[child.name] = child

    def insert(self, child):
        self.append(child)
        self.node.append(child.node)

    def delete(self, child):
        del self.child_map[child.name]
        child.node.remove()
        child.remove()

class TestManifestItem(ExpectedManifest):
    def set_expected(self, other_manifest):
        for item in self.node.children:
            if isinstance(item, KeyValueNode) and item.data == "expected":
                assert "expected" in self._data
                item.remove()
                del self._data["expected"]
                break

        for item in other_manifest.node.children:
            if isinstance(item, KeyValueNode) and item.data == "expected":
                assert "expected" in other_manifest._data
                item.remove()
                self.node.children.insert(0, item)
                self._data["expected"] = other_manifest._data.pop("expected")
                break

def data_cls_getter(output_node, visited_node):
    # visited_node is intentionally unused
    if output_node is None:
        return ExpectedManifest
    if isinstance(output_node, ExpectedManifest):
        return TestManifestItem
    raise ValueError

def compile(stream, data_cls_getter=None, **kwargs):
    return base.compile(Compiler, stream, data_cls_getter=data_cls_getter, **kwargs)

def get_manifest(manifest_path):
    """Get the ExpectedManifest for a particular manifest path"""
    try:
        with open(manifest_path, "rb") as f:
            try:
                return compile(f, data_cls_getter=data_cls_getter)
            except Exception:
                f.seek(0)
                sys.stderr.write("Error parsing:\n%s" % f.read().decode("utf8"))
                raise
    except IOError:
        return None

def indent(str_data, indent=2):
    rv = []
    for line in str_data.splitlines():
        rv.append("%s%s" % (" " * indent, line))
    return "\n".join(rv)

class Differences(object):
    def __init__(self):
        self.added = []
        self.deleted = []
        self.modified = []

    def __nonzero__(self):
        return bool(self.added or self.deleted or self.modified)

    def __str__(self):
        modified = []
        for item in self.modified:
            if isinstance(item, TestModified):
                modified.append(
                    "  %s\n    %s\n%s" % (item[0], item[1], indent(str(item[2]), 4))
                )
            else:
                assert isinstance(item, ExpectedModified)
                modified.append("  %s\n    %s %s" % item)
        return "Added:\n%s\nDeleted:\n%s\nModified:\n%s\n" % (
            "\n".join("  %s:\n    %s" % item for item in self.added),
            "\n".join("  %s" % item for item in self.deleted),
            "\n".join(modified),
        )

TestModified = namedtuple("TestModified", ["test", "test_manifest", "differences"])

ExpectedModified = namedtuple(
    "ExpectedModified", ["test", "ancestor_manifest", "new_manifest"]
)

def compare_test(test, ancestor_manifest, new_manifest):
    changes = Differences()

    compare_expected(changes, None, ancestor_manifest, new_manifest)

    for subtest, ancestor_subtest_manifest in ancestor_manifest.child_map.items():
        compare_expected(
            changes,
            subtest,
            ancestor_subtest_manifest,
            new_manifest.child_map.get(subtest),
        )

    for subtest, subtest_manifest in new_manifest.child_map.items():
        if subtest not in ancestor_manifest.child_map:
            changes.added.append((subtest, subtest_manifest))

    return changes

def compare_expected(changes, subtest, ancestor_manifest, new_manifest):
    if not (
        ancestor_manifest and ancestor_manifest.has_key("expected")  # noqa W601
    ) and (
        new_manifest and new_manifest.has_key("expected")  # noqa W601
    ):
        changes.modified.append(
            ExpectedModified(subtest, ancestor_manifest, new_manifest)
        )
    elif (
        ancestor_manifest
        and ancestor_manifest.has_key("expected")  # noqa W601
        and not (new_manifest and new_manifest.has_key("expected"))  # noqa W601
    ):
        changes.deleted.append(subtest)
    elif (
        ancestor_manifest
        and ancestor_manifest.has_key("expected")  # noqa W601
        and new_manifest
        and new_manifest.has_key("expected")  # noqa W601
    ):
        old_expected = ancestor_manifest.get("expected")
        new_expected = new_manifest.get("expected")
        if expected_values_changed(old_expected, new_expected):
            changes.modified.append(
                ExpectedModified(subtest, ancestor_manifest, new_manifest)
            )

def expected_values_changed(old_expected, new_expected):
    if len(old_expected) != len(new_expected):
        return True

    old_dict = {}
    new_dict = {}
    for dest, cond_lines in [(old_dict, old_expected), (new_dict, new_expected)]:
        for cond_line in cond_lines:
            if isinstance(cond_line, tuple):
                condition, value = cond_line
            else:
                condition = None
                value = cond_line
            dest[condition] = value

    return new_dict != old_dict

def record_changes(ancestor_manifest, new_manifest):
    changes = Differences()

    for test, test_manifest in new_manifest.child_map.items():
        if test not in ancestor_manifest.child_map:
            changes.added.append((test, test_manifest))
        else:
            ancestor_test_manifest = ancestor_manifest.child_map[test]
            test_differences = compare_test(test, ancestor_test_manifest, test_manifest)
            if test_differences:
                changes.modified.append(
                    TestModified(test, test_manifest, test_differences)
                )

    for test, test_manifest in ancestor_manifest.child_map.items():
        if test not in new_manifest.child_map:
            changes.deleted.append(test)

    return changes

def apply_changes(current_manifest, changes):
    for test, test_manifest in changes.added:
        if test in current_manifest.child_map:
            current_manifest.delete(current_manifest.child_map[test])
        current_manifest.insert(test_manifest)

    for test in changes.deleted:
        if test in current_manifest.child_map:
            current_manifest.delete(current_manifest.child_map[test])

    for item in changes.modified:
        if isinstance(item, TestModified):
            test, new_manifest, test_changes = item
            if test in current_manifest.child_map:
                apply_changes(current_manifest.child_map[test], test_changes)
            else:
                current_manifest.insert(new_manifest)
        else:
            assert isinstance(item, ExpectedModified)
            subtest, ancestor_manifest, new_manifest = item
            if not subtest:
                current_manifest.set_expected(new_manifest)
            elif subtest in current_manifest.child_map:
                current_manifest.child_map[subtest].set_expected(new_manifest)
            else:
                current_manifest.insert(new_manifest)

def get_parser():
    parser = argparse.ArgumentParser()
    parser.add_argument("ancestor")
    parser.add_argument("current")
    parser.add_argument("new")
    parser.add_argument("dest", nargs="?")
    return parser

def get_parser_mergetool():
    parser = argparse.ArgumentParser()
    parser.add_argument("--no-overwrite", dest="overwrite", action="store_false")
    return parser

def make_changes(ancestor_manifest, current_manifest, new_manifest):
    changes = record_changes(ancestor_manifest, new_manifest)
    apply_changes(current_manifest, changes)

    return serialize(current_manifest.node)

def run(ancestor, current, new, dest):
    ancestor_manifest = get_manifest(ancestor)
    current_manifest = get_manifest(current)
    new_manifest = get_manifest(new)

    updated_current_str = make_changes(
        ancestor_manifest, current_manifest, new_manifest
    )

    if dest != "-":
        with open(dest, "wb") as f:
            f.write(updated_current_str.encode("utf8"))
    else:
        print(updated_current_str)


from mollie.api.objects.capture import Capture

from .utils import assert_list_object

PAYMENT_ID = "tr_7UhSN1zuXS"
SETTLEMENT_ID = "stl_jDk30akdN"
CHARGEBACK_ID = "chb_n9z0tp"

def test_get_settlement_captures_by_capture_id(oauth_client, response):
    """Get captures relevant to settlement by settlement id."""
    response.get(f"https://api.mollie.com/v2/settlements/{SETTLEMENT_ID}/captures", "captures_list")

    captures = oauth_client.settlement_captures.with_parent_id(SETTLEMENT_ID).list()
    assert_list_object(captures, Capture)

def test_list_settlement_captures_by_capture_object(oauth_client, response):
    """Get a list of captures relevant to settlement object."""
    response.get(f"https://api.mollie.com/v2/settlements/{SETTLEMENT_ID}/captures", "captures_list")
    response.get(f"https://api.mollie.com/v2/settlements/{SETTLEMENT_ID}", "settlement_single")

    settlement = oauth_client.settlements.get(SETTLEMENT_ID)
    captures = oauth_client.settlement_captures.on(settlement).list()
    assert_list_object(captures, Capture)