# /dust/src/structs/SimpleNeuralNetwork.py

# Built-ins
import os
import pickle
import zipfile

# Package
import __init__

# Additional Packages
import numpy as np
from scipy.special import expit

def sigmoid(s): return expit(s)
def sigmoid_prime(s): return s*(1-s)

class nLayerNeuralNetwork():

    def __init__(self, inputlayers, outputlayers, hiddenlayers=(7,7,7),
                 activation_fn = sigmoid, activationprime_fn = sigmoid_prime,
                 learning_rate = 0.01,
                 trainX = None, trainY = None, epochs=1000):

        self.inputlayers = inputlayers
        self.outputlayers = outputlayers
        self.hiddenlayers = hiddenlayers

        self.activation_fn = activation_fn
        self.activationprime_fn = activationprime_fn
        self.learning_rate = learning_rate

        weight_ih0 = np.random.randn(self.inputlayers, self.hiddenlayers[0])
        hidden_weights = [np.random.randn(self.hiddenlayers[i], self.hiddenlayers[i+1]) \
                               for i in range(len(self.hiddenlayers[0:-1]))]
        weight_hNo = np.random.randn(self.hiddenlayers[-1], self.outputlayers)

        self.weights = [weight_ih0, *hidden_weights, weight_hNo]

        if(trainX is not None and trainY is not None):
            self.train(trainX, trainY, N=epochs)

    def forward_prop(self, X):
        activations = []
        p = X
        for i in range(len(self.weights)):
            p = self.activation_fn(np.dot(p, self.weights[i]))
            activations.append(p)
        self.activations = activations
        return activations[-1]

    def backward_prop(self, X, output, expected_output):

        errors, deltas = [], []

        errors.append(expected_output - output)
        deltas.append(errors[-1] * self.activationprime_fn(output))

        for i in range(len(self.weights)-1, 0, -1):
            e = errors[-1].dot(self.weights[i].T)
            d = e*self.activationprime_fn(self.activations[i-1])

            errors.append(e)
            deltas.append(d)

        errors.reverse()
        deltas.reverse()

        a = [X] + self.activations
        for i in range(len(self.weights)):
            self.weights[i] += self.learning_rate * a[i].T.dot(deltas[i])

        self.errors = errors
        self.deltas = deltas

    def train(self, X, expected_outputs, N=1, prune_rate = 1/10000):
        for _ in range(N):
            try:
                p_error = sum(self.errors[-1])
            except AttributeError: p_error = np.zeros(self.outputlayers)

            output = self.forward_prop(X)
            self.backward_prop(X, output, expected_outputs)

            c_error = sum(self.errors[-1])

            ientropy = sum(abs(p_error - c_error))
            if(ientropy < prune_rate):
                print('Entropy = {}'.format(ientropy))
                print('No new info gained; pruning at {}'.format(_))
                break

def save(model, path):

    os.makedirs(path, exist_ok=True)
    for file in os.listdir(path):
        try:
            os.remove(os.path.join(path, file))
        except Exception as e:
            print(e)
            continue

    def _savenp(arrs, type_):
        for i, arr in enumerate(arrs):
            outfile = os.path.join(path,'{}_{}.pickle'.format(type_,i))
            with open(outfile, 'wb') as f: np.save(f, arr)

    _savenp(model.activations, 'activation')
    _savenp(model.weights,     'weight')
    _savenp(model.deltas,      'delta')
    _savenp(model.errors,      'error')

    main_copy = nLayerNeuralNetwork(model.inputlayers, model.outputlayers,
                                    hiddenlayers = model.hiddenlayers,
                                    activation_fn = model.activation_fn,
                                    activationprime_fn = model.activationprime_fn,
                                    learning_rate = model.learning_rate)
    main_copy.weights = []
    with open(os.path.join(path,'main.pickle'), 'wb') as f:
        pickle.dump(main_copy, f)

    z = zipfile.ZipFile('{}.model'.format(path),mode='w')
    for file in os.listdir(path):
        z.write(os.path.join(path,file), arcname=os.path.basename(file))
    z.close()

    for file in os.listdir(path):
        try:
            os.remove(os.path.join(path, file))
        except Exception as e: print(e)
    os.rmdir(path)

def load(path):

    with zipfile.ZipFile(path) as z:
        path = path.replace('.model','')
        z.extractall(path)

    files = tuple(map(lambda x: os.path.join(path, x), os.listdir(path)))

    def _loadnp(files, list_):
        files = sorted(files, key=lambda x: \
                       int(os.path.basename(x).split('_')[1].replace('.pickle','')))
        for file in files:
            with open(file, 'rb') as f:
                list_.append(np.load(f))

    with open(os.path.join(path,'main.pickle'), 'rb') as f:
        main = pickle.load(f)

    main.activations = []
    main.weights = []
    main.deltas = []
    main.errors = []

    _loadnp(filter(lambda x: 'activation_' in x, files), main.activations)
    _loadnp(filter(lambda x: 'weight_'     in x, files), main.weights)
    _loadnp(filter(lambda x: 'deltas_'     in x, files), main.deltas)
    _loadnp(filter(lambda x: 'errors_'     in x, files), main.errors)

    for file in os.listdir(path):
        try:
            os.remove(os.path.join(path, file))
        except Exception as e: print(e)
    try: os.rmdir(path)
    except Exception as e: print(e)

    return main


# %% Importing
#########################################################
#########################################################

import os
import sys
import matplotlib.pyplot as plt
from mpl_toolkits import mplot3d
# get path of this script
path_this_script = os.getcwd()
path_this_script = os.path.realpath(__file__)

# add the ./src/ path to the search path
path_this_script_splitted = os.path.split(path_this_script)
this_script_filename = path_this_script_splitted[1]
path_this_script_splitted = os.path.split(path_this_script_splitted[0])
path_to_src = os.path.join(path_this_script_splitted[0], 'src')
sys.path.append(path_to_src)  # I could have used sys.path.append('../src/'), but it didn't work with the debugger
path_to_cache = os.path.join(path_this_script_splitted[0], 'cache')
from SphTools import *

# %%
A=pd.read_csv('/home/melzouka/elzouka_codes_library/Python/SphEM/results/database/scratch/simulation.csv')
B=A['simulation_result_filepath']

# %%
root_path='/home/melzouka/elzouka_codes_library/Python/SphEM/'
csv_path='/home/melzouka/elzouka_codes_library/Python/SphEM/results/database/scratch/'
processed_data_dir='/home/zhw/SphEM-1/Processed_data'
path_to_cache = os.path.join(root_path, B[8])
get_data_from_SCUFF_TMATRIX_file(path_to_cache)

# %%

def pd_resize(pd_array,id_key,search_id='id'):
    pd_properties=pd_array.columns
   # print(pd_properties)
    arv_array=[]
    mis_array=[]
    for i in range(0,len(id_key)):
        check_id=pd_array[search_id]==id_key[i]
        s=pd_array[check_id]
        if s.empty:
            mis_array.append(id_key[i])

        arv_array.append(s)
       # print(s)

    arv_array=pd.concat(arv_array)

    return arv_array ,mis_array
    #print('--')
            #print(pd_array[pd_properties[i]])

def prop_finder(simu_type,prop,file_path):
    try:
        new_prop_array=[]
        if simu_type=='neq':
            NEQ_df=get_data_from_SCUFF_NEQ_file(file_path)

            try:
                prop_neq=NEQ_df.columns
                NEQ_df_prop=[]
                for j in range(0,len(prop)):
                    check_in=prop[j] in prop_neq
                    #print(check_in)
                    if check_in:
                        NEQ_df_prop.append(NEQ_df[prop[j]])
                    else:
                        print('s')
                return NEQ_df_prop
            except:
                pass

        elif simu_type=='scatter':
            Scatter_df=get_data_from_SCUFF_SCATTER_file(file_path)
            prop_scat=Scatter_df.columns
            try:
                Scat_df_prop=[]

                for j in range(0,len(prop)):
                    check_in=prop[j] in prop_scat
                   # print(len(prop))
                   # print(check_in)
                    if check_in:
                        Scat_df_prop=Scatter_df[prop]

                    else:
                        print('pp')
                return Scat_df_prop
            except:
                pass

        elif simu_type=='tmatrix':
            T_matrix_df=get_data_from_SCUFF_TMATRIX_file(file_path)

            prop_tmat=T_matrix_df.columns
            try:
                tmat_df_prop=[]
                for j in range(0,len(prop)):
                    check_in=prop[j] in prop_tmat
                    #print(check_in)
                    if check_in:
                        tmat_df_prop=T_matrix_df[prop]

                    else:
                        pass
                return tmat_df_prop
            except:
                pass

    except:
        raise TypeError

def path_gen(root_path,data_pd):
    file_path_ralative=data_pd['simulation_result_filepath'].to_numpy()
    sim_id=data_pd['simulation_id'].to_numpy()
    mesh_id=data_pd['mesh_id'].to_numpy()
    real_path_array=[]
    simu_id_array=[]
    mesh_id_array=[]
    for i in range(0,len(file_path_ralative)):
        #print(file_path_ralative[i])
        current_path=os.path.join(root_path, file_path_ralative[i])
        real_path_array.append(current_path)

    return real_path_array,sim_id,mesh_id

def search_id(mesh,Simulation,
                geom_id,mesh_id,material_id,prop_id):
    geo_id_array_at_mesh_file=pd_resize(pd_array=mesh,id_key=geom_id,search_id='geometry_id')[0]
    # check mesh id
    mesh_id_array_at_mesh_file=pd_resize(pd_array=geo_id_array_at_mesh_file,id_key=mesh_id,search_id='mesh_id')
    # check simulation id
    vali_mesh_id=mesh_id_array_at_mesh_file[0]['mesh_id'].to_numpy()

    simu_id_array_at_simu_file=pd_resize(pd_array=Simulation,id_key=vali_mesh_id,search_id='mesh_id')

    vali_simu_id=np.unique(simu_id_array_at_simu_file[0]['material_id'].to_numpy())
    material_id_array_at_simu_file=pd_resize(pd_array=simu_id_array_at_simu_file[0],id_key=vali_simu_id,search_id='material_id')
    print(material_id_array_at_simu_file)

    #print(vali_mesh_id)
    #print(vali_simu_id)
    simu_array_path=material_id_array_at_simu_file[0]['simulation_result_filepath']
    neq_type_simu=material_id_array_at_simu_file[0][material_id_array_at_simu_file[0]['simulation_type']=='neq']
    scatter_type_simu=material_id_array_at_simu_file[0][material_id_array_at_simu_file[0]['simulation_type']=='scatter']
    tmatrix_type_simu=material_id_array_at_simu_file[0][material_id_array_at_simu_file[0]['simulation_type']=='tmatrix']
    neq_p=path_gen(root_path,neq_type_simu)
    scatter_p=path_gen(root_path,scatter_type_simu)
    tmatrix_p=path_gen(root_path,tmatrix_type_simu)
    target_path_neq=neq_p[0]
    target_path_scatter=scatter_p[0]
    target_path_tmatrix=tmatrix_p[0]

    scatter_mesh_id=scatter_p[2]
    scatter_simu_id=scatter_p[1]

    #print(len(target_path))
    #print(neq_type_simu['simulation_result_filepath'])
    #print(neq_type_simu)
    neq_prop=[]
    for n in range(0,len(target_path_neq)):
        file_path=target_path_neq[n]
        K=prop_finder('neq',prop_id,file_path)
        neq_prop.append(K)

    scatter_prop=[]
    for n in range(0,len(target_path_scatter)):
        file_path=target_path_scatter[n]

        K=prop_finder('scatter',prop_id,file_path)

        scatter_prop.append(K)
    #scatter_prop=pd.concat(scatter_prop,axis=1)
    #id_ver_scatter_prop=pd.concat([scatter_prop,scatter_mesh_id,scatter_simu_id],axis=1)

    tmatrix_prop=[]
    for n in range(0,len(target_path_tmatrix)):
        file_path=target_path_tmatrix[n]

        K=prop_finder('tmatrix',prop_id,file_path)
        tmatrix_prop.append(K)
    print(scatter_prop)

    #simu_type=simu_type
    #for i in range(0,len(vali_mesh_id)):
    #    prop_finder(simu_type,prop_name=0,file_path)

    #except:

    return mesh
#%%
geom_id=['wireSphEnds/Lx_0.3/D_0.04/','wire/Lx_0.4/D_0.08/','spheroid/rx_0.04/rz_0.08/']
mesh_id=['wireSphEnds/Lx_0.3/D_0.04//clscale_1_clmax_1','wireSphEnds/Lx_0.3/D_0.04//clscale_0.95_clmax_1.2','spheroid2/rx_0.04/rz_0.08//clscale_0.95_clmax_1','adhihadih']
simu_id=[2,3,4,5,34,1,2,4,5]
material_id=['CONST_EPS_1.8//Au']
root_path='/home/melzouka/elzouka_codes_library/Python/SphEM/'
csv_path='/home/melzouka/elzouka_codes_library/Python/SphEM/results/database/scratch/'
processed_data_dir='/home/zhw/SphEM-1/Processed_data'
prop_id=['scatter_CrossSection_m2','freq_radpers']
mesh=F[0]
Simulation=F[1]
AA=search_id(mesh,Simulation,geom_id,mesh_id,material_id,prop_id)
file_look(mesh,Simulation,geom_id)
#%%

class pointer():
    def __init__(self,root_path,csv_path,
                    processed_data_dir):
        self.root_path=root_path
        self.csv_path=csv_path
        self.processed_data_dir=processed_data_dir
        sim_csv_path=os.path.join(csv_path,  'simulation.csv')
        geo_csv_path=os.path.join(csv_path, 'geometry.csv')
        mater_csv_path=os.path.join(csv_path, 'material.csv')
        msh_csv_path=os.path.join(csv_path, 'mesh.csv')

        Simulation=pd.read_csv(sim_csv_path)
        self.Geometry=pd.read_csv(geo_csv_path)
        self.materials=pd.read_csv(mater_csv_path)
        self.mesh=pd.read_csv(msh_csv_path)
        Simulation["material_id"] = Simulation["material_surr"] +'//'+ Simulation["material_particle"]
        self.Simulation=Simulation

        self.geo_id_list=np.array(self.mesh['geometry_id'])
        self.mesh_id_list=np.array(self.mesh['mesh_id'])
        self.simu_id_list=np.array(self.Simulation['simulation_id'])
        self.material_id_list=np.array(self.Simulation['material_id'])
        print(self.Simulation.columns)

    def meshsimu(self):
        return self.mesh, self.Simulation

    def get_id(self):
        id_list={'geo_id_list':self.geo_id_list,
        'mesh_id_list':self.mesh_id_list,
        'simu_id_list':self.simu_id_list,'material_id_list':self.material_id_list}
        import csv
        with open(os.path.join(self.processed_data_dir,  'id_list.csv'), 'w') as f:  # Just use 'w' mode in 3.x
            w = csv.DictWriter(f, id_list.keys())
            w.writeheader()
            w.writerow(id_list)
        return id_list

        #print(geo_id_list)
    def data_init(self, geom_id,mesh_id,simu_type,prop_id):
        # getting all files
        root_path=self.root_path
        csv_path=self.csv_path
        processed_data_dir=self.processed_data_dir

        Simulation=self.Simulation
        Geometry=self.Geometry
        materials=self.materials
        mesh=self.mesh

        geo_check=len(geom_id)
        mesh_check=len(mesh_id)
        simu_check=len(simu_id)
        if  (geo_check<=mesh_check):
            # check geo id
            #try:
            geo_id_array_at_mesh_file=pd_resize(pd_array=mesh,id_key=geom_id,search_id='geometry_id')[0]
            # check mesh id
            mesh_id_array_at_mesh_file=pd_resize(pd_array=geo_id_array_at_mesh_file,id_key=mesh_id,search_id='mesh_id')
            # check simulation id
            vali_mesh_id=mesh_id_array_at_mesh_file[0]['mesh_id'].to_numpy()

            simu_id_array_at_simu_file=pd_resize(pd_array=Simulation,id_key=vali_mesh_id,search_id='mesh_id')

            #print(simu_id_array_at_simu_file)
            simu_array_path=simu_id_array_at_simu_file[0]['simulation_result_filepath']
            neq_type_simu=simu_id_array_at_simu_file[0][simu_id_array_at_simu_file[0]['simulation_type']=='neq']
            scatter_type_simu=simu_id_array_at_simu_file[0][simu_id_array_at_simu_file[0]['simulation_type']=='scatter']
            tmatrix_type_simu=simu_id_array_at_simu_file[0][simu_id_array_at_simu_file[0]['simulation_type']=='tmatrix']
            neq_p=path_gen(self.root_path,neq_type_simu)
            scatter_p=path_gen(self.root_path,scatter_type_simu)
            tmatrix_p=path_gen(self.root_path,tmatrix_type_simu)
            target_path_neq=neq_p[0]
            target_path_scatter=scatter_p[0]
            target_path_tmatrix=tmatrix_p[0]
            print(target_path_tmatrix)

            scatter_mesh_id=scatter_p[2]
            scatter_simu_id=scatter_p[1]

            #print(len(target_path))
            #print(neq_type_simu['simulation_result_filepath'])
            #print(neq_type_simu)
            neq_prop=[]
            for n in range(0,len(target_path_neq)):
                file_path=target_path_neq[n]
                K=prop_finder('neq',prop_id,file_path)
                neq_prop.append(K)

            scatter_prop=[]
            for n in range(0,len(target_path_scatter)):
                file_path=target_path_scatter[n]

                K=prop_finder('scatter',prop_id,file_path)

                scatter_prop.append(K)
            #scatter_prop=pd.concat(scatter_prop,axis=1)
            #id_ver_scatter_prop=pd.concat([scatter_prop,scatter_mesh_id,scatter_simu_id],axis=1)

            tmatrix_prop=[]
            for n in range(0,len(target_path_tmatrix)):
                file_path=target_path_tmatrix[n]

                K=prop_finder('tmatrix',prop_id,file_path)
                tmatrix_prop.append(K)
            #print(scatter_prop)

            #simu_type=simu_type
            #for i in range(0,len(vali_mesh_id)):
            #    prop_finder(simu_type,prop_name=0,file_path)

            #except:

            return mesh

        elif geo_check==1 and (not mesh_check) and ( not simu_check):
            pass
        elif geo_check==1 and (geo_check<=mesh_check) and ( not simu_check):
            pass
        elif (not geo_check) and (not mesh_check) and ( simu_check):
            pass
        elif (not geo_check) and (mesh_check) and ( not simu_check):
            pass

        #path_to_cache = os.path.join(root_path, B[8])

    #def add_branch(self,)

# %%
R=pointer(root_path,csv_path,processed_data_dir)
geom_id=['wireSphEnds/Lx_0.3/D_0.04/','wire/Lx_0.4/D_0.08/','spheroid/rx_0.04/rz_0.08/']
mesh_id=['wireSphEnds/Lx_0.3/D_0.04//clscale_1_clmax_1','wireSphEnds/Lx_0.3/D_0.04//clscale_0.95_clmax_1.2','spheroid2/rx_0.04/rz_0.08//clscale_0.95_clmax_1','adhihadih']
simu_id=[2,3,4,5,34,1,2,4,5]
material_id=['CONST_EPS_1.8//Au']
root_path='/home/melzouka/elzouka_codes_library/Python/SphEM/'
csv_path='/home/melzouka/elzouka_codes_library/Python/SphEM/results/database/scratch/'
processed_data_dir='/home/zhw/SphEM-1/Processed_data'
prop_id=['scatter_CrossSection_m2','freq_radpers']
K=R.data_init(geom_id,mesh_id,simu_id,prop_id)
F=R.meshsimu()

# %%
file_csv_path='/home/melzouka/elzouka_codes_library/Python/SphEM/results/database/scratch/geometry.csv'
master_name='geometry'

# %%
R.get_id()

# %%

# %%
# a function identify files
def pd_resize_n(pd_array,id_key,search_id='id'):
    pd_properties=pd_array.columns
   # print(pd_properties)
    arv_array=[]
    mis_array=[]
    for i in range(0,len(id_key)):
        check_id=pd_array[search_id]==id_key[i]
        s=pd_array[check_id]
        if s.empty:
            mis_array.append(id_key[i])

        arv_array.append(s)
       # print(s)

    arv_array=pd.concat(arv_array)
    '''
    neq_type_simu=arv_array[arv_array['simulation_type']=='neq']
    scatter_type_simu=arv_array[arv_array['simulation_type']=='scatter']
    tmatrix_type_simu=arv_array[arv_array['simulation_type']=='tmatrix']

    neq_full_path=path_gen(root_path,neq_type_simu)
    scatter_full_path=path_gen(root_path,scatter_type_simu)
    tmatrix_full_path=path_gen(root_path,tmatrix_type_simu)
    print(reduced_files)
    '''
    return arv_array ,mis_array
def full_path(root_path,relative_path_contained_data_array):
    check_len=len(relative_path_contained_data_array)

    relative_path_contained_data_array['full_path']=root_path+relative_path_contained_data_array['simulation_result_filepath']
    return relative_path_contained_data_array

def full_path_concat(full_path_data_array):
    # find path
    neq_type_simu=full_path_data_array[full_path_data_array['simulation_type']=='neq']
    scatter_type_simu=full_path_data_array[full_path_data_array['simulation_type']=='scatter']
    tmatrix_type_simu=full_path_data_array[full_path_data_array['simulation_type']=='tmatrix']

    return full_path_data_array,neq_type_simu, scatter_type_simu,tmatrix_type_simu

    #except:
      #  print('s')
def file_look(root_path,mesh,Simulation,geom_id=[],mesh_id=[],material_id=[],simulation_id=[]):
    # getting the size of the input
    geo_size=len( geom_id)
    mesh_szie=len(mesh_id)
    material_size=len(material_id)
    simulation_size=len(simulation_id)

    # check from back to start

    if simulation_size==0:
        No_found_geo_id=[]
        No_found_simu_id=[]
        No_found_material_id=[]
        No_found_mesh_id=[]
        # check material and mesh size
        if  mesh_szie!=0:
            # look mesh id
            Full_files_log=pd_resize_n(Simulation,mesh_id,search_id='mesh_id')

            reduced_files=Full_files_log[0]
            No_found_mesh_id=Full_files_log[1]
            No_found_geo_id=[]
            No_found_simu_id=[]
            No_found_material_id=[]
            # path gene
            full_path_array=full_path(root_path,reduced_files)
            # file display

            if material_size!=0:
                # checking material from the reduced array
                Full_files_log=pd_resize_n(full_path_array,material_id,search_id='material_id')
                #print(Full_files_log)
                material_reduced_files=Full_files_log[0]
                No_found_material_id=Full_files_log[1]

                Selected_property_files=full_path_concat(material_reduced_files)# (NEQ, Scatter, Tmatrix)

            else:
                Selected_property_files=full_path_concat(full_path_array)
                No_found_material_id=[]

            # print out the log file
            full_path_data_array=Selected_property_files[0]
            return_file_log=pd.DataFrame([
                full_path_data_array['simulation_id'],
                full_path_data_array['material_id'],
                full_path_data_array['simulation_type'],
                full_path_data_array['full_path']
            ]).T

        elif mesh_szie==0 and material_size!=0:
            Full_files_log=pd_resize_n(Simulation,material_id,search_id='material_id')
            reduced_files=Full_files_log[0]
            No_found_material_id=Full_files_log[1]
            No_found_mesh_id=[]

            # path gene
            full_path_array=full_path(root_path,reduced_files)
            Selected_property_files=full_path_concat(full_path_array)

            # print out the log file
            full_path_data_array=Selected_property_files[0]
            return_file_log=pd.DataFrame([
                full_path_data_array['simulation_id'],
                full_path_data_array['material_id'],
                full_path_data_array['simulation_type'],
                full_path_data_array['full_path']
            ]).T

        elif mesh_szie==0 and material_size==0 and geo_size!=0:
            # checking file
            Full_files_log_mesh=pd_resize_n(mesh,geom_id,search_id='geometry_id')
            reduced_files_mesh_from_geo=Full_files_log_mesh[0]
            No_found_mesh_id=[]
            No_found_material_id=[]
            No_found_geo_id=Full_files_log_mesh[1]

            # grab mesh id from geo
            mesh_id_from_geo=reduced_files_mesh_from_geo['mesh_id'].to_numpy()

            # Checking new mesh file. Same process from above
            Full_files_log=pd_resize_n(Simulation,mesh_id_from_geo,search_id='mesh_id')
            reduced_files=Full_files_log[0]
            No_found_mesh_id=Full_files_log[1]
            # path gene
            full_path_array=full_path(root_path,reduced_files)
            Selected_property_files=full_path_concat(full_path_array)

            # print out the log file
            full_path_data_array=Selected_property_files[0]
            return_file_log=pd.DataFrame([
                full_path_data_array['simulation_id'],
                full_path_data_array['material_id'],
                full_path_data_array['simulation_type'],
                full_path_data_array['full_path']
            ]).T

        else:
            Selected_property_files=([],[],[],[])
            No_found_mesh_id=[]
            No_found_material_id=[]
            No_found_geo_id=[]

    else:
            Full_files_log=pd_resize_n(Simulation,simulation_id,search_id='simulation_id')
            reduced_files=Full_files_log[0]
            No_found_simu_id=Full_files_log[1]
            # path gene
            full_path_array=full_path(root_path,reduced_files)
            Selected_property_files=full_path_concat(full_path_array)

            # print out the log file
            full_path_data_array=Selected_property_files[0]
            return_file_log=pd.DataFrame([
                full_path_data_array['simulation_id'],
                full_path_data_array['material_id'],
                full_path_data_array['simulation_type'],
                full_path_data_array['full_path']
            ]).T
            No_found_mesh_id=[]
            No_found_geo_id=[]
            No_found_material_id=[]

            #print(Selected_property_files)

    return Selected_property_files,return_file_log

# Checking the properties of three files
def full_properties_look(neq_pandas,
                    scatter_pandas,
                    tmatrix_pandas):
    # get the file path

    neq_file_path=neq_pandas['full_path'].reset_index(drop=True)
    neq_len=len(neq_file_path)
    #neq_prop_len=len(neq_properties)
    neq_simu_id=neq_pandas['simulation_id'].reset_index(drop=True)

    scatter_file_path=scatter_pandas['full_path'].reset_index(drop=True)
    scatter_len=len(scatter_file_path)
    #scatter_prop_len=len(scatter_properties)
    scatter_simu_id=scatter_pandas['simulation_id'].reset_index(drop=True)

    tmatrix_file_path=tmatrix_pandas['full_path'].reset_index(drop=True)
    tmatrix_len=len(tmatrix_file_path)
    #tmatrix_prop_len=len(tmatrix_properties)
    tmatrix_simu_id=tmatrix_pandas['simulation_id'].reset_index(drop=True)

    # EXPAND the file
    if neq_len!=0:
        NEQ_properties=[]
        for i in range(0,neq_len):
            # define the current simu_id
            current_simu_path=neq_file_path[i]
            current_simu_id=neq_simu_id[i]
            Current_NEQ_file=get_data_from_SCUFF_NEQ_file(current_simu_path)
            Current_NEQ_file['simulation_id']=current_simu_id
            NEQ_properties.append(Current_NEQ_file)
            # combine into one pandas frame
        NEQ_properties=pd.concat(NEQ_properties)
    else:
        NEQ_properties=pd.DataFrame()

    if scatter_len!=0:
        Scatter_properties=[]
        for i in range(0,scatter_len):

            # define the current simu_id
            current_simu_path=scatter_file_path[i]
            current_simu_id=scatter_simu_id[i]
            Current_SCATTER_file=get_data_from_SCUFF_SCATTER_file(current_simu_path)
            Current_SCATTER_file['simulation_id']=current_simu_id
            Scatter_properties.append(Current_SCATTER_file)
            # combine into one pandas frame
        Scatter_properties=pd.concat(Scatter_properties)

    else:
        Scatter_properties=pd.DataFrame()

    if tmatrix_len!=0:
        Tmatrix_properties=[]
        for i in range(0,scatter_len):

            # define the current simu_id
            current_simu_path=tmatrix_file_path[i]
            current_simu_id=tmatrix_simu_id[i]
            Current_TMATRIX_file=get_data_from_SCUFF_TMATRIX_file(current_simu_path)
            Current_TMATRIX_file['simulation_id']=current_simu_id
            Tmatrix_properties.append(Current_TMATRIX_file)
            # combine into one pandas frame
        Tmatrix_properties=pd.concat(Tmatrix_properties)

        #print(Tmatrix_properties)
    else:
        Tmatrix_properties=pd.DataFrame()

    return NEQ_properties, Scatter_properties, Tmatrix_properties

def properties_look(NEQ_properties, neq_properties,
                    Scatter_properties,scatter_properties,
                    Tmatrix_properties,tmatrix_properties):
    # check is empty
    neq_len=len(NEQ_properties)
    scatter_len=len(Scatter_properties)
    tmatrix_len=len(Tmatrix_properties)

    try:
        if neq_len!=0:
            # append simu id
            neq_properties+=['simulation_id']
            Target_NEQ=NEQ_properties[neq_properties]
            Target_NEQ.set_index(['simulation_id'],inplace=True)
        else:
            Target_NEQ=pd.DataFrame()
    except:
        Target_NEQ=['wrong properties name']

    try:
        if scatter_len!=0:
            # append simu id

            scatter_properties+=['simulation_id']

            Target_Scatter=Scatter_properties[scatter_properties]
            Target_Scatter.set_index(['simulation_id'],inplace=True)
        else:
            Target_Scatter=pd.DataFrame()
    except:
        Target_Scatter=['wrong properties name']

    try:
        if tmatrix_len!=0:
            # append simu id
            tmatrix_properties+=['simulation_id']
            Target_Tmatrix=Tmatrix_properties[tmatrix_properties]
            Target_Tmatrix.set_index(['simulation_id'],inplace=True)
        else:
            Target_Tmatrix=pd.DataFrame()
    except:
        Target_Tmatrix=['wrong properties name']

    return Target_NEQ, Target_Scatter, Target_Tmatrix

geom_id=[]#['wireSphEnds/Lx_0.3/D_0.04/','wire/Lx_0.4/D_0.08/','spheroid/rx_0.04/rz_0.08/']
mesh_id=['wireSphEnds/Lx_0.3/D_0.04//clscale_1_clmax_1','wireSphEnds/Lx_0.3/D_0.04//clscale_0.95_clmax_1.2','spheroid/rx_0.04/rz_0.08//clscale_0.95_clmax_1','adhihadih']
simulation_id=[]#['wireSphEnds/Lx_0.3/D_0.04//clscale_0.9_clmax_1.2/CONST_EPS_1.8/Au/scatter/tol_1e-05']#[2,3,4,5,34,1,2,4,5]
material_id=['CONST_EPS_1.8//Au']
mesh=F[0]
Simulation=F[1]

S=file_look(root_path,mesh,Simulation,geom_id,mesh_id,material_id,simulation_id)
Prop=S[0]
neq_pandas=Prop[1]
scatter_pandas=Prop[2]
tmatrix_pandas=Prop[3]
neq_prop=[]
scatter_prop=['scatter_CrossSection_m2','freq_radpers']
tmatrix_prop=[]
P=full_properties_look(neq_pandas,
                    scatter_pandas,
                    tmatrix_pandas)
NEQ_properties=P[0]
Scatter_properties=P[1]
Tmatrix_properties=P[2]

SS=properties_look(NEQ_properties, neq_prop,
                    Scatter_properties,scatter_prop,
                    Tmatrix_properties,tmatrix_prop)
# %%

# %%


from django.core.exceptions import ValidationError
from django.db import models

# Create your models here.
from cities.models import City

class Train(models.Model):
    name = models.CharField(max_length=25, unique=True, verbose_name='Номер поезда')
    travel_time = models.PositiveSmallIntegerField(verbose_name='Время в пути')
    from_city = models.ForeignKey(City, on_delete=models.CASCADE,
                                  related_name='from_city_set',
                                  verbose_name='Откуда'
                                  )
    to_city = models.ForeignKey('cities.City', on_delete=models.CASCADE,
                                  related_name='to_city_set',
                                  verbose_name='Куда'
                                  )

    def __str__(self):
        return f'Поезд № {self.name} из г. {self.from_city} в г. {self.to_city}'

    class Meta:
        verbose_name='Поезд'
        verbose_name_plural = 'Поезда'
        ordering = ['name', 'travel_time']

    def clean(self):
        if self.from_city == self.to_city:
            raise ValidationError('Измените город прибытия !')
        qs = Train.objects.filter(
            from_city = self.from_city,
            to_city = self.to_city,
            travel_time = self.travel_time
        ).exclude(pk=self.pk)
        # Train == self.__class__
        if qs.exists():
            raise ValidationError('Измените время в пути')

    def save(self, *args, **kwargs):
        self.clean()
        super(Train, self).save(*args, **kwargs)


import re
from collections import namedtuple

ASCII_BYTE = " !\"#\$%&\'\(\)\*\+,-\./0123456789:;<=>\?@ABCDEFGHIJKLMNOPQRSTUVWXYZ\[\]\^_`abcdefghijklmnopqrstuvwxyz\{\|\}\\\~\t"

String = namedtuple("String", ["s", "offset"])

def ascii_strings(buf, n=4):
    reg = "([%s]{%d,})" % (ASCII_BYTE, n)
    ascii_re = re.compile(reg)
    for match in ascii_re.finditer(buf):
        yield String(match.group().decode("ascii"), match.start())

def unicode_strings(buf, n=4):
    reg = b"((?:[%s]\x00){%d,})" % (ASCII_BYTE, n)
    uni_re = re.compile(reg)
    for match in uni_re.finditer(buf):
        try:
            yield String(match.group().decode("utf-16"), match.start())
        except UnicodeDecodeError:
            pass

def main():
    import sys

    with open(sys.argv[1], 'rb') as f:
        b = f.read()

    for s in ascii_strings(b, n=4):
        print('0x{:x}: {:s}'.format(s.offset, s.s))

    for s in unicode_strings(b):
        print('0x{:x}: {:s}'.format(s.offset, s.s))

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
"""
Created on Tue Oct 13 04:37:31 2020
Security Analytics - task1 - Isolation Tree
@author: b0350
"""

import pandas as pd
import numpy as np
from numpy import quantile, where, random
import matplotlib.pyplot as plt
import time
from tqdm import tqdm
#%% read the encoded data from csv
stime = time.time()

# Feature 1
'''
test_data = pd.read_csv('CSV_T1/test_data_encoded_new.csv')
training_data = pd.read_csv('CSV_T1/training_data_encoded_new.csv')
valid_data_with_labels = pd.read_csv('CSV_T1/valid_data_with_labels_encoded_new.csv')
'''
# Feature 3
'''
test_data = pd.read_csv('CSV_T1/test_data_pca_IPcount_port.csv')
training_data = pd.read_csv('CSV_T1/training_data_pca_IPcount_port.csv')
valid_data_with_labels = pd.read_csv('CSV_T1/valid_data_with_labels_pca_IPcount_port.csv')
'''
# Feature 2
test_data = pd.read_csv('CSV_T1/test_data_encoded_new_addFeature.csv')
training_data = pd.read_csv('CSV_T1/training_data_encoded_new_addFeature.csv')
valid_data_with_labels = pd.read_csv('CSV_T1/valid_data_with_labels_encoded_new_addFeature.csv')

print("Time spent on loading data is: " + str(time.time()-stime))

#%% drop state
'''
dropList=['srcPort','dstPort']
training_data = training_data.drop(dropList, axis=1)
test_data = test_data.drop(dropList, axis=1)
valid_data_with_labels = valid_data_with_labels.drop(dropList, axis=1)
'''
valid_data = valid_data_with_labels.drop('label', axis=1)

# sample the training_data
##training_data = training_data.sample(n=100000, axis=0, replace=False, random_state = 1)

# %%Normalize
from sklearn.preprocessing import normalize

training_data = normalize(training_data)
test_data = normalize(test_data)
valid_data = normalize(valid_data)

# %% Building model
###from sklearn.cluster import DBSCAN
from sklearn.ensemble import IsolationForest

from sklearn import metrics
from sklearn.datasets import make_blobs
from sklearn.preprocessing import StandardScaler
import pickle

stime = time.time()

# Setting the OCSVM parameters
clf = IsolationForest(random_state=0, n_jobs = -1)
# Fit the model
clf.fit(training_data)
print("Time spent on fit ocsvm is: "+str(time.time()-stime))
#%% save the model to disk
filename = 'iTree_1013.sav'
pickle.dump(clf, open(filename, 'wb'))

# %% predit on valid_set
#training_pred = clf.predict(training_data)

stime = time.time()
valid_score = clf.score_samples(valid_data)
valid_pred = clf.predict(valid_data)
print("Time spent on predit valid_data is: "+str(time.time()-stime))

# %% Save the predicted valid result
df_valid_pred = pd.DataFrame(columns=['valid_pred', 'valid_score'])
df_valid_pred['valid_pred'] = valid_pred
df_valid_pred['valid_score'] = valid_score
#df_valid_pred.to_csv('CSV_T1/valid_pred_iTree.csv',header=True,index=False)
#df_valid_pred.to_csv('Useful_Model/Feature1-encoded/iTree/valid_pred_score_iTree.csv',header=True,index=False)
#df_valid_pred.to_csv('Useful_Model/Feature3-PCA/iTree/valid_predicted_Score.csv',header=True,index=False)
df_valid_pred.to_csv('Useful_Model/Featrue2-normalize/iTree/valid_predicted_Score.csv',header=True,index=False)

# %% Encode the label of valid set
valid_label = valid_data_with_labels['label']
valid_label=valid_label.str.contains("Botnet") #True means anomaly
valid_label = valid_label.apply(lambda x: -1 if x==True else 1) #-1 means anomaly

#%% Next, we'll obtain the threshold value from the scores by using the quantile function. Here, we'll get the lowest 3 percent of score values as the anomalies.
'''
Reference:
    https://www.datatechnotes.com/2020/04/anomaly-detection-with-one-class-svm.html

'''
'''
thresh = quantile(scores, 0.03)
print(thresh)

#Next, we'll extract the anomalies by comparing the threshold value and identify the values of elements.

index = where(scores<=thresh)
values = x[index]
'''

#%% Evaluate the model by confusion matrix
from sklearn import metrics
print("Accuracy:", metrics.accuracy_score(valid_pred,valid_label))

#Confusion Matrix
import seaborn as sns
cm = metrics.confusion_matrix(valid_pred,valid_label,labels=[1,-1])
sns.heatmap(cm, annot = True, fmt='g')
tn,fp,fn,tp = cm.ravel()
#cm.set(xlabel="predicted label",ylabel="true label")

precision = tp/(tp+fp) #predicted postive中有多少是真的positive
recall = tp/(tp+fn) #true positive rate (true positive有多少被正確找出來)
fp_rate = fp/(fp+tn)
print("Precision = "+str(precision))
print("Recall = "+str(recall))
print("False Positive rate = "+str(fp_rate))

#%% Classification Report
from sklearn.metrics import classification_report

#target_names = ["-1","1"]
target_names = ['anomaly','normal']
print(classification_report(valid_label, valid_pred, target_names=target_names))

# %% ROC Curve & AUC
fpr,tpr, thresholds = metrics.roc_curve(valid_pred, valid_label, pos_label = -1)
auc_sc = metrics.auc(fpr, tpr)
#print("True positive rate is: "+str(tpr))
#print("Flase positive rate is "+str(fpr))

plt.plot(fpr, tpr, color='navy',label='ROC curve (area = %0.2f)' % auc_sc)
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic with test data')
plt.legend()
plt.show()

#%%
'''
#%% load the model from disk
loaded_model = pickle.load(open(filename, 'rb'))
result = loaded_model.score(X_test, Y_test)
print(result)
'''

'''
#%% Read the result
predicted_valid = pd.read_csv('CSV_T1/predicted_valid.csv')
predicted_label = predicted_valid['predicted_label']
valid_pred = predicted_label
'''

'''
recall
Out[2]: 0.10590053585772825

precision
Out[3]: 0.35347021890378416

fp_rate
Out[4]: 0.05125321509669504

Acc = 77.24%
AUC = 0.53
'''
'''
# export the final attack report
result = pd.read_csv('CSV_T1/valid_data_with_labels.csv')
# get the index of botnet
anomaly_index_list = df_valid_pred.index[df_valid_pred['valid_pred']==-1].tolist()
#result = pd.DataFrame()

result = result.loc[anomaly_index_list]

result.to_csv('CSV_T1/attack_log_iTree_Port_no-normalize.csv')
'''

# %% predict on test_set
stime = time.time()
test_score = clf.score_samples(test_data)
test_pred = clf.predict(test_data)
print("Time spent on predict test_data is: "+str(time.time()-stime))
df_test_pred = pd.DataFrame(columns=['test_pred', 'test_score'])
df_test_pred['test_pred'] = test_pred
df_test_pred['test_score'] = test_score
#df_test_pred.to_csv('CSV_T1/test_pred_iTree_addFeature_normalize.csv',header=True,index=False)
#df_test_pred.to_csv('Useful_Model/Feature1-encoded/iTree/test_pred_score.csv',header=True,index=False)
#df_test_pred.to_csv('Useful_Model/Feature3-PCA/iTree/test_pred_score.csv',header=True,index=False)
df_test_pred.to_csv('Useful_Model/Featrue2-normalize/iTree/test_pred_score.csv',header=True,index=False)

# export the final attack report (for the test_data)
result = pd.read_csv('CSV_T1/test_data.csv')
# get the index of botnet
anomaly_index_list = df_test_pred.index[df_test_pred['test_pred']==-1].tolist()
#result = pd.DataFrame()

result = result.loc[anomaly_index_list]

#result.to_csv('CSV_T1/attack_log_iTree_Port_no-normalize.csv',header=True,index=False)
#result.to_csv('Useful_Model/Feature1-encoded/iTree/test_attack_log.csv',header=True,index=False)
#result.to_csv('Useful_Model/Feature3-PCA/iTree/test_attack_log.csv',header=True,index=False)
result.to_csv('Useful_Model/Featrue2-normalize/iTree/test_attack_log.csv',header=True,index=False)


# Made by Kerberos v1.0 on 2009/21/02
# this script is part of the Official L2J Datapack Project.
# Visit http://www.l2jdp.com/forum for more details.

from l2server.gameserver.instancemanager import QuestManager
from l2server.gameserver.model.quest import State
from l2server.gameserver.model.quest.jython import QuestJython as JQuest

qn = "183_Relic_Exploration"

# NPCs
Kusto = 30512
Lorain = 30673
Nikola = 30621

class Quest(JQuest):
    def __init__(self, id, name, descr):
        JQuest.__init__(self, id, name, descr)

    def onAdvEvent(self, event, npc, player):
        st = player.getQuestState(qn)
        if not st: return
        htmltext = event
        if event == "30512-03.htm":
            st.playSound("ItemSound.quest_accept")
            st.set("cond", "1")
            st.setState(State.STARTED)
        elif event == "30673-04.htm":
            st.set("cond", "2")
            st.playSound("ItemSound.quest_middle")
        elif event == "30621-02.htm":
            if player.getLevel() < 50:
                st.addExpAndSp(60000, 3000)
            st.giveItems(57, 18100)
            st.exitQuest(False)
            st.playSound("ItemSound.quest_finish")
        elif event == "Contract":
            q1 = QuestManager.getInstance().getQuest("184_Nikolas_Cooperation_Contract")
            if q1:
                qs1 = q1.newQuestState(player)
                qs1.setState(State.STARTED)
                q1.notifyEvent("30621-01.htm", npc, player)
            return
        elif event == "Consideration":
            q2 = QuestManager.getInstance().getQuest("185_Nikolas_Cooperation_Consideration")
            if q2:
                qs2 = q2.newQuestState(st.getPlayer())
                qs2.setState(State.STARTED)
                q2.notifyEvent("30621-01.htm", npc, player)
            return
        return htmltext

    def onTalk(self, npc, player):
        htmltext = Quest.getNoQuestMsg(player)
        st = player.getQuestState(qn)
        if not st: return htmltext
        npcId = npc.getNpcId()
        id = st.getState()
        cond = st.getInt("cond")
        if id == State.COMPLETED:
            if npcId == Kusto:
                htmltext = Quest.getAlreadyCompletedMsg(player)
            elif npcId == Nikola:
                qs1 = player.getQuestState("184_Nikolas_Cooperation_Contract")
                qs2 = player.getQuestState("185_Nikolas_Cooperation_Consideration")
                if not qs1 and not qs2:
                    htmltext = "30621-03.htm"
                else:
                    htmltext = Quest.getAlreadyCompletedMsg(player)
        elif npcId == Kusto:
            if id == State.CREATED:
                if player.getLevel() < 40:
                    htmltext = "30512-00.htm"
                else:
                    htmltext = "30512-01.htm"
            else:
                htmltext = "30512-04.htm"
        elif npcId == Lorain:
            if cond == 1:
                htmltext = "30673-01.htm"
            else:
                htmltext = "30673-05.htm"
        elif npcId == Nikola:
            if cond == 2:
                htmltext = "30621-01.htm"
        return htmltext

QUEST = Quest(183, qn, "Relics Exploration")

QUEST.addStartNpc(Kusto)
QUEST.addStartNpc(Nikola)
QUEST.addTalkId(Kusto)
QUEST.addTalkId(Lorain)
QUEST.addTalkId(Nikola)


from django.urls import path, re_path
from product import views

app_name = 'product'

urlpatterns = [
    path('', views.index, name='admin-product'),
    path('import/', views.import_page, name='admin-product-import'),
    path('list/', views.list_page, name='admin-product-list'),

    # apis
    path('api/get-product-types/', views.getProductTypee),
    path('api/get-items-paginate', views.getItemsPaginate),
    path('api/save-product-manually/', views.saveManually),
    path('api/update-product/', views.updateItem),
    path('api/upload-file-product/', views.saveFile),
]


import requests
from bs4 import BeautifulSoup
import re

class Goodparse:
    def __init__(self, id_goodreads):
        self.params = {"id": id_goodreads,
                       "shelf": "to-read",
                       "sort": "date_added",
                       "order": "d"}
        self.url = "https://www.goodreads.com/review/list?id={}?shelf={}?sort={}?order={}"
        self.url = self.url.format(self.params["id"], self.params["shelf"], self.params["sort"],
                             self.params["order"])

    def parse_want_to_read(self):
        page = requests.get(self.url)
        soup = BeautifulSoup(page.content, 'html.parser')
        print("Parsing Goodreads...")
        html_lines = soup.find_all('td', class_='field isbn13')
        return html_lines

    def get_isbns13(self):
        html_lines = self.parse_want_to_read()
        isbns13 = []
        for element in html_lines:
            isbns13.append(element.text)
        # isbns13 = [clean_string(item) for item in isbns13]
        for i in range(len(isbns13)):
            if re.search('(\d{13})', isbns13[i]):
                isbns13[i] = re.search('(\d{13})', isbns13[i]).group(1)

        # isbns13 = [re.search('(\d{13})', item).group(1) for item in isbns13]
        # isbns13 = BeautifulSoup.find_all("td", )
        # for element in html_lines:
        #     isbns13.append()
        # for element in str(html_lines):
        #     isbns13.append(re.search('(\d){13}', element).group(1))
        # html_lines = [BeautifulSoup.__str__(x) for x in html_lines]
        # isbns13 = re.findall('(\d){13}', html_lines)
        # isbns13 = [x for x in isbns13 if not exists_in_db(x)]
        return isbns13

def clean_string(s):
    return ' '.join(s.split())


import pytest
pytestmark = [pytest.mark.setone, pytest.mark.working, pytest.mark.schema]

@pytest.fixture
def genomic_region_w_onlyendloc(testapp, lab, award):
    item = {
        "genome_assembly": "dm6",
        "end_coordinate": 3,
        'award': award['@id'],
        'lab': lab['@id']
    }
    return testapp.post_json('/genomic_region', item).json['@graph'][0]

@pytest.fixture
def dt4genomic_regions(genomic_region_w_onlyendloc, some_genomic_region, basic_genomic_region,
                       vague_genomic_region, vague_genomic_region_w_desc):
    return {
        'dm6': genomic_region_w_onlyendloc,
        'GRCh38:1:17-544': some_genomic_region,
        'GRCh38': basic_genomic_region,
        'GRCm38:5': vague_genomic_region,
        'gene X enhancer': vague_genomic_region_w_desc
    }

@pytest.fixture
def targets(target_w_desc, target_w_region, target_w_genes):
    return {'target_w_desc': target_w_desc,
            'target_w_region': target_w_region,
            'target_w_genes': target_w_genes
            }

def test_calculated_target_summaries(testapp, targets):
    for name in targets:
        summary = targets[name]['target_summary']
        if name == 'target_w_genes':
            assert summary == 'Gene:eeny,meeny'
        if name == 'target_w_regions' in targets:
            assert summary == 'GRCh38:X:1-3'
        if name == 'target_w_desc':
            assert summary == 'no target'

@pytest.fixture
def protocol_data(lab, award):
    return {
        'lab': lab['@id'],
        'award': award['@id'],
        'protocol_type': 'Experimental protocol',
        'description': 'Test Protocol'
    }

@pytest.fixture
def protocol_w_attach(testapp, protocol_data, attachment):
    protocol_data['attachment'] = attachment
    return testapp.post_json('/protocol', protocol_data).json['@graph'][0]

def test_protocol_display_title_w_attachment(testapp, protocol_w_attach):
    assert protocol_w_attach['display_title'] == 'red-dot.png'

def test_protocol_display_title_wo_attachment(testapp, protocol_data):
    from datetime import datetime
    protocol = testapp.post_json('/protocol', protocol_data).json['@graph'][0]
    assert protocol['display_title'] == 'Experimental protocol from ' + str(datetime.now())[:10]

def test_protocol_other_display_title_wo_attachment(testapp, protocol_data):
    from datetime import datetime
    protocol_data['protocol_type'] = 'Other'
    protocol = testapp.post_json('/protocol', protocol_data).json['@graph'][0]
    assert protocol['display_title'] == 'Protocol from ' + str(datetime.now())[:10]

@pytest.fixture
def vendor_data(lab, award):
    return {"title": "WorTHington Biochemical", 'lab': lab['@id'], 'award': award['@id']}

def test_vendor_update_name_no_caps(testapp, vendor_data):
    res = testapp.post_json('/vendor', vendor_data, status=201)
    assert res.json['@graph'][0]['name'] == "worthington-biochemical"

def test_vendor_update_name_no_punctuation_or_space(testapp, vendor_data):
    vendor_data['title'] = "Eeny, = Meeny!  # -miny?"
    res = testapp.post_json('/vendor', vendor_data, status=201)
    assert res.json['@graph'][0]['name'] == "eeny-meeny-miny"

def test_vendor_name_updates_on_patch(testapp, vendor_data):
    res = testapp.post_json('/vendor', vendor_data, status=201)
    assert res.json['@graph'][0]['name'] == "worthington-biochemical"
    res = testapp.patch_json(res.json['@graph'][0]['@id'], {'title': 'WaHoo'}, status=200)
    assert res.json['@graph'][0]['name'] == "wahoo"

@pytest.fixture
def vendor_data_alias(lab, award):
    return {
        'title': 'Wrong Alias Biochemical',
        'lab': lab['@id'],
        'award': award['@id'],
        'aliases': ['my_lab:this_is_correct_one',
                    'my_lab:this/is_wrong',
                    'my_lab:this\is_wrong_too']}

def test_vendor_alias_wrong_format(testapp, vendor_data_alias):
    res = testapp.post_json('/vendor', vendor_data_alias, status=422)
    response = res.json
    print(res.json)
    assert response['status'] == 'error'
    assert response['code'] == 422
    problematic_aliases = 0
    for an_error in response['errors']:
        if an_error['name'][0] == 'aliases':
            problematic_aliases += 1
    assert problematic_aliases == 2

def test_genomic_region_display_title(testapp, dt4genomic_regions):
    for dt, region in dt4genomic_regions.items():
        assert region.get('display_title') == dt


#! /usr/bin/env python

import rospy
from ros_beginner.srv import Kyolo,KyoloResponse
from ros_beginner.msg import Kpub
from sync_sub.msg import some_position

def calculate(request):
    rospy.loginfo('called!')
    rospy.loginfo(request)
     #rospy.loginfo(request.a)
     #rospy.loginfo(request.b)
     #rospy.loginfo(request.words.split())
     #rospy.loginfo(len(request.words.split()))
     #result = request.a + request.b

    #position = [sub.position]
    position =[1,1,1]
    Class = 'Process End!!'#'Detect Started!!'
    #print sub.position
    #print sub.Class
    pub = rospy.Publisher('chatter3', some_position, queue_size=10)

    r = rospy.Rate(10) # 10hz
    #while not rospy.is_shutdown():
    for num in range(101):
        msg = some_position()

        msg.header.stamp = rospy.Time.now()

        msg.position = [7,8,9]
        pub.publish(msg)
        r.sleep()

    return KyoloResponse(position,Class)

rospy.init_node('bbox_server')

#sub = rospy.Subscriber('chatter5', Kpub)
#sub = rospy.Subscriber('chatter3', some_position)
#print(sub)

#position1=sub.position
#Class1=sub.Class

service = rospy.Service('bbox', Kyolo , calculate)

#print(service)
#rospy.loginfo(service)
rospy.spin()


"""
Created on November 09 16:40:40 2018
@author: carlos enrique gutierrez  carlosengutierrez@gmail.com
DOYA UNIT - OIST
Okinawa Institute of Science and TEchnology
"""

# Training data set
BRAINS = ['R01_0070_CM1180F','R01_0072_CM1176F','R01_0078_CM1347F','R01_0071_CM1178F',
          'R01_0029_CM696F','R01_0030_CM690F','R01_0054_CM1060F','R01_0034_CM521F',
          'R01_0039_CM703F','R01_0033_CM694F']

# Test data set
#BRAINS = ['R01_0026_CM692F','R01_0040_CM710M','R01_0048_CM1011F',
#          'R01_0043_CM628F','R01_0053_CM1061F','R01_0046_CM1023M']

# Options for jobs building (change this as needed)
SLURM_OPTIONS_1 = ['#!/bin/bash \n\n',
                    '#SBATCH --time=7-00:00:00 \n',
                    '#SBATCH --mem 4gb \n',
                    '#SBATCH -c 1 \n',
                    '#SBATCH --partition=postproc1 \n',
                    '#SBATCH --input=none \n']
SLURM_OPTIONS_1_1 = ['module load matlab/R2015a \n',
                    'module load python/3.5.0 \n',
                    'export PATH=\"/work/DoyaU/mrtrix3/bin:/work/DoyaU/deap:${PATH}\" \n',
                    'module load gcc/6.2.0 \n',
                    'module load eigen/3.3.1 \n',
                    'module load zlib/1.2.8 \n']
COMMAND_1 = 'sbatch '

SLURM_OPTIONS_2 = ['#!/bin/bash \n\n',
                    '#SBATCH --time=7-00:00:00 \n',
                    '#SBATCH --mem 30gb \n',
                    '#SBATCH -c 2 \n',
                    '#SBATCH --partition=compute \n',
                    '#SBATCH --input=none \n']
SLURM_OPTIONS_2_1 = ['module load matlab/R2015a \n',
                    'module load python/3.5.0 \n',
                    'export PATH=\"/work/DoyaU/mrtrix3/bin:/work/DoyaU/deap:${PATH}\" \n',
                    'module load gcc/6.2.0 \n',
                    'module load eigen/3.3.1 \n',
                    'module load zlib/1.2.8 \n']
COMMAND_2 = 'matlab -r \"addpath  /work/DoyaU/marcogt/;'

# parameters exploration ranges
grid={}
grid['width']= [0.01,0.15]  # -> [0.01,0.10]
grid['length']= [0.24,0.65] # -> [0.32,0.65]
grid['weight']= [0.01,0.22] # -> [0.01,0.13]
grid['chemPot2']=[0.05,0.6] # -> [0.01,0.22]
grid['connlike']=[0.5,6.0]  # -> [0.1,3.0]

my_path = '/work/DoyaU/carlos/dwi_pip/database/' #dwi database path
mask = 'auto-brain_map_org_mask.nii' #whole-brain mask at standard brain space
lod = 'suggestSparse'
my_mode = 'optimize' #'test' will run 5 times,  #'optimize' will run optimization

# how many runs of Global tracking per brain for the same parameters (results are averaged)
n_runs = 2 #1


# coding: utf-8
from src.parse_tools import list_directory_files
from src.parse_tools import get_node_match
import re
import unittest
import os

class ParserTest(unittest.TestCase):

    def setUp(self):
        self.foo_c = "test_data/dummy_directory/foo.c"
        self.foo_cs = "test_data/dummy_directory/foo.cs"
        self.dummy_lst = sorted([self.foo_c, self.foo_cs])
        self.bar_c = "test_data/dummy_directory/dummy_child/bar.c"
        self.bar_cs = "test_data/dummy_directory/dummy_child/bar.cs"
        self.dummy_child_lst = sorted([self.bar_c, self.bar_cs])

        self.dummy_dir = "test_data/dummy_directory"
        self.dummy_child = "src/test_data/dummy_directory/dummy_child"

    def list_abs_path(self, lst):
        return sorted(map(lambda fn: os.path.abspath(fn), lst))

    def test_listdir_no_recurse(self):
        result_lst = list_directory_files(self.dummy_dir, False, '*')
        self.assertEqual(sorted(result_lst), sorted(self.dummy_lst))

    def test_listdir_recurse(self):
        result_lst = sorted(list_directory_files(self.dummy_dir, True, '*'))
        self.assertEqual(result_lst, sorted(self.dummy_lst+self.dummy_child_lst))

    def test_get_node_match(self):
        test_iterable = ["match", "asdf asdf"]
        test_node = {'depth': 0, 'initial': lambda text: re.search('match', text) or None}
        result_match = get_node_match(test_node, test_iterable[0], test_iterable, False)
        return self.assertEqual(result_match.string, "match")

if __name__ == '__main__':
    unittest.main()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
from django.contrib.postgres.fields import JSONField
from django.utils.translation import ugettext_lazy as _

class Migration(migrations.Migration):

    dependencies = [
        ('auditlog', '0008_changes_to_psql_jsonfield'),
    ]

    operations = [
        migrations.RemoveField(
            model_name='logentry',
            name='additional_data',
        ),
        migrations.AddField(
            model_name='logentry',
            name='additional_data',
            field=JSONField(null=True, verbose_name=_("additional data"))
        ),
    ]


"""
REVISIT: Explanation of the backtracking algorithm
Question: https://leetcode.com/problems/word-search

Given a 2D board and a word, find if the word exists in the grid.

The word can be constructed from letters of sequentially adjacent cell, where "adjacent" cells are those horizontally
or vertically neighboring. The same letter cell may not be used more than once.

Example:

board =
[
  ['A','B','C','E'],
  ['S','F','C','S'],
  ['A','D','E','E']
]

Given word = "ABCCED", return true.
Given word = "SEE", return true.
Given word = "ABCB", return false.

Constraints:

board and word consists only of lowercase and uppercase English letters.
1 <= board.length <= 200
1 <= board[i].length <= 200
1 <= word.length <= 10^3
"""
from typing import List

class Solution:
    def exist(self, board: List[List[str]], word: str) -> bool:
        return self.first_implementation(board, word)

    def first_implementation(self, board: List[List[str]], word: str) -> bool:
        """
        Each cell in the grid has to be visited in the worst case, or maybe only up til the letters that remain are
        of smaller size than the length of the word being sought?
        """
        num_rows = len(board)
        num_cols = len(board[0])

        # if num_rows * num_cols < len(word):
        #     return False

        def backtrack(row: int, col: int, suffix):
            if len(suffix) == 0:
                # All letters have been found for the word being sought.
                return True

            # Do some bounds checking of the directions the
            # Indices won''t cause out of bounds errors, and that the current element is the one being sought
            if row < 0 or row == num_rows or col < 0 or col == num_cols \
                    or board[row][col] != suffix[0]:
                return False

            # the current letter being sought is matched.
            # Continue the operations marking that this cell should not be revisited.
            board[row][col] = '#'
            for rowOffset, colOffset in [(0, 1), (1, 0), (0, -1), (-1, 0)]:
                if backtrack(row + rowOffset, col + colOffset, suffix[1:]):
                    return True
            # unmark the work that's been done so another attempt could visit this cell
            board[row][col] = suffix[0]
            return False

        for row_start in range(len(board)):
            for col_start in range(len(board[0])):
                if backtrack(row_start, col_start, word):
                    return True
        return False


# -*- coding: utf-8 -*-
"""
通过pyspark RDD索引数据至es
"""
from pyspark import SparkContext
import json

sc = SparkContext(master="spark://spark001:7077",
                  appName="index_via_pyspark_rdd")

rdd = sc.parallelize([{"num": i} for i in range(10)])

def remove_id(doc):
    """
    剔除doc中的_id字段
    :param doc:
    :return:
    """
    doc.pop("_id", "")
    return doc

new_rdd = rdd.map(remove_id).map(json.dumps).map(lambda x: ('key', x))

new_rdd.saveAsNewAPIHadoopFile(
    path='-',
    outputFormatClass="org.elasticsearch.hadoop.mr.EsOutputFormat",
    keyClass="org.apache.hadoop.io.NullWritable",
    valueClass="org.elasticsearch.hadoop.mr.LinkedMapWritable",
    conf={
        "es.nodes": "192.168.0.111",
        "es.port": "9288",
        "es.resource": "spark/doc",
        "es.input.json": "true"
    }
)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
This returns the flavors based on kamaki flavor list.

@author: Ioannis Stenos, Nick Vrionis
'''

import os
import sys
import logging
from okeanos_utils import *
from kamaki.clients.cyclades import CycladesClient
from backend.models import Status

# Definitions of return value errors
error_flavor_list = -23
error_user_quota = -22
error_syntax_logging_level = -24
error_syntax_auth_token = -25

# Global constants
Bytes_to_GB = 1073741824  # Global to convert bytes to gigabytes
Bytes_to_MB = 1048576  # Global to convert bytes to megabytes
auth_url='https://accounts.okeanos.grnet.gr/identity/v2.0'

def check_quota(token):
    '''
    Checks if user available quota .
    Available = limit minus (used and pending).Also divides with 1024*1024*1024
    to transform bytes to gigabytes.
     '''

    auth = check_credentials(token, auth_url)

    try:
        dict_quotas = auth.get_quotas()
    except Exception:
        logging.exception('Could not get user quota')
        sys.exit(error_user_quota)
    limit_cd = dict_quotas['system']['cyclades.disk']['limit'] / Bytes_to_GB
    usage_cd = dict_quotas['system']['cyclades.disk']['usage'] / Bytes_to_GB
    pending_cd = dict_quotas['system']['cyclades.disk']['pending'] / Bytes_to_GB
    available_cyclades_disk_GB = (limit_cd-usage_cd-pending_cd)

    limit_cpu = dict_quotas['system']['cyclades.cpu']['limit']
    usage_cpu = dict_quotas['system']['cyclades.cpu']['usage']
    pending_cpu = dict_quotas['system']['cyclades.cpu']['pending']
    available_cpu = limit_cpu - usage_cpu - pending_cpu

    limit_ram = dict_quotas['system']['cyclades.ram']['limit'] / Bytes_to_MB
    usage_ram = dict_quotas['system']['cyclades.ram']['usage'] / Bytes_to_MB
    pending_ram = dict_quotas['system']['cyclades.ram']['pending'] / Bytes_to_MB
    available_ram = (limit_ram-usage_ram-pending_ram)

    limit_vm = dict_quotas['system']['cyclades.vm']['limit']
    usage_vm = dict_quotas['system']['cyclades.vm']['usage']
    pending_vm = dict_quotas['system']['cyclades.vm']['pending']
    available_vm = limit_vm-usage_vm-pending_vm

    quotas = {'cpus' : { 'limit' : limit_cpu , 'available' : available_cpu } ,
              'ram' : {'limit' : limit_ram , 'available' : available_ram } ,
              'disk' : {'limit' : limit_cd , 'available' : available_cyclades_disk_GB } ,
              'cluster_size' : {'limit' : limit_vm , 'available' : available_vm }}
    logging.info(quotas)
    return quotas

def get_flavor_id(token):
    '''From kamaki flavor list get all possible flavors '''
    auth = check_credentials(token, auth_url)
    endpoints, user_id = endpoints_and_user_id(auth)
    cyclades = init_cyclades(endpoints['cyclades'], token)
    try:
        flavor_list = cyclades.list_flavors(True)
    except Exception:
        logging.exception('Could not get list of flavors')
        sys.exit(error_flavor_list)
    cpu_list = []
    ram_list = []
    disk_list = []
    disk_template_list = []

    for flavor in flavor_list:
        if flavor['vcpus'] not in cpu_list:
            cpu_list.append(flavor['vcpus'])
        if flavor['ram'] not in ram_list:
            ram_list.append(flavor['ram'])
        if flavor['disk'] not in disk_list:
            disk_list.append(flavor['disk'])
        if flavor['SNF:disk_template'] not in disk_template_list:
            disk_template_list.append(flavor['SNF:disk_template'])
    cpu_list = sorted (cpu_list)
    ram_list = sorted(ram_list)
    disk_list = sorted(disk_list)
    flavors = {'cpus' : cpu_list, 'ram' : ram_list , 'disk' : disk_list , 'disk_template' : disk_template_list}
    logging.info(flavors)
    return flavors

def updateStatus(user):
    '''
    Method that retrieves user quotas and flavor list from kamaki
    using get_flavor_id and check_quota methods and returns the updated
    Status model.
    '''
    i = 0
    j = 1
    vms_av = []
    okeanos_token = user.okeanos_token
    flavors = get_flavor_id(okeanos_token)
    quotas = check_quota(okeanos_token)

    vms_max = quotas['cluster_size']['limit']
    vms_available = quotas['cluster_size']['available']
    for i in range(vms_available):
        vms_av.append(j)
        j = j +1
    cpu_max = quotas['cpus']['limit']
    cpu_av = quotas['cpus']['available']
    mem_max = quotas['ram']['limit']
    mem_av = quotas['ram']['available']
    disk_max = quotas['disk']['limit']
    disk_av = quotas['disk']['available']
    cpu_p = flavors['cpus']
    mem_p = flavors['ram']
    disk_p = flavors['disk']
    disk_template = flavors['disk_template']
    os_p = ['Debian Base']

    new_status = Status(user_id=user, vms_max=vms_max, vms_av=vms_av,
                        cpu_max=cpu_max, cpu_av=cpu_av, mem_max=mem_max,
                        mem_av=mem_av, disk_max=disk_max, disk_av=disk_av,
                        cpu_p=cpu_p, mem_p=mem_p,
                        disk_p=disk_p, disk_template=disk_template, os_p=os_p)

    return new_status


entrada = open("human_18s").read()
saida = open("human.html", "w")

# Cria um dicionário
contador = {}

# Percorre os valores, fazendo a combinação em PAR
for i in ['A', 'T', 'C', 'G']:
    for j in ['A', 'T', 'C', 'G']:
        # inicia contador para cada par de letra
        contador[i+j] = 0

# Retira a quebra de linha, trocando o \n por nada
entrada = entrada.replace("\n", "")

# Percorre o tamanho do arquivo -1, faz o somatório para cada PAR
for k in range(len(entrada)-1):
    # Realiza a soma no par de letra
    contador[entrada[k]+entrada[k+1]] =+ 1

# HTML
saida.write("
")

i = 1
for k in contador:
    # Transparencia recebe o par de valor dividido
    # pelo valor máximo armazenado em contador
    transparencia = contador[k]/max(contador.values())
    saida.write("
"+k+"
")

    if i%4 == 0:
        saida.write("

")

    i+=1

saida.close()


#!/usr/bin/python

import sys
import json

'''- name: force_create
type: Bool
- name: containerslist
type: ContainersList
- name: container_name
type: String'''

inputs = json.load(sys.stdin)

if not inputs['force_create'][0]['value']:
    if inputs['container_name'][0]['value'] in inputs['containerslist'][0]['containers']:
        sys.stderr.write("ERROR: container {} already runs on target".format(
            inputs['container_name'][0]['value']) + "\n")
        sys.exit(1)


from typing import List

from sharpy.plans.acts import *
from sharpy.plans.acts.terran import *
from sharpy.plans.require import *
from sharpy.plans.tactics import *
from sharpy.plans.tactics.terran import *
from sharpy.plans import BuildOrder, Step, SequentialList, StepBuildGas
from sc2 import BotAI, UnitTypeId, AbilityId, Race
from sc2.ids.upgrade_id import UpgradeId

from sharpy.knowledges import Knowledge, KnowledgeBot

class CycloneBot(KnowledgeBot):
    def __init__(self):
        super().__init__("Rusty Locks")

    async def create_plan(self) -> BuildOrder:
        buildings = [
            Step(RequiredSupply(13), GridBuilding(UnitTypeId.SUPPLYDEPOT, 1)),
            Step(RequiredSupply(16), ActExpand(2)),

            Step(RequiredSupply(18), GridBuilding(UnitTypeId.BARRACKS, 1)),
            StepBuildGas(1),
            Step(RequiredSupply(20), GridBuilding(UnitTypeId.SUPPLYDEPOT, 2)),
            Step(None, StepBuildGas(2), skip_until=RequiredUnitExists(UnitTypeId.MARINE, 2)),
            Step(None, GridBuilding(UnitTypeId.FACTORY, 1),
                 skip_until=RequiredUnitReady(UnitTypeId.BARRACKS, 1)),
            GridBuilding(UnitTypeId.FACTORY, 1),
            ActBuildAddon(UnitTypeId.FACTORYTECHLAB, UnitTypeId.FACTORY, 1),
            StepBuildGas(4),
            Step(None, ActExpand(3)),
            GridBuilding(UnitTypeId.FACTORY, 2),
            ActBuildAddon(UnitTypeId.FACTORYTECHLAB, UnitTypeId.FACTORY, 2),
            Step(None, ActTech(UpgradeId.CYCLONELOCKONDAMAGEUPGRADE, UnitTypeId.FACTORYTECHLAB),
                 skip_until=RequiredUnitReady(UnitTypeId.FACTORYTECHLAB, 1)),
            StepBuildGas(5),
            Step(None, ActTech(UpgradeId.HIGHCAPACITYBARRELS, UnitTypeId.FACTORYTECHLAB),
                 skip_until=RequiredUnitReady(UnitTypeId.FACTORYTECHLAB, 2)),
            StepBuildGas(6, None, RequiredGas(100)),
            Step(RequiredMinerals(400), GridBuilding(UnitTypeId.FACTORY, 4)),
            Step(None, ActBuildAddon(UnitTypeId.FACTORYREACTOR, UnitTypeId.FACTORY, 1)),
            ActBuildAddon(UnitTypeId.FACTORYTECHLAB, UnitTypeId.FACTORY, 3),
            Step(RequiredMinerals(400), ActExpand(4)),
            GridBuilding(UnitTypeId.ENGINEERINGBAY, 1),
            StepBuildGas(8),
            GridBuilding(UnitTypeId.FACTORY, 6),
            Step(RequiredMinerals(400), GridBuilding(UnitTypeId.FACTORY, 8)),
            ActBuildAddon(UnitTypeId.FACTORYTECHLAB, UnitTypeId.FACTORY, 7),
            Step(RequiredSupply(120), GridBuilding(UnitTypeId.ARMORY, 2)),

        ]

        upgrades = [
            Step(RequiredUnitReady(UnitTypeId.ARMORY, 1), ActTech(UpgradeId.TERRANVEHICLEWEAPONSLEVEL1, UnitTypeId.ARMORY)),
            ActTech(UpgradeId.TERRANVEHICLEANDSHIPARMORSLEVEL1, UnitTypeId.ARMORY),
            ActTech(UpgradeId.TERRANVEHICLEWEAPONSLEVEL2, UnitTypeId.ARMORY),
            ActTech(UpgradeId.TERRANVEHICLEANDSHIPARMORSLEVEL2, UnitTypeId.ARMORY),
            ActTech(UpgradeId.TERRANVEHICLEWEAPONSLEVEL3, UnitTypeId.ARMORY),
            ActTech(UpgradeId.TERRANVEHICLEANDSHIPARMORSLEVEL3, UnitTypeId.ARMORY),
        ]

        self.attack = PlanZoneAttack(40)

        worker_scout = Step(None, WorkerScout(), skip_until=RequiredUnitExists(UnitTypeId.SUPPLYDEPOT, 1))
        self.distribute_workers = PlanDistributeWorkers()

        tactics = [
            PlanCancelBuilding(),
            LowerDepots(),
            PlanZoneDefense(),
            worker_scout,
            Step(None, CallMule(50), skip=RequiredTime(5 * 60)),
            Step(None, CallMule(100), skip_until=RequiredTime(5 * 60)),
            Step(None, ScanEnemy(), skip_until=RequiredTime(5 * 60)),

            self.distribute_workers,
            ManTheBunkers(),
            Repair(),
            ContinueBuilding(),
            PlanZoneGatherTerran(),
            Step(RequiredTechReady(UpgradeId.CYCLONELOCKONDAMAGEUPGRADE, 0.95), self.attack),
            PlanFinishEnemy(),
        ]

        return BuildOrder([
            Step(RequiredUnitExists(UnitTypeId.BARRACKS, 1), SequentialList(self.depots)),
            [
                Step(RequiredUnitExists(UnitTypeId.COMMANDCENTER, 2), MorphOrbitals(3), skip_until=RequiredUnitReady(UnitTypeId.BARRACKS, 1)),
                Step(None, MorphPlanetary(2), skip_until=RequiredUnitReady(UnitTypeId.ENGINEERINGBAY, 1)),
            ],
            [
                Step(None, ActUnit(UnitTypeId.SCV, UnitTypeId.COMMANDCENTER, 40)),
                Step(RequiredUnitExists(UnitTypeId.COMMANDCENTER, 3), ActUnit(UnitTypeId.SCV, UnitTypeId.COMMANDCENTER, 70))
            ],
            upgrades,
            ActUnit(UnitTypeId.MARINE, UnitTypeId.BARRACKS, 4),
            [
                ActUnit(UnitTypeId.CYCLONE, UnitTypeId.FACTORY, 4),
                ActUnitOnce(UnitTypeId.HELLION, UnitTypeId.FACTORY, 1),
                ActUnit(UnitTypeId.CYCLONE, UnitTypeId.FACTORY, 4),
                ActUnit(UnitTypeId.CYCLONE, UnitTypeId.FACTORY, 120, priority=True),
            ],
            Step(RequiredUnitReady(UnitTypeId.FACTORYREACTOR, 1),
                 ActUnit(UnitTypeId.HELLION, UnitTypeId.FACTORY, 60),
                 skip_until=RequiredMinerals(300),
                 ),

            buildings,
            SequentialList(tactics)
        ])

    @property
    def depots(self) -> List[Step]:
        return [
            Step(RequiredUnitReady(UnitTypeId.SUPPLYDEPOT, 1), None),
            Step(RequiredSupplyLeft(6), GridBuilding(UnitTypeId.SUPPLYDEPOT, 2)),
            Step(RequiredUnitReady(UnitTypeId.SUPPLYDEPOT, 2), None),
            Step(RequiredSupplyLeft(14), GridBuilding(UnitTypeId.SUPPLYDEPOT, 4)),
            Step(RequiredUnitReady(UnitTypeId.SUPPLYDEPOT, 4), None),
            Step(RequiredSupplyLeft(20), GridBuilding(UnitTypeId.SUPPLYDEPOT, 6)),
            Step(RequiredUnitReady(UnitTypeId.SUPPLYDEPOT, 5), None),
            Step(RequiredSupplyLeft(20), GridBuilding(UnitTypeId.SUPPLYDEPOT, 7)),
            Step(RequiredUnitReady(UnitTypeId.SUPPLYDEPOT, 6), None),
            Step(RequiredSupplyLeft(20), GridBuilding(UnitTypeId.SUPPLYDEPOT, 10)),
            Step(RequiredUnitReady(UnitTypeId.SUPPLYDEPOT, 8), None),
            Step(RequiredSupplyLeft(20), GridBuilding(UnitTypeId.SUPPLYDEPOT, 12)),
            Step(RequiredUnitReady(UnitTypeId.SUPPLYDEPOT, 10), None),
            Step(RequiredSupplyLeft(20), GridBuilding(UnitTypeId.SUPPLYDEPOT, 14)),
            Step(RequiredUnitReady(UnitTypeId.SUPPLYDEPOT, 13), None),
            Step(RequiredSupplyLeft(20), GridBuilding(UnitTypeId.SUPPLYDEPOT, 16)),
            Step(RequiredUnitReady(UnitTypeId.SUPPLYDEPOT, 16), GridBuilding(UnitTypeId.SUPPLYDEPOT, 20)),
        ]

class LadderBot(CycloneBot):
    @property
    def my_race(self):
        return Race.Terran


import re

import voluptuous as vol

from esphome.automation import ACTION_REGISTRY, LambdaAction
import esphome.config_validation as cv
from esphome.const import CONF_ARGS, CONF_BAUD_RATE, CONF_FORMAT, CONF_HARDWARE_UART, CONF_ID, \
    CONF_LEVEL, CONF_LOGS, CONF_TAG, CONF_TX_BUFFER_SIZE
from esphome.core import CORE, EsphomeError, Lambda
from esphome.cpp_generator import Pvariable, RawExpression, add, process_lambda, statement
from esphome.cpp_types import App, Component, esphome_ns, global_ns, void
from esphome.py_compat import text_type

LOG_LEVELS = {
    'NONE': global_ns.ESPHOME_LOG_LEVEL_NONE,
    'ERROR': global_ns.ESPHOME_LOG_LEVEL_ERROR,
    'WARN': global_ns.ESPHOME_LOG_LEVEL_WARN,
    'INFO': global_ns.ESPHOME_LOG_LEVEL_INFO,
    'DEBUG': global_ns.ESPHOME_LOG_LEVEL_DEBUG,
    'VERBOSE': global_ns.ESPHOME_LOG_LEVEL_VERBOSE,
    'VERY_VERBOSE': global_ns.ESPHOME_LOG_LEVEL_VERY_VERBOSE,
}

LOG_LEVEL_TO_ESP_LOG = {
    'ERROR': global_ns.ESP_LOGE,
    'WARN': global_ns.ESP_LOGW,
    'INFO': global_ns.ESP_LOGI,
    'DEBUG': global_ns.ESP_LOGD,
    'VERBOSE': global_ns.ESP_LOGV,
    'VERY_VERBOSE': global_ns.ESP_LOGVV,
}

LOG_LEVEL_SEVERITY = ['NONE', 'ERROR', 'WARN', 'INFO', 'DEBUG', 'VERBOSE', 'VERY_VERBOSE']

UART_SELECTION_ESP32 = ['UART0', 'UART1', 'UART2']

UART_SELECTION_ESP8266 = ['UART0', 'UART0_SWAP', 'UART1']

HARDWARE_UART_TO_UART_SELECTION = {
    'UART0': global_ns.UART_SELECTION_UART0,
    'UART0_SWAP': global_ns.UART_SELECTION_UART0_SWAP,
    'UART1': global_ns.UART_SELECTION_UART1,
    'UART2': global_ns.UART_SELECTION_UART2,
}

HARDWARE_UART_TO_SERIAL = {
    'UART0': 'Serial',
    'UART0_SWAP': 'Serial',
    'UART1': 'Serial1',
    'UART2': 'Serial2',
}

# pylint: disable=invalid-name
is_log_level = cv.one_of(*LOG_LEVELS, upper=True)

def uart_selection(value):
    if CORE.is_esp32:
        return cv.one_of(*UART_SELECTION_ESP32, upper=True)(value)
    if CORE.is_esp8266:
        return cv.one_of(*UART_SELECTION_ESP8266, upper=True)(value)
    raise NotImplementedError

def validate_local_no_higher_than_global(value):
    global_level = value.get(CONF_LEVEL, 'DEBUG')
    for tag, level in value.get(CONF_LOGS, {}).items():
        if LOG_LEVEL_SEVERITY.index(level) > LOG_LEVEL_SEVERITY.index(global_level):
            raise EsphomeError(u"The local log level {} for {} must be less severe than the "
                               u"global log level {}.".format(level, tag, global_level))
    return value

LogComponent = esphome_ns.class_('LogComponent', Component)

CONFIG_SCHEMA = vol.All(cv.Schema({
    cv.GenerateID(): cv.declare_variable_id(LogComponent),
    vol.Optional(CONF_BAUD_RATE, default=115200): cv.positive_int,
    vol.Optional(CONF_TX_BUFFER_SIZE, default=512): cv.validate_bytes,
    vol.Optional(CONF_HARDWARE_UART, default='UART0'): uart_selection,
    vol.Optional(CONF_LEVEL): is_log_level,
    vol.Optional(CONF_LOGS): cv.Schema({
        cv.string: is_log_level,
    })
}), validate_local_no_higher_than_global)

def to_code(config):
    rhs = App.init_log(config.get(CONF_BAUD_RATE),
                       config.get(CONF_TX_BUFFER_SIZE),
                       HARDWARE_UART_TO_UART_SELECTION[config.get(CONF_HARDWARE_UART)])
    log = Pvariable(config[CONF_ID], rhs)
    if CONF_LEVEL in config:
        add(log.set_global_log_level(LOG_LEVELS[config[CONF_LEVEL]]))
    for tag, level in config.get(CONF_LOGS, {}).items():
        add(log.set_log_level(tag, LOG_LEVELS[level]))

def required_build_flags(config):
    flags = []
    if CONF_LEVEL in config:
        flags.append(u'-DESPHOME_LOG_LEVEL={}'.format(str(LOG_LEVELS[config[CONF_LEVEL]])))
        this_severity = LOG_LEVEL_SEVERITY.index(config[CONF_LEVEL])
        verbose_severity = LOG_LEVEL_SEVERITY.index('VERBOSE')
        very_verbose_severity = LOG_LEVEL_SEVERITY.index('VERY_VERBOSE')
        is_at_least_verbose = this_severity >= verbose_severity
        is_at_least_very_verbose = this_severity >= very_verbose_severity
        has_serial_logging = config.get(CONF_BAUD_RATE) != 0
        if CORE.is_esp8266 and has_serial_logging and is_at_least_verbose:
            debug_serial_port = HARDWARE_UART_TO_SERIAL[config.get(CONF_HARDWARE_UART)]
            flags.append(u"-DDEBUG_ESP_PORT={}".format(debug_serial_port))
            flags.append(u"-DLWIP_DEBUG")
            DEBUG_COMPONENTS = {
                'HTTP_CLIENT',
                'HTTP_SERVER',
                'HTTP_UPDATE',
                'OTA',
                'SSL',
                'TLS_MEM',
                'UPDATER',
                'WIFI',
            }
            for comp in DEBUG_COMPONENTS:
                flags.append(u"-DDEBUG_ESP_{}".format(comp))
        if CORE.is_esp32 and is_at_least_verbose:
            flags.append('-DCORE_DEBUG_LEVEL=5')
        if CORE.is_esp32 and is_at_least_very_verbose:
            flags.append('-DENABLE_I2C_DEBUG_BUFFER')

    return flags

def maybe_simple_message(schema):
    def validator(value):
        if isinstance(value, dict):
            return cv.Schema(schema)(value)
        return cv.Schema(schema)({CONF_FORMAT: value})

    return validator

def validate_printf(value):
    # https://stackoverflow.com/questions/30011379/how-can-i-parse-a-c-format-string-in-python
    # pylint: disable=anomalous-backslash-in-string
    cfmt = u"""\
    (                                  # start of capture group 1
    %                                  # literal "%"
    (?:                                # first option
    (?:[-+0 #]{0,5})                   # optional flags
    (?:\d+|\*)?                        # width
    (?:\.(?:\d+|\*))?                  # precision
    (?:h|l|ll|w|I|I32|I64)?            # size
    [cCdiouxXeEfgGaAnpsSZ]             # type
    ) |                                # OR
    %%)                                # literal "%%"
    """  # noqa
    matches = re.findall(cfmt, value[CONF_FORMAT], flags=re.X)
    if len(matches) != len(value[CONF_ARGS]):
        raise vol.Invalid(u"Found {} printf-patterns ({}), but {} args were given!"
                          u"".format(len(matches), u', '.join(matches), len(value[CONF_ARGS])))
    return value

CONF_LOGGER_LOG = 'logger.log'
LOGGER_LOG_ACTION_SCHEMA = vol.All(maybe_simple_message({
    vol.Required(CONF_FORMAT): cv.string,
    vol.Optional(CONF_ARGS, default=list): cv.ensure_list(cv.lambda_),
    vol.Optional(CONF_LEVEL, default="DEBUG"): cv.one_of(*LOG_LEVEL_TO_ESP_LOG, upper=True),
    vol.Optional(CONF_TAG, default="main"): cv.string,
}), validate_printf)

@ACTION_REGISTRY.register(CONF_LOGGER_LOG, LOGGER_LOG_ACTION_SCHEMA)
def logger_log_action_to_code(config, action_id, template_arg, args):
    esp_log = LOG_LEVEL_TO_ESP_LOG[config[CONF_LEVEL]]
    args_ = [RawExpression(text_type(x)) for x in config[CONF_ARGS]]

    text = text_type(statement(esp_log(config[CONF_TAG], config[CONF_FORMAT], *args_)))

    lambda_ = yield process_lambda(Lambda(text), args, return_type=void)
    rhs = LambdaAction.new(template_arg, lambda_)
    type = LambdaAction.template(template_arg)
    yield Pvariable(action_id, rhs, type=type)


from bs4 import BeautifulSoup
from pprint import pprint
from .utils import *
import re

class Company(object):
    """Linkedin Profile object

    Attributes:
        - personal_info
        - experiences
        - skills
        - accomplishments
        - interests
        - to_dict: return dictionary of all properties listed above
    """

    def __init__(self, body):
        """Initializes a profile with beautifulsoup body"""
        self.soup = BeautifulSoup(body, 'html.parser')

    def top_card_info(self):
        """Return dict of personal info about the user"""
        info = {}
        top_card = self.soup.find('div',{'class':'org-top-card-module__container'})
        Employees = top_card.findAll('a',{'class':'org-company-employees-snackbar__details-highlight'})
        if len(Employees) > 1:
            liEmployees = Employees[1]
        else:
            liEmployees = top_card.find('a',{'class':'org-company-employees-snackbar__details-highlight'})
        comp_name = ' '.join(top_card.find('h1',{'class':'org-top-card-module__name'})['title'].split()).encode('ascii', errors='ignore').decode('utf-8', errors='ignore')
        industry = ' '.join(top_card.find('span',{'class':'company-industries'}).text.split()).encode('ascii', errors='ignore').decode('utf-8', errors='ignore')
        city = ' '.join(top_card.find('span',{'class':'org-top-card-module__location'}).text.split(',')[0].split()).encode('ascii', errors='ignore').decode('utf-8', errors='ignore')
        state = ' '.join(top_card.find('span',{'class':'org-top-card-module__location'}).text.split(',')[1].split()).encode('ascii', errors='ignore').decode('utf-8', errors='ignore')
        liEmployees = ''.join(re.findall(r'\d+',liEmployees.text)).encode('ascii', errors='ignore').decode('utf-8', errors='ignore')
        top_card_info = {
            'Company Name':comp_name,
            'Industry': industry,
            'City': city,
            'State': state,
            'LIEmployees': liEmployees
        }

        return top_card_info

    def company_info(self):
        """
        Returns:
            dict of person's professional experiences.  These include:
                - Jobs
                - Education
                - Volunteer Experiences
        """

        other_comp = self.soup.find('div', {'class': 'org-about-company-module__org-info'})
        comp_type = other_comp.find('p', {'class': 'org-about-company-module__company-type'}).text
        desc = other_comp.find('p', {'class': 'org-about-us-organization-description__text'}).text.strip('\n').strip('\t').strip('\r\n').strip('\r\n\r\n')
        web = other_comp.find('a', {'class': 'org-about-us-company-module__website'})['href']
        spec = str(list(other_comp.find('p',{'class':'org-about-company-module__specialities'}).text.strip('\n').strip().split(',')))

        other_comp_info = {
            'Company Type': comp_type,
            'Description': desc,
            'Website': web,
            'Specialties': spec
        }
        return other_comp_info
    def testUS(self):
        states = ["Alaska",
                  "Alabama",
                  "Arkansas",
                  "American Samoa",
                  "Arizona",
                  "California",
                  "Colorado",
                  "Connecticut",
                  "District of Columbia",
                  "Delaware",
                  "Florida",
                  "Georgia",
                  "Guam",
                  "Hawaii",
                  "Iowa",
                  "Idaho",
                  "Illinois",
                  "Indiana",
                  "Kansas",
                  "Kentucky",
                  "Louisiana",
                  "Massachusetts",
                  "Maryland",
                  "Maine",
                  "Michigan",
                  "Minnesota",
                  "Missouri",
                  "Mississippi",
                  "Montana",
                  "North Carolina",
                  "North Dakota",
                  "Nebraska",
                  "New Hampshire",
                  "New Jersey",
                  "New Mexico",
                  "Nevada",
                  "New York",
                  "Ohio",
                  "Oklahoma",
                  "Oregon",
                  "Pennsylvania",
                  "Puerto Rico",
                  "Rhode Island",
                  "South Carolina",
                  "South Dakota",
                  "Tennessee",
                  "Texas",
                  "Utah",
                  "Virginia",
                  "Virgin Islands",
                  "Vermont",
                  "Washington",
                  "Wisconsin",
                  "West Virginia",
                  "Wyoming"]
        Location = self.soup.find('span',{'class':'org-top-card-module__location'}).text.split(',')[1]
        print(Location)
        if Location in states:
            print('YAY')
            return False
        else:
            return True

    def to_dict(self):
        """Return full dict of person's profile
        """
        info = {}
        info['top_card_info'] = self.top_card_info
        info['other_comp_info'] = self.other_comp_info
        print(info)
        return info

    def __dict__(self):
        return self.to_dict()

    def __eq__(self, that):
        return self.to_dict() == that.to_dict()


from src import position

class Troop():

    def __init__(self, id, attributes, terrain, player, acted=True):
        self.id = id
        self.attributes = attributes
        self.position = None
        self.terrain = None
        self.player = player
        self.acted = acted

        self.move(terrain)

    def getId(self):
        return self.id

    def getPosition(self):
        return self.position

    def calcMove(self):
        movesList = []

        minX = self.position.x - self.attributes.speed
        maxX = self.position.x + self.attributes.speed + 1
        minY = self.position.y - self.attributes.speed
        maxY = self.position.y + self.attributes.speed + 1

        for x2 in range(minX, maxX):
            for y2 in range(minY, maxY):
                new_pos = position.Position(x2,y2)
                if self.validMove(new_pos):
                    movesList.append(new_pos)

        return movesList

    def validMove(self, position2):
        if (abs(self.position.x - position2.x) + abs(self.position.y - position2.y) <= self.attributes.speed):
            return True
        return False

    def move(self, terrain):
        if self.position:
            self.terrain.deallocate()
        self.terrain = terrain
        self.position = self.terrain.getPosition()
        self.terrain.allocate(self)


import numpy as np
import os
import csv

"""Robot Colour"""
#region
robot_colour = 2                 # 0 - red, 1 - green, 2- blue
other_robot_colour = 0
#endregion

"""Define Waypoints and Home"""
#region
red_base    = [1,1]
red_standby = [1,0.7]
blue_base   = [1,-1]
blue_standby = [1,-0.7]
home        = blue_base
standby     = blue_standby
other_robot_coordinates = red_base

path = np.array([home,home])
og_path = [home,home]
a = 0       # Path index
previous_coordinates = path[0]

dirname = os.path.dirname(__file__)
filename = os.path.join(dirname, '../paths/horizontal_traverse.txt')

with open(filename, 'r') as csvfile:
    csvreader = csv.reader(csvfile,  delimiter = ',')
    for row in csvreader:
        for j in range(len(row)):
            row[j] = float(row[j])
        point = (np.array(row))
        list_point = list(point)
        path = np.vstack([path, point])
        og_path.append(list_point)

timeout = 270    # Simulation time in seconds when robot quits everything to go home
#endregion
"""Path Planning Variables"""

# start and goal position
sx = 100  # [cm]
sy = 100  # [cm]
gx = -100  # [cm]
gy = 100 # [cm]

# set obstacle positions
ox, oy = [], []
exceed= 10 #12.5 - 7.5
#wall boundaries
for i in range(-120-exceed, 121+exceed):
    ox.append(i)
    oy.append(-120-exceed)
for i in range(-120-exceed, 121+exceed):
    ox.append(120+exceed)
    oy.append(i)
for i in range(-120-exceed, 121+exceed):
    ox.append(i)
    oy.append(120+exceed)
for i in range(-120-exceed, 121+exceed):
    ox.append(-120-exceed)
    oy.append(i)

"""State Variables"""
#region
unloading   = False         # Unloading state
obstacle    = False         # Obstacle detection state
obstacle1   = False
obstacle2   = False
goinghome   = False         # Going Home state
atHome      = True          # At Home state
blockcoords_sent = False
colour_determined = False
getting_away = False
other_robot_done = False
started_collecting = False
end_of_sweep_message_sent = False
#endregion

"""Miscellaneous Variables"""
#region
deg2rad     = 3.14159/180
#endregion

"""Constant Variables"""
#region
block_width = 5
wall_list_x = [-1.2, 1.2]
wall_list_z = [-1.2, 1.2]

MAX_SPEED = 10
#endregion

"""Obstacle variables"""
#region
#initialise 'active block coordinates'
list_of_blocks = []
#initialise block list
other_colour_blocks = []
indetermined_obs_blocks = []
#declare last cartesian bearing
cartesian_bearing = 0
#declare last known point
last_known_point = np.array([])
#declare turn variable to decide on path home
path_turns = 0
turnpoints = np.array([[-1, 1.0], [-1, 0.6],[5,5]]) #have a dummy at the end
#endregion