from bs4 import BeautifulSoup
import requests

#miner site
website = requests.get("https://www.asicminervalue.com/miners")

#get content
soup = BeautifulSoup(website.content, "html.parser")

#find id for table
#id = soup.find(id = 'datatable_profitability')

#for tr in soup.find_all('tr')[2:]:
#    tds = tr.find_all('td')
#    print ("Model: %s, Power: %s, ProfitPerDay: %s" % \
#          (tds[0].text, tds[3].text, tds[6].text))

with open('output.txt', 'w') as f:
    for tr in soup.find_all('tr')[1:]:
        tds = tr.find_all('td')
        f.write("%s, %s, %s\n" % \
              (tds[0].text, tds[3].text.strip("W"), tds[6].text.strip("/day")))

#print(id.text)
#print(soup.prettify())


import copy
import math
import os
import random

import numpy as np
from tqdm import tqdm

import tensorflow.compat.v1 as tf
from tensorflow.keras.models import Model, load_model
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.layers import *
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau
from sklearn.metrics import accuracy_score, f1_score, confusion_matrix

from model import baseModel

eps = 1e-5

def sigmoid(x):
    return 1 / (1 + math.exp(-x))

class RLe2e:

    def __init__(self, init_alpha, data_dir):
        self.data_dir = data_dir
        self.policy = self.build_model()

        # test_action is used to check whether the effect of the selector is consistent with the ideal effect
        noise_data = np.load(os.path.join(data_dir, 'data/noise_train_data.npz'))
        clean_data = np.load(os.path.join(data_dir, 'data/clean_test_data.npz'))
        self.train_data, self.train_label_, self.train_face_label = noise_data['data'], noise_data['label'], noise_data[
            'face_label']
        self.test_data, self.test_label = clean_data['data'], clean_data['label']
        self.test_action_data, self.test_action_label = noise_data['data'], noise_data['face_label']
        self.test_action_label[self.test_action_label <= 1] = 1
        self.test_action_label[self.test_action_label > 1] = 0
        # del noise_data
        del clean_data
        self.train_label_ -= 1
        self.test_label -= 1
        self.train_label = to_categorical(self.train_label_)
        self.test_label = to_categorical(self.test_label)
        self.test_action_label = to_categorical(self.test_action_label)

        self.num_data = self.train_label.shape[0]
        self.num_test_action_data = self.test_action_label.shape[0]
        self.num_labels = self.train_label.shape[1]
        self.num_each_label = np.sum(self.train_label, axis=0).astype(int)
        print("Distribution over labels: {}".format(self.num_each_label))

        self.classifier = load_model(os.path.join(data_dir, "shallow_cnn_noise.h5"))
        #         self.classifier = baseModel(self.data_shape)

        dense = self.classifier.layers[12]
        self.img_emb_dim = dense.output_shape[-1]

        self.data_shape = self.train_data[0].shape
        self.get_img_emb = tf.keras.backend.function([self.classifier.input], [dense.output])
        self.state_feat = np.zeros((self.num_data, self.img_emb_dim))
        self.test_emb = np.zeros((self.num_test_action_data, self.img_emb_dim))
        batch_size = 128
        lo, hi = 0, 0
        while lo < self.num_data:
            if lo + batch_size < self.num_data:
                hi = lo + batch_size
            else:
                hi = self.num_data
            label_emb = self.get_img_emb([self.train_data[lo: hi]])[0]
            self.state_feat[lo: hi] = label_emb
            lo += batch_size

        lo, hi = 0, 0
        while lo < self.num_test_action_data:
            if lo + batch_size < self.num_test_action_data:
                hi = lo + batch_size
            else:
                hi = self.num_test_action_data
            label_emb = self.get_img_emb([self.test_action_data[lo: hi]])[0]
            self.test_emb[lo: hi] = label_emb
            lo += batch_size

        self.score = np.zeros(self.num_data, dtype=np.float32)
        score = self.classifier.predict(self.train_data)

        self.label = np.argmax(self.train_label, axis=-1)
        self.pred_label = np.argmax(score, axis=-1)
        f1 = f1_score(self.label, self.pred_label, average='macro')
        for i in range(self.num_data):
            label = self.label[i]
            self.score[i] = score[i][label] - np.mean(score[i])

        self.tot_avg_score = np.mean(self.score)
        total_score = np.sum(self.score)
        self.total_score = []
        self.total_f1 = []
        self.total_score.append(total_score)
        self.total_f1.append(f1)
        print('avg_score:{} total_score:{}'.format(self.tot_avg_score, total_score))
        self.sample_time = 1

        self.best_ratio = 0.0
        self.best_f1 = 0.0

        self.final_selected_img = []
        self.selected_img_set = []
        self.selected_img = []
        self.rewards = []
        self.max_avg_score = self.tot_avg_score
        self.alpha = init_alpha / 16

        self.best_acc = 0

    def build_model(self):
        '''
        policy model
        '''
        inputs = Input(shape=(128,))
        dense1 = Dense(64, activation='relu')(inputs)
        dense2 = Dense(32, activation='relu')(dense1)
        dense3 = Dense(16, activation='relu')(dense2)
        dense4 = Dense(8, activation='relu')(dense3)
        output = Dense(2, activation='sigmoid')(dense4)

        model = Model(inputs=inputs, outputs=output)
        return model

    def update_score(self):
        best_path = os.path.join(self.data_dir, "shallow_cnn_noise_e2e.h5")
        self.classifier = load_model(best_path)
        score = self.classifier.predict(self.train_data)
        self.pred_label = np.argmax(score, axis=-1)
        dense = self.classifier.layers[12]
        self.get_img_emb = tf.keras.backend.function([self.classifier.input], [dense.output])
        batch_size = 128
        lo, hi = 0, 0
        while lo < self.num_data:
            if lo + batch_size < self.num_data:
                hi = lo + batch_size
            else:
                hi = self.num_data
            label_emb = self.get_img_emb([self.train_data[lo: hi]])[0]
            self.state_feat[lo: hi] = label_emb
            lo += batch_size

        lo, hi = 0, 0
        while lo < self.num_test_action_data:
            if lo + batch_size < self.num_test_action_data:
                hi = lo + batch_size
            else:
                hi = self.num_test_action_data
            label_emb = self.get_img_emb([self.test_action_data[lo: hi]])[0]
            self.test_emb[lo: hi] = label_emb
            lo += batch_size

    def get_action(self):
        action = np.zeros(self.num_data)
        for i in range(self.num_data):
            if self.pred_label[i] == self.label[i]:
                action[i] = 1
            else:
                action[i] = 0
        return action

    def decide_action(self):
        prob = self.policy.predict(self.state_feat)[:, 1]
        action = np.zeros(self.num_data)
        for i in range(self.num_data):
            if prob[i] > 0.49:
                action[i] = 1
            else:
                action[i] = 0
        return action

    def update_state_feat(self, label, k, before_selection):
        if before_selection:
            self.state_feat[:self.img_emb_dim] = self.train_emb_bags[label][k]
        else:
            num_selected = len(self.selected_img)
            tmp_feat = self.state_feat[self.img_emb_dim:]
            tmp_feat = (num_selected * tmp_feat + self.train_emb_bags[label][k]) / (num_selected + 1)
            self.state_feat[self.img_emb_dim:] = tmp_feat

    def selection(self, epochs=100):
        last_update_epoch = 0

        self.action_label = np.zeros(self.num_data)
        self.policy.compile(loss='binary_crossentropy', optimizer=Adam(lr=0.01), metrics=['accuracy'])
        for epoch in range(epochs):
            print("Epoch {}".format(epoch))
            if epoch > 0:
                self.update_score()
            rl_score = 0
            total_face = 0
            half_face = 0
            none_face = 0
            shuffle_idx = list(range(self.num_data))
            random.shuffle(shuffle_idx)
            self.final_selected_img.clear()
            self.rewards.clear()
            for j in range(self.sample_time):
                self.selected_img.clear()
                self.action_label = self.get_action()
                self.action_label = to_categorical(self.action_label, 2)
                distribute = np.sum(self.action_label, axis=0).astype(int)
                print(f"distribute: {distribute}")

                policy_name = 'pre-train-policy-new.h5'
                callbacks = [
                    EarlyStopping(monitor='val_loss', patience=5, verbose=0),
                    ModelCheckpoint(os.path.join(self.data_dir, policy_name), monitor='val_accuracy', mode='max',
                                    save_best_only=True)]
                print('policy_train...')
                self.policy.fit(x=self.state_feat, y=self.action_label, batch_size=16, epochs=10,
                                validation_data=(self.test_emb, self.test_action_label), verbose=2, callbacks=callbacks,
                                class_weight='auto')

            self.policy = load_model(os.path.join(self.data_dir, policy_name))
            self.final_action = self.decide_action()
            for k in shuffle_idx:
                face_label = self.train_face_label[k]
                if self.final_action[k] == 1:
                    self.final_selected_img.append(k)
                    if face_label == 0:
                        total_face += 1
                    elif face_label == 1:
                        half_face += 1
                    else:
                        none_face += 1
                    rl_score += self.score[k]

            if self.final_selected_img:
                rl_score /= len(self.final_selected_img)
                num_selected_img = len(self.final_selected_img)
                ratio = num_selected_img / sum(self.num_each_label)
                print("Selected: {} images ratio:{}".format(num_selected_img, ratio))

                train_data = np.zeros((num_selected_img, *self.data_shape))
                train_label = np.zeros(num_selected_img)
                for k, id_ in enumerate(self.final_selected_img):
                    train_data[k] = self.train_data[id_]
                    train_label[k] = self.train_label_[id_]
                opt = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999)
                model_path = os.path.join(self.data_dir, "../shallow_cnn_noise_e2e.h5")
                # model_path = os.path.join(self.data_dir, "shallow_cnn_noise_e2e_new.h5")
                callbacks = [
                    EarlyStopping(monitor='val_loss', patience=5, verbose=0),
                    ModelCheckpoint(model_path, monitor='val_accuracy', mode='max', save_best_only=True)]
                self.classifier.compile(optimizer=opt,
                                        loss='categorical_crossentropy',
                                        metrics=['accuracy'])

                train_label = to_categorical(train_label, 7)
                print('classifier training')
                self.classifier.fit(x=train_data, y=train_label,
                                    batch_size=16,
                                    validation_data=(self.test_data, self.test_label),
                                    epochs=100,
                                    verbose=2,
                                    callbacks=callbacks)
            else:
                rl_score = 1
                ratio = 0
            print("score: {:.4f}\tratio:{:.4f}".format(rl_score, ratio))
            score = self.classifier.predict(self.train_data)
            self.pred_label = np.argmax(score, axis=-1)
            for i in range(self.num_data):
                label = self.label[i]
                self.score[i] = score[i][label] - np.mean(score[i])

            self.tot_avg_score = np.mean(self.score)
            total_score = np.sum(self.score)
            self.total_score.append(total_score)
            f1 = f1_score(self.label, self.pred_label, average='macro')
            print(f'total_reward:{total_score}')
            self.total_f1.append(f1)

            if epoch - last_update_epoch > 16:
                break
            self.update_score()
            self.classifier = load_model(model_path)
            pred = self.classifier.predict(self.test_data, batch_size=32, verbose=1)
            y_true = np.argmax(self.test_label, axis=-1)
            y_pred = np.argmax(pred, axis=-1)
            val_acc = accuracy_score(y_true, y_pred)
            f1 = f1_score(y_true, y_pred, average='macro')

            if self.best_acc <= val_acc:
                print("Weight updated!")
                last_update_epoch = epoch
                self.best_acc = val_acc
                self.best_f1 = f1
                self.best_ratio = ratio
                self.classifier.save('shallow_cnn_nise_best_e2e.h5')
                self.policy.save('policy_noise_e2e_best.h5')
            print("val_acc: {:.4f}\tmax val_acc: {:.4f}\t f1: {:.4f}\t ratio:{:.4f}".format(val_acc, self.best_acc,
                                                                                self.best_f1,
                                                                                self.best_ratio))

if __name__ == '__main__':
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = "1"
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    session = tf.Session(config=config)
    tf.keras.backend.get_session(session)

    data_dir = '/home/lihuadong/workspace/RL/'
    alpha = 0.003
    model = RLe2e(alpha, data_dir)
    model.selection()


import json
from glob import glob
import os
import shutil

import pandas as pd

ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
METADATA_FILENAME = "metadata.json"
SRC_DIR = os.path.join(ROOT_DIR, "src")

DATAFRAMES_DIRECTORY = os.path.join(ROOT_DIR, "dataframes")
PAIR_DATAFRAMES_UPDATED_DIRECTORY = os.path.join(DATAFRAMES_DIRECTORY, "dataframes_pairs_updated")
PAIR_REAL_DATAFRAME = os.path.join(PAIR_DATAFRAMES_UPDATED_DIRECTORY, "real_crops.csv")
PAIR_FAKE_DATAFRAME = os.path.join(PAIR_DATAFRAMES_UPDATED_DIRECTORY, "fake_crops.csv")

MODELS_DIRECTORY = os.path.join(ROOT_DIR, "trained_models")

VALIDATION_DATAFRAME_PATH = os.path.join(DATAFRAMES_DIRECTORY, "faces_validation_new.csv")
VALIDATION_FACES_DIRECTORY = os.path.join(ROOT_DIR, "validation_faces")
TESTING_DIRECTORY = os.path.join(ROOT_DIR, "data", "test")
TESTING_LABELS = os.path.join(TESTING_DIRECTORY, "labels.csv")

TRAIN_DIRECTORY = os.path.join(ROOT_DIR, "data", "train")
TRAIN_FAKE_FACES_DIRECTORY = os.path.join(ROOT_DIR, "training_fake_faces")
TRAIN_REAL_FACES_DIRECTORY = os.path.join(ROOT_DIR, "training_real_faces")

RESNET_FOLDER = os.path.join(ROOT_DIR, "special_models", "resnet_model")

MASKS_FOLDER = os.path.join(ROOT_DIR, "mask_xray_fake_new")
MASK_REAL_PATH = os.path.join(ROOT_DIR, "mask_xray_real", "black_mask.png")

HRNET_CONFIG_FILE = os.path.join(SRC_DIR, "xray_config", "hrnet_seg.yaml")

PRIVATE_TESTING_DIRECTORY = os.path.join(ROOT_DIR, "aws")
PRIVATE_TESTING_LABELS_PATH = os.path.join(PRIVATE_TESTING_DIRECTORY, "labels_updated.csv")

SEQUENCE_DATAFRAMES_FOLDER = os.path.join(DATAFRAMES_DIRECTORY, "dataframes_3dcnn")
SEQUENCE_DATAFRAME_PATH = os.path.join(SEQUENCE_DATAFRAMES_FOLDER, "3dcnn_dataframe_final.csv")
SEQUENCE_DATAFRAME_TESTING_PATH = os.path.join(SEQUENCE_DATAFRAMES_FOLDER, "3dtesting_dataframe_final.csv")

COVIAR_DATAFRAME_FOLDER = os.path.join(DATAFRAMES_DIRECTORY, "coviar_dataframes")
COVIAR_DATAFRAME_PATH = os.path.join(COVIAR_DATAFRAME_FOLDER, "coviar_dataframe.csv")
COVIAR_TEST_DATAFRAME_PATH = os.path.join(COVIAR_DATAFRAME_FOLDER, "coviar_test_dataframe.csv")

FACE_XRAY_FULL_DATAFRAME = os.path.join(DATAFRAMES_DIRECTORY, "full_facexray_df.csv")

PREVIEW_MODELS = os.path.join(ROOT_DIR, "preview", "trained_models")
PREVIEW_DATAFRAMES = os.path.join(ROOT_DIR, "preview", "dataframes")
PREVIEW_TEST = os.path.join(PREVIEW_DATAFRAMES, "test_dataframe.csv")
PREVIEW_TRAIN = os.path.join(PREVIEW_DATAFRAMES, "training_dataframe.csv")

MEAN = [0.485, 0.456, 0.406]
STD = [0.229, 0.224, 0.225]

def get_specific_video_names(folder, number=20, label='FAKE'):
    path = os.path.join(folder, METADATA_FILENAME)
    names = []
    with open(path) as f:
        data = json.load(f)
        for id in data:
            if data[id]["label"] == label:
                names.append(id)

            if len(names) == number:
                return names

    return names

def get_number_of_real_videos_in_folders():
    directories_to_traverse = glob(os.path.join(TRAIN_DIRECTORY, '*', '*') + r'\\')
    list_real_in_folders = []
    for directory in directories_to_traverse:
        path = os.path.join(directory, METADATA_FILENAME)
        current_real = 0
        with open(path) as f:
            data = json.load(f)
            for id in data:
                if data[id]["label"] == "REAL":
                    current_real += 1

        list_real_in_folders.append(current_real)

    return list_real_in_folders

def get_original_videos_of_fake_videos(fake_videos, metadata_folder):
    metadata_path = os.path.join(metadata_folder, METADATA_FILENAME)
    videos_to_fakes = dict()
    with open(metadata_path) as f:
        data = json.load(f)
        for video in fake_videos:
            if data[video]["original"]:
                videos_to_fakes[video] = data[video]["original"]

    return videos_to_fakes

def copy_specific_videos(src_folder, src_names, destination_folder):
    for name in src_names:
        path = os.path.join(src_folder, name)
        destination_path = os.path.join(destination_folder, name)
        shutil.copyfile(path, destination_path)

def write_list_to_file(list, dest, label):
    path = os.path.join(dest, label)
    with open(path, 'w') as f:
        for name in list:
            f.write("%s\n" % name)

def read_txt_as_list(filename):
    with open(filename) as f:
        lines = f.read().splitlines()

    return lines

def list_to_dataframe(video_names, column_name="videoname"):
    return pd.DataFrame(video_names, columns=[column_name])

def get_all_files_with_extension_from_folder(folder, extension):
    video_files = []
    for file in os.listdir(folder):
        if file.endswith(extension):
            video_files.append(file)

    return video_files


import pypylon.pylon as py

cam = py.InstantCamera(py.TlFactory.GetInstance().CreateFirstDevice())
cam.Open()
print(cam.PixelFormat.Value)

# cam.StartGrabbing()
# with cam.RetrieveResult(1000) as img:
with cam.GrabOne(10000) as img:
	image=img.Array
	import numpy as np
	mean=np.mean(image)
	print(mean)

import matplotlib.pyplot as plt
plt.imshow(image)
plt.show()

# import cv2
# rgb_image=cv2.cvtColor(image,cv2.COLOR_BayerRG2BGR)
# plt.imshow(rgb_image)
# plt.show()
# cam.Close()


import sqlite3
import tkinter as tk
import customtkinter as ctk
import smtplib
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
from email.mime.base import MIMEBase
from email import encoders

ctk.set_appearance_mode("dark")  # Modes: system (default), light, dark
ctk.set_default_color_theme("dark-blue")  # Themes: blue (default), dark-blue, green

root = ctk.CTk()
root.title("Nihongo Personal Dictionary")
root.geometry("650x300") #(x,y)
root.wm_iconphoto(False,tk.PhotoImage(file="images/window_icon.png"))
font_data = ("Raleway",9)
status_label = ctk.CTkLabel(master=root, text="[STATUS]", text_font=font_data)
status_label.place(relx=0.5,y=20,anchor=tk.CENTER)

file_name="Nihongo_Personal_Dictionary"
status=""
clr_red="#ba0502"
clr_green="#02a12c"
sort_by_var=1 # 1=by English, 0=by Romaji

def sort_by_english_func():
    sort_by_var=1

def sort_by_romaji_func():
    sort_by_var=0

def display_status(status,color):
    if color=="g":
        text_clr=clr_green
    elif color=="r":
        text_clr=clr_red
    status_label.configure(text=status,text_color=text_clr)

try:
    sqliteConnection = sqlite3.connect('nihongo.db')
    cursor = sqliteConnection.cursor()
    cursor.execute("CREATE TABLE IF NOT EXISTS words(english VARCHAR(20), romaji VARCHAR(50));")
    status="Connected to the database"
    display_status(status,"g")
except:
    status="Could not connect to the database"
    display_status(status,"r")

def sendEmail():
    try:
        write_file()
        credentials=[]
        with open("credentials.txt","r") as f:
            credentials=f.readline().split(",")
        fromaddr = credentials[0]
        toaddr = "paramjangale@gmail.com"

        # instance of MIMEMultipart
        msg = MIMEMultipart()

        # storing the senders email address
        msg['From'] = fromaddr

        # storing the receivers email address
        msg['To'] = toaddr

        # storing the subject
        msg['Subject'] = "Nihongo Personal Dictionary"

        # open the file to be sent
        filename = "Japanese_Personal_Dictionary.txt"
        attachment = open("Japanese_Personal_Dictionary.txt", "rb")

        # instance of MIMEBase and named as p
        p = MIMEBase('application', 'octet-stream')

        # To change the payload into encoded form
        p.set_payload((attachment).read())

        # encode into base64
        encoders.encode_base64(p)

        p.add_header('Content-Disposition', "attachment; filename= %s" % filename)

        # attach the instance 'p' to instance 'msg'
        msg.attach(p)

        # creates SMTP session
        s = smtplib.SMTP('smtp.gmail.com', 587)

        # start TLS for security
        s.starttls()

        # Authentication
        sending_status=s.login(fromaddr, credentials[1])

        # Converts the Multipart msg into a string
        text = msg.as_string()

        # sending the mail
        s.sendmail(fromaddr, toaddr, text)

        # terminating the session
        s.quit()
        if sending_status[0]==235:
            status="Sent successfully"
            display_status(status,"g")
        else:
            status="Couldn't send Email"
            display_status(status,"r")

    except:
        status="Something went wrong"
        display_status(status,"r")

def addToDB():
    try:
        english_val=english_var.get()
        romaji_val=romaji_var.get()
        cursor.execute("Insert into words (english,romaji) values (?,?);",(english_val,romaji_val))
        sqliteConnection.commit()
        status="Word added successfully"
        display_status(status,"g")
        english_text.delete(0,"end")
        romaji_text.delete(0,"end")
        english_text.focus_set()
    except:
        status="Something went wrong"
        display_status(status,"r")

def clearDB():
    def clearDB_yes():
        cursor.execute("DELETE from words;")
        sqliteConnection.commit()
        confirm_window.destroy()
        status="Dictionary cleared successfully"
        display_status(status,"g")

    def clearDB_no():
        confirm_window.destroy()

    confirm_window=ctk.CTkToplevel(root)
    confirm_window.geometry("300x120")
    confirm_window.title("Clear Dictionary?")
    confirm_window.wm_iconphoto(False,tk.PhotoImage(file="images/window_icon.png"))
    label=ctk.CTkLabel(master=confirm_window,text="Do you want to clear the dictionary?", text_font=font_data)
    label.place(relx=0.5,y=40,anchor=tk.CENTER)
    yes_btn = ctk.CTkButton( master=confirm_window,
                            text="Yes",
                            compound=ctk.RIGHT,
                            command=clearDB_yes,
                            fg_color=clr_green,
                            height=30,
                            width=40)
    yes_btn.place(relx=0.4,y=80,anchor=tk.CENTER)
    no_btn = ctk.CTkButton( master=confirm_window,
                            text="No",
                            compound=ctk.RIGHT,
                            command=clearDB_no,
                            fg_color=clr_red,
                            height=30,
                            width=40)
    no_btn.place(relx=0.6,y=80,anchor=tk.CENTER)

def write_file():
    try:
        with open(file_name+".txt",'w') as f:
            sort_by_val=sort_by.get()
            if sort_by_val:
                cursor.execute("SELECT * FROM words ORDER BY english;")
            else:
                cursor.execute("SELECT * FROM words ORDER BY romaji;")

            rows=cursor.fetchall()
            for row in rows:
                f.write(row[0].ljust(25)+"- "+row[1]+"\n")
        status="Added to file successfully"
        display_status(status,"g")
    except:
        status="Something went wrong"
        display_status(status,"r")

text_frame=ctk.CTkFrame(root,bg_color="#ba0502")

romaji_label=ctk.CTkLabel(master=root,text="Romaji -", text_font=font_data)
romaji_label.place(relx=0.35,y=74,anchor=tk.CENTER)

english_label=ctk.CTkLabel(master=root,text="English -", text_font=font_data)
english_label.place(relx=0.35,y=47,anchor=tk.CENTER)

enter_btn = ctk.CTkButton( master=root,
                            text="Enter",
                            compound=ctk.RIGHT,
                            command=addToDB,
                            height=30,
                            width=95 )
enter_btn.place(relx=0.5,y=130,anchor=tk.CENTER)

send_email_btn = ctk.CTkButton( master=root,
                            text="Send Email",
                            compound=ctk.RIGHT,
                            command=sendEmail,
                            height=30,
                            width=120 )
send_email_btn.place(relx=0.5,y=170,anchor=tk.CENTER)

root.bind('<Return>', lambda e: addToDB())

clear_db_btn = ctk.CTkButton( master=root,
                            text="Clear Dictionary",
                            compound=ctk.RIGHT,
                            command=clearDB,
                            height=30,
                            width=120 )
clear_db_btn.place(relx=0.7,y=130,anchor=tk.CENTER)

write_file_btn = ctk.CTkButton(master=root,
                            text="Add to File",
                            compound=ctk.RIGHT,
                            command=write_file,
                            height=30,
                            width=120 )
write_file_btn.place(relx=0.3,y=130,anchor=tk.CENTER)

english_var=tk.StringVar()
english_text = tk.Entry(root, textvariable=english_var, width = 20)
english_text.place(relx=0.5,y=70,anchor=tk.CENTER)

romaji_var=tk.StringVar()
romaji_text = tk.Entry(root, textvariable=romaji_var, width = 20)
romaji_text.place(relx=0.5,y=110, anchor=tk.CENTER)

sort_by = tk.IntVar()
sort_by_english = ctk.CTkRadioButton(root, text='Sort by English words',variable=sort_by, value=1)
sort_by_romaji = ctk.CTkRadioButton(root, text='Sort by Romaji words', variable=sort_by, value=0)
sort_by_english.place(relx=0.15,y=200,anchor=tk.CENTER)
sort_by_romaji.place(relx=0.15,y=230,anchor=tk.CENTER)

root.mainloop()


"""
Indexation model using the structural approach

StructuralModel.gms: Indexation model using the structural approach
Consiglio, Nielsen and Zenios.
PRACTICAL FINANCIAL OPTIMIZATION: A Library of GAMS Models, Section 7.2.1
Last modified: Apr 2008.
"""
from pathlib import Path

import numpy as np

from gamspy import Alias
from gamspy import Container
from gamspy import Equation
from gamspy import Model
from gamspy import Number
from gamspy import Parameter
from gamspy import Set
from gamspy import Sum
from gamspy import Variable

def main():
    gdx_file = str(Path(__file__).parent.absolute()) + "/InputData.gdx"
    m = Container(load_from=gdx_file)

    # SETS #
    BB, EE, BxE = m.getSymbols(["BB", "EE", "BxE"])

    # PARAMETERS #
    data = m.getSymbols("data")[0]

    # SETS
    DD = Set(
        m,
        name="DD",
        records=["LOW", "MEDIUM", "HIGH"],
        description="Duration levels",
    )
    BxD = Set(
        m, name="BxD", domain=[BB, DD], description="Bonds by Duration levels"
    )

    i = Alias(m, name="i", alias_with=BB)
    e = Alias(m, name="e", alias_with=EE)
    k = Alias(m, name="k", alias_with=DD)

    # Partition the set of bonds by duration levels

    BxD[i, "LOW"] = data[i, "Duration"] <= 3

    BxD[i, "MEDIUM"] = (data[i, "Duration"] > 3) & (data[i, "Duration"] <= 7)

    BxD[i, "HIGH"] = data[i, "Duration"] > 7

    # The index dollar duration

    # SCALARS #
    IndexDollarDuration = Parameter(m, name="IndexDollarDuration", records=820)

    # PARAMETERS #
    DurationWeights = Parameter(
        m,
        name="DurationWeights",
        domain=[k],
        records=np.array([0.3, 0.2, 0.5]),
    )
    CurrencyWeights = Parameter(
        m,
        name="CurrencyWeights",
        domain=[e],
        records=np.array([0.6, 0.3, 0.1]),
    )

    # VARIABLE #
    z = Variable(m, name="z", type="free")
    x = Variable(m, name="x", type="positive", domain=[i])

    # EQUATIONS #
    ObjDef = Equation(
        m,
        name="ObjDef",
        type="regular",
        description="Linear approximation of the portfolio yield",
    )
    ObjDefTwo = Equation(
        m,
        name="ObjDefTwo",
        type="regular",
        description="NonLinear approximation of the portfolio yield",
    )
    NormalCon = Equation(
        m,
        name="NormalCon",
        type="regular",
        description="Equation defining the budget contraint",
    )
    DurationMatch = Equation(
        m,
        name="DurationMatch",
        type="regular",
        description=(
            "Equation matching the dollar duration of the portfolio and of the"
            " index"
        ),
    )
    CurCons = Equation(
        m,
        name="CurCons",
        type="regular",
        domain=[e],
        description="Equation matching the index currency allocation",
    )
    DurCons = Equation(
        m,
        name="DurCons",
        type="regular",
        domain=[k],
        description="Equation matching the index duration allocation",
    )

    ObjDef.expr = (
        z
        == Number(-1)
        * Sum(
            i, data[i, "Duration"] * data[i, "Price"] * data[i, "Yield"] * x[i]
        )
        / IndexDollarDuration
    )

    DurationMatch.expr = (
        Sum(i, data[i, "Duration"] * data[i, "Price"] * x[i])
        == IndexDollarDuration
    )

    CurCons[e] = Sum(i.where[BxE[i, e]], x[i]) == CurrencyWeights[e]

    DurCons[k] = Sum(i.where[BxD[i, k]], x[i]) == DurationWeights[k]

    NormalCon.expr = Sum(i, x[i]) == 1.0

    IndexFund = Model(
        m,
        name="IndexFund",
        equations=[ObjDef, DurationMatch, CurCons, DurCons, NormalCon],
        problem="LP",
        sense="MAX",
        objective=z,
    )

    IndexFund.solve()

    x_l = {i: round(j, 3) for i, j in x.toDict().items() if j != 0}
    print("\nSolving 'IndexFund' Model:")
    print("x: ", x_l)
    print("z: ", round(z.toList()[0], 3))

    # If we let the duration of the portfolio unconstrained, the objective
    # function turns to nonlinear as the variable x[i] will appear in the denominator

    ObjDefTwo.expr = z == Number(-1) * Sum(
        i, data[i, "Duration"] * data[i, "Price"] * data[i, "Yield"] * x[i]
    ) / Sum(i, data[i, "Duration"] * data[i, "Price"] * x[i])

    NonLinearIndexFund = Model(
        m,
        name="NonLinearIndexFund",
        equations=[ObjDefTwo, CurCons, DurCons, NormalCon],
        problem="NLP",
        sense="MAX",
        objective=z,
    )

    NonLinearIndexFund.solve()

    x_l = {i: round(j, 3) for i, j in x.toDict().items() if j != 0}
    print(f"\n{'-'*25}\n")
    print("Solving 'NonLinearIndexFund' Model:")
    print("x: ", x_l)
    print("z: ", round(z.toList()[0], 3), "\n")

if __name__ == "__main__":
    main()


import numpy as np

from experiments.geneticProgramming.tree import Tree
from experiments.geneticProgramming.packing import Packing

from experiments.geneticProgramming.operations import add, multipy, devide, square, power, subtract

class ObjectiveFunction(object):
    def __init__(self):
        self.funcSet = [
            [add, 2, '+'],
            [subtract, 2, '-'],
            [multipy, 2, '*'],
            [devide, 2, '/'],
            [square, 1, '^2'],
            [power, 2, '^'],
        ]
        self.termSet = [None, None, None, None, None, None, None]

        self.population = []
        self.size = 30
        self.maxDepth = 5
        self.minDepth = 2
        self.tournamentSize = 3
        self.generations = 30

        self.problems = [
            ['experiments/problems/bin1data/N1C1W1_A.BPP', 25],
            ['experiments/problems/bin1data/N1C1W1_O.BPP', 32],
            ['experiments/problems/bin1data/N1C1W1_P.BPP', 26],
            ['experiments/problems/bin1data/N1C1W1_T.BPP', 28]
        ]

        self.testproblems = [
            ['experiments/problems/bin1data/N1C1W1_G.BPP', 25],
            ['experiments/problems/bin1data/N1C1W1_K.BPP', 26],
            ['experiments/problems/bin1data/N1C1W1_S.BPP', 28],
            ['experiments/problems/bin1data/N1C1W1_F.BPP', 27]
        ]

    def createPopulation(self):
        perlayer = int(self.size / (self.maxDepth - self.minDepth) / 2)

        for x in range(self.minDepth, self.maxDepth):
            for y in range(perlayer):
                fullTree = Tree(x, x, self.funcSet, self.termSet)
                fullTree.full()
                self.population.append(fullTree)

                growTree = Tree(x, x, self.funcSet, self.termSet)
                growTree.grow()
                self.population.append(growTree)

    def tournamentSelector(self, reverse=False):
        # Make the tournament
        random_indicies = np.random.randint(len(self.population), size=self.tournamentSize).tolist()
        tournament = []
        for idx, val in np.ndenumerate(random_indicies):
            tournament.append(self.population[val])

        # Run the tournament
        results = []
        bins = []
        random_problem = np.random.randint(len(self.problems))
        for val in tournament:
            result, bin = self.getFitness(val, self.problems[random_problem])
            results.append(result)
            bins.append(bin)

        # Find the worst or best
        bins = np.array(bins)

        bestTorPos = np.argmin(bins)
        worstTorPos = np.argmax(bins)

        best = self.population[random_indicies[bestTorPos]]
        bestPos = random_indicies[bestTorPos]
        bestResult = bins[bestTorPos]

        worst = self.population[random_indicies[worstTorPos]]
        worstPos = random_indicies[worstTorPos]
        worstResult = bins[worstTorPos]

        return (best, bestPos, bestResult), (worst, worstPos, worstResult), bins / self.problems[random_problem][1]

    def getFitness(self, candidate, problem):
        packing = Packing()

        return packing.run(candidate, problem[0], problem[1])

    def addToList(self, results, arr):
        for i in arr:
            results.append(i)
        return results

    def doCrossover(self):
        results = []
        bestDetails1, worstDetails1, arr = self.tournamentSelector()
        results = self.addToList(results, arr)
        bestDetails2, worstDetails2, arr = self.tournamentSelector()
        results = self.addToList(results, arr)

        c1 = bestDetails1[0].copy()
        c2 = bestDetails2[0].copy()

        c1branch = c1.getBranch(c2)
        c2branch = c2.getBranch(c1)

        c1.setBranch(c2branch)
        c2.setBranch(c1branch)

        self.population[worstDetails1[1]] = c1
        self.population[worstDetails2[1]] = c2

        results = np.array(results)

        return results.mean(), results.std()

    def run(self):
        self.createPopulation()

        meanList = []
        for i in range(self.generations):
            mean, std = self.doCrossover()

            meanList.append(mean)

            print(mean)
            print('('+ str(std) +')')

        meanList = np.array(meanList)
        print("meanlist mean: ", meanList.mean())
        print("meanlist std: ", meanList.std())

        import matplotlib.pyplot as plt

        iterations = []
        for i in range(self.generations):
            iterations.append(i)

        plt.close()

        fig = plt.figure()
        plt.grid(1)
        plt.xlim([0, self.generations])
        plt.ion()
        plt.xlabel('Generations')
        plt.ylabel('Fitness')

        plots = []
        descriptions = []

        plots.append(plt.plot(iterations, meanList, 'b-', linewidth=1, markersize=3)[0])
        descriptions.append("Solve ratio")

        plt.legend(plots, descriptions)
        fig.savefig('Result.png')
        plt.show()

        plt.close()

        results = []
        bins = []
        random_problem = np.random.randint(len(self.testproblems))
        for val in self.population:
            result, bin = self.getFitness(val, self.testproblems[random_problem])
            results.append(result)
            bins.append(bin)

        bins = np.array(bins)
        pos = np.argmin(bins)

        print("testing mean: ", meanList.mean())
        print("testing std: ", meanList.std())

        print(self.population[pos].toString())
        print(pos)
        print(bins[pos], '/', self.problems[random_problem][1])

        return 0

oj = ObjectiveFunction()

oj.run()


# -*- coding: UTF-8 -*-
from behave import *
from nose.tools import assert_equals
from time import sleep
from selenium.webdriver.common.by import By
from commonSteps import*
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

@when(u'点击顶部页面元素{topEle}，进入对应页面{topPage}')
def verify_top_tab(context, topEle, topPage):
    # 获取顶部标签，并点击进入对应页面
    find_element_by_link_text(context, topEle).click()
    # 不同页面标志文本，目前首页和技术文档sidebar_head标记，关于我们是sidebar_title
    if topEle in ['产品', '技术文档']:
        headText = find_element_by_class_name(context, "sidebar_head").text
    elif topEle == '关于我们':
        headText = find_element_by_class_name(context, "sidebar-title").text
    elif topEle == '解决方案':
        headText = find_element_by_xpath(
            context, "//div[@class='resBanner_cont']/h1").text
    elif topEle == '技术支持':
        headText = find_element_by_class_name(
            context, "site-header__title").text
    assert_equals(headText, topPage)

@when(u'点击滚动页面{scrollBanner}，进入对应页面{scrollPage}')
def verify_scollBanner(context, scrollBanner, scrollPage):
    # 回到首页
    find_element_by_class_name(context, "top_logo").click()
    # sleep(2)
    if scrollBanner == 'dns':
        locator = (By.CLASS_NAME, "banner_item1")
        bannerItem = find_element_by_class_name(context, "banner_item1")
    elif scrollBanner == 'ndc':
        locator = (By.CLASS_NAME, "banner_item2")
        bannerItem = find_element_by_class_name(context, "banner_item2")
    elif scrollBanner == 'ddos':
        locator = (By.CLASS_NAME, "banner_item3")
        bannerItem = find_element_by_class_name(context, "banner_item3")
    elif scrollBanner == 'wsa':
        locator = (By.CLASS_NAME, "banner_item4")
        bannerItem = find_element_by_class_name(context, "banner_item4")
    # 等待元素出现
    is_visible(context, locator)
    # 进入产品详情页
    bannerItem.click()
    sleep(1)
    # 窗口句柄跳到当前页面
    switch_to_window(context, 1)
    productDesc = find_element_by_xpath(
        context, "//div[@class='product_banner']//h2").text
    assert_equals(productDesc, scrollPage)

@when(u'点击推荐产品{hotPro}了解详情，进入对应详情页面')
def verify_hotPro_detail(context, hotPro):
    # 回到首页
    find_element_by_class_name(context, "top_logo").click()
    sleep(1)
    hotProDetailItems = find_elements_by_class_name(
        context, "hotProduct_info")
    if hotPro == '授权DNS':
        hotProDetailItems[0].click()
    elif hotPro == '云防御IP':
        hotProDetailItems[2].click()
    elif hotPro == '安全检测':
        hotProDetailItems[3].click()

    sleep(2)
    switch_to_window(context, 1)
    productDesc = find_element_by_xpath(
        context, "//div[@class='product_banner']//h2").text
    assert_equals(productDesc, hotPro)

@when(u'点击咨询或者购买{hotPro}，进入工单页面或者购买页面{hotProBuyPage}')
def verify_hotPro_Buy(context, hotPro, hotProBuyPage):
    # 回到首页
    find_element_by_class_name(context, "top_logo").click()
    sleep(1)
    hotProBuyItems = find_elements_by_class_name(
        context, "hotProduct_buy")
    if hotPro == '授权DNS':
        hotProBuyItems[0].click()
        sleep(2)
        switch_to_window(context, 2)
        hotProBuy = find_element_by_class_name(context, "buy_head").text
    elif hotPro == '云防御IP':
        hotProBuyItems[2].click()
        hotProBuy = find_element_by_class_name(context, "buy_head").text
    elif hotPro == '安全检测':
        hotProBuyItems[3].click()
        hotProBuy = find_element_by_xpath(
            context, "//div[@class='product_banner']//h2").text

    # hotProBuy = find_element_by_class_name(context, "buy_head")
    assert_equals(hotProBuy, hotProBuyPage)

@when(u'点击行业{solutionItem}和了解详情，均进入对应详情页面{solutionPage}')
def solution(context, solutionItem, solutionPage):
    if solutionItem == '金融行业':
        solutionBg = find_element_by_class_name(
            context, "solution_mainBg1")
        # 找到了解详情元素
        # solutionLink = find_element_by_xpath(
        #     context,
        #     "//section[@class='solution']//a[@href='/resolve/finance']/a[@class='solution_link']")
        # locator = (
        #     By.XPATH,
        #     "//section[@class='solution']//a[@href='/resolve/finance']/a[@class='solution_link']")
    elif solutionItem == '企业门户':
        solutionBg = find_element_by_class_name(
            context, "solution_mainBg2")
    elif solutionItem == '教育行业':
        solutionBg = find_element_by_class_name(
            context, "solution_mainBg3")
    elif solutionItem == '游戏行业':
        solutionBg = find_element_by_class_name(
            context, "solution_mainBg4")
    # 获取解决方案元素，点击进入解决方案详情页
    solutionBg.click()
    sleep(3)
    switch_to_window(context, 1)
    headText = find_element_by_xpath(
        context, "//div[@class='resBanner_cont']/h1").text
    assert_equals(headText, solutionPage)

    # # 点击首页，回到首页，然后点击了解详情
    # find_element_by_link_text(context, "首页").click()
    # # time.slepp(3)
    # is_visible(context, locator)
    # solutionLink.click()
    # # sleep(3)
    # switch_to_window(context, 1)
    # headText = find_element_by_xpath(
    #     context, "//div[@class='resBanner_cont']/h1").text
    # assert_equals(headText, "")

@when(u'点击行业动态下面的查看全部，将进入行业动态页面')
def check_news(context):
    find_element_by_link_text(context, "查看全部").click()
    # sleep(1)
    switch_to_window(context, 1)
    sleep(3)
    newsPage = find_elements_by_xpath(
        context, "//span[@itemprop='name']")[1].text
    assert_equals(newsPage, "行业动态")

@when(u'点击底部导航栏标签{footerItem}, 进入对应页面{footerPage}')
def footer_page(context, footerItem, footerPage):
    xpathEle = "//ul[@class='footer_list']//a[text()='" + footerItem + "']"
    find_element_by_xpath(
        context, xpathEle).click()
    switch_to_window(context, 1)
    sleep(1)

    if footerItem in ['解析入门', '功能介绍', '常见故障', '账号相关', '销售相关', 'API相关']:
        # sleep(3)
        # 因为解析入门页面打开太慢，所以等帮助与文档显示出来之后再捕捉想捕捉的元素
        locator = (By.CLASS_NAME, "mysite-header__title")
        is_visible(context, locator)
        # switch_to_window(context, 1)
        footerPageHeader = find_elements_by_xpath(
            context, "//span[@itemprop='name']")[1].text
    elif footerItem in ['公司简介', '加入我们', '联系我们']:
        # switch_to_window(context, 1)
        footerPageHeader = find_element_by_xpath(
            context, "//div[@class='comm-cont']//h4").text
    else:
        # switch_to_window(context, 1)
        footerPageHeader = find_element_by_xpath(
            context, "//div[@class='product_banner']//h2").text
    assert_equals(footerPageHeader, footerPage)


from __future__ import print_function

import config
import os
import subprocess

# Find the makefile/blueprint based on the benchmark, and make a copy of
# it for restoring later.
def backup_file(bench, file_type):
    mk_file = os.path.join(config.android_home, config.bench_dict[bench],
                           'Android.' + file_type)
    try:
        # Make a copy of the makefile/blueprint so that we can recover it after
        # building the benchmark
        subprocess.check_call([
            'cp', mk_file,
            os.path.join(config.android_home, config.bench_dict[bench],
                         'tmp_makefile')
        ])
    except subprocess.CalledProcessError():
        raise OSError('Cannot backup Android.%s file for %s' % (file_type,
                                                                bench))

# Insert lines to add LOCAL_CFLAGS/LOCAL_LDFLAGS to the benchmarks
# makefile/blueprint
def replace_flags(bench, android_type, file_type, cflags, ldflags):
    # Use format ["Flag1", "Flag2"] for bp file
    if file_type == 'bp':
        if cflags:
            cflags = '\", \"'.join(cflags.split())
        if ldflags:
            ldflags = '\", \"'.join(ldflags.split())

    if not cflags:
        cflags = ''
    else:
        cflags = '\"' + cflags + '\",'
    if not ldflags:
        ldflags = ''
    else:
        ldflags = '\"' + ldflags + '\",'

    # Two different diffs are used for aosp or internal android repo.
    if android_type == 'aosp':
        bench_diff = bench + '_flags_aosp.diff'
    else:
        bench_diff = bench + '_flags_internal.diff'

    # Replace CFLAGS_FOR_BENCH_SUITE marker with proper cflags
    output = ''
    with open(bench_diff) as f:
        for line in f:
            line = line.replace('CFLAGS_FOR_BENCH_SUITE', cflags)
            line = line.replace('LDFLAGS_FOR_BENCH_SUITE', ldflags)
            output += line

    with open('modified.diff', 'w') as f:
        f.write(output)

def apply_patches(bench):
    bench_dir = os.path.join(config.android_home, config.bench_dict[bench])
    bench_diff = 'modified.diff'
    flags_patch = os.path.join(
        os.path.dirname(os.path.realpath(__file__)), bench_diff)
    try:
        subprocess.check_call(['git', '-C', bench_dir, 'apply', flags_patch])
    except subprocess.CalledProcessError:
        raise OSError('Patch for adding flags for %s does not succeed.' % bench)

def replace_flags_in_dir(bench, cflags, ldflags):
    bench_mk = os.path.join(config.android_home, config.bench_dict[bench],
                            'Android.mk')

    if not cflags:
        cflags = ''
    if not ldflags:
        ldflags = ''

    output = ''
    with open(bench_mk) as f:
        for line in f:
            line = line.replace('$(CFLAGS_FOR_BENCH_SUITE)', cflags)
            line = line.replace('$(LDFLAGS_FOR_BENCH_SUITE)', ldflags)
            output += line
    with open(bench_mk, 'w') as f:
        f.write(output)

def add_flags_Panorama(cflags, ldflags):
    backup_file('Panorama', 'mk')
    replace_flags_in_dir('Panorama', cflags, ldflags)

def add_flags_Synthmark(cflags, ldflags):
    backup_file('Synthmark', 'mk')
    replace_flags_in_dir('Synthmark', cflags, ldflags)

def add_flags_Skia(cflags, ldflags):
    backup_file('Skia', 'bp')
    replace_flags('Skia', config.android_type, 'bp', cflags, ldflags)
    apply_patches('Skia')

def add_flags_Binder(cflags, ldflags):
    backup_file('Binder', 'bp')
    replace_flags('Binder', config.android_type, 'bp', cflags, ldflags)
    apply_patches('Binder')

def add_flags_Hwui(cflags, ldflags):
    backup_file('Hwui', 'bp')
    replace_flags('Hwui', config.android_type, 'bp', cflags, ldflags)
    apply_patches('Hwui')

def add_flags_Dex2oat(cflags, ldflags):
    backup_file('Dex2oat', 'bp')
    replace_flags('Dex2oat', config.android_type, 'bp', cflags, ldflags)
    apply_patches('Dex2oat')


from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as ec
from selenium.webdriver.support.select import Select
import time

#any drop down not having select tag,like jquery/angular/react drop down

def selecrdrp(options_list,value):
    if not value[0]=='all':

        for ele in drop_down_list:
            print(ele.text)
            for i in range(len(value)):
                if ele.text==value[i]:
                    ele.click()
                    break

    else:
        try:
            for ele in options_list:
                ele.click()
        except Exception as e:
            print(e)

driver=webdriver.Chrome(ChromeDriverManager().install())
driver.get("https://www.jqueryscript.net/demo/Drop-Down-Combo-Tree/")
driver.maximize_window()
time.sleep(2)

driver.find_element(By.ID,"justAnInputBox").click()
time.sleep(3)
drop_down_list=driver.find_elements(By.CSS_SELECTOR,"span.comboTreeItemTitle")
#value_list=['choice 1','choice 2','choice 3','choice 6 2 1']

#value_list=['choice 1']
value_list=['all']
#selecrdrp(drop_down_list,'choice 6 2 1')
selecrdrp(drop_down_list,value_list)

'''for ele in drop_down_list:
    print(ele.text)
    if ele.text=='choice 6 2 3':
        ele.click()
        break'''


import asyncio
from client import Client
from app_msgs import (
    EchoRequest,
    EchoResponse,
    SumRequest,
    SumResponse,
)
from schema import (
    encode,
    decode,
    Message,
    Payload,
    EmptyPayload,
    UnhandledApplicationError,
)

class TestClient:
    def __init__(self):
        self.ipc = Client("ipc://server.sock")
        self.ipc.set_error_handler(self.handle_error)

    def connect(self):
        self.ipc.connect()

    # async def dispatch(self, func_name, msg):
    # return await self.ipc.dispatch(msg)
    #
    async def handle_error(self, request: Message, error: Message) -> None:
        print("CUSTOM ERROR HANDLER")
        print(f"got error for {request}")
        print(f"error: {error.payload}")

    async def echo(self, m):
        return (await self.ipc.dispatch("handle_echo", EchoRequest(m))).echo

    async def add(self, a: int, b: int) -> int:
        return (
            await self.ipc.dispatch(
                "handle_sum", SumRequest(a, b)  # , error_handler=special_error_handler
            )
        ).result

    async def gogo(self) -> None:
        return await self.ipc.dispatch("handle_gogo")

async def main():
    client = TestClient()
    client.connect()
    r = await client.echo("ping")
    assert r == "ping"
    print(f"Received echo: {r}")
    r = await client.add(1, 5)
    assert r == 6
    print(f"Received sum: {r}")
    await client.gogo()
    print("Went")

if __name__ == "__main__":
    asyncio.run(main())


# Databricks notebook source
# MAGIC %run ../../test_helpers

# COMMAND ----------

# MAGIC %run ../../../notebooks/common/spark_helpers

# COMMAND ----------

# MAGIC %run ../../../notebooks/scraper_stages/aggregation/field_summarisers

# COMMAND ----------

from datetime import datetime

from pyspark.sql import functions as F

# region Prevent downstream linting highlights
spark = spark
FunctionTestSuite = FunctionTestSuite
field_summariser = field_summariser
columns_equal = columns_equal
columns_approx_equal = columns_approx_equal
with_retry = with_retry
# endregion

# COMMAND ----------

def long_format_df_fixture():
  # Multiplier to the right is the expected weight of each field/value combination
  frequency_field_data = [
    *[["table1", "field1", "CATEGORICAL", "x",   None,                None] for _ in range(49)],
    *[["table1", "field1", "CATEGORICAL", "y",   None,                None] for _ in range(45)],
    *[["table1", "field1", "CATEGORICAL", "z",   None,                None] for _ in range(82)],
    *[["table1", "field1", "CATEGORICAL", None,  None,                None] for _ in range(7)],  # Null value categorical
    *[["table1", "field2", "CATEGORICAL", "x",   None,                None] for _ in range(41)],
    *[["table1", "field2", "CATEGORICAL", "y",   None,                None] for _ in range(67)],
    *[["table1", "field2", "CATEGORICAL", "z",   None,                None] for _ in range(97)],
    *[["table1", "field2", "CATEGORICAL", None,  None,                None] for _ in range(37)],  # Null value categorical
    *[["table1", "field3", "DATE",        None, datetime(2022, 1, 1), None] for _ in range(4)],
    *[["table1", "field3", "DATE",        None, datetime(2022, 1, 2), None] for _ in range(60)],
    *[["table1", "field3", "DATE",        None, datetime(2022, 1, 3), None] for _ in range(76)],
    *[["table1", "field3", "DATE",        None, None,                 None] for _ in range(38)],  # Null value date
    *[["table1", "field4", "DATE",        None, datetime(2022, 1, 1), None] for _ in range(7)],
    *[["table1", "field4", "DATE",        None, datetime(2022, 1, 2), None] for _ in range(37)],
    *[["table1", "field4", "DATE",        None, datetime(2022, 1, 3), None] for _ in range(61)],
    *[["table1", "field4", "DATE",        None, None,                 None] for _ in range(33)],  # Null value date
    *[["table1", "field5", "DISCRETE",    None, None,                 1.  ] for _ in range(69)],
    *[["table1", "field5", "DISCRETE",    None, None,                 2.  ] for _ in range(48)],
    *[["table1", "field5", "DISCRETE",    None, None,                 3.  ] for _ in range(56)],
    *[["table1", "field5", "DISCRETE",    None, None,                 None] for _ in range(46)],  # Null value discrete
    *[["table1", "field6", "DISCRETE",    None, None,                 1.  ] for _ in range(94)],
    *[["table1", "field6", "DISCRETE",    None, None,                 2.  ] for _ in range(67)],
    *[["table1", "field6", "DISCRETE",    None, None,                 3.  ] for _ in range(24)],
    *[["table1", "field6", "DISCRETE",    None, None,                 None] for _ in range(85)],
  ]
  schema = "TABLE_NAME: string, FIELD_NAME: string, VALUE_TYPE: string, VALUE_STRING: string, VALUE_DATE: date, VALUE_NUMERIC: double"
  frequency_long_df = spark.createDataFrame(frequency_field_data, schema)

  continuous_long_df = (
    spark.range(10000)
    .withColumn("TABLE_NAME", F.lit("table1"))
    .withColumn("FIELD_NAME", F.when(F.rand() > 0.5, "field7").otherwise("field8"))
    .withColumn("VALUE_TYPE", F.lit("CONTINUOUS"))
    .withColumn("VALUE_STRING", F.lit(None).cast("string"))
    .withColumn("VALUE_DATE", F.lit(None).cast("date"))
    .withColumn("VALUE_NUMERIC", F.when(F.col("FIELD_NAME") == "field7", F.rand() * 100).otherwise(F.rand() * 200))
    .drop("id")
  )

  return frequency_long_df.unionByName(continuous_long_df)

# COMMAND ----------

field_summariser_test_suite = FunctionTestSuite()

@field_summariser_test_suite.add_test
def test_correct_frequencies_for_categorical_fields():
  input_df = long_format_df_fixture()
  result_df = field_summariser(input_df, 1, 1).filter(F.col("VALUE_TYPE") == "CATEGORICAL")

  expected_frequency_col = (
    F
    .when((F.col("FIELD_NAME") == "field1") & (F.col("VALUE_STRING") == "x"), 49.)
    .when((F.col("FIELD_NAME") == "field1") & (F.col("VALUE_STRING") == "y"), 45.)
    .when((F.col("FIELD_NAME") == "field1") & (F.col("VALUE_STRING") == "z"), 82.)
    .when((F.col("FIELD_NAME") == "field1") & F.isnull("VALUE_STRING"),       7.)
    .when((F.col("FIELD_NAME") == "field2") & (F.col("VALUE_STRING") == "x"), 41.)
    .when((F.col("FIELD_NAME") == "field2") & (F.col("VALUE_STRING") == "y"), 67.)
    .when((F.col("FIELD_NAME") == "field2") & (F.col("VALUE_STRING") == "z"), 97.)
    .when((F.col("FIELD_NAME") == "field2") & F.isnull("VALUE_STRING"),       37.)
  )

  result_df = result_df.withColumn("EXPECTED_WEIGHT", expected_frequency_col)

  assert columns_equal(result_df, "WEIGHT", "EXPECTED_WEIGHT"), "Weights for categorical fields did not match expectations"
  assert result_df.filter(F.col("VALUE_TYPE") == "CATEGORICAL").count() == 8, "Too many rows found for categorical fields"

@field_summariser_test_suite.add_test
def test_correct_frequencies_for_date_fields():
  input_df = long_format_df_fixture()
  result_df = field_summariser(input_df, 1, 1).filter(F.col("VALUE_TYPE") == "DATE")

  expected_frequency_col = (
    F
    .when((F.col("FIELD_NAME") == "field3") & (F.col("VALUE_DATE") == datetime(2022, 1, 1)), 4.)
    .when((F.col("FIELD_NAME") == "field3") & (F.col("VALUE_DATE") == datetime(2022, 1, 2)), 60.)
    .when((F.col("FIELD_NAME") == "field3") & (F.col("VALUE_DATE") == datetime(2022, 1, 3)), 76.)
    .when((F.col("FIELD_NAME") == "field3") & F.isnull("VALUE_DATE"),                        38.)
    .when((F.col("FIELD_NAME") == "field4") & (F.col("VALUE_DATE") == datetime(2022, 1, 1)), 7.)
    .when((F.col("FIELD_NAME") == "field4") & (F.col("VALUE_DATE") == datetime(2022, 1, 2)), 37.)
    .when((F.col("FIELD_NAME") == "field4") & (F.col("VALUE_DATE") == datetime(2022, 1, 3)), 61.)
    .when((F.col("FIELD_NAME") == "field4") & F.isnull("VALUE_DATE"),                        33.)
  )

  result_df = result_df.withColumn("EXPECTED_WEIGHT", expected_frequency_col)

  assert columns_equal(result_df, "WEIGHT", "EXPECTED_WEIGHT"), "Weights for date fields did not match expectations"
  assert result_df.filter(F.col("VALUE_TYPE") == "DATE").count() == 8, "Too many rows found for date fields"

@field_summariser_test_suite.add_test
def test_correct_frequencies_for_discrete_fields():
  input_df = long_format_df_fixture()
  result_df = field_summariser(input_df, 1, 1).filter(F.col("VALUE_TYPE") == "DISCRETE")

  expected_frequency_col = (
    F
    .when((F.col("FIELD_NAME") == "field5") & (F.col("VALUE_NUMERIC") == 1), 69.)
    .when((F.col("FIELD_NAME") == "field5") & (F.col("VALUE_NUMERIC") == 2), 48.)
    .when((F.col("FIELD_NAME") == "field5") & (F.col("VALUE_NUMERIC") == 3), 56.)
    .when((F.col("FIELD_NAME") == "field5") & F.isnull("VALUE_NUMERIC"),     46.)
    .when((F.col("FIELD_NAME") == "field6") & (F.col("VALUE_NUMERIC") == 1), 94.)
    .when((F.col("FIELD_NAME") == "field6") & (F.col("VALUE_NUMERIC") == 2), 67.)
    .when((F.col("FIELD_NAME") == "field6") & (F.col("VALUE_NUMERIC") == 3), 24.)
    .when((F.col("FIELD_NAME") == "field6") & F.isnull("VALUE_NUMERIC"),     85.)
  )

  result_df = result_df.withColumn("EXPECTED_WEIGHT", expected_frequency_col)

  assert columns_equal(result_df, "WEIGHT", "EXPECTED_WEIGHT"), "Weights for discrete fields did not match expectations"
  assert result_df.filter(F.col("VALUE_TYPE") == "DISCRETE").count() == 8, "Too many rows found for discrete fields"

@field_summariser_test_suite.add_test
def test_value_field_populated_according_to_value_type():
  input_df = long_format_df_fixture()
  result_df = field_summariser(input_df, 1, 1)

  row_where_wrong_field_populated = (
    result_df
    .filter(
      F.expr("VALUE_TYPE == 'CATEGORICAL'  AND (isnotnull(VALUE_DATE)    OR isnotnull(VALUE_NUMERIC) OR isnotnull(VALUE_STRING_ARRAY) OR isnotnull(VALUE_NUMERIC_ARRAY))")
      | F.expr("VALUE_TYPE == 'DATE'       AND (isnotnull(VALUE_NUMERIC) OR isnotnull(VALUE_STRING)  OR isnotnull(VALUE_STRING_ARRAY) OR isnotnull(VALUE_NUMERIC_ARRAY))")
      | F.expr("VALUE_TYPE == 'DISCRETE'   AND (isnotnull(VALUE_STRING)  OR isnotnull(VALUE_DATE)    OR isnotnull(VALUE_STRING_ARRAY) OR isnotnull(VALUE_NUMERIC_ARRAY))")
      | F.expr("VALUE_TYPE == 'CONTINUOUS' AND (isnotnull(VALUE_STRING)  OR isnotnull(VALUE_DATE)    OR isnotnull(VALUE_NUMERIC)      OR isnotnull(VALUE_STRING_ARRAY))" )
    )
    .first()
  )

  assert row_where_wrong_field_populated is None

@field_summariser_test_suite.add_test
def test_frequencies_are_rounded_to_the_nearest_given_number():
  input_df = long_format_df_fixture()

  for frequency_rounding in [1, 5, 10]:
    result_df = field_summariser(input_df, frequency_rounding, 0)

    result_df = result_df.withColumn("REMAINDER", F.col("WEIGHT") % frequency_rounding)
    assert result_df.first() is not None  # Some rows

    # No rows with non-zero remainder
    assert result_df.filter("REMAINDER != 0.0").first() is None, f"Frequencies were not rounded to the nearest {frequency_rounding}"

@field_summariser_test_suite.add_test
def test_values_are_excluded_with_frequency_below_threshold():
  input_df = long_format_df_fixture()

  for min_frequency in [0, 5, 10]:
    result_df = field_summariser(input_df, 1, min_frequency)
    assert result_df.first() is not None  # Some rows

    # No rows with weight below threshold
    assert result_df.filter(F.col("WEIGHT") < min_frequency).first() is None, f"Found frequencies below minimum {min_frequency}"

@field_summariser_test_suite.add_test
def test_values_are_excluded_with_frequency_below_threshold_within_rounding():
  # Need to make sure frequencies aren't rounded up so that they pass the threshold!

  data = [
    *[["table1", "field1", "CATEGORICAL", "x",   None,                None] for _ in range(7)],  # Excluded
    *[["table1", "field1", "CATEGORICAL", "y",   None,                None] for _ in range(10)], # Included
    *[["table1", "field2", "DATE",        None, datetime(2022, 1, 1), None] for _ in range(8)],  # Excluded
    *[["table1", "field2", "DATE",        None, datetime(2022, 1, 2), None] for _ in range(13)], # Included
    *[["table1", "field3", "DISCRETE",    None, None,                 1.  ] for _ in range(9)],  # Excluded
    *[["table1", "field3", "DISCRETE",    None, None,                 2.  ] for _ in range(16)], # Included
  ]
  schema = "TABLE_NAME: string, FIELD_NAME: string, VALUE_TYPE: string, VALUE_STRING: string, VALUE_DATE: date, VALUE_NUMERIC: double"
  input_df = spark.createDataFrame(data, schema)

  frequency_rounding = 5
  min_frequency = 10
  result_df = field_summariser(input_df, frequency_rounding, min_frequency)

  expected_exclusions = (
    F
    .when(F.col("FIELD_NAME") == "field1", F.col("VALUE_STRING") == "x")
    .when(F.col("FIELD_NAME") == "field2", F.col("VALUE_DATE") == datetime(2022, 1, 1))
    .when(F.col("FIELD_NAME") == "field3", F.col("VALUE_NUMERIC") == 1)
    .otherwise(False)
  )

  assert result_df.filter(expected_exclusions).first() is None, "Found values that should have been suppressed!"
  assert result_df.count() == 3, "Too many rows found"

  expected_frequency_col = (
    F
    .when((F.col("FIELD_NAME") == "field1") & (F.col("VALUE_STRING") == "y"), 10.)
    .when((F.col("FIELD_NAME") == "field2") & (F.col("VALUE_DATE") == datetime(2022, 1, 2)), 15.)
    .when((F.col("FIELD_NAME") == "field3") & (F.col("VALUE_NUMERIC") == 2.), 15.)
  )

  result_df = result_df.withColumn("EXPECTED_WEIGHT", expected_frequency_col)
  assert columns_equal(result_df, "WEIGHT", "EXPECTED_WEIGHT"), "Weights did not match expectations"

@field_summariser_test_suite.add_test
@with_retry(n_retries=10)
def test_correct_percentiles_for_continuous_fields():
  input_df = long_format_df_fixture()

  result_df = field_summariser(input_df, 1, 1)

  expected_percentile_col = (
    F
    .when(F.col("FIELD_NAME") == "field7", F.sequence(F.lit(2), F.lit(99), F.lit(1)).cast("array<double>"))
    .when(F.col("FIELD_NAME") == "field8", F.sequence(F.lit(4), F.lit(198), F.lit(2)).cast("array<double>"))
  )
  result_df = (
    result_df
    .withColumn("EXPECTED_VALUE_NUMERIC_ARRAY", expected_percentile_col)
    .withColumn("PAIRED_ACTUAL_EXPECTED", F.expr("zip_with(VALUE_NUMERIC_ARRAY, EXPECTED_VALUE_NUMERIC_ARRAY, (x, y) -> struct(x as ACTUAL, y as EXPECTED))"))
    .select("FIELD_NAME", F.expr("inline(PAIRED_ACTUAL_EXPECTED)"))
  )

  assert columns_approx_equal(result_df, "EXPECTED", "ACTUAL", 0.1), "Difference between ACTUAL and EXPECTED column values"

field_summariser_test_suite.run()


import matplotlib.pyplot as plt
import cv2
from PIL import Image
import numpy as np
from scipy.ndimage import rotate

def make_rectangle(img):
    height, width = img.shape[:2]
    res_list = []

    res1 = img[:height//2, :width//2]
    res1 = rotate(res1, angle=270)

    res2 = img[height//2:, :width//2]
    res2 = rotate(res2, angle=180)

    res3 = img[height//2:, width//2:]
    res3 = rotate(res3, angle=90)

    res4 = img[:height//2, width//2:]

    res_list= [res1, res2, res3, res4]
    return res_list

def find_circle_crop(path:str):
    img = cv2.imread(path)
    height, width = img.shape[:2]
    mask = np.zeros((height, width))
    res = np.zeros((height, width, 3))
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
    # detect circles in the image
    circles = cv2.HoughCircles(gray, cv2.HOUGH_GRADIENT,
                            1,
                            100,
                            param1=50,param2=200,
                            minRadius=200, maxRadius=600)
    # ensure at least some circles were found
    if circles is not None:
        # convert the (x, y) coordinates and radius of the circles to integers
        circles = np.round(circles[0, :]).astype("int")
        # loop over the (x, y) coordinates and radius of the circles
        for (x, y, r) in circles:
            # draw the circle in the output image, then draw a rectangle
            # corresponding to the center of the circle
            cv2.circle(mask, (x, y), r, 255, cv2.FILLED)
            mask=mask.astype('uint8')
            _,thresh = cv2.threshold(mask,1,255,cv2.THRESH_BINARY)
            #Find Contour
            contours = cv2.findContours(thresh,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)
            #Find Reectangle
            x,y,w,h = cv2.boundingRect(np.array(contours[0][0]))
            #Crop masked_data
            crop = img[y:y+h,x:x+w]
            return make_rectangle(crop)

def seperate_image(img):
    print('seperate')
    res_list = find_circle_crop(img)

    for i in range(len(res_list)):
        res_list[i] = cv2.resize(res_list[i], (256, 256), interpolation=cv2.INTER_AREA)

    #res_list = np.array(res_list, np.dtype(object))
    print('seperate complete')
    return res_list

if __name__ == "__main__":
    #ax = [plt.subplot(2,2,i+1) for i in range(4)]
    res = find_circle_crop('./Untitled Folder/2.bmp')
    for idx, img in enumerate(res):
        size = 220
        plt.subplot(size+1+idx)
        plt.imshow(res[idx])
    plt.xticks([]), plt.yticks([])
    plt.show()

    print(res)


from copy import deepcopy

import torch.nn as nn

from oslo.torch.distributed import ParallelContext, ParallelMode
from oslo.torch.nn import RowParallelLinear
from oslo.torch.nn.parallel.tensor_parallel._1d._ops import gather_1d
from oslo.torch.nn.parallel.tensor_parallel._1d._wrapper import (
    _TensorParallel1D,
)

tp_size = 4
tp_depth = 2

# parallel context 생성
parallel_context = ParallelContext.from_torch(
    data_parallel_size=1,
    pipeline_parallel_size=1,
    tensor_parallel_size=tp_size,
    tensor_parallel_mode=ParallelMode.TENSOR_1D,
    tensor_parallel_depth=tp_depth,
)

world_size = parallel_context.get_world_size(ParallelMode.TENSOR_1D)
rank = parallel_context.get_local_rank(ParallelMode.TENSOR_1D)
fusion_degree = 3

linear = nn.Linear(4, fusion_degree * 4).cuda()
w = deepcopy(linear.weight.data)
b = deepcopy(linear.bias.data)

dim = 1

weight_list = w.t().chunk(fusion_degree * world_size, dim=dim)
bias_list = b.chunk(fusion_degree * world_size, dim=0)

# [t][f*t]
weight_list = _TensorParallel1D._deconstruct_combined_qkv(
    weight_list, world_size, fusion_degree, dim
)
chunked_w = weight_list[rank].contiguous()
bias_list = _TensorParallel1D._deconstruct_combined_qkv(
    bias_list, world_size, fusion_degree, 0
)
chunked_b = bias_list[rank].contiguous()

linear_1d = RowParallelLinear(
    4, fusion_degree * 4, parallel_context=parallel_context, bias=True
)
if parallel_context.get_global_rank() == 0:
    print(chunked_w.size())
    print(linear_1d.weight.data.size())
linear_1d.weight.data = chunked_w
linear_1d.bias.data = chunked_b

recon_chunked_w = gather_1d(parallel_context, linear_1d.weight.data, world_size, dim)
recon_chunked_b = gather_1d(parallel_context, linear_1d.bias.data, world_size, 0)
# reshaped_w = recon_chunked_w.view(-1, tesseract_dim, 4)
# recon_w = torch.cat([reshaped_w[:3], reshaped_w[3:]], 1).view(-1, 4).contiguous()
print(recon_chunked_w.shape)
recon_w = _TensorParallel1D._reconstruct_combined_qkv(
    recon_chunked_w, world_size, fusion_degree, dim
)
recon_b = _TensorParallel1D._reconstruct_combined_qkv(
    recon_chunked_b, world_size, fusion_degree, 0
)

if parallel_context.get_global_rank() == 0:
    print(f"original w: \n{w}\n")
    print(f"reconstruct w: \n{recon_w}\n")

    print(f"original b: \n{b}\n")
    print(f"reconstruct b: \n{recon_b}\n")


#The goal of this project is to be able to edit and simulate a solar system. This system should be custom and creatable.
#Good luck Jake, you're going to need it...
import pygame as p
import random as r
import simclasses as s
import sys
p.init()
buttons = []

#This function creates a screen for the simulation to use.
def create_screen(width,height):
    screen_init = p.display.set_mode(size=(width,height))
    p.display.set_caption("Space Simulator")
    return screen_init

def button_pressed():
    print("Button Pressed")

def add_star():
    new_star = s.Star(game_screen)
    new_star.get_star_attributes()
    new_star.create()

def add_planet():
    new_planet = s.Planet(game_screen)
    new_planet.get_planet_attribute()
    new_planet.create()

#Here is where the simulation is run

#Initialize the screen
game_screen = create_screen(750,750)
game_screen.fill((0,0,0))

running = True
while running:
    mouse = p.mouse.get_pos()
    button1 = s.Button(game_screen,(10,600), 150, 30, button_list=buttons, on_click_function=add_star,one_press=True, text="Add Star")
    button2 = s.Button(game_screen,(10,650), 185, 30, button_list=buttons, on_click_function=add_planet,one_press=True, text="Add Planet")
    button3 = s.Button(game_screen,(10,700), 75, 30,button_list=buttons, on_click_function=quit,one_press=True,text="Quit")
    #Searchiong through events to set certain conditions
    for event in p.event.get():
        if event.type == p.QUIT: #If the x is hit on the screen, quit the program.
            running = False
        if event.type == p.MOUSEBUTTONUP:
            s.pressed = False
        if event.type == p.MOUSEBUTTONDOWN:
            pass

    #Searching through the objects to see if they are being executed
    for obj in buttons:
        obj.process()

    p.display.update()


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="eego2lsl", # Replace with your own username
    version="0.1.1",
    author="Johan Medrano",
    author_email="johan.medrano@umontpellier.fr",
    description="A driver to connect the EEGO devices to the LabStreamingLayer.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    packages=setuptools.find_packages(),
    entry_points = {
        'console_scripts': [
            'eego2lsl=eego2lsl.eego2lsl:main'
        ]
    },
    classifiers=[
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ],
    package_data = {
	'eego2lsl': ['extra/*.txt', 'extra/*.elc']
    },
    install_requires=['pylsl'
    ],
    python_requires='>=2.7',
)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Aug 24 10:32:51 2018

@author: daniel
"""

import pandas as pd
import matplotlib.pyplot as plt

dataset = pd.read_csv('./data/Position_Salaries.csv')

X = dataset.iloc[:, 1:2].values
y = dataset.iloc[:,2].values

# Como o dataset é muito pequeno e de exemplo, não iremos dividir ele em dados
# de treinamento e de teste

from sklearn.preprocessing import PolynomialFeatures
poli = PolynomialFeatures(degree=4)

from sklearn.linear_model import LinearRegression
predictor = LinearRegression()

# Adequando as características do polinomio

X_poli = poli.fit_transform(X)
predictor.fit(X_poli, y)

# visualizar os resultados

plt.scatter(X,y,color='red')
plt.plot(X, predictor.predict(X_poli), color='blue')
plt.title('Regressão Polinomial')
plt.xlabel('Posição')
plt.ylabel('Salário')
plt.show()


# A class to represent a tredly partition
import builtins
import os.path
import re
import yaml

from includes.util import *
from includes.output import *

class ConfigFile:

    # Constructor
    def __init__(self, filePath = None):
        if (filePath is None):
            filePath = builtins.tredlyConfDirectory + "/tredly-host.conf"

        # TODO: the following is deprecated and will be removed in a future release
        self.filePath = filePath
        self.required = []
        self.wif = None
        self.lif = None
        self.wifPhysical = None
        self.lifNetwork = None
        self.lifCIDR = None
        self.dns = []
        self.httpProxyIP = None
        self.tld = None
        self.vnetDefaultRoute = None
        self.firewallEnableLogging = None

        # for YAML
        self.json = None

    # Action: process a config file - populates this class
    #
    # Pre:
    # Post: Config file has been processed
    #
    # Params:
    #
    # Return: True if succeeded, False otherwise
    def process(self):
        # check if the file exists at the given path
        if (not self.fileExists()):
            e_error("Config file " + self.filePath + " does not exist.")
            return False

        # file exists so process it
        with open(self.filePath) as configFile:
            for line in configFile:
                # strip off leading and following whitespace
                line = line.strip()

                # if this line has data then process it
                if ((len(line) > 0) and (not line.startswith('#'))):
                    key, value = line.partition("=")[::2]

                    # populate only if there was a value
                    if (len(value) > 0):
                        # populate our variables
                        if (key == "required"):
                            # split out the required values
                            values = value.split(',')
                            # strip any whitespace
                            map(str.strip, values)

                            self.required = values

                        elif (key == "wif"):
                            self.wif = value

                        elif (key == "lif"):
                            self.lif = value

                        elif (key == "wifPhysical"):
                            self.wifPhysical = value

                        elif (key == "lifNetwork"):
                            values = value.split('/')
                            # split it into network and cidr
                            self.lifNetwork = values[0]
                            self.lifCIDR = values[1]

                        elif (key == "dns"):
                            self.dns.append(value)

                        elif (key == "httpproxy"):
                            self.httpProxyIP = value

                        elif (key == "tld"):
                            self.tld = value

                        elif (key == "vnetdefaultroute"):
                            self.vnetDefaultRoute = value

                        elif (key == "firewallEnableLogging"):
                            self.firewallEnableLogging = value
                        else:
                            e_warning("Unrecognised config definition: " + line)

        return True

    # Action: parses a YAML config file
    #
    # Pre:
    # Post: this object has been populated with data from the config file
    #
    # Params:
    #
    # Return: True if exists, False otherwise
    def processYaml(self):
        file = open(self.filePath, 'r')

        self.json = yaml.load(file.read())

    # Action: checks whether the path to the tredlyfile exists or not
    #
    # Pre:
    # Post:
    #
    # Params:
    #
    # Return: True if exists, False otherwise
    def fileExists(self):
        return os.path.isfile(self.filePath)

    # TODO: validate YAML

    # Action: validates this object
    #
    # Pre: this object exixsts
    # Post:
    #
    # Params:
    #
    # Return: True if valid, False otherwise
    def validate(self):
        if (self.wif is None):
            return False

        if (self.lif is None):
            return False

        if (self.wifPhysical is None):
            return False

        if (self.lifNetwork is None):
            return False

        if (self.dns is None):
            return False

        if (self.httpProxyIP is None):
            return False

        if (self.tld is None):
            return False

        if (self.vnetDefaultRoute is None):
            return False

        if (self.firewallEnableLogging is None):
            return False

        # make sure network interfaces exist
        if (not networkInterfaceExists(self.wif)):
            e_error('Interface "' + self.wif + '" from "tredly-host.conf" does not exist on this host.')
            return False
        if (not networkInterfaceExists(self.lif)):
            e_error('Interface "' + self.lif + '" from "tredly-host.conf" does not exist on this host.')
            return False
        if (not networkInterfaceExists(self.wifPhysical)):
            e_error('Interface "' + self.wifPhysical + '" from "tredly-host.conf" does not exist on this host.')
            return False

        # everything passed, so return true
        return True


#!/usr/bin/python

import csv
import json
import codecs
from os import walk

# Export results
# text = json.dumps(pdf.summary, indent=4)
# with codecs.open(fileName, "w", "utf-8") as f:
    # f.write(text)

# with open('pdfsToCheck.txt', 'r') as file:
    # files = [line.replace('\n', '') for line in file.readlines()]

csvFile = open('results.csv', 'w')
writer = csv.writer(csvFile)
writer.writerow(['name', 'url', '200', '404', 'others'])

# Get files in output directory
files = []
for (_, _, filenames) in walk("output"):
    files.extend(filenames)
    break

for filename in files:
    with open('output/' + filename, 'r') as file:
        data = json.loads(file.read())
        name = data['source']['filename']
        location = data['source']['location']

        refCheck = data['refCheck']
        if '200' in refCheck:
            valid = '\n'.join(refCheck['200'])
        else:
            valid = ''

        if '404' in refCheck:
            missing = '\n'.join(refCheck['404'])
        else:
            missing = ''

        output = [name, location, valid, missing]
        otherKeys = [k for k in refCheck.keys() if k not in ['200', '404']]
        for key in otherKeys:
            output.append(key + ': \n\n' + '\n'.join(refCheck[key]))
        print(output)
        writer.writerow(output)

csvFile.close()


import sys
import glob
import serial

def serial_ports():
    """ Lists serial port names

        :raises EnvironmentError:
            On unsupported or unknown platforms
        :returns:
            A list of the serial ports available on the system
    """
    if sys.platform.startswith('win'):
        ports = ['COM%s' % (i + 1) for i in range(256)]
    elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
        # this excludes your current terminal "/dev/tty"
        ports = glob.glob('/dev/tty[A-Za-z]*')
    elif sys.platform.startswith('darwin'):
        ports = glob.glob('/dev/tty.*')
    else:
        raise EnvironmentError('Unsupported platform')

    result = []
    ports += ["01A377A5"]
    for port in ports:
        try:
            s = serial.Serial(port)
            s.close()
            result.append(port)
        except (OSError, serial.SerialException):
            pass
    return result

if __name__ == '__main__':
    print(serial_ports())

# exit()
import serial.tools.list_ports

ports = list(serial.tools.list_ports.grep("USB"))
print(ports)
print(len(ports))
print("aaaaaaaaaaaaaaaaa")
for p in ports:
    print(p)
    print("device (", type(p.device), "): ", p.device)
    print("name (", type(p.name), "): ", p.name)
    print("description (", type(p.description), "): ", p.description)
    print("hwid (", type(p.hwid), "): ", p.hwid)
    print("vid (", type(p.vid), "): ", p.vid)
    print("pid (", type(p.pid), "): ", p.pid)
    print("serial_number (", type(p.serial_number), "): ", p.serial_number)
    print("location (", type(p.location), "): ", p.location)
    print("manufacturer (", type(p.manufacturer), "): ", p.manufacturer)
    print("product (", type(p.product), "): ", p.product)
    print("interface (", type(p.interface), "): ", p.interface)
    print()


# -*- coding: utf-8 -*-

import urllib2
import urllib
import httplib
import socket
import time
import logging
import verbix_parser

class VerbixScraper:

    __languages = {
        'ro': 5,
        'es': 1,
        'pt': 2,
        'fr': 3,
        'ca': 7,
        'de': 13,
        'en': 20
    }

    __retries = 5

    def get_infinitive(self, language, verb):
        base_url = 'http://www.verbix.com/find-verb/'
        params = {
            'verb': verb,
            'Submit': 'Find'
        }

        response = self.__request(self.__post, base_url, params)

        if response is None:
            return

        content = self.__get_response_content(response)

        parser = verbix_parser.VerbixParser()
        return parser.get_infinitive(VerbixScraper.__languages[language], content)

    def get_verb_info(self, language, verb):
        base_url = 'http://www.verbix.com/webverbix/go.php'
        params = {
            'T1': verb,
            'D1': VerbixScraper.__languages[language]
        }

        response = self.__request(self.__get, base_url, params)

        if response is None:
            return

        content = self.__get_response_content(response)

        parser = verbix_parser.VerbixParser()
        return parser.parse(content)

    def __get_response_content(self, response):
        try:
            return response.read()
        except httplib.IncompleteRead as e:
            return e.partial

    def __request(self, make_request, url, params):
        num_attempts = 0

        while num_attempts < VerbixScraper.__retries:
            try:
                (success, response) = self.__try_request(make_request, url, params)

                if success:
                    return response
                else:
                    logging.warning('Request %d/%d failed' % (num_attempts + 1, VerbixScraper.__retries))
                    num_attempts += 1
                    time.sleep(2)

            except UnicodeEncodeError:
                logging.error('Could not encode parameters')
                return None

        logging.error('Too many attempts, giving up')

    def __try_request(self, make_request, url, params):
        try:
            return make_request(url, params)
        except (socket.error,httplib.BadStatusLine, urllib2.HTTPError, urllib2.URLError):
            return (False, None)

    def __get(self, url, params):
        final_url = '%s?%s' % (url, urllib.urlencode(params))
        logging.info('get: %s' % final_url)
        response = urllib2.urlopen(final_url)
        return (True, response)

    def __post(self, url, params):
        logging.info('post: %s' % url)
        request = urllib2.Request(url, urllib.urlencode(params))
        response = urllib2.urlopen(request)
        return (True, response)


"""
Tools for working with files and directories in namer.
"""

import argparse
import gzip
from dataclasses import dataclass
import os
import shutil
import sys
from pathlib import Path
from platform import system
from typing import Iterable, List, Optional, Sequence, Tuple

import jsonpickle
from loguru import logger

from namer.configuration import NamerConfig
from namer.configuration_utils import default_config
from namer.ffmpeg import FFProbeResults
from namer.fileinfo import parse_file_name, FileInfo
from namer.comparison_results import ComparisonResults, LookedUpFileInfo

# noinspection PyDataclass
@dataclass(init=False, repr=False, eq=True, order=False, unsafe_hash=True, frozen=False)
class Command:
    input_file: Path
    """
    This is the original user/machine input of a target path.
    If this path is a directory a movie is found within it (recursively).
    If this file is a the movie file itself, the parent directory is calculated.
    """
    target_movie_file: Path
    """
    The movie file this name is targeting.
    """
    target_directory: Optional[Path] = None
    """
    The containing directory of a File.  This may be the immediate parent directory, or higher up, depending
    on whether a directory was selected as the input to a naming process.
    """
    parsed_dir_name: bool
    """
    Was the input file a directory and is parsing directory names configured?
    """
    parsed_file: Optional[FileInfo] = None
    """
    The parsed file name.
    """

    inplace: bool = False
    """
    Was the command told to keep the files in place.
    """

    write_from_nfos: bool = False
    """
    Should .nfo files be used as a source of metadata and writen into file tag info and used for naming.
    """

    tpdb_id: Optional[str] = None
    """
    The _id used to identify video in tpdb
    """

    is_auto: bool = True
    """
    If False then it means command was from web ui
    """

    config: NamerConfig

    def get_command_target(self):
        return str(self.target_movie_file.absolute())

def move_command_files(target: Optional[Command], new_target: Path, is_auto: bool = True) -> Optional[Command]:
    if not target:
        return

    if target.target_directory and target.input_file == target.target_directory:
        working_dir = Path(new_target) / target.target_directory.name
        logger.info('Moving {} to {} for processing', target.target_directory, working_dir)
        shutil.move(target.target_directory, working_dir)
        output = make_command(working_dir, target.config, is_auto=is_auto)
    else:
        working_file = Path(new_target) / target.target_movie_file.name
        shutil.move(target.target_movie_file, working_file)
        logger.info('Moving {} to {} for processing', target.target_movie_file, working_file)
        output = make_command(working_file, target.config, is_auto=is_auto)

    if output:
        output.tpdb_id = target.tpdb_id
        output.inplace = target.inplace
        output.write_from_nfos = target.write_from_nfos

    return output

def write_log_file(movie_file: Optional[Path], match_attempts: Optional[ComparisonResults], namer_config: NamerConfig) -> Optional[Path]:
    """
    Given porndb scene results sorted by how closely they match a file,  write the contents
    of the result matches to a log file with json pickle, the ui could reconstitute the results
    """
    log_name = None
    if movie_file:
        log_name = movie_file.with_name(movie_file.stem + '_namer.json.gz')
        logger.info('Writing log to {}', log_name)
        with open(log_name, 'wb') as log_file:
            if match_attempts:
                for result in match_attempts.results:
                    del result.looked_up.original_query
                    del result.looked_up.original_response

            json_out = jsonpickle.encode(match_attempts, separators=(',', ':'))
            if json_out:
                json_out = json_out.encode('UTF-8')
                json_out = gzip.compress(json_out)
                log_file.write(json_out)

        set_permissions(log_name, namer_config)

    return log_name

def _set_perms(target: Path, config: NamerConfig):
    file_perm: Optional[int] = int(str(config.set_file_permissions), 8) if config.set_file_permissions else None
    dir_perm: Optional[int] = int(str(config.set_dir_permissions), 8) if config.set_dir_permissions else None

    if config.set_gid:
        os.lchown(target, uid=-1, gid=config.set_gid)

    if config.set_uid:
        os.lchown(target, uid=config.set_uid, gid=-1)

    if target.is_dir() and dir_perm:
        target.chmod(dir_perm)
    elif target.is_file() and file_perm:
        target.chmod(file_perm)

def set_permissions(file: Optional[Path], config: NamerConfig):
    """
    Given a file or dir, set permissions from NamerConfig.set_file_permissions,
    NamerConfig.set_dir_permissions, and uid/gid if set for the current process recursively.
    """
    if system() != 'Windows' and file and file.exists() and config.update_permissions_ownership:
        _set_perms(file, config)
        if file.is_dir():
            for target in file.rglob('**/*'):
                _set_perms(target, config)

def extract_relevant_attributes(ffprobe_results: Optional[FFProbeResults], config: NamerConfig) -> Tuple[float, int, int]:
    if not ffprobe_results:
        return 0, 0, 0

    stream = ffprobe_results.get_default_video_stream()
    if not stream:
        return 0, 0, 0

    return stream.duration, stream.height if stream.height else 0, get_codec_value(stream.codec_name.lower(), config)

def get_codec_value(codec: str, config: NamerConfig) -> int:
    desired_codecs = list(config.desired_codec)
    desired_codecs.reverse()
    if codec in desired_codecs:
        return desired_codecs.index(codec) + 1

    return 0

def greater_than(seq1: Sequence, seq2: Sequence) -> bool:
    for val in zip(seq1, seq2):
        if val[0] > val[1]:
            return True

        if val[0] == val[1]:
            continue
        else:
            return False

    return False  # equal

def selected_best_movie(movies: List[str], config: NamerConfig) -> Optional[Path]:
    # This could use a lot of work.
    if movies:
        selected = Path(movies[0])
        selected_values = extract_relevant_attributes(config.ffmpeg.ffprobe(selected), config)
        for current_movie_str in movies:
            current_movie = Path(current_movie_str)
            current_values = extract_relevant_attributes(config.ffmpeg.ffprobe(current_movie), config)
            if current_values[1] <= config.max_desired_resolutions or config.max_desired_resolutions == -1:  # noqa: SIM102
                if greater_than(current_values, selected_values):
                    selected_values = current_values
                    selected = current_movie

        return selected

def move_to_final_location(command: Command, new_metadata: LookedUpFileInfo) -> Command:
    """
    Moves a file or directory to its final location after verifying there is no collision.
    Should a collision occur, the file is appropriately renamed to avoid collision.
    """

    # determine where we will move the movie, and how we will name it.
    # if in_place is False we will move it to the config defined destination dir.
    # if a directory name was passed in we will rename the dir with the relative_path_name from the config
    # else we will just rename the movie in its current location (as all that was defined in the command was the movie file.)
    name_template = command.config.inplace_name
    target_dir = command.target_movie_file.parent
    if command.target_directory:
        name_template = command.config.new_relative_path_name
        target_dir = command.target_directory.parent

    if not command.inplace:
        name_template = command.config.new_relative_path_name
        target_dir = command.config.dest_dir

    infix = 0
    # Find non-conflicting movie name.
    movies: List[str] = []
    while True:
        relative_path = Path(new_metadata.new_file_name(name_template, command.config, f'({infix})'))
        movie_name = target_dir / relative_path
        movie_name = movie_name.resolve()
        infix += 1
        if not movie_name.exists():
            break

        movies.append(str(movie_name))
        if command.target_movie_file.samefile(movie_name):
            break

    # Create the new dir if needed and move the movie file to it.
    movie_name.parent.mkdir(exist_ok=True, parents=True)
    shutil.move(command.target_movie_file, movie_name)
    movies.append(str(movie_name))

    # Now that all files are in place we'll see if we intend to minimize duplicates
    if not command.config.preserve_duplicates and movies:
        # Now set to the final name location since -- will grab the metadata requested
        # incase it has been updated.
        relative_path = Path(new_metadata.new_file_name(name_template, command.config, '(0)'))

        # no move best match to primary movie location.
        final_location = (target_dir / relative_path).resolve()
        selected_movie = selected_best_movie(movies, command.config)
        if selected_movie:
            movies.remove(str(selected_movie))
            if str(selected_movie.absolute()) != str(final_location.absolute()):
                movies.remove(str(final_location))
                final_location.unlink()
                shutil.move(selected_movie, final_location)
                movie_name = final_location

            for movie in movies:
                Path(movie).unlink()

    containing_dir: Optional[Path] = None
    if relative_path.parts:
        containing_dir = target_dir / relative_path.parent

    # we want to retain files if asked and if a directory will exist.
    if command.target_directory and not command.config.del_other_files and containing_dir:
        containing_dir.mkdir(parents=True, exist_ok=True)
        logger.info(f'moving other files to new dir: {containing_dir} from {command.target_directory}')
        # first remove namer log if exists
        possible_log = command.target_movie_file.parent / (command.target_movie_file.stem + '_namer.json.gz')
        if possible_log.exists():
            possible_log.unlink()

        # move directory contents
        for file in command.target_directory.iterdir():
            if file != command.target_movie_file:
                dest_file = containing_dir / file.name
                dest_file.parent.mkdir(parents=True, exist_ok=True)
                shutil.move(file, dest_file)

    if command.target_directory and containing_dir:
        set_permissions(containing_dir, command.config)
    else:
        set_permissions(movie_name, command.config)

    output = Command()
    if movie_name:
        output.target_movie_file = movie_name
        output.input_file = movie_name

    if containing_dir:
        output.target_directory = containing_dir
        output.input_file = containing_dir

    if command.target_directory and not is_relative_to(output.target_directory, command.target_directory):
        shutil.rmtree(command.target_directory)

    return output

def is_relative_to(potential_sub: Optional[Path], potential_parent: Optional[Path]) -> bool:
    try:
        if potential_sub and potential_parent:
            potential_sub.absolute().relative_to(potential_parent.absolute())
            return True
        return False
    except ValueError:
        return False

def is_interesting_movie(path: Optional[Path], config: NamerConfig) -> bool:
    if not path:
        return False

    exists = path.exists()
    suffix = path.suffix.lower()[1:] in config.target_extensions
    size = path.stat().st_size / (1024 * 1024) >= config.min_file_size if path.is_file() else False

    return exists and size and suffix

def gather_target_files_from_dir(dir_to_scan: Path, config: NamerConfig) -> Iterable[Command]:
    """
    Find files to process in a target directory.
    """
    if dir_to_scan and dir_to_scan.is_dir() and dir_to_scan.exists():
        logger.info('Scanning dir {} for sub-dirs/files to process', dir_to_scan)
        mapped: Iterable = map(lambda file: make_command((dir_to_scan / file), config), dir_to_scan.iterdir())
        filtered: Iterable[Command] = filter(lambda file: file is not None, mapped)  # type: ignore
        return filtered

    return []

def __exact_command(target_movie_file: Path, target_dir: Optional[Path], config: NamerConfig) -> Command:
    """
    Given a target movie file and a target containing directory, parse appropriate names as determined by
    config, aka, "prefer_dir_name_if_available".
    """
    command = Command()
    command.target_directory = target_dir
    command.target_movie_file = target_movie_file
    command.parsed_dir_name = bool(target_dir and config.prefer_dir_name_if_available)
    command.config = config
    name = target_movie_file.name

    parsed_dir_name = False
    if target_dir and config.prefer_dir_name_if_available:
        name = target_dir.name + target_movie_file.suffix
        parsed_dir_name = True

    command.parsed_file = parse_file_name(name, config)
    command.parsed_dir_name = parsed_dir_name

    return command

def find_target_file(root_dir: Path, config: NamerConfig) -> Optional[Path]:
    """
    returns largest matching file
    """
    list_of_files = list(root_dir.rglob('**/*.*'))
    file = None
    if list_of_files:
        for target_ext in config.target_extensions:
            filtered = list(filter(lambda o, ext=target_ext: o.suffix and o.suffix.lower()[1:] == ext, list_of_files))
            if not file and filtered:
                file = max(filtered, key=lambda x: x.stat().st_size)

    return file

def make_command(input_file: Path, config: NamerConfig, nfo: bool = False, inplace: bool = False, uuid: Optional[str] = None, ignore_file_restrictions: bool = False, is_auto: bool = True) -> Optional[Command]:
    """
    after finding target directory and target movie from input, returns file name descriptors.
    """
    target_dir = input_file if input_file.is_dir() else None
    target_movie = input_file if not input_file.is_dir() else find_target_file(input_file, config)
    if not target_movie:
        return

    target_file = __exact_command(target_movie, target_dir, config)
    target_file.input_file = input_file
    target_file.tpdb_id = uuid
    target_file.write_from_nfos = nfo
    target_file.inplace = inplace
    target_file.is_auto = is_auto

    output = target_file if is_interesting_movie(target_file.target_movie_file, config) or ignore_file_restrictions else None

    return output

def make_command_relative_to(input_dir: Path, relative_to: Path, config: NamerConfig, nfo: bool = False, inplace: bool = False, uuid: Optional[str] = None, is_auto: bool = True) -> Optional[Command]:
    """
    Ensure we are going to handle the directory relative to another directory, rather than just the file
    specified
    """
    if is_relative_to(input_dir, relative_to):
        relative_path = input_dir.absolute().relative_to(relative_to.absolute())
        if relative_path:
            target_file = relative_to / relative_path.parts[0]
            return make_command(target_file, config, nfo, inplace, uuid, is_auto=is_auto)

def main(arg_list: List[str]):
    """
    Attempt to parse a name.
    """
    description = 'You are using the file name parser of the Namer project. Expects a single input, and will output the contents of FileInfo, which is the internal input to the namer_metadatapi.py script. Output will be the representation of that FileInfo.\n'
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-f', '--file', help='String to parse for name parts', required=True)
    parser.add_argument('-c', '--configfile', help='override location for a configuration file.', type=Path)
    args = parser.parse_args(arg_list)
    target = Path(args.file).absolute()
    config_file = Path(args.configfile).absolute()
    target_file = make_command(target, default_config(config_file))
    if target_file:
        print(target_file.parsed_file)

if __name__ == '__main__':
    main(arg_list=sys.argv[1:])