import os
threads = '1'
os.environ["OMP_NUM_THREADS"] = threads
os.environ["OPENBLAS_NUM_THREADS"] = threads
os.environ["MKL_NUM_THREADS"] = threads
os.environ["VECLIB_MAXIMUM_THREADS"] = threads
os.environ["NUMEXPR_NUM_THREADS"] = threads

import sys
import time
import pickle
import shutil
import argparse
import traceback
import numpy as np
import pandas as pd
import random
import cv2
import dlib
import scipy.io as sio
from skimage import io
from datetime import datetime

from sklearn.neighbors import NearestNeighbors
from sklearn.cluster import KMeans
from sklearn.decomposition import PCA
from sklearn.metrics.pairwise import euclidean_distances
import matplotlib.pyplot as plt
from matplotlib import style
from mpl_toolkits.mplot3d import Axes3D
from sklearn.manifold import TSNE
from utils import *
from multiprocessing import Pool
from multiprocessing_logging import install_mp_handler

import torch
import torchvision.transforms as transforms
from torch.utils.data import Dataset, DataLoader
import torch.backends.cudnn as cudnn

## detect -> label -> recognize, align & compose: save aligned, composed faces and frame information
def recognize_align_compose(frame_dir, align_dir, compo_dir, score_dir, filt_info_file,
                            cluster_n=2, thres_dist=0.6, min_faces=10,
                            min_targets=10, min_variation=0.2, min_dist=0.6, verbose=True):
    '''
    (1) Detect all face locations, 68 2d landmarks in all frames and encode faces.
        Record information of each frame:
        {'num_faces': 0/1/2, 'has_target': 0/1, 'target_idx': None/0/1/2, 'locations': [(top, right, bottom, left)], '2d_landmarks': [ np.ndarray(68, 2)], 'encodings': [ np.ndarray(128,)]}
    (2) Label images as known target reference images.
    (3) Preprocess: record videos should be excluded.
    (4) Recognize target faces. Along the way, align and compose faces then save images.
    (5) Save frame information: {'frame001': {...}, 'frame002': {...}, ...},
        save table: ['frame_id', 'num_faces', 'has_target'],
        save possibly excluded video ids.
    :param frame_dir: directory containing frames
    :param align_dir: directory for saving aligned face images
    :param compo_dir: directory for saving composite face images
    :param filt_info_file: file for saving non-candidate video information
    :param cluster_n: number of known images chosen in each cluster, default is 2.
    :param thres_dist: threshold distance for recogizing face as target, default is 0.6.
    :param min_faces: minimum number of faces detected from all frames, default is 10. Will no recognize target if very few faces.
    :param min_targets: miminum number of target faces, default is 10.
    :param min_variation: minimum variation of all target faces, smaller videos will be discarded, default is 0.2.
    :param mim_dist: minimum distance between two clusters (only target faces), larger videos will be discarded, default is 0.6.
    :param verbose: verbose the process or not
    :return: no return
    '''

    ## extract video id
    vid_id = frame_dir.split('/')[-1]

    ## save aligned and composite faces to
    if align_dir and not os.path.exists(align_dir):
        os.makedirs(align_dir)
    if compo_dir and not os.path.exists(compo_dir):
        os.makedirs(compo_dir)
    if score_dir and not os.path.exists(score_dir):
        os.makedirs(score_dir)

    ## frames
    frame_files = os.listdir(frame_dir)
    frame_files = [x for x in frame_files if '.jpg' in x]
    frame_files = sorted(frame_files, key=lambda x: int(x.split('_')[0].replace('frame', '')))
    frame_ids = [x.split('.')[0] for x in frame_files]

    ## record
    frame_dict = dict.fromkeys(frame_ids)
    info_df = pd.DataFrame(columns=['frame_id', 'num_faces', 'has_target'])
    comment = ''

    # print(f'Begin detection {vid_id}')
    ## detect and encode
    for i, image_file in enumerate(frame_files):
        frame_info = dict.fromkeys(['num_faces', 'has_target', 'target_idx', 'locations', '2d_landmarks', 'encodings'])
        frame_id = image_file.split('.')[0]
        # original frame
        image = cv2.imread(os.path.join(frame_dir, image_file))
        # face located in the rectangle (0.2-0.3s): list of dlib rectangles
        face_locations = face_detector(image)
        # coordinates of 68 landmarks (0.004s): list of dlib full_object_detection objects
        landmarks = [pose_predictor_68_point(image, face_location) for face_location in face_locations]
        # face encodings (0.6-0.8s): list of (128,) arrays
        encodings = [np.array(face_encoder.compute_face_descriptor(image, landmark)) for landmark in landmarks]
        # record
        save_locations = [rect_to_css(x) for x in face_locations] # convert rectangle to (top, right, bottom, left)
        save_2d_landmarks = [landmarks_to_np(x) for x in landmarks] # convert dlib full_object_detection object to (68, 2) np.ndarray
        frame_info['num_faces'], frame_info['locations'], frame_info['2d_landmarks'], frame_info['encodings'] = \
            len(face_locations), save_locations, save_2d_landmarks, encodings
        if verbose and i % 100 == 0:
            print(f"[{datetime.today().strftime('%Y-%m-%d %H:%M:%S')}] Encoded {i} / {len(frame_files)}")
        frame_dict[frame_id] = frame_info
    if verbose:
        print(f"[{datetime.today().strftime('%Y-%m-%d %H:%M:%S')}] Finish face detection.")

    ## label images: closest to mean + k closest to mean of each cluster
    tot_num_faces = sum([v['num_faces'] for _, v in frame_dict.items()])
    if tot_num_faces >= min_faces:
        known_encodings, all_encodings = [], []
        for k, v in frame_dict.items():
            all_encodings.extend(v['encodings'])
        X = np.array(all_encodings)
        kmeans = KMeans(n_clusters=2).fit(X)
        y = kmeans.labels_
        large_id = 1 if sum(y) > len(y) / 2 else 0
        small_id = 1 - large_id
        mean_encoding = np.mean(X, axis=0)
        mean_encoding_l = np.mean(X[np.where(y == large_id)[0]], axis=0)
        mean_encoding_s = np.mean(X[np.where(y == small_id)[0]], axis=0)
        known_encodings.extend(X[np.argsort(np.linalg.norm(X[np.where(y == large_id)[0]] - mean_encoding_s, axis=1))[:cluster_n].tolist()])

        X_l = X[np.where(y == large_id)[0], :]
        X_s = X[np.where(y == small_id)[0], :]
        X_cluster = np.vstack([X_l, X_s])
        dist_cluster = euclidean_distances(X_cluster, X_cluster)
        min_between_dist_cluster = np.min(dist_cluster[X_l.shape[0]:, :X_l.shape[0]]) if X_l.shape[0] < X.shape[0] else 0
        # only use larger cluster if minimum distance between two clusters is greater than min_dist
        if min_between_dist_cluster <= min_dist:
            known_encodings.append(X[np.argmin(np.linalg.norm(X - mean_encoding, axis=1))])
            known_encodings.extend(X[np.argsort(np.linalg.norm(X[np.where(y == small_id)[0]] - mean_encoding_l, axis=1))[:cluster_n].tolist()])
        known_encodings = np.array(known_encodings)

        ## recognize
        encoding_list = []
        n = 0
        for i, (k, v) in enumerate(frame_dict.items()):
            # no faces detected
            if v['num_faces'] == 0:
                v['has_target'] = 0
            # have faces detected
            else:
                closest_dists, closest_dists_l, closest_dists_s = [], [], []
                for j, encoding in enumerate(v['encodings']):
                    if y[n+j] == large_id:
                        closest_dists_l.append(np.min(np.linalg.norm(known_encodings - encoding, axis=1)))
                    else:
                        closest_dists_s.append(np.min(np.linalg.norm(known_encodings - encoding, axis=1)))
                    closest_dists.append(np.min(np.linalg.norm(known_encodings - encoding, axis=1)))
                n += j + 1
                if len(closest_dists_l) > 0:
                    v['has_target'] = 1
                    v['target_idx'] = closest_dists.index(min(closest_dists_l))
                    encoding_list.append(np.insert(encoding, 0, 1))
                elif len(closest_dists_l) == 0 and len(closest_dists_s) > 0:
                    v['has_target'] = 1
                    v['target_idx'] = closest_dists.index(min(closest_dists_s))
                    encoding_list.append(np.insert(encoding, 0, 1))
                else:
                    v['has_target'] = 0
                    v['target_idx'] = None
                    encoding_list.append(np.insert(encoding, 0, 0))
                if v['target_idx'] is not None:
                    # align
                    image = cv2.imread(os.path.join(frame_dir, f'{k}.jpg'))
                    landmarks = v['2d_landmarks'][v['target_idx']]  # (68, 2)
                    aligned_face = face_align(image, landmarks)
                    # save aligned face
                    if align_dir:
                        cv2.imwrite(os.path.join(align_dir, f"{k}.jpg"), aligned_face)
                    # compose
                    left2, right2 = face_composite(aligned_face)
                    # save left2 and right2
                    if compo_dir:
                        cv2.imwrite(os.path.join(compo_dir, f"{k}_left.jpg"), left2)
                        cv2.imwrite(os.path.join(compo_dir, f"{k}_right.jpg"), right2)
            # record
            info_df.loc[i] = [k, v['num_faces'], v['has_target']]
        # non-candidate video info
        if sum(info_df['has_target']) < min_targets:
            comment = f"Less than {min_faces} target faces in all extracted frames."

        X_target = np.array(encoding_list)
        y_target = kmeans.labels_[np.where(X_target[:, 0] == 1)]
        X_target = X_target[np.where(X_target[:, 0] == 1)[0], 1:]
        X0_target = X_target[np.where(y_target == 0)[0], :]
        X1_target = X_target[np.where(y_target == 1)[0], :]
        X_cluster = np.vstack([X0_target, X1_target])
        dist_cluster = euclidean_distances(X_cluster, X_cluster)
        min_between_dist_cluster = np.min(dist_cluster[X0_target.shape[0]:, :X0_target.shape[0]]) if X0_target.shape[0] < X_target.shape[0] else 0
        if min_between_dist_cluster > min_dist:
            comment = f"Minimum distance between two target encodings is larger than {min_dist}"
        if np.max(dist_cluster) < min_variation:
            comment = f"Maximum distance between target encodings is smaller than {min_variation}"

    else:
        # some videos may not have faces detected
        for i, (k, v) in enumerate(frame_dict.items()):
            info_df.loc[i] = [k, v['num_faces'], None]
        # non-candidate video info
        comment = f"Less than {min_faces} faces in all extracted frames."

    # save frame info
    info_df.to_csv(os.path.join(score_dir, 'frame_info.csv'), sep='\t', index=False)
    # save all info
    with open(os.path.join(score_dir, 'frame_2d_68pts_encodings.pkl'), 'wb') as f:
        pickle.dump(frame_dict, f)
    # save non-candidate video info
    if len(comment) > 0:
        filt_info = f'{vid_id}: {comment}\n'
        with open(filt_info_file, 'a+') as f:
            f.write(filt_info)

    print(f"[{datetime.today().strftime('%Y-%m-%d %H:%M:%S')}] Recognized and aligned video {frame_dir.split('/')[-1]}")
    logging.info(f"Recognized and aligned video {frame_dir.split('/')[-1]} (PID: {os.getpid()})")

if __name__ == '__main__':

    ## parse arguments
    parser = argparse.ArgumentParser()
    # directories
    parser.add_argument('--video-root', type=str, help='Specify the directory containing the videos.')
    parser.add_argument('--frame-root', type=str, help='Specify the directory for saving extracted frames.')
    parser.add_argument('--align-root', type=str, help='Specify the directory for saving the aligned face images.')
    parser.add_argument('--compo-root', type=str, help='Specify the directory for saving the composite face images.')
    parser.add_argument('--score-root', type=str, help='Specify the directory for saving information and scores.')
    parser.add_argument('--pre-filter-file', type=str, help='Specify the file for saving the pre-filtered video information.')
    # global setting
    parser.add_argument('-v', '--verbose', action='store_true', help='Verbose inside each step.')
    parser.add_argument('-q', '--quiet', action='store_true', help='Quiet inside each step.')

    args = parser.parse_args()

    video_root = args.video_root
    frame_root = args.frame_root
    align_root = args.align_root
    compo_root = args.compo_root
    score_root = args.score_root
    pre_filter_file = args.pre_filter_file
    open(pre_filter_file, 'w').close()
    if args.verbose:
        verbose = True
    elif args.quiet:
        verbose = False

    if not os.path.exists(align_root):
        os.makedirs(align_root)
    if not os.path.exists(compo_root):
        os.makedirs(compo_root)
    if not os.path.exists(score_root):
        os.makedirs(score_root)
    if not os.path.exists('logs'):
        os.makedirs('logs')
    log_file = f"logs/{frame_root.split('/')[1].split('_')[0]}_recognition_align.log"
    open(log_file, 'w').close()
    logging.basicConfig(filename=log_file,
                        level=logging.INFO,
                        format='[%(asctime)s] [%(levelname)s] %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S',
                        filemode='a+')
    install_mp_handler()

    video_ids = os.listdir(video_root)
    video_ids = sorted([x.split('.')[0] for x in video_ids])

    ## 2. recognize, align and compose
    # if os.path.exists(pre_filter_file):
    #     os.remove(pre_filter_file)
    inp_args = []
    for idx, vid_id in enumerate(video_ids):
        frame_dir = os.path.join(frame_root, vid_id)
        align_dir = os.path.join(align_root, vid_id)
        compo_dir = os.path.join(compo_root, vid_id)
        score_dir = os.path.join(score_root, vid_id)
        # if os.path.exists(os.path.join(score_dir, 'frame_2d_68pts_encodings.pkl')):
        #     continue
        inp_args.append((frame_dir, align_dir, compo_dir, score_dir, pre_filter_file,
                         2, 0.6, 10, 10, 0.2, 0.6, verbose))
    if len(inp_args) > 0:
        try:
            with Pool(os.cpu_count()) as pool:
                pool.starmap(recognize_align_compose, inp_args)
                pool.close()
                pool.join()
        except Exception as e:
            print(e)
            logging.error(e)
        print(f"[{datetime.today().strftime('%Y-%m-%d %H:%M:%S')}] ********** Finish step 2: recognize, align and compose **********")
        logging.info('********** Finish step 2: recognize, align and compose **********')


import unittest
from cStringIO import StringIO
import sys, logging

sys.path.insert(0, '../')
from models.poems.parseLines import parse, parse_all, is_english

class TestParseFunctions(unittest.TestCase):
    def test_invalid(self):
        ''' Tests that invalid lines are properly ignored '''
        while True:
            line = raw_input(r"Enter invalid line (or 'quit' to quit): ")
            if line == 'quit':
                return
            self.assertEqual(parse(line), {})

    def test_valid(self):
        ''' Tests that valid lines are not ignored '''
        while True:
            line = raw_input(r"Enter valid line (or 'quit' to quit): ")
            if line == 'quit':
                return
            self.assertNotEqual(parse(line), {})

if __name__ == '__main__':
    logging.basicConfig(stream=sys.stderr, level=logging.INFO)
    unittest.main()


import tkinter as tk
import tkinter
import customtkinter as ct
from tkinter import simpledialog
from tkinter.messagebox import askyesno, showinfo, showerror
from cryptography.fernet import Fernet
import cryptography
import pyperclip
import keyboard
import win10toast
from tkinter.filedialog import askopenfilename, asksaveasfile
import os
conversations = {}
global current_value
current_value = 0
class EncryptionSoftware():
    def __init__(self, key, enc, dec):
        self.key = key
        self.enc = enc
        self.dec = dec
        self.fern = Fernet(self.key)
        self.notifier=win10toast.ToastNotifier()
    def files_enc(self):
        file_r = askopenfilename()
        if os.path.splitext(file_r)[-1] == '.txt':
            with open(os.path.abspath(file_r), 'r') as encfile:
                dat =  encfile.read().encode('utf-8')
        else:
            with open(os.path.abspath(file_r), 'rb') as encfile:
                dat =  encfile.read()
        file_w = asksaveasfile(mode='wb', defaultextension='.crypt')
        file_w.write(self.fern.encrypt(dat))
        file_w.close()

    def files_dec(self):
        try:
            file_r = askopenfilename()
            with open(os.path.abspath(file_r), 'rb') as encfile:
                dat =  encfile.read()
            file_w = asksaveasfile(mode='wb')
            file_w.write(self.fern.decrypt(dat))
        except:
                showerror(title="File Permission denied", message="Could not open file! (maybe you closed the file explorer window?)")

    def encrypt(self, hotkey:bool = False, file:bool = False):
        data = pyperclip.paste().encode('utf-8') if hotkey else self.enc.get().encode('utf-8')
        pyperclip.copy(self.fern.encrypt(data).decode('utf-8'))

        self.notifier.show_toast(title="Encrypt and chat", msg="message encrypted.. paste to send encrypted message", threaded=True)
    def decrypt(self, hotkey:bool = False):
        try:
            data = pyperclip.paste().encode('utf-8')  if hotkey else self.dec.get().encode('utf-8')
            dec = str(self.fern.decrypt(data).decode('utf-8'))
        except cryptography.fernet.InvalidToken:
            showerror('An error occurred', f"On decrypting data, the following exception occured (probably the key or the message to be decrypted is invalid)")
            dec = "ERROR: <an error occured while decrypting this message>"
        pyperclip.copy(dec)
        self.notifier.show_toast(title="Encrypt and chat", msg=f"decrypted message reads: {dec}", threaded=True)
        showinfo('decrypted message', f"decrypted message: \n {dec}")
ct.set_appearance_mode("dark")
root = ct.CTk()
root.geometry("800x400")
root.title("Encrypt and chat")
l1 = ct.CTkLabel(root, text="Encrypt and chat", font=("Segoe UI Variable", 25))
l1.place(relx=0.5,rely=0.1,anchor=tkinter.CENTER)
tabview = ct.CTkTabview(root, width=800)
tabview.pack(padx=20, pady=60)
showinfo("How to use?", """
Instructions on how to use the program:\n
Key is copied to your clipboard after you either ask for a new key or type the new key into dialog box.\n
To use the program. share the program with your friends and ask them to download and set it up. once thats done, run the program\n
this time when the dialog asks if u require a new key. select "yes" and then the new key will be copied to your clipboard.\n
share the new key with your friends over a secure channel\n
Once the deired recipient recieves the key, they have to run the program and select "no" at the prompt asking you "if u require a new key".\n
once they select "no" they will have to type in the key sent by you and finally run the program by clicking "ok".
 and then use the following steps to encrypt or decrypt.\n
to encrypt: select and copy (ctrl+c) the text to encrypt and press ctrl+e to encrypt.. press ctrl+V to paste the encrypted data\n
to decrypt: select and paste (ctrl+c) the text to decrypt and press ctrl+d to decrypt.. then press ctrl+v to see the decrypted text\n
decrypted text also appears on the notification shown after decyption\n
NEW!: You can now add new conversations by presing the new tab button""")
def keyret():
    keystr = ""
    answer = askyesno("new key", "do you require a new key?")
    if answer:
        key = Fernet.generate_key()
    else:
        while keystr is None or keystr == "":
            keydial = ct.CTkInputDialog(text="Type in your key:", title="Enter Key")
            keystr = keydial.get_input()
            if keystr == "" or keystr is None:
                showerror("No key entered!", "Please enter an appropriate key!")
        else:
            key = keystr.decode("utf-8")
    pyperclip.copy(str(key.decode('utf-8')))
    n=win10toast.ToastNotifier()
    n.show_toast(title="key copied", msg="key copied to clipboard!", duration = 3, threaded=True)
    return key
def changekeys():
    n_key = keyret()
    conversations[tabview.get()][0] = n_key
    conversations[tabview.get()][1].key = n_key
def checkbox():
    global current_value
    current_value = 1 if current_value == 0 else 0
    print(current_value)
def enc_handler():
    if current_value == 0: conversations[tabview.get()][1].encrypt()
    else: conversations[tabview.get()][1].files_enc()
def dec_handler():
    if current_value == 0: conversations[tabview.get()][1].decrypt()
    else: conversations[tabview.get()][1].files_dec()
def ctab():
    if len(conversations) == 1:
        quit()
    else:
        conversations.pop(tabview.get())
        tabview.delete(tabview.get())
def ntab():
    conv = ct.CTkInputDialog(title="Conversation name", text="Enter a name for this conversation: ")
    conv_ret = conv.get_input()
    conv_name = conv_ret if not conv_ret == '' else f"Untitled conv {len(conversations)}"
    tabview.add(conv_name)
    kb = keyret()
    l1 = ct.CTkLabel(tabview.tab(conv_name), text="Data to Encrypt")
    l1.place(relx=0.5,rely=0.1,anchor=tkinter.CENTER)
    enc = ct.CTkEntry(master=tabview.tab(conv_name),
                               placeholder_text="Data to Encrypt",
                               width=240,
                               height=25,
                               border_width=2,
                               corner_radius=10)
    enc.place(relx=0.5, rely=0.3, anchor=tkinter.CENTER)
    l2 = ct.CTkLabel(tabview.tab(conv_name), text="Data to Decrypt")
    l2.place(relx=0.5,rely=0.5,anchor=tkinter.CENTER)
    dec = ct.CTkEntry(master=tabview.tab(conv_name),
                               placeholder_text="Data to Decrypt",
                               width=240,
                               height=25,
                               border_width=2,
                               corner_radius=10)
    dec.place(relx=0.5, rely=0.7, anchor=tkinter.CENTER)
    conversations[conv_name] = (kb, EncryptionSoftware(kb, enc, dec))
    button_1 = ct.CTkButton(tabview.tab(conv_name), text="New conversation", command=ntab)
    button_1.place(relx=0.9, rely=0.9, anchor=tkinter.CENTER)
    check1 = ct.CTkCheckBox(master=tabview.tab(conv_name), text="File", onvalue=1, offvalue=0, command=checkbox)
    check1.place(relx=0.1, rely=0.9, anchor=tkinter.CENTER)
    print(check1.get())
    button_2 = ct.CTkButton(tabview.tab(conv_name), text="Decrypt", command=dec_handler)
    button_2.place(relx=0.7, rely=0.9, anchor=tkinter.CENTER)
    button_3 = ct.CTkButton(tabview.tab(conv_name), text="Encrypt", command=enc_handler)
    button_3.place(relx=0.5, rely=0.9, anchor=tkinter.CENTER)
    button_4 = ct.CTkButton(tabview.tab(conv_name), text="Regenerate key", command=changekeys)
    button_4.place(relx=0.3, rely=0.9, anchor=tkinter.CENTER)
    button_5 = ct.CTkButton(tabview.tab(conv_name), text="Close Chat", command=ctab)
    button_5.place(relx=0.9, rely=0.1, anchor=tkinter.CENTER)
    button_6 = ct.CTkButton(tabview.tab(conv_name), text="Copy Key", command=lambda: pyperclip.copy(str(kb.decode('utf-8'))))
    button_6.place(relx=0.1, rely=0.1, anchor=tkinter.CENTER)
ntab()
keyboard.add_hotkey("ctrl+e", conversations[tabview.get()][1].encrypt, args=(True, ))
keyboard.add_hotkey("ctrl+d", conversations[tabview.get()][1].decrypt, args=(True, ))
keyboard.add_hotkey("ctrl+shift+e", conversations[tabview.get()][1].files_enc)
keyboard.add_hotkey("ctrl+shift+d", conversations[tabview.get()][1].files_dec)
keyboard.add_hotkey("ctrl+n+c", ntab)
keyboard.add_hotkey("ctrl+s+c", ctab)
root.mainloop()
#  TODO: once we make RE encryption algo, change this algo to that


import json

from collections import OrderedDict

from django.forms import Media, widgets
from django.utils.functional import cached_property

from wagtail import VERSION as WAGTAIL_VERSION
from wagtail.admin.rich_text.converters.editor_html import EditorHTMLConverter
from wagtail.admin.staticfiles import versioned_static

if WAGTAIL_VERSION >= (3, 0):
    from wagtail.admin.panels import FieldPanel
    from wagtail.rich_text import features
    from wagtail.telepath import register
    from wagtail.widget_adapters import WidgetAdapter
else:
    from wagtail.admin.edit_handlers import RichTextFieldPanel
    from wagtail.core.rich_text import features
    from wagtail.core.telepath import register
    from wagtail.core.widget_adapters import WidgetAdapter

from .plugins import CORE_HALLO_PLUGINS

class HalloRichTextArea(widgets.Textarea):
    template_name = "wagtail_hallo/widgets/hallo_rich_text_area.html"

    # this class's constructor accepts a 'features' kwarg
    accepts_features = True

    def get_panel(self):
        return FieldPanel if WAGTAIL_VERSION >= (3, 0) else RichTextFieldPanel

    def __init__(self, *args, **kwargs):
        self.options = kwargs.pop("options", None)

        self.features = kwargs.pop("features", None)
        if self.features is None:
            self.features = features.get_default_features()

        self.converter = EditorHTMLConverter(self.features)

        # construct a list of plugin objects, by querying the feature registry
        # and keeping the non-null responses from get_editor_plugin
        self.plugins = CORE_HALLO_PLUGINS + list(
            filter(
                None,
                [
                    features.get_editor_plugin("hallo", feature_name)
                    for feature_name in self.features
                ],
            )
        )
        self.plugins.sort(key=lambda plugin: plugin.order)

        super().__init__(*args, **kwargs)

    def format_value(self, value):
        # Convert database rich text representation to the format required by
        # the input field
        value = super().format_value(value)

        if value is None:
            return None

        return self.converter.from_database_format(value)

    def get_context(self, name, value, attrs):
        context = super().get_context(name, value, attrs)

        if self.options is not None and "plugins" in self.options:
            # explicit 'plugins' config passed in options, so use that
            plugin_data = self.options["plugins"]
        else:
            plugin_data = OrderedDict()
            for plugin in self.plugins:
                plugin.construct_plugins_list(plugin_data)
        context["widget"]["plugins_json"] = json.dumps(plugin_data)

        return context

    def value_from_datadict(self, data, files, name):
        original_value = super().value_from_datadict(data, files, name)
        if original_value is None:
            return None
        return self.converter.to_database_format(original_value)

    @cached_property
    def media(self):
        media = Media(
            js=[
                versioned_static("js/vendor/hallo.js"),
                versioned_static("js/hallo-editor.js"),
                versioned_static("js/hallo-telepath.js"),
            ],
            css={"all": [versioned_static("css/hallo.css")]},
        )

        for plugin in self.plugins:
            media += plugin.media

        return media

class HalloRichTextAreaAdapter(WidgetAdapter):
    js_constructor = "wagtail.widgets.HalloRichTextArea"

register(HalloRichTextAreaAdapter(), HalloRichTextArea)


from zope.component import provideUtility
from agx.core.interfaces import IScope
from agx.core.metaconfigure import _chkregistered
from scope import (
    XMIScope,
    StereotypeScope,
)

def registerXMIScope(name, transform, tags, type, class_=XMIScope):
    name = '%s.%s' % (transform, name)
    _chkregistered(IScope, name=name)
    scope = class_(name, tags, type)
    provideUtility(scope, provides=IScope, name=name)

def registerStereotypeScope(name, transform, class_=StereotypeScope):
    name = '%s.%s' % (transform, name)
    _chkregistered(IScope, name=name)
    scope = class_()
    provideUtility(scope, provides=IScope, name=name)


import sys
import subprocess
import psycopg2
import datetime
import os
import shutil
import sys
import argparse

def main(args):

	try:
		conn = psycopg2.connect("dbname=%s password=%s host=%s" % ('mosi2_crawl', 'multicomp', 'localhost'))
	except:
		print("Database connection error")
		exit(0)
	cur = conn.cursor()

	input_file = open(args.input,"r")
	output_file = open(args.output,"w+")

	channel_set = set([])

	one_line = input_file.readline()
	line_list = one_line.split(',')
	while((one_line != '') and (one_line != "\n")):
		vid_id = line_list[0][0:-4]
		print(vid_id)
		search_command = """SELECT channel_id
							FROM videos_video
							WHERE id=%(id)s"""

		try:
			cur.execute(search_command, {"id":vid_id})
			channel_ids = [a for (a,) in cur.fetchall()]
			channel_id = channel_ids[0]
			if(channel_id[-2:] == '_m') or (channel_id[-2:] == '_f'):
				channel_set.add(channel_id)
				output_file.write(channel_id + ',' + ','.join(line_list))
		except:
			pass

		one_line = input_file.readline()
		line_list = one_line.split(",")

	output_file.close()
	output_chans = open(args.output_chans,'w+')
	for chan in channel_set:
		output_chans.write(chan + '\n')

	output_chans.close()

def parse_args():
	parser = argparse.ArgumentParser(description="""Take in files of transcript
		sentences and p2fa files and return text files of relevant videos and
		channels""")

	parser.add_argument("-i", required=True, dest="input", action="store")
	parser.add_argument("-c", required=True, dest="output_chans", action="store")
	parser.add_argument("-o", required=True, dest="output", action="store")
	parser.add_argument('-v', action='store_true', required = False,
		help='enable verbose')

	args = parser.parse_args()

	main(args)

parse_args()


import csv
from datetime import datetime
import os

def open_csv_file():
    csv_file = open('data.csv', 'w', encoding='utf-8', newline='')
    wr = csv.writer(csv_file)
    return (csv_file, wr)

def close_csv_file(csv_file):
    csv_file.close()

def make_new_list(newlist, list, wr, head, content):
    content = content.strip()
    newlist.insert(0, head)
    newlist.insert(0, content)
    wr.writerow(newlist)
    del newlist[1] ,newlist[0]

def parsing(filename,wr,header):
    file = open(filename, 'r')
    dic = {}

    while True:
        line = file.readline()
        if line == '\n':
            break
        if line != '':
            if line.find(':') == -1:
                value = dic[key]
                dic[key] = value + line.strip()
                continue

            # key parsing
            key = line.split(':')[0]
            # print(key)

            # value parsing
            value = line.replace(key, "")[:-1]
            value = value[1:].lstrip()
            # print(value)
            dic[key] = value

    list = []

    for head in header:
        if head in ('To', 'Cc', 'Bcc'):
            continue

        if dic.get(head) != None:
            if head == "Date":
                t = datetime.strptime(dic.get(head)[:-6],
                                      '%a, %d %b %Y %H:%M:%S %z')
                list.append(t.strftime("%Y-%m-%d %H:%M:%S"))
            else:
                list.append(dic.get(head))
        else:
            list.append("")

    for head in header:
        if head in ('To', 'Cc', 'Bcc'):
            content = dic.get(head)
            if content != None:
                if content.find(',') == None:
                    newlist = list
                    make_new_list(newlist, list, wr, head, content)
                else :
                    contentlist = content.split(',')
                    for content in contentlist:
                        newlist = list
                        make_new_list(newlist, list, wr, head, content)
    file.close()

def search(dirname,wr,header):
    filenames = os.listdir(dirname)
    for filename in filenames:
        full_filename = os.path.join(dirname, filename)
        # print (full_filename)
        # wr.writerow([filename])
        parsing(full_filename,wr,header)

if __name__ == "__main__":

    csv_file, wr = open_csv_file()

    header = ["Message-ID", "Date", "From", "To", "Subject", "Cc",
              "Mime-Version", "Content-Type", "Content-Transfer-Encoding",
              "Bcc", "X-From", "X-To", "X-cc", "X-bcc", "X-Folder",
              "X-Origin", "X-FileName"]

    newheader = ["Recipient", "Recipient-type", "Message-ID", "Date", "From",
                 "Subject", "Mime-Version", "Content-Type",
                 "Content-Transfer-Encoding", "X-From", "X-To", "X-cc",
                 "X-bcc", "X-Folder", "X-Origin", "X-FileName"]

    wr.writerow(newheader)
    search("c:/enron_email/emails/",wr, header)
    close_csv_file(csv_file)


from __future__ import print_function
import pickle

# import os.path, os.listdir
import os
import io
import shutil
import requests
from typing import List
from mimetypes import MimeTypes
from dateutil.parser import parse
from datetime import datetime
from googleapiclient.discovery import build
from google_auth_oauthlib.flow import InstalledAppFlow
from google.auth.transport.requests import Request
from googleapiclient.http import MediaIoBaseDownload, MediaFileUpload
from apiclient import errors

global SCOPES
SCOPES = ["https://www.googleapis.com/auth/drive"]

def __init__(self,apikey):
        path = os.path.expanduser("~/google_pickle")
        with open(path, "rb") as token:
            self.creds = pickle.load(token)

        # Connect to the API service
        self.service = build("drive", "v3", credentials=self.creds)

def GoogleFileUpload(self, filepath: str, folderid: str) -> None:

    name = filepath.split("/")[-1]

    mimetype = MimeTypes().guess_type(name)[0]

    file_metadata = {"name": name, "parents": [folderid]}
    try:
        media = MediaFileUpload(filepath, mimetype=mimetype)
        file = (
            self.service.files()
            .create(
                body=file_metadata,
                media_body=media,
                fields="id",
            )
            .execute()
        )
        print(f"{filepath} Uploaded.")

    except Exception as e:
        print(e)
        raise Exception("Can't Upload File.")

def update_file(self, file_id, new_title, new_description, new_mime_type, new_filename, new_revision,folderid):
        """Update an existing file's metadata and content.

        Args:
            service: Drive API service instance.
            file_id: ID of the file to update.
            new_title: New title for the file.
            new_description: New description for the file.
            new_mime_type: New MIME type for the file.
            new_filename: Filename of the new content to upload.
            new_revision: Whether or not to create a new revision for this file.
        Returns:
            Updated file metadata if successful, None otherwise.
        """
        name = file_id
        file_metadata = {"name": name, "parents": [folderid]}
        try:
            # First retrieve the file from the API.
            file = self.service.files().get(fileId=file_id).execute()

            # File's new metadata.
            file['title'] = new_title
            file['description'] = new_description
            file['mimeType'] = new_mime_type

            # File's new content.
            media_body = MediaFileUpload(
                new_filename, mimetype=new_mime_type, resumable=True)

            return self.files().update(fileId=file_id, body=file_metadata, newRevision=new_revision, media_body=media_body).execute()

        except errors.HttpError as error:
            print(f"An error occurred: {error}")
            return None

def delete_gfile(self,filepath,folderid):
    file_id = self.get_fileid(filepath,folderid)
    try:
        self.service.files().delete(fileId=file_id).execute()
    except Exception as e:
        print(e)


from sklearn.ensemble import RandomForestClassifier
import random
import import_data as impd
import pdb

#logic
def create_random_forest(num_trees,criterion,bootstrap,vectors,classes,max_depth):
    rfc=RandomForestClassifier(n_estimators=num_trees,bootstrap=bootstrap,criterion=criterion,max_depth=max_depth)
    rfc.fit(vectors,classes)
    return rfc

def predict_data(testing_data,clf):
    fvs=testing_data["vectors"]
    class_results=testing_data["classes"]
    predictions=clf.predict(fvs)
    total_vectors=len(fvs)
    idx=0
    correct_results=0
    results_by_class={
        "exp": {},
        "total": {}
    }
    for item in set(class_results):
        results_by_class["total"][item]=0
    for item in set(predictions):
        results_by_class["exp"][item]=0

    for _ in predictions:
        results_by_class["total"][class_results[idx]] += 1
        if(class_results[idx]==predictions[idx]):
            correct_results += 1
            results_by_class["exp"][predictions[idx]] += 1

        #print("Actual: " + class_results[idx] + ", result: " + predictions[idx])
        idx += 1
    return correct_results,total_vectors,results_by_class

def split_data(feature_vectors,classes):
    percent_split = 1
    training_vectors = list()
    training_classes=list()

    testing_vectors = list()
    testing_classes = list()
    idx=0
    for _ in feature_vectors:
        z=random.randint(1,10)
        if z <= percent_split:
            testing_vectors.append(feature_vectors[idx])
            testing_classes.append(classes[idx])
        else:
            training_vectors.append(feature_vectors[idx])
            training_classes.append(classes[idx])
        idx += 1
    return {"vectors":training_vectors,"classes":training_classes}, {"vectors":testing_vectors,"classes":testing_classes}


from PyQt5.QtWidgets import (QWidget,QMainWindow, QApplication, QPushButton,
QRadioButton, QLabel, QGridLayout, QCheckBox, QLineEdit, QFileDialog, QFontDialog, QVBoxLayout, QHBoxLayout, QGroupBox, QFileDialog)
import sys
from PyQt5.QtGui import QFont
from PyQt5.QtCore import QSize
from XImageClass import QHRadioGroupBox, QRadioButtonCustom

class App(QWidget):

    label_font = QFont('Consolas',10)

    def __init__(self):
        super().__init__()
        self.setWindowTitle('new-XImage')
        # self.setGeometry(500,200,700,500)
        # self.setWindowIcon()
        self.setupUI()

    def setupUI(self):

        self.grid_layout = QGridLayout(self)
        self.setLayout(self.grid_layout)

        self.label_find = QLabel('Find:', self)
        self.grid_layout.addWidget(self.label_find,0,0)

        self.line_find = QLineEdit(self)
        self.line_find.setPlaceholderText('What do you want to find?')
        self.grid_layout.addWidget(self.line_find,0,1)

        self.label_page = QLabel('Page:', self)
        self.grid_layout.addWidget(self.label_page,1,0)

        self.line_page = QLineEdit(self)
        self.grid_layout.addWidget(self.line_page,1,1)

        self.label_per_page = QLabel('Per page:', self)
        self.grid_layout.addWidget(self.label_per_page,2,0)

        self.line_per_page = QLineEdit(self)
        self.grid_layout.addWidget(self.line_per_page,2,1)

        self.label_save_at = QLabel('Save at:', self)
        self.grid_layout.addWidget(self.label_save_at,3,0)

        self.btt_save_at_browse = QPushButton('Browse...')
        self.btt_save_at_browse.clicked.connect(self.browse)
        self.grid_layout.addWidget(self.btt_save_at_browse,3,2)

        self.line_save_at = QLineEdit(self)
        self.grid_layout.addWidget(self.line_save_at,3,1)

        self.radio_group_box_quality = QHRadioGroupBox('Quality')
        self.rd_quality_raw = QRadioButtonCustom('Raw', 1, self)
        self.rd_quality_full = QRadioButtonCustom('Full', 2, self)
        self.rd_quality_regular = QRadioButtonCustom('Regular', 3, self)
        self.rd_quality_small = QRadioButtonCustom('Small', 4, self)
        self.rd_quality_thumnail = QRadioButtonCustom('Thumnail', 5, self)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_raw)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_full)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_regular)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_small)
        self.radio_group_box_quality.addRadioButtonCustom(self.rd_quality_thumnail)
        self.grid_layout.addWidget(self.radio_group_box_quality,4,0,1,3)

        self.rd_group_box_order_by = QHRadioGroupBox('Order by',parent=self)
        self.rd_latest = QRadioButtonCustom('Latest', 1, self.rd_group_box_order_by)
        self.rd_oldest = QRadioButtonCustom('Oldest', 2, self.rd_group_box_order_by)
        self.rd_popular = QRadioButtonCustom('Popular', 3, self.rd_group_box_order_by)
        self.rd_group_box_order_by.addRadioButtonCustom(self.rd_latest)
        self.rd_group_box_order_by.addRadioButtonCustom(self.rd_oldest)
        self.rd_group_box_order_by.addRadioButtonCustom(self.rd_popular)
        self.grid_layout.addWidget(self.rd_group_box_order_by,5,0,1,3)

        self.checkbox_random = QCheckBox('Random')
        self.checkbox_random.setToolTip('Get random images, maximum is 20 images')
        self.checkbox_random.stateChanged.connect(self.randomIsTrue)
        self.grid_layout.addWidget(self.checkbox_random,6,0)

        self.btt_start = QPushButton('Start')
        self.btt_start.clicked.connect(self.download)
        self.grid_layout.addWidget(self.btt_start,7,0,1,3)

        self.show()

    def download(self):
        # get info download
        pass

    def randomIsTrue(self,state):
        if state > 0:
            self.rd_group_box_order_by.setEnabled(False)
            self.line_per_page.setEnabled(False)
            self.line_page.setEnabled(False)
        else:
            self.rd_group_box_order_by.setEnabled(True)
            self.line_per_page.setEnabled(True)
            self.line_page.setEnabled(True)

    def browse(self):
        file = QFileDialog.getExistingDirectory(self,"Select a folder")
        self.line_save_at.setText(file)

if __name__ == "__main__":
    a = QApplication(sys.argv)
    app = App()
    sys.exit(a.exec_())


import option_keyboard.envs
from option_keyboard.option_keyboard.ok import option_keyboard
from option_keyboard.core.utils import set_global_seed
from option_keyboard.core.value_function import ValueFunction
import argparse
import gym
import torch
import numpy as np
import os
import pickle

parser = argparse.ArgumentParser('test_ok')
parser.add_argument('-e', '--env-name', default='ForagingWorld-v0',
                    help='Name of environment')
parser.add_argument('-s', '--seed', default=0, type=int,
                    help='Random seed')
parser.add_argument('--exp-name', required=True,
                    help='Name of experiment')
parser.add_argument('--n-test-episodes', default=100,
                    help='Number of test episodes')
parser.add_argument('--visualize', action='store_true',
                    help='Flag for visualization')
parser.add_argument('--saved-models', required=True,
                    help='Path to saved models')
parser.add_argument('--save-path', default='',
                    help='Path to file to which results are to be saved')
parser.add_argument('--w', default=[1, 1], nargs=2, type=int,
                    help='Weight vector')

def main():
    args = parser.parse_args()
    env = gym.make(args.env_name)
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    set_global_seed(args.seed)

    d = env.num_resources()
    w = np.array(args.w)
    env.set_learning_options(w=w, flag=True)

    hyperparams_file = open(os.path.join(args.saved_models.split('saved_models')[0],
                            'hyperparams'), 'rb')

    # Loading saved models and constant values
    returns = []
    if args.save_path:
        fp = open(args.save_path, 'a+b')

    hyperparams = pickle.load(hyperparams_file)
    gamma = hyperparams.gamma_ok
    max_ep_steps = hyperparams.max_steps_agent

    value_fns = [ValueFunction(input_dim=env.observation_space.shape[0] + d,
                               action_dim=(env.action_space.n + 1),
                               n_options=d,
                               hidden=[64, 128],
                               batch_size=hyperparams.ok_batch_size,
                               gamma=gamma,
                               alpha=hyperparams.alpha_ok)
                 for _ in range(d)]

    for i in range(env.num_resources()):
        if not torch.cuda.is_available():
            checkpoint = torch.load(os.path.join(args.saved_models,
                                                 'value_fn_%d.pt' %
                                                 (i + 1)),
                                    map_location=torch.device('cpu'))
        else:
            checkpoint = torch.load(os.path.join(args.saved_models,
                                                 'value_fn_%d.pt' %
                                                 (i + 1)))

        value_fns[i].q_net.load_state_dict(checkpoint['Q'])
        value_fns[i].q_net.to(device)

    # Testing
    for _ in range(args.n_test_episodes):
        s = env.reset()
        done = False
        s = torch.from_numpy(s).float().to(device)
        n_steps = 0
        ret = 0

        while not done:
            (s_next, done, _, _, n_steps, info) = option_keyboard(env, s, w,
                                                                  value_fns,
                                                                  gamma,
                                                                  n_steps,
                                                                  max_ep_steps,
                                                                  device,
                                                                  args.visualize)

            ret += sum(info['rewards'])
            s = torch.from_numpy(s_next).float().to(device)

        print('Episode return:', ret)
        returns.append(ret)

    returns = np.array(returns)
    print('Mean: %f, Std. dev: %f' % (returns.mean(), returns.std()))
    pickle.dump({'Seed': args.seed, 'Returns': returns}, fp)
    fp.close()

if __name__ == '__main__':
    main()


import torch
import torchvision
from PIL import Image
import os
import matplotlib.pyplot as plt
import numpy as np
import torch
from torch import nn
from torch import optim
import torch.nn.functional as F
from torchvision import datasets, transforms, models
import torchvision.models as models
from PIL import Image
import json
from matplotlib.ticker import FormatStrFormatter
import os
import cv2
import argparse
from pprint import pprint
parser = argparse.ArgumentParser()
parser.add_argument("t", type=str, help= 'specify a car type; type none if you do not have one')
args = parser.parse_args()
type = args.t
print(type + ' Car will be now searched for')
model = models.resnet34(pretrained=True)
num_ftrs = model.fc.in_features
model.fc = nn.Linear(num_ftrs, 196)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
lrscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=3, threshold = 0.9)
def load_checkpoint(filepath):
    checkpoint = torch.load(filepath, map_location=torch.device('cpu'))

    # model.load_state_dict(checkpoint['state_dict'])
    model.load_state_dict(checkpoint['state_dict'], strict=False)
    model.class_to_idx = checkpoint['class_to_idx']

    return model

model = load_checkpoint('my_checkpoint1.pth')
model = torch.nn.DataParallel(model)

def process_image(image):
    # Process a PIL image for use in a PyTorch model

    # Converting image to PIL image using image file path
    pil_im = Image.open(image, mode='r').convert('RGB')

    # Building image transform
    transform = transforms.Compose([transforms.Resize((244, 244)),
                                    # transforms.CenterCrop(224),
                                    transforms.ToTensor(),
                                    transforms.Normalize([0.485, 0.456, 0.406],
                                                         [0.229, 0.224, 0.225])])

    # Transforming image for use with network
    pil_tfd = transform(pil_im)

    # Converting to Numpy array
    array_im_tfd = np.array(pil_tfd)

    return array_im_tfd

def predict(image_path, model, topk=5):
    # Implement the code to predict the class from an image file

    # Loading model - using .cpu() for working with CPUs
    loaded_model = load_checkpoint(model).cpu()
    # Pre-processing image
    img = process_image(image_path)
    # Converting to torch tensor from Numpy array
    img_tensor = torch.from_numpy(img).type(torch.FloatTensor)
    # Adding dimension to image to comply with (B x C x W x H) input of model
    img_add_dim = img_tensor.unsqueeze_(0)

    # Setting model to evaluation mode and turning off gradients
    loaded_model.eval()
    with torch.no_grad():
        # Running image through network
        output = loaded_model.forward(img_add_dim)

    # conf, predicted = torch.max(output.data, 1)
    probs_top = output.topk(topk)[0]
    predicted_top = output.topk(topk)[1]

    # Converting probabilities and outputs to lists
    conf = np.array(probs_top)[0]
    predicted = np.array(predicted_top)[0]

    # return probs_top_list, index_top_list
    return conf, predicted
def most_frequent(List):
    return max(set(List), key = List.count)
def find_classes(dir):
    classes = os.listdir(dir)
    classes.sort()
    class_to_idx = {classes[i]: i for i in range(len(classes))}
    return classes, class_to_idx
classes, c_to_idx = find_classes("car_data/train")
model_path = 'my_checkpoint1.pth'
classes_detect = []
count = 0
frame_Car = {}
for d in [d for d in os.listdir("Video/")]:
    count= count + 1
    img_path = "Video/" + d

    if (img_path != 'Video/.DS_Store'):
        if (count < 500):
            conf1, predicted1 = predict(img_path, model_path, topk=5)
            classes_detect.append(predicted1[0])
            frame_Car[count]= {}
            frame_Car[count]['file'] = img_path
            frame_Car[count]['ID'] = predicted1[0]

    print(img_path)
    print(count)
    print(classes[(most_frequent(classes_detect))])
pprint(frame_Car)
#Sedan
#SUV
#Coupe
#Convertible
#Cab
#Van
#Wagon
#Hatchback
#Minivan
img_file_arr = []
img_arr=[]
predicted_car = classes[(most_frequent(classes_detect))]
predicted_class = most_frequent(classes_detect)
for i in frame_Car.values():
    if(i['ID'] == predicted_class):
        img_file_arr.append([i['file']])
if (type != 'none'):
    if (type in predicted_car):
        print('Found type: ' + type + '   ')
        print(predicted_car)
    else:
        print('Did not find a ' + type + ', but found ' + ' ' + predicted_car)
else:
    print('found ' + predicted_car)
str1 = ""

for i in range(len(img_file_arr)):
    file = str1.join(img_file_arr[i])
    img = cv2.imread(file)
    height, width, layers = img.shape
    size = (width,height)
    img_arr.append(img)
fourcc = cv2.VideoWriter_fourcc(*'MP4V')

out = cv2.VideoWriter('project.mp4',fourcc, 3, size)
for i in range(len(img_arr)):
    out.write(img_arr[i])
out.release()


#!/usr/bin/env python3

from asyncio import all_tasks, current_task, gather, get_event_loop as loop, create_task
import websockets
from signal import SIGTERM, SIGINT

from .client import Client
from .game import Game

class Server:
    def __init__(self, port, updating_command=None, update_cmd=None, ping_every=120):
        self.port = port
        self.client_id = 1
        self.game = Game()
        if updating_command and not update_cmd:
            raise ValueError('update_cmd not specified')
        self.updating_command = updating_command
        self.update_cmd = update_cmd
        self.ping_every = ping_every

    async def caught(self, ws, path):
        cc = Client(self, self.game, ws, self.client_id)
        self.client_id += 1
        self.game.add(cc)
        await cc.handle()

    async def shutdown(self, sig=None):
        if sig:
            print(f'[SIGNAL] Got {sig}, closing connections...')
        await self.game.kill()
        tasks = [t for t in all_tasks() if t is not current_task()]
        [task.cancel() for task in tasks]
        print(f'[SIGNAL] Cancelling tasks...')
        results = await gather(*tasks, return_exceptions=True)
        for r in results:
            if isinstance(r, Exception):
                print(f'[SIGNAL] Got exception: {r}')
        print(f'[SIGNAL] Stopping...')
        loop().stop()

    def start(self):
        start_server = websockets.serve(self.caught, "", self.port)

        for sig in (SIGTERM, SIGINT):
            loop().add_signal_handler(sig, lambda sig=sig: create_task(self.shutdown(sig)))

        loop().run_until_complete(start_server)
        loop().run_forever()


import cv2
import os
import random

dataset_path = "/Users/jingwenshi/Desktop/Dataset/"

# Create Train, Test, and Valid folders with subfolders
folders = ['Train', 'Test', 'Valid']
subfolders = ['12%', '37%', '62%', '87%']

for folder in folders:
    for subfolder in subfolders:
        os.makedirs(os.path.join(folder, subfolder), exist_ok=True)

# Loop through all files in the directory
for video in os.listdir(dataset_path):
    # Check if the file has a .MOV extension
    if video.endswith(".MOV"):
        path = os.path.join(dataset_path, video)
        cam = cv2.VideoCapture(path)

        video_name = video[:-4]

        try:
            if not os.path.exists("temp_data"):
                os.makedirs("temp_data")
        except OSError:
            print('Error: Creating directory of data')

        currentframe = 0
        images = []

        while True:
            ret, frame = cam.read()

            if ret:
                if currentframe % 15 == 0:
                    name = f'./temp_data/{video_name}_' + str(currentframe) + '_' + video_name[-3:] + '.jpg'
                    print('Creating...' + name)
                    cv2.imwrite(name, frame)
                    images.append(name)
                currentframe += 1

            else:
                break

        cam.release()

        # Shuffle the images and assign them to Train, Test, and Valid folders
        random.shuffle(images)
        num_images = len(images)
        test_images = images[:num_images // 5]
        valid_images = images[num_images // 5:2 * num_images // 5]
        train_images = images[2 * num_images // 5:]

        for img_path in train_images + test_images + valid_images:
            img_name = os.path.basename(img_path)
            folder = "Train" if img_path in train_images else "Test" if img_path in test_images else "Valid"
            subfolder = img_name.split('_')[2][:3]  # Extract the percentage from the image name
            print(subfolder)
            dest_path = os.path.join(folder, subfolder, img_name)
            os.rename(img_path, dest_path)

# Remove the temp_data folder
os.rmdir("temp_data")


# -*- coding:utf-8 -*-
# Created by shellbye on 2016/12/27.

from PIL import Image

def grey(src_path, des_path):
    pil_im = Image.open(src_path).convert('L')
    pil_im.save(des_path)

def thumbnail(src_path, des_path, size_tuple):
    pil_im = Image.open(src_path)
    pil_im.thumbnail(size_tuple)
    pil_im.save(des_path)

def crop_paste(src_path, des_path, box_tuple):
    # 四元组的坐标依次是(左，上，右，下)。PIL 中指定 坐标系的左上角坐标为(0，0)
    pil_im = Image.open(src_path)
    region = pil_im.crop(box_tuple)
    region.save('../pic/region.jpg')
    region = region.transpose(Image.ROTATE_180)
    pil_im.paste(region, box_tuple)
    pil_im.save(des_path)

def resize(src_path, des_path, size_tuple):
    pil_im = Image.open(src_path)
    pil_im_resized = pil_im.resize(size_tuple)
    pil_im_resized.save(des_path)

def rotate(src_path, des_path, degree):
    pil_im = Image.open(src_path)
    pil_im_rotated = pil_im.rotate(degree)
    pil_im_rotated.save(des_path)

if __name__ == '__main__':
    empire = '../pic/empire.jpg'
    # grey(empire, '../pic/empire_grey.jpg')
    # thumbnail(empire, '../pic/empire_thumb.jpg', (128, 128))
    # crop_paste(empire, '../pic/empire_crop_paste.jpg',
    #            (100, 100, 400, 400))
    # resize(empire, '../pic/empire_resize.jpg', (50, 50))
    rotate(empire, '../pic/empire_rotate.jpg', 180)
    pass


'''
Filename: save_to_hdf5.py
Python Version: 3.6.5
Project: Neutrophil Identifier
Author: Yang Liu
Created date: Sep 5, 2018 4:13 PM
-----
Last Modified: Oct 3, 2018 3:57 PM
Modified By: Yang Liu
-----
License: MIT
http://www.opensource.org/licenses/MIT
'''

import h5py
import os
import glob
from PIL import Image
import numpy as np
from random import shuffle
import tables as tb

def save_to_hdf5(path):
    """
    """
    TRAINING_DATA_PECENTAGE = 0.8
    TESTING_DATA_PECENTAGE = 0.1

    src_path = os.path.abspath(path)
    des_path = os.path.join(src_path, "..")
    des_path = os.path.join(des_path, "76_79_80.hdf5")

    files_path = os.path.join(src_path, "*.png")
    files = glob.glob(files_path)
    labels = [1 if "pos" in file_name else 0 for file_name in files]

    c = list(zip(files, labels))
    shuffle(c)
    files, labels = zip(*c)

    m = len(files)
    part_train = int(m * TRAINING_DATA_PECENTAGE)
    part_test = part_train + int(m * TESTING_DATA_PECENTAGE)
    x_train = files[0: part_train]
    x_test = files[part_train: part_test]
    x_val = files[part_test:]
    y_train = labels[0: part_train]
    y_test = labels[part_train: part_test]
    y_val = labels[part_test:]

    train_shape = (len(x_train), 299, 299, 3)
    test_shape = (len(x_test), 299, 299, 3)
    val_shape = (len(x_val), 299, 299, 3)

    hdf5_file = h5py.File(des_path, mode='w')
    m_train = len(x_train)
    m_test = len(x_test)
    m_val = len(x_val)

    hdf5_file.create_dataset("train_img", train_shape, np.uint8)
    hdf5_file.create_dataset("test_img", test_shape, np.uint8)
    hdf5_file.create_dataset("val_img", val_shape, np.uint8)

    hdf5_file.create_dataset("train_labels", (m_train,), np.uint8)
    hdf5_file.create_dataset("test_labels", (m_test,), np.uint8)
    hdf5_file.create_dataset("val_labels", (m_val,), np.uint8)

    hdf5_file.create_dataset("train_mean", train_shape[1:], np.float32)

    hdf5_file["train_labels"][...] = y_train
    hdf5_file["test_labels"][...] = y_test
    hdf5_file["val_labels"][...] = y_val

    mean = np.zeros(train_shape[1:], np.float32)

    for i in range(m_train):
        if i % 500 == 0 and i > 1:
            print("Train data: {}/{}".format(i, m_train))

        x = x_train[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        hdf5_file["train_img"][i, ...] = img
        mean += img / float(m_train)

    for i in range(m_test):
        if i % 100 == 0 and i > 1:
            print("Test data: {}/{}".format(i, m_test))

        x = x_test[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        hdf5_file["test_img"][i, ...] = img

    for i in range(m_val):
        if i % 100 == 0 and i > 1:
            print("Validation data: {}/{}".format(i, m_val))

        x = x_val[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        hdf5_file["val_img"][i, ...] = img

    hdf5_file["train_mean"][...] = mean
    hdf5_file.close()

def save_to_hdf5_with_tables(path):
    """
    """
    TRAINING_DATA_PECENTAGE = 0.8
    TESTING_DATA_PECENTAGE = 0.1

    src_path = os.path.abspath(path)
    des_path = os.path.join(src_path, "..")
    des_path = os.path.join(des_path, "76_79_80.hdf5")

    files_path = os.path.join(src_path, "*.png")
    files = glob.glob(files_path)
    labels = [1 if "pos" in file_name else 0 for file_name in files]

    c = list(zip(files, labels))
    shuffle(c)
    files, labels = zip(*c)

    m = len(files)
    part_train = int(m * TRAINING_DATA_PECENTAGE)
    part_test = part_train + int(m * TESTING_DATA_PECENTAGE)
    x_train = files[0: part_train]
    x_test = files[part_train: part_test]
    x_val = files[part_test:]
    y_train = labels[0: part_train]
    y_test = labels[part_train: part_test]
    y_val = labels[part_test:]

    data_shape = (0, 299, 299, 3)

    hdf5_file = tb.open_file(des_path, mode='w')
    m_train = len(x_train)
    m_test = len(x_test)
    m_val = len(x_val)

    train_storage = hdf5_file.create_earray(
        hdf5_file.root, "train_img", tb.UInt8Atom(), shape=data_shape
    )
    test_storage = hdf5_file.create_earray(
        hdf5_file.root, "test_img", tb.UInt8Atom(), shape=data_shape
    )
    val_storage = hdf5_file.create_earray(
        hdf5_file.root, "val_img", tb.UInt8Atom(), shape=data_shape
    )

    hdf5_file.create_array(hdf5_file.root, "train_labels", y_train)
    hdf5_file.create_array(hdf5_file.root, "test_labels", y_test)
    hdf5_file.create_array(hdf5_file.root, "val_labels", y_val)

    mean_storage = hdf5_file.create_earray(
        hdf5_file.root, "train_mean", tb.Float32Atom(), shape=data_shape
    )

    mean = np.zeros(data_shape[1:], np.float32)

    for i in range(m_train):
        if i % 500 == 0 and i > 1:
            print("Train data: {}/{}".format(i, m_train))

        x = x_train[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        train_storage.append(img[None])
        mean += img / float(m_train)

    for i in range(m_test):
        if i % 100 == 0 and i > 1:
            print("Test data: {}/{}".format(i, m_test))

        x = x_test[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        test_storage.append(img[None])

    for i in range(m_val):
        if i % 100 == 0 and i > 1:
            print("Validation data: {}/{}".format(i, m_val))

        x = x_val[i]
        img = Image.open(x)
        img = np.array(img)
        img = img[:, :, 0:3]
        val_storage.append(img[None])

    mean_storage.append(mean[None])
    hdf5_file.close()

if __name__ == "__main__":
    import timeit
    start_time = timeit.default_timer()
    home = os.path.abspath("..")
    pool_path = os.path.join(home, "data", "pool")
    save_to_hdf5_with_tables(pool_path)
    end_time = timeit.default_timer()

    print("Time consumed: {}".format(end_time - start_time))


import cv2
import numpy as np
import torch

def torch_fix_seed(seed = 42):
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
    torch.use_deterministic_algorithms = True

torch_fix_seed()

def integrate_edges(
    filepath_line: str,
    filepath_triangle: str,
    filepath_output: str,
) -> str:
    """ 三角形の輪郭と直線を合わせる

    Args:
        filepath_line:      CSDの直線の二値画像
        filepath_triangle:  CSDの三角形の二値画像

    """

    img_line = cv2.imread(filepath_line, cv2.IMREAD_GRAYSCALE)
    img_triangle = cv2.imread(filepath_triangle, cv2.IMREAD_GRAYSCALE)

    edges_triangle = cv2.Canny(img_triangle, 50, 100)

    output = cv2.add(img_line, edges_triangle)

    cv2.imwrite(filepath_output, output)

    return filepath_output

if __name__ == "__main__":
    integrate_edges("./outputs/infer/pred_class1.png", "./outputs/infer/pred_class2.png", filapath_output="./outputs/infer/integrated_edge.png")


#!/usr/bin/env python
# coding: utf-8

# In[15]:

import numpy as np
from sklearn import tree,metrics,svm,ensemble,neighbors
import pandas as pd
from sklearn.model_selection import train_test_split,KFold
from sklearn import linear_model
import xgboost as xgb
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import Pipeline
import matplotlib.pyplot as plt

# ### 去掉缺失值较多的列

# In[20]:

nume = ['转炉终点温度', '转炉终点C', '转炉终点S',
        '钢水净重', '氮化钒铁FeV55N11-A', '低铝硅铁',
       '钒氮合金(进口)', '钒铁(FeV50-A)', '钒铁(FeV50-B)', '钒铁(FeV50-B).1', '硅铝钙',
       '硅铝合金FeAl30Si25', '硅铝锰合金球', '硅锰面（硅锰渣）', '硅铁(合格块)', '硅铁FeSi75-B',
       '石油焦增碳剂', '锰硅合金FeMn64Si27(合格块)', '锰硅合金FeMn68Si18(合格块)', '碳化硅(55%)',
       '硅钙碳脱氧剂']

# ### 缺失值处理、归一化

# In[31]:

CTrain = pd.read_excel('q1_1_收得率.xls')[nume+['C收得率']]
# CTrain = CTrain.dropna()
CTrain = CTrain.drop(CTrain[CTrain['C收得率'].isnull()].index)
# 离群值处理
outs=['转炉终点温度', '转炉终点C','钢水净重']
for out in outs:
    CTrain.drop(CTrain[(CTrain[out]>(CTrain[out].mean()+3*CTrain[out].std()))|(CTrain[out]<(CTrain[out].mean()-3*CTrain[out].std()))].index,inplace=True)

CTrain = CTrain.fillna(CTrain.mean())
X = CTrain[nume] #[['钢水净重','连铸正样C', '连铸正样Ceq_val', '低铝硅铁','石油焦增碳剂']]
X = X[nume].apply(lambda x: (x - np.min(x)) / (np.max(x) - np.min(x)+1e-10)).values
Y = CTrain[['C收得率']].values
len(CTrain.values)

# In[32]:

# cget = (CTrain['C收得率']-CTrain['C收得率'].min())/(CTrain['C收得率'].max()-CTrain['C收得率'].min())
plt.plot(X['钢水净重'][::50])
plt.plot(CTrain['C收得率'][::50])

# In[33]:

# #缺失值数量统计
# CTrain.isnull().sum()

# In[34]:

CTrain.head(1)

# In[35]:

# train_data, test_data, train_target, test_target = train_test_split(X.values,Y.values,test_size=0.2)

# ### 决策树回归

# In[36]:

clf = tree.DecisionTreeRegressor()
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index])
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# 以上三个值分别为R2决定系数、平均绝对误差、平均平方误差

# ### 线性回归

# In[49]:

clf = linear_model.LinearRegression()
ki = 4
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index])
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# In[50]:

mapk={}
for i in range(len(nume)):
    mapk[nume[i]]=np.round(clf.coef_[0][i],4)
    print(nume[i],' & ',np.round(clf.coef_[0][i],4))
# sorted(mapk.items(),key=lambda item:abs(item[1]))

# ### SVM

# In[115]:

clf = svm.SVR(gamma='auto')
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# ### 贝叶斯

# In[116]:

clf = linear_model.BayesianRidge()
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# ### 集成

# In[117]:

clf = ensemble.GradientBoostingRegressor(n_estimators=100, learning_rate=0.1,max_depth=1, random_state=0, loss='ls')
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# ### 多项式回归（emm）

# In[142]:

clf = Pipeline([('poly', PolynomialFeatures(degree=2)),('linear', linear_model.LinearRegression(fit_intercept=False))])
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# # Mn元素预测

# ### 数据预处理

# In[123]:

nume = ['转炉终点温度', '转炉终点C', '转炉终点Mn', '转炉终点S',
       '转炉终点Si', '钢水净重', '连铸正样Mn',
#         '连铸正样C','连铸正样S', '连铸正样P', '连铸正样Si',
#        '连铸正样Ceq_val', '连铸正样Cr', '连铸正样Ni_val', '连铸正样Cu_val',
#        '连铸正样V_val', '连铸正样Alt_val', '连铸正样Als_val', '连铸正样Mo_val', '连铸正样Ti_val',
        '氮化钒铁FeV55N11-A', '低铝硅铁',
       '钒氮合金(进口)', '钒铁(FeV50-A)', '钒铁(FeV50-B)', '钒铁(FeV50-B).1', '硅铝钙',
       '硅铝合金FeAl30Si25', '硅铝锰合金球', '硅锰面（硅锰渣）', '硅铁(合格块)', '硅铁FeSi75-B',
       '石油焦增碳剂', '锰硅合金FeMn64Si27(合格块)', '锰硅合金FeMn68Si18(合格块)', '碳化硅(55%)',
       '硅钙碳脱氧剂']

# In[131]:

MnTrain = pd.read_excel('data1.xlsx')[nume]
# Mn初始含量较低，对反应收得率影响小，使用均值替换缺失值
MnTrain[['转炉终点Mn']] = MnTrain[['转炉终点Mn']].fillna(MnTrain[['转炉终点Mn']].mean())
# 去掉为对合金化后合金钢采样的数据
MnTrain = MnTrain.drop(MnTrain[MnTrain['连铸正样Mn'].isnull()].index)
# 使用均值替代少量的缺失值
MnTrain[nume] = MnTrain[nume].fillna(MnTrain[nume].mean())
# 重新计算Mn收得率
mn_t=['硅铝锰合金球','硅锰面（硅锰渣）','锰硅合金FeMn64Si27(合格块)','锰硅合金FeMn68Si18(合格块)']
mn_p = [0.3,0.664,0.664,0.664]
mn_total=(MnTrain[mn_t]*mn_p).sum(axis=1)
mn_comsu=(MnTrain['连铸正样Mn']-MnTrain['转炉终点Mn'])*MnTrain['钢水净重']
MnTrain['Mn收得率']=mn_comsu/(mn_total)
# 删掉异常数据
MnTrain = MnTrain.drop(MnTrain[MnTrain['Mn收得率']>1.5].index)

# 构造训练数据并对自变量值进行归一化
X = MnTrain[nume].apply(lambda x: (x - np.min(x)) / (np.max(x) - np.min(x)+1e-9))
X.drop('连铸正样Mn',axis=1,inplace=True)
X=X.values
Y = MnTrain[['Mn收得率']].values
# train_data, test_data, train_target, test_target = train_test_split(X.values,Y.values,test_size=0.2)
len(Y)

# In[132]:

# MnTrain #.isnull().sum()
# Y.sort_values('Mn收得率')

# ### 多项式拟合

# In[133]:

clf = Pipeline([('poly', PolynomialFeatures(degree=2)),('linear', linear_model.LinearRegression(fit_intercept=False))])
ki = 10
kf = KFold(n_splits=ki)
r2,mae,mse=0,0,0
for train_index,test_index in kf.split(X):
    clf = clf.fit(X[train_index],Y[train_index].flatten().astype(np.float32))
    pre_y = clf.predict(X[test_index])
    r2+=metrics.r2_score(pre_y,Y[test_index])
    mae+=metrics.mean_absolute_error(pre_y,Y[test_index])
    mse+=metrics.mean_squared_error(pre_y,Y[test_index])
r2/ki,mae/ki,mse/ki

# In[134]:

clf = linear_model.BayesianRidge()
clf = clf.fit(train_data,train_target.flatten())
pre_y = clf.predict(test_data)
metrics.r2_score(pre_y,test_target),metrics.mean_absolute_error(pre_y,test_target),metrics.mean_squared_error(pre_y,test_target)

# In[135]:

clf = linear_model.LinearRegression()
clf = clf.fit(train_data,train_target)
pre_y = clf.predict(test_data)
metrics.r2_score(pre_y,test_target),metrics.mean_absolute_error(pre_y,test_target),metrics.mean_squared_error(pre_y,test_target)

# In[ ]: