#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
import json
import logging
import re
import shutil
import sys

import os
import zipfile
import Config as Config

import jpype

try:
    import xml.etree.cElementTree as ET
except ImportError:
    import xml.etree.ElementTree as ET

from libs.modules.BaseModule import BaseModule

logging.basicConfig(stream=sys.stdout, format="%(levelname)s: %(message)s", level=logging.INFO)
log = logging.getLogger(__name__)

class BufanApp(BaseModule):
    def doSigCheck(self):
        if self.host_os == "android":
            return self._find_main_activity("com.bufan.app")
        elif self.host_os == "ios":
            log.error("not support yet.")
            return False
        return False

    def doExtract(self, working_folder):

        extract_folder = self._format_working_folder(working_folder)

        if os.access(extract_folder, os.R_OK):
            shutil.rmtree(extract_folder)
        os.makedirs(extract_folder, exist_ok=True)
        tmp_folder = os.path.join(os.getcwd(), extract_folder, "tmp")
        os.makedirs(tmp_folder, exist_ok=True)

        zf = zipfile.ZipFile(self.detect_file, 'r')
        appUrl = ""
        if "assets/source/dconfig.json" in zf.namelist():
            config_file = zf.extract("assets/source/dconfig.json", tmp_folder)
            fo = open(config_file, "r+")
            config_json = json.load(fo)
            appUrl = config_json["url"]
            shutil.copy(config_file, extract_folder)
        else:
            """
            get keys from smali code
            filename:  com.bufan.util.JwtUtils.smali
            """
            self._apktool(tmp_folder)
            JwtUtilsPath = os.path.join(tmp_folder, "smali/com/bufan/utils/JwtUtils.smali")
            JwtUtilsFile = open(JwtUtilsPath, "r+")
            JwtUtilsStr = JwtUtilsFile.read()
            keys = []
            patterns = [".field public static final TAG:Ljava/lang/String; = \"(.*?)\"",
                        ".field public static final key1:Ljava/lang/String; = \"(.*?)\"",
                        ".field public static final key2:Ljava/lang/String; = \"(.*?)\"",
                        ".field public static final key3:Ljava/lang/String; = \"(.*?)\"",
                        ".field public static final key4:Ljava/lang/String; = \"(.*?)\"",
                        ".field public static final key5:Ljava/lang/String; = \"(.*?)\"",
                        ".field public static final key6:Ljava/lang/String; = \"(.*?)\"",
                        ".field public static final key7:Ljava/lang/String; = \"(.*?)\"",
                        ".field public static final key8:Ljava/lang/String; = \"(.*?)\""]
            for pattern in patterns:
                pt = re.findall(pattern, JwtUtilsStr)
                keys.extend(pt)
            JwtUtilsFile.close()

            config_file = zf.extract("assets/source/config.json", tmp_folder)
            shutil.copy(config_file, extract_folder)
            jvmPath = jpype.getDefaultJVMPath()
            if not jpype.isJVMStarted():
                jpype.startJVM(jvmPath, '-ea',
                               '-Djava.class.path={0}'.format(Config.Config["decrypt_jar"]),
                               convertStrings=False)

            jclass = jpype.JClass("com.ResDecode.Main")()
            """
            get appUrl
                method: com.ResDecode.Main.get_appUrl
                input: (String) encoded JSONString, (String[]) keys
                output: (String) url
            """
            try:
                appUrl = str(jclass.get_appUrl(config_file, keys))  # cast to str
            except:
                self._log_error(os.path.basename(__file__), self.detect_file, "foo")

        self._dump_info(extract_folder, appUrl)
        # jpype.shutdownJVM()

        # clean env
        shutil.rmtree(tmp_folder)

        return extract_folder, appUrl

def main():
    # f = "./test_case/bugs/14b03ed97138c6f99f71f5bd633548c4.apk"  # need decode
    f = "./test_case/BufanApp/世耀国际_bufan.apk"
    f = "./test_case/BufanApp/惠普分期_bufan.apk"  # need decode
    # f = "./test_case/BufanApp/淘客_bufan.apk"
    bufan = BufanApp(f, "android")
    if bufan.doSigCheck():
        logging.info("BufanApp signature Match")
        extract_folder, launch_path = bufan.doExtract("./working_folder")
        log.info("{} is extracted to {}, the start page is {}".format(f, extract_folder, launch_path))
    return

if __name__ == "__main__":
    sys.exit(main())


from collections import Counter
import pyspark.sql.functions as F

def task_5(df):
    task_5_res_df = df.groupBy("channel_title").count().orderBy('count', ascending=False). \
        limit(10).join(df, 'channel_title', 'left').groupBy('channel_title').agg(
        F.first('count').alias("total_trending_days"),
        F.collect_list('video_id').alias("video_id_list"),
        F.collect_list('title').alias("video_title_list")
    )
    query_result = task_5_res_df.collect()

    json_result = {
        "channels": [
            {
                "channel_name": row.channel_title,
                "total_trending_days": row.total_trending_days,
                "videos_days": [
                    {
                        "video_id": video_id,
                        "video_title": row.video_title_list[row.video_id_list.index(video_id)],
                        "trending_days": count
                    }

                    for video_id, count in Counter(row.video_id_list).items()
                ]
            }
            for row in query_result
        ]
    }
    return json_result


from typing import List, Optional, Tuple
from tmsproviderapisdk.tms_extended_model import TmsExtendedModel

class TmsDevice(TmsExtendedModel):
    _path_url = "/devices/"

    def __init__(self, unique_id: str, account: int, device_id: int = None, ipaddr: str = None, mac: str = None,
                 remote_custom_field: str = None, comment: str = None, last_online: str = None, last_fw_ver: str = None,
                 first_online: str = None, use_nat: bool = False, operation_system: str = None, udpxy_addr: str = None,
                 device_type: int = None, provider: int = None):

        self.unique_id = unique_id
        self.account = account
        self.id = device_id
        self.ipaddr = ipaddr
        self.mac = mac
        self.remote_custom_field = remote_custom_field
        self.comment = comment
        self.last_online = last_online
        self.last_fw_ver = last_fw_ver
        self.first_online = first_online
        self.use_nat = use_nat
        self.operation_system = operation_system
        self.udpxy_addr = udpxy_addr
        self.device_type = device_type
        self.provider = provider

    @staticmethod
    def _dict_to_object(device_dict: dict) -> object:
        device = TmsDevice(
            unique_id=device_dict["unique_id"],
            device_id=device_dict["id"],
            ipaddr=device_dict["ipaddr"],
            mac=device_dict["mac"],
            remote_custom_field=device_dict["remote_custom_field"],
            comment=device_dict["comment"],
            last_online=device_dict["last_online"],
            last_fw_ver=device_dict["last_fw_ver"],
            first_online=device_dict["first_online"],
            use_nat=device_dict["use_nat"],
            operation_system=device_dict["operation_system"],
            udpxy_addr=device_dict["udpxy_addr"],
            device_type=device_dict["device_type"],
            provider=device_dict["provider"],
            account=device_dict["account"]
        )

        return device

    @classmethod
    def get_list(cls, account: int = None, device_type: int = None, limit: int = 50, provider: int = None,
                 quick_search: str = "", remote_custom_field: str = None, sort: str = "", start: int = 0,
                 unique_id: str = "") -> Optional[Tuple[List[object], int]]:

        devices = super().get_list(start=start, limit=limit, account=account, device_type=device_type,
                                   provider=provider, quick_search=quick_search,
                                   remote_custom_field=remote_custom_field, sort=sort, unique_id=unique_id)

        return devices

    def __str__(self):
        return """id:{}, ipaddr:{}, mac:{}, unique_id:{}, remote_custom_field: {}, comment: {}, last_online: {}, \
        last_fw_ver: {}, first_online: {}, use_nat: {}, operation_system: {}, \
        udpxy_addr: {}, device_type: {}, provider: {}, account: {}""".format(
            self.id,
            self.ipaddr,
            self.mac,
            self.unique_id,
            self.remote_custom_field,
            self.comment,
            self.last_online,
            self.last_fw_ver,
            self.first_online,
            self.use_nat,
            self.operation_system,
            self.udpxy_addr,
            self.device_type,
            self.provider,
            self.account
        )


import numpy as np
from sklearn import metrics, model_selection, multioutput
import lightgbm
from sklearn import ensemble
import tensorflow as tf
from tensorflow import keras
import tensorflow_addons as tfa
import matplotlib.pyplot as plt
from .utils import labels
import pandas as pd

def train_model(x, y, cv_folds, approach_name):

    if approach_name == 'independent':
        # Initialize an empty list to store f1 scores for each fold
        _scores = []

        # Iterate through each fold using its index and the corresponding training/validation indices
        for fold_idx, (idxT, idxV) in enumerate(cv_folds):

            # Define a multioutput classifier model using LGBMClassifier with a random seed of 42
            model = multioutput.MultiOutputClassifier(lightgbm.LGBMClassifier(random_seed=42))

            # Train the model on the training data
            model.fit(x.iloc[idxT], y.iloc[idxT])

            # Generate predictions using the validation data
            p = model.predict(x.iloc[idxV])

            # Calculate the average f1 score for each label using the validation data and the corresponding predictions
            avg_f1_score = np.mean([metrics.f1_score(y.iloc[idxV].iloc[:, i].to_numpy().astype(int), p[:, i].astype(int)) for i in range(4)])

            # Append the average f1 score to the scores list
            _scores.append(avg_f1_score)

            # Print the fold index and the average f1 score
            print("%d\t%.4f" % (fold_idx, avg_f1_score))

        # Return the mean f1 score across all folds
        return np.mean(_scores), model

    if approach_name == 'classifier_chains':
        _scores = []  # create a list to store F1 scores for each fold

        for fold_idx, (idxT, idxV) in enumerate(cv_folds):  # iterate through each fold
            p_list = []  # create a list to store predictions for each repetition
            avg_f1_score_list = []  # create a list to store F1 scores for each repetition

            for rep in range(3):  # repeat the process 3 times
                # create a Classifier Chain model with LightGBM as the base classifier
                model = multioutput.ClassifierChain(lightgbm.LGBMClassifier(random_seed=42), order="random", cv=5)
                # fit the model on training data
                model.fit(x.iloc[idxT].fillna(0.0), y.iloc[idxT])
                # make predictions on validation data
                p_rep = model.predict(x.iloc[idxV].fillna(0.0))
                # store predictions and F1 score for each repetition
                p_list.append(p_rep)
                avg_f1_score_list.append(np.mean([metrics.f1_score(y.iloc[idxV].iloc[:, i].to_numpy().astype(int), p_rep[:, i].astype(int)) for i in range(4)]))

            # calculate average predictions and F1 score across repetitions
            p = np.stack(p_list, axis=0).mean(0)
            avg_f1_score = np.mean([metrics.f1_score(y.iloc[idxV].iloc[:, i].to_numpy().astype(int), p[:, i].astype(int)) for i in range(4)])
            _scores.append(avg_f1_score)
            # print fold index and F1 scores for each repetition and overall F1 score for the fold
            print("%d\t%.4f | %.4f %.4f %.4f" % (fold_idx, avg_f1_score, avg_f1_score_list[0], avg_f1_score_list[1], avg_f1_score_list[2]))

        # Return the mean f1 score across all folds
        return np.mean(_scores), model

    if approach_name == 'native_extra_trees':

        _scores = [] # initialize a list to store f1 score of each fold

        # loop over each fold of the cross-validation
        for fold_idx, (idxT, idxV) in enumerate(cv_folds):

            # initialize an ExtraTreesClassifier model
            model = ensemble.ExtraTreesClassifier()

            # fit the model on training data and target labels
            model.fit(x.iloc[idxT].fillna(-100), y.iloc[idxT])

            # predict the target labels on validation data
            p = model.predict(x.iloc[idxV].fillna(-100))

            # calculate f1 score for each label and take average
            avg_f1_score = np.mean([metrics.f1_score(y.iloc[idxV].iloc[:, i].to_numpy().astype(int), p[:, i].astype(int)) for i in range(4)])

            # append the f1 score to the list
            _scores.append(avg_f1_score)

        # Return the mean f1 score across all folds
        return np.mean(_scores), model

    if approach_name == 'native_neural_net':

        _scores = []
        for fold_idx, (idxT, idxV) in enumerate(cv_folds):
            norm = keras.layers.Normalization()
            norm.adapt(x.iloc[idxT].fillna(0.0))

            model = keras.Sequential([
                norm,
                keras.layers.Dense(512, activation='relu'),
                keras.layers.Dropout(0.5),
                keras.layers.Dense(256, activation='relu'),
                keras.layers.Dropout(0.5),
                keras.layers.Dense(128, activation='relu'),
                keras.layers.Dense(y.shape[-1])
            ])
            model.compile(
                optimizer=keras.optimizers.Adam(5e-4),
                loss=keras.losses.BinaryCrossentropy(from_logits=True),
            )

            x_train = x.iloc[idxT].fillna(0.0).values.astype('float32')
            y_train = y.iloc[idxT].values.astype('int32')
            x_valid = x.iloc[idxV].fillna(0.0).values.astype('float32')
            y_valid = y.iloc[idxV].values.astype('int32')

            hist = model.fit(x_train, y_train, validation_data=(x_valid, y_valid), epochs=50, batch_size=32, verbose=0)
            plt.plot(hist.history['loss'], color='tab:blue')
            plt.plot(hist.history['val_loss'], color='tab:red')
            plt.title('Training and Validation Loss')
            plt.xlabel('Epoch')
            plt.ylabel('Loss')
            plt.legend(['Training Loss', 'Validation Loss'], loc='upper right')
            plt.savefig('training_validation_loss.png')

            p_valid = model.predict(x_valid, verbose=0)
            p_valid = (p_valid > 0.0).astype(int)
            avg_f1_score = np.mean([metrics.f1_score(y.iloc[idxV].iloc[:, i].to_numpy().astype(int), p_valid[:, i].astype(int)) for i in range(4)])
            _scores.append(avg_f1_score)
            print("%d\t%.4f" % (fold_idx, avg_f1_score))

        return np.mean(_scores), model

    if approach_name == 'multilabel_to_multiclass':
        y_ = y.copy()
        y_['code'] = y.apply(lambda x: '-'.join([str(xi) for xi in x]), axis=1)
        y_codes = y_.drop_duplicates()
        y_codes['idx'] = np.arange(y_codes.shape[0])
        y_class = y_.merge(y_codes, on='code', how='left')['idx']
        print(f"{y_codes.shape[0]} classes")

        _scores = []
        for fold_idx, (idxT, idxV) in enumerate(cv_folds):
            model = lightgbm.LGBMClassifier(random_seed=42)
            model.fit(x.iloc[idxT], y_class.iloc[idxT])

            p = model.predict(x.iloc[idxV])
            p = pd.DataFrame({'idx': p}).merge(y_codes, on='idx', how='left')[labels]
            avg_f1_score = np.mean([metrics.f1_score(y.iloc[idxV].iloc[:, i].to_numpy().astype(int), p.iloc[:, i].astype(int)) for i in range(4)])
            _scores.append(avg_f1_score)
            print("%d\t%.4f" % (fold_idx, avg_f1_score))

        return np.mean(_scores), model

def train_and_evaluate_model(x, y, cv_folds, approach_name):

    print("------------------------")
    print(f"Results for the approach {approach_name}:")
    # Train and evaluate the model using the cross-validation folds
    mean_f1_score, model = train_model(x, y, cv_folds, approach_name)

    # Print the mean f1 score across all folds
    print("--------------")
    print(" \t%.4f" % mean_f1_score)

    return mean_f1_score, model


from db import  connection
from flask import jsonify,abort
class TreinamentoService():

    def get_all_treinamentos(self):
        try:
            query = """SELECT t.id, t.nome, t.descricao, t.duracao, t.caminho from treinamento t;"""
            cursor = connection.cursor()
            cursor.execute(query)
            treinamentos = cursor.fetchall()
            cursor.close()
        except Exception as ex:
            return jsonify({"message":str(ex),"code":500})
        if treinamentos is None:
            abort(404)
        return jsonify({"data": treinamentos})

    def post_treinamento(self, treinamento):
        try:
            query = """INSERT INTO treinamento
                    (nome, descricao, duracao)
                    VALUES (%s, %s, %s);"""
            cursor = connection.cursor()
            cursor.execute(query, (treinamento["nome"], treinamento["descricao"], treinamento["duracao"], treinamento["caminho"]))
            id_treinamento = cursor.lastrowid
            connection.commit()
            cursor.close()
        except Exception as ex:
            print(str(ex))
            return jsonify({"message": str(ex)})
        return jsonify({"success": True, "id": id_treinamento, "message": "Treinamento registrado com sucesso"}), 200

    def get_one_treinamentos(self, id):
        try:
            query = """SELECT t.id, t.nome , t.descricao, t.duracao, t.caminho
                    FROM treinamento t
                    where t.id=%s"""
            cursor = connection.cursor()
            cursor.execute(query, id)
            treinamentos = cursor.fetchall()
            cursor.close()
        except Exception as ex:
            return jsonify({"message":ex})
        if treinamentos is None:
            abort(404)
        return jsonify({"data":treinamentos[0]})

    def put_treinamento(self, treinamento, id):

        try:
            query = """UPDATE treinamento SET nome = %s, descricao = %s, duracao = %s,
            WHERE treinamento.id = %s"""
            cursor = connection.cursor()
            cursor.execute(query, (treinamento["nome"],
                                  treinamento["descricao"],
                                  treinamento["duracao"], id))
            connection.commit()
            cursor.close()

        except Exception as ex:
            print(str(ex))
            return jsonify({"message": str(ex)})
        return jsonify({"success":True, "message":"Treinamento actualizado com sucesso"}), 201

    def desativar_treinamento(self, id):
            try:
                cursor = connection.cursor()
                query = "UPDATE treinamento SET estado = false WHERE id = %s"
                cursor.execute(query, (id,))
                connection.commit()
                cursor.close()
            except Exception as ex:
                return jsonify({"message": str(ex)})
            return jsonify({"message": "Treinamento desativado com sucesso."})


#!/usr/bin/env python3.10
import matplotlib.pyplot as plt
import pandas as pd

df = pd.read_csv("/home/klimpert/Documents/research/Afrothismia/figures/dNdS/dnds_data_20221104.csv")

AFGE_paml = df[["Gene", "AFGE_bg", "AFGE_fg"]]
AFHY_paml = df[["Gene", "AFHY_bg", "AFHY_fg"]]
AFWI_paml = df[["Gene", "AFWI_bg", "AFWI_fg"]]
AFGE_relax = df[["Gene", "AFGE_relax"]]
AFHY_relax = df[["Gene", "AFHY_relax"]]
AFWI_relax = df[["Gene", "AFWI_relax"]]

fig, axes = plt.subplots(nrows=3, ncols=2, sharex=True)

AFGE_paml.plot.bar(ax=axes[0,0], x='Gene', ylim=(0,0.5), ylabel="dN/dS")
AFGE_relax.plot.bar(ax=axes[0,1], x='Gene', ylim=(0,2.5), ylabel="k-value")
AFHY_paml.plot.bar(ax=axes[1,0], x='Gene', ylim=(0,0.5), ylabel="dN/dS")
AFHY_relax.plot.bar(ax=axes[1,1], x='Gene', ylim=(0,2.5), ylabel="k-value")
AFWI_paml.plot.bar(ax=axes[2,0], x='Gene', ylim=(0,0.5), ylabel="dN/dS")
AFWI_relax.plot.bar(ax=axes[2,1], x='Gene', ylim=(0,2.5), ylabel="k-value")

axes[0,1].axhline(y=1.0, color='grey', linestyle='--')
axes[1,1].axhline(y=1.0, color='grey', linestyle='--')
axes[2,1].axhline(y=1.0, color='grey', linestyle='--')

plt.show()


from prettytable import PrettyTable
from copy import copy

def priority(processes):

	temp_store, idle, time = [], [], 0
	while processes:
		temp = [copy(process) for process in processes if process[1] <= time]
		# Lower number -> higher priority
		temp.sort(key=lambda x: x[3], reverse=False)

		if temp:
			# If same priority then check AT
			curr_proc = temp[0]
			for item in temp:
				if item[3] == curr_proc[3]:
					if item[1] < curr_proc[1] or (item[1] == curr_proc[1] and item[0] < curr_proc[0]):
						curr_proc = item
				else:
					break

			temp_store.append([curr_proc[0], time, time + 1])
			time += 1
			curr = copy(curr_proc)
			curr_proc[2] -= 1
			if curr_proc[2]:
				processes[processes.index(curr)] = curr_proc
			else:
				processes.remove(curr)
		else:
			idle.append((time, time+1))
			time += 1

	ct = [0] * n
	at, bt, tat, wt, rt = [], [], [], [], []
	for i in range(len(temp_store)):
		ct[temp_store[i][0]-1] = temp_store[i][2]

	for i in range(n):
		at.append(dict_proc[i+1][0])
		bt.append(dict_proc[i+1][1])
		tat.append(ct[i] - at[-1])
		wt.append(tat[-1] - bt[-1])

	total = temp_store[-1][2]

	return temp_store, at, bt, ct, tat, wt, total, idle

def sasta_printer(at, bt, ct, tat, wt, total, idle,prior):

    table = PrettyTable(['PID', 'AT', 'BT', 'Priority','CT', 'TAT', 'WT'])
    for i in range(n):
        table.add_row([i+1, at[i], bt[i], prior[i],ct[i], tat[i], wt[i]])
    print(table)
    for time in idle:
        print('CPU idle from {} to {}'.format(time[0], time[1]))
    print('Total time:', total)
    print("Idle time of CPU: ",sum(idle[i][1]-idle[i][0] for i in range(len(idle))))
    print('Average Turnaround time:', sum(tat) / n)
    print('Average Waiting time:', sum(wt) / n)

n = int(input('Enter no of processes: '))

dict_proc = {}
processes,priorities = [],[]
for i in range(n):
    at, bt, prior = map(int, input('Enter AT, BT and priority for {}: '.format(i+1)).split())
    priorities.append(prior)
    # format -> PID, AT, BT, Priority
    processes.append([i+1, at, bt, prior])
    dict_proc[i+1] = (at, bt, prior)

print('\nPriority non-preemptive:')
_, at, bt, ct, tat, wt, total, idle = priority(processes[::])
sasta_printer(at, bt, ct, tat, wt, total, idle,priorities)

'''
5
0 15 6
5 9 3
6 3 7
8 6 9
10 12 4
'''


from rest_framework import serializers
from .models import Comment

class CommentSerializer(serializers.ModelSerializer):
    class Meta:
        model = Comment
        fields = [
            'author',
            'author_flair_text',
            'author_flair_text_color',
            'body',
            'id',
            'permalink',
            'score',
            'subreddit',
        ]


import asyncio
import functools

def callback(arg, *, kwarg='default'):
    print(arg, kwarg)

async def proc(loop):
    loop.call_soon(callback, 1)
    wrapped = functools.partial(callback, kwarg='nope')
    loop.call_soon(wrapped, 2)
    await asyncio.sleep(0.1)

def main():
    el = asyncio.get_event_loop()
    try:
        el.run_until_complete(proc(el))
    finally:
        el.close()

if __name__ == '__main__':
    main()


from utils.helper_functions import *
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from utils.constants import *
from sklearn.utils import class_weight
import numpy as np

df = read_data()
df = clean_up(df)
df = drop_head(drop_tail(df))
df = df[~(df['skill_name']=='Attempts to upsell')]
df.replace(to_replace={'skill_name':{'Probing question':'Fact-gathering question'}},inplace=True)

X_train, X_test, Y_train, Y_test= create_train_test(df,VALIDATION_SPLIT)
Y_train_unique = np.unique(Y_train)
class_weight = class_weight.compute_class_weight('balanced', Y_train_unique, Y_train)

Y_train=get_dummies(Y_train)
Y_test=get_dummies(Y_test)

tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
tokenizer.fit_on_texts(list(df['content']))
word_index = tokenizer.word_index

train_sequences_1 = tokenizer.texts_to_sequences(list(X_train))
train_data_1 = pad_sequences(train_sequences_1, maxlen=MAX_SEQUENCE_LENGTH)

test_sequences_1 = tokenizer.texts_to_sequences(list(X_test))
test_data_1 = pad_sequences(test_sequences_1, maxlen=MAX_SEQUENCE_LENGTH)

Y_train=np.matrix(Y_train)
Y_test=np.matrix(Y_test)

print("Done with data processing")


import numpy as np
from pyfft import Fouriert

# f-plane version two-layer baroclinic primitive equation model
# same as dry version of mc2RSW model of Lambaerts 2011 (doi:10.1063/1.3582356)

# run on command line to generate an animation.
# needs pyfftw and matplotlib with qt5agg backend for animation.

class TwoLayer(object):

    def __init__(self,ft,dt,theta1=300,theta2=320,f=1.e-4,\
                 zmid=5.e3,ztop=10.e3,diff_efold=6*3600.,diff_order=8,\
                 div2_diff_efold=1.e30,tdrag1=10*86400,tdrag2=10.*86400,tdiab=15*86400,umax=8):
        dtype = ft.precision
        # setup model parameters
        self.dtype = dtype
        self.theta1 = np.array(theta1,dtype) # lower layer pot. temp. (K)
        self.theta2 = np.array(theta2,dtype) # upper layer pot. temp. (K)
        self.delth = np.array(theta2-theta1,dtype) # difference in potential temp between layers
        self.grav = 9.80616 # gravity
        self.zmid = np.array(zmid,dtype) # resting depth of lower layer (m)
        self.ztop = np.array(ztop,dtype) # resting depth of both layers (m)
        self.umax = np.array(umax,dtype) # equilibrium jet strength
        self.ft = ft # Fouriert instance (defines domain size, grid resolution)
        self.dt = np.array(dt,dtype) # time step (secs)
        self.tdiab = np.array(tdiab,dtype) # interface relaxation time scalee (secs)
        self.tdrag = np.array([tdrag1,tdrag2],dtype) # linear drag time scale in each layer (secs)
        self.f = np.array(f,dtype) # coriolis parameter
        # hyperdiffusion parameters
        self.diff_order = np.array(diff_order, dtype)  # hyperdiffusion order
        self.diff_efold = np.array(diff_efold, dtype)  # hyperdiff time scale (secs)
        self.div2_diff_efold = np.array(div2_diff_efold, dtype) # extra laplacian div damping
        ktot = np.sqrt(self.ft.ksqlsq)
        pi = np.array(np.pi,dtype)
        ktotcutoff = np.array(pi * self.ft.N / self.ft.L, dtype)
        # integrating factor for hyperdiffusion
        self.hyperdiff = -(1./self.diff_efold)*(ktot/ktotcutoff)**self.diff_order
        if div2_diff_efold < 1.e10:
            # extra laplacian diffusion of divergence to damp gravity waves
            self.divlapdiff = -(1./self.div2_diff_efold)*(ktot/ktotcutoff)**2
        else:
            self.divlapdiff = 0.
        x = np.arange(0, self.ft.L, self.ft.L / self.ft.Nt, dtype=dtype)
        y = np.arange(0, self.ft.L, self.ft.L / self.ft.Nt, dtype=dtype)
        x, y = np.meshgrid(x,y)
        self.x = x; self.y = y
        # set equilibrium layer thicknes profile (cos(2*pi*y/L structure).
        l = np.array(2*np.pi,self.dtype) / self.ft.L
        self.dzref = (self.f*self.theta1*umax*np.cos(l*self.y)/(l*self.grav*self.delth))+self.zmid
        if self.dzref.min() < 0:
            raise ValueError('negative layer thickness! adjust model parameters')
        self.t = 0.
        self.timesteps = 1
        self.masstendvar = 0

    def gettend(self,vrtspec,divspec,dzspec,masstend_diag=False):
        # compute tendencies.
        # first, transform fields from spectral space to grid space.
        vrt = self.ft.spectogrd(vrtspec)
        u,v = self.ft.getuv(vrtspec,divspec)
        dz = self.ft.spectogrd(dzspec)
        # diabatic mass flux due to interface relaxation.
        massflux = (self.dzref - dz[1])/self.tdiab
        # horizontal vorticity flux
        tmpg1 = u*(vrt+self.f); tmpg2 = v*(vrt+self.f)
        # add linear drag contribution
        tmpg1 += v/self.tdrag[:,np.newaxis,np.newaxis]
        tmpg2 += -u/self.tdrag[:,np.newaxis,np.newaxis]
        # add diabatic momentum flux contribution
        # (this version averages vertical flux at top
        # and bottom of each layer)
        # same as 'improved' mc2RSW model (DOI: 10.1002/qj.3292)
        tmpg1 += 0.5*(v[1]-v[0])*massflux/dz
        tmpg2 -= 0.5*(u[1]-u[0])*massflux/dz
        # compute vort flux contributions to vorticity and divergence tend.
        ddivdtspec, dvrtdtspec = self.ft.getvrtdivspec(tmpg1,tmpg2)
        dvrtdtspec *= -1
        # horizontal mass flux contribution to continuity
        tmpg1 = u*dz; tmpg2 = v*dz
        tmpspec, ddzdtspec = self.ft.getvrtdivspec(tmpg1,tmpg2)
        if masstend_diag:
            # mean abs total mass tend (meters/hour)
            self.masstendvar = 3600.*np.abs((self.ft.spectogrd(ddzdtspec)).sum(axis=0)).mean()
            #print(self.t,self.masstendvar)
        ddzdtspec *= -1
        # diabatic mass flux contribution to continuity
        tmpspec = self.ft.grdtospec(massflux)
        ddzdtspec[0] -= tmpspec; ddzdtspec[1] += tmpspec
        # pressure gradient force contribution to divergence tend
        mstrm = np.empty(dz.shape, dtype=self.dtype) # montgomery streamfunction
        mstrm[0] = self.grav*(dz[0] + dz[1])
        mstrm[1] = mstrm[0] + (self.grav*self.delth/self.theta1)*dz[1]
        ddivdtspec += -self.ft.lap*self.ft.grdtospec(mstrm+0.5*(u**2+v**2))
        # hyperdiffusion of vorticity and divergence
        dvrtdtspec += self.hyperdiff*vrtspec
        # extra laplacian diffusion of divergence to suppress gravity waves
        ddivdtspec += self.hyperdiff*divspec + self.divlapdiff*divspec
        return dvrtdtspec,ddivdtspec,ddzdtspec

    def rk4step(self,vrtspec,divspec,dzspec):
        # update state using 4th order runge-kutta
        dt = self.dt
        k1vrt,k1div,k1thk = \
        self.gettend(vrtspec,divspec,dzspec,masstend_diag=True)
        #masstendspec = k1thk.sum(axis=0)
        # parameter measuring vertically integrated mass tend amplitude (external mode imbalance)
        #self.masstendvar = ((masstendspec*np.conjugate(masstendspec)).real).sum()
        k2vrt,k2div,k2thk = \
        self.gettend(vrtspec+0.5*dt*k1vrt,divspec+0.5*dt*k1div,dzspec+0.5*dt*k1thk)
        k3vrt,k3div,k3thk = \
        self.gettend(vrtspec+0.5*dt*k2vrt,divspec+0.5*dt*k2div,dzspec+0.5*dt*k2thk)
        k4vrt,k4div,k4thk = \
        self.gettend(vrtspec+dt*k3vrt,divspec+dt*k3div,dzspec+dt*k3thk)
        vrtspec += dt*(k1vrt+2.*k2vrt+2.*k3vrt+k4vrt)/6.
        divspec += dt*(k1div+2.*k2div+2.*k3div+k4div)/6.
        dzspec += dt*(k1thk+2.*k2thk+2.*k3thk+k4thk)/6.
        self.t += dt
        return vrtspec,divspec,dzspec

    def advance(self,vrt,div,dz,grid=False):
        # advance forward number of timesteps given by 'timesteps' instance var.
        # if grid==True, inputs and outputs  are u,v,dz on grid, otherwise
        # spectral coeffs of vorticity,divergence and layer thickness
        if grid :
            vrtspec, divspec = self.ft.getvrtdivspec(vrt,div)
            dzspec = self.ft.grdtospec(dz)
        else:
            vrtspec=vrt;divspec=div;dzspec=dz
        for n in range(self.timesteps):
            vrtspec, divspec, dzspec = self.rk4step(vrtspec,divspec,dzspec)
        if grid:
            u, v = self.ft.getuv(vrtspec,divspec)
            dz = self.ft.spectogrd(dzspec)
            return u,v,dz
        else:
            return vrtspec, divspec, dzspec

    def nlbalance(self,vrtspec):
        """computes balanced layer thickness given vorticity (from nonlinear bal eqn)"""
        dzspec = np.zeros(vrtspec.shape, vrtspec.dtype)
        psispec = self.ft.invlap*vrtspec
        psixx = self.ft.spectogrd(-self.ft.k**2*psispec)
        psiyy = self.ft.spectogrd(-self.ft.l**2*psispec)
        psixy = self.ft.spectogrd(self.ft.k*self.ft.l*psispec)
        tmpspec = self.f*vrtspec + 2.*self.ft.grdtospec(psixx*psiyy - psixy**2)
        mspec = self.ft.invlap*tmpspec
        # alternate form of NBE
        #divspec = np.zeros(vrtspec.shape, vrtspec.dtype)
        #vrt = self.ft.spectogrd(vrtspec)
        #u,v = self.ft.getuv(vrtspec,divspec)
        #tmpg1 = u*(vrt+self.f); tmpg2 = v*(vrt+self.f)
        #tmpspec1, tmpspec2 = self.ft.getvrtdivspec(tmpg1,tmpg2)
        #tmpspec2 = self.ft.grdtospec(0.5*(u**2+v**2))
        #mspec = self.ft.invlap*tmpspec1 - tmpspec2
        dzspec[0,...] = mspec[0,...]/self.theta1
        dzspec[1,...] = (mspec[1,:]-mspec[0,...])/self.delth
        dzspec[0,...] = dzspec[0,...] - dzspec[1,...]
        dzspec = (self.theta1/self.grav)*dzspec # convert from exner function to height units (m)
        # set area mean in grid space to state of rest value
        dz = self.ft.spectogrd(dzspec)
        dz[0,...] = dz[0,...] - dz[0,...].mean() + self.zmid
        dz[1,...] = dz[1,...] - dz[1,...].mean() + self.ztop - self.zmid
        return dz

# simple driver function suitable for mulitprocessing (easy to serialize)
def run_model(u,v,dz,N,L,dt,timesteps,theta1=300,theta2=320,f=1.e-4,div2_diff_efold=1.e30,\
              zmid=5.e3,ztop=10.e3,diff_efold=6.*3600.,diff_order=8,tdrag1=10*86400,tdrag2=10.*86400,tdiab=15*86400,umax=8):
    ft = Fouriert(N,L,threads=1)
    model=TwoLayer(ft,dt,theta1=theta1,theta2=theta2,f=f,div2_diff_efold=div2_diff_efold,\
    zmid=zmid,ztop=ztop,diff_efold=diff_efold,diff_order=diff_order,tdrag1=tdrag1,tdrag2=tdrag2,tdiab=tdiab,umax=umax)
    vrtspec, divspec = ft.getvrtdivspec(u,v)
    dzspec = ft.grdtospec(dz)
    for n in range(timesteps):
        vrtspec, divspec, dzspec = model.rk4step(vrtspec,divspec,dzspec)
    u, v = ft.getuv(vrtspec,divspec)
    dz = ft.spectogrd(dzspec)
    return u,v,dz,model.masstendvar

if __name__ == "__main__":
    import matplotlib
    matplotlib.use('qt5agg')
    import matplotlib.pyplot as plt
    import matplotlib.animation as animation
    import os

    # grid, time step info
    N = 64
    L = 20000.e3
    dt = 600 # time step in seconds

    # get OMP_NUM_THREADS (threads to use) from environment.
    threads = int(os.getenv('OMP_NUM_THREADS','1'))
    ft = Fouriert(N,L,threads=threads)

    # create model instance.
    # symmetric jet (linear drag in both layers, eddy stats same in both layers)
    model=TwoLayer(ft,dt,theta1=300,theta2=320,umax=8,tdrag1=10*86400,tdrag2=10.*86400,tdiab=15*86400)
    # asymmetric jet (linear drag only in lower layer)
    #model=TwoLayer(ft,dt,theta1=300,theta2=320,umax=8,tdrag1=5*86400,tdrag2=1.e30,tdiab=15*86400)

    # vort, div initial conditions
    dtype = model.dtype
    l = np.array(2*np.pi,dtype) / model.ft.L
    uref = np.zeros((2,model.ft.Nt,model.ft.Nt),dtype=dtype)
    vref = np.zeros((2,model.ft.Nt,model.ft.Nt),dtype=dtype)
    if model.tdrag[1] < 1.e10:
        uref[0] = -0.5*model.umax*np.sin(l*model.y)
        uref[1] = 0.5*model.umax*np.sin(l*model.y)
    else:
        uref[1] = model.umax*np.sin(l*self.y)
    vrtspec, divspec = model.ft.getvrtdivspec(uref, vref)
    vrt = model.ft.spectogrd(vrtspec)
    ampvrt = 2.e-6
    vrt += np.random.normal(0,ampvrt,size=(2,ft.Nt,ft.Nt)).astype(dtype)
    # add isolated blob to upper layer
    nexp = 20
    x = np.arange(0,2.*np.pi,2.*np.pi/ft.Nt); y = np.arange(0.,2.*np.pi,2.*np.pi/ft.Nt)
    x,y = np.meshgrid(x,y)
    x = x.astype(dtype); y = y.astype(dtype)
    vrt[1] = vrt[1]+ampvrt*(np.sin(x/2)**(2*nexp)*np.sin(y)**nexp)
    vrtspec = model.ft.grdtospec(vrt)
    divspec = np.zeros(vrtspec.shape, vrtspec.dtype)
    dz = model.nlbalance(vrtspec)
    dzspec = model.ft.grdtospec(dz)
    u,v = model.ft.getuv(vrtspec,divspec)
    vrtspec, divspec = model.ft.getvrtdivspec(u,v)
    if dz.min() < 0:
        raise ValueError('negative layer thickness! adjust jet parameters')

    # run model, animate pv
    nout = int(6.*3600./model.dt) # plot interval
    nsteps = int(200*86400./model.dt)//nout-2 # number of time steps to animate

    fig = plt.figure(figsize=(16,8))
    ax = fig.add_subplot(121); ax.axis('off')
    pv = (0.5*model.zmid/model.f)*(vrt + model.f)/dz
    vmin = 0; vmax = 1.75
    plt.tight_layout()
    im1=ax.imshow(pv[0],cmap=plt.cm.jet,vmin=vmin,vmax=vmax,interpolation="nearest")
    td = model.t/86400.
    txt1=ax.text(0.5,0.95,'Lower Layer PV day %6.3f' % td,\
                ha='center',color='w',fontsize=18,transform=ax.transAxes)
    ax = fig.add_subplot(122); ax.axis('off')
    plt.tight_layout()
    im2=ax.imshow(pv[1],cmap=plt.cm.jet,vmin=vmin,vmax=vmax,interpolation="nearest")
    txt2=ax.text(0.5,0.95,'Upper Layer PV day %6.3f' % td,\
                ha='center',color='w',fontsize=18,transform=ax.transAxes)

    def updatefig(*args):
        global vrtspec, divspec, dzspec
        for n in range(nout):
            vrtspec, divspec, dzspec = model.rk4step(vrtspec, divspec, dzspec)
        vrt = model.ft.spectogrd(vrtspec)
        dz = model.ft.spectogrd(dzspec)
        #print(dz[0].mean(), dz[1].mean())
        pv = (0.5*model.zmid/model.f)*(vrt + model.f)/dz
        td = model.t/86400.
        im1.set_data(pv[0])
        txt1.set_text('Lower Layer PV day %7.3f' % td)
        im2.set_data(pv[1])
        txt2.set_text('Upper Layer PV day %7.3f' % td)
        return im1,txt1,im2,txt2,

    ani = animation.FuncAnimation(fig,updatefig,interval=0,frames=nsteps,repeat=False,blit=True)
    plt.show()


# Import necessary libraries
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn import svm
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from ML_Pipeline.utils import max_val_index
from ML_Pipeline.model_evaluation import evaluate_model
import warnings
warnings.simplefilter(action='ignore')

# Create a function to train multiple models and evaluate their performance
def train_model(X_train, y_train, X_test, y_test):
     # Dictionary mapping model names to their corresponding scikit-learn classifiers
     model_dict = {
         'logistic_reg' : LogisticRegression(solver="liblinear"),
         'naive_bayes'  : GaussianNB(),
         'svm_model'    : svm.SVC(gamma=0.25, C=10),
         'decision_tree': DecisionTreeClassifier(criterion='gini', max_depth=5, random_state=1),
         'rfcl'         : RandomForestClassifier(random_state=1),
     }

     fitted_model = []  # List to store trained models
     score = []  # List to store accuracy scores

     for model_name in list(model_dict.keys()):
        model = model_dict[model_name]
        fitted_model.append(model.fit(X_train, y_train))
        score.append(evaluate_model(y_test, model.predict(X_test), 'accuracy_score'))

     # Find the model with the highest accuracy
     max_test = max_val_index(score)
     max_score = max_test[0]
     max_score_index = max_test[1]
     final_model = fitted_model[max_score_index]

     return final_model, max_score  # Return the best model with the highest accuracy


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jul  9 08:30:12 2021

@author: q

GOAL : Test TA-lib

Try 1:
a) Move talib.tar to $HOME : mv talib.tar $HOME
b) Extract .tar : sudo tar -xvf ta-lib-0.4.0-src.tar.gz
c) cd ta-lib
d) ./configure --prefix=/home/q/anaconda3/lib/python3.7/site-packages/
   *** Use print(module) to get the address where the libraries are installed
e) make
f) sudo make install
Result : Not working

Try 2:
conda install -c conda-forge ta-lib
Result : Working !

"""

# import
import talib as ta
import yfinance as yf

# dowload data
aapl = yf.download('AAPL', '2019-1-1','2019-12-27')

# create indicators
aapl['Simple MA'] = ta.SMA(aapl['Close'],14)
aapl['EMA'] = ta.EMA(aapl['Close'], timeperiod = 14)

# check df tail
print(aapl.tail())

# plot
aapl[['Adj Close', 'EMA']].plot()


import discord
from discord.ext import commands
from discord_slash import cog_ext
prefixint = "n!"

class Help(commands.Cog):

    def __init__(self, client):
        self.client = client

    @cog_ext.cog_slash(name="help", description="📚 Afficher la liste de toutes les commandes du bot.")
    async def help(self,ctx):
        page1 = discord.Embed(
            title = f"Panneau d'aide • Prefix : `{prefixint}`",
            description = "Voici toutes les catégories de commandes.",
            color=0xe4a3f3
        )
        page1.set_image(url="https://i.imgur.com/6CkbQkt.gif")
        page1.add_field(name="Commandes principales", value=f":gear:",inline=True)
        page1.add_field(name="Commandes d'images", value=f":frame_photo:",inline=True)
        page1.add_field(name="Commandes drôles", value=f":clown:",inline=True)
        page1.add_field(name="Commandes de staff", value=f"⚠",inline=True)
        page1.set_footer(text=f"Support: n!invite")

        page2 = discord.Embed (
                title = f"Panneau d'aide • Prefix : `{prefixint}`",
                description = "Les commandes principales.",
                color=0xe4a3f3)
        page2.set_image(url="https://i.imgur.com/6CkbQkt.gif")
        page2.add_field(name=f"{prefixint}say",value="Pour faire parler le bot.")
        page2.add_field(name=f"{prefixint}ping",value="Pour savoir le ping du bot.")
        page2.add_field(name=f"{prefixint}invite",value="Spécialement pour invité le bot sur ton serveur.")
        page2.add_field(name=f"** **",value="** **",inline=False)
        page2.add_field(name=f"{prefixint}serverinfo",value="Pour obtenir des informations sur le serveur")
        page2.add_field(name=f"{prefixint}userinfo",value="Pour obtenir des informations sur toi.")
        page2.add_field(name=f"{prefixint}botinfo",value="Pour obtenir des informations sur le bot.")
        page2.add_field(name=f"{prefixint}bypasslink",value="Pour bypass un lien adfly ou linkvertise par exemple.")
        #page2.add_field(name=f"{prefixint}suggest",value="To send a suggestion to the bot maker.",inline=False)
        page2.add_field(name=f"** **",value="** **",inline=False)
        page2.add_field(name=f"{prefixint}setmsg",value=f"Commande pour définir un message qui pourras être utilisé avec la commande interne au serveur")
        page2.add_field(name=f"{prefixint}sendmsg",value="Commande pour envoyer le message définie par l'administrateur")

        page3 = discord.Embed (
            title = f"Panneau d'aide • Prefix : `{prefixint}`",
            description = "Les commandes d'images.",
            color=0xe4a3f3)
        page3.set_image(url="https://i.imgur.com/6CkbQkt.gif")
        page3.add_field(name=f"{prefixint}pp",value="Pour obtenir ta pdp ou celle d'une personne mentionée.")
        page3.add_field(name=f"{prefixint}servericon",value="Pour récuperer la photo de profil du serveur.")
        page3.add_field(name=f"{prefixint}gif",value="Pour envoyer un gif.")
        page3.add_field(name=f"{prefixint}nekosimg",value="Pour envoyer une image aléatoire grâce à l'api Nekos Life.")

        page4 = discord.Embed (
            title = f"Panneau d'aide • Prefix : `{prefixint}`",
            description = "Les commandes droles.",
            color=0xe4a3f3)
        page4.set_image(url="https://i.imgur.com/6CkbQkt.gif")
        #page4.add_field(name=f"{prefixint}osu",value="Pour avoir tes stats osu!.",inline=False)
        page4.add_field(name=f"{prefixint}fight",value="Pour te battre avec la personne mentionée.")
        page4.add_field(name=f"{prefixint}pof",value=f"Pour faire un pile ou face, un super moyen de décider.")

        page5 = discord.Embed (
            title = f"Panneau d'aide • Prefix : `{prefixint}`",
            description = "Les commandes de staff.",
            color=0xe4a3f3)
        page5.set_image(url="https://i.imgur.com/6CkbQkt.gif")
        page5.add_field(name=f"{prefixint}ban",value="Pour pouvoir ban les méchants de ton serveur")
        page5.add_field(name=f"{prefixint}kick",value="Pour pouvoir kick les méchants de ton serveur")
        page5.add_field(name=f"{prefixint}purge",value="Pour pouvoir supprimer les messages des channels en masse")

        pages = [page1, page2, page3, page4, page5]

        message = await ctx.send(embed = page1)
        await message.add_reaction('⬅')
        await message.add_reaction('⚙')
        await message.add_reaction('🖼')
        await message.add_reaction('🤡')
        await message.add_reaction('⚠')

        def check(reaction, user):
            return user == ctx.author

        i = 0
        reaction = None

        while True:
            if str(reaction) == '⚙':
                i = 1
                await message.edit(embed = pages[i])
            elif str(reaction) == '🖼':
                i = 2
                await message.edit(embed = pages[i])
            elif str(reaction) == '🤡':
                i = 3
                await message.edit(embed = pages[i])
            elif str(reaction) == '⬅':
                i = 0
                await message.edit(embed = pages[i])
            elif str(reaction) == '⚠':
                i = 4
                await message.edit(embed = pages[i])

            try:
                reaction, user = await self.client.wait_for('reaction_add', timeout = 30.0, check = check)
                await message.remove_reaction(reaction, user)
            except:
                break

        await message.clear_reactions()

def setup(client):
    client.add_cog(Help(client))


import transform as tf
import scipy.linalg
import numpy as np
import rosbag
import rospy
import math
import sys

args = sys.argv

if len(args) < 2:
    print("Please insert RosVive data file")
    sys.exit(0)

bag = rosbag.Bag(args[1], "r")

# Dictionary of trackers with poses in vive frame
alphaH = dict()
alphaV = dict()
for i in range(0,24):
  alphaH[i] = list()
  alphaV[i] = list()

for topic, msg, t in bag.read_messages(topics=["/loc/vive/light"]):
  if msg.axis == 1:
    for sample in msg.samples:
      if sample.angle > math.pi/3 or sample.angle < -math.pi / 3:
        continue
      # if (sample.sensor == 21):
        # print(str(msg.axis) + " - " + str(sample.angle))
      alphaV[sample.sensor].append(sample.angle)
  elif msg.axis == 0:
    for sample in msg.samples:
      if sample.angle > math.pi/3 or sample.angle < -math.pi / 3:
        continue
      # if (sample.sensor == 21):
        # print(str(msg.axis) + " - " + str(sample.angle))
      alphaH[sample.sensor].append(sample.angle)

counter = 0.0
stds = 0.0;
print("Horizontal")
for i in range(0,24):
  if len(alphaH[i]) < 10:
    continue
  # print("Sensor " + str(i) + " --- " + str(alphaH[i]))
  # print("\n\n\n\n\n")
  print("Sensor " + str(i) + " " + str(np.std(alphaH[i])))
  counter += 1
  stds += np.std(alphaH[i])

print("Vertical")
for i in range(0,24):
  if len(alphaV[i]) < 10:
    continue
  # print("Sensor " + str(i) + " --- " + str(alphaV[i]))
  # print("\n\n\n\n\n")
  print("Sensor " + str(i) + " " + str(np.std(alphaV[i])))
  counter += 1
  stds += np.std(alphaV[i])

print(stds/counter)

# import matplotlib.pyplot as plt
# for i in range(0, 24):
#   plt.plot(alphas[i])
#   break
# plt.show()


import click
from api import get_definition

@click.command()
@click.option('--word', '-w', prompt=True, help='Include a word to look up.')
@click.option('--target', '-t', help='Select target language by language code.', default='en')
@click.option('--origin', '-o', help='Select origin language by language code.', default='')

def cli(word, origin, target):
    get_definition(word, tr_from=origin, tr_to=target)


import datetime

car_all = [
    {
        'Mobil':'Avanza',
        'Warna': 'Putih',
        'Kapasitas': 7,
        'Transmisi': 'Matic',
        'Stok Unit': 3,
        'Harga':290000
    },
    {
        'Mobil':'Calya',
        'Warna': 'Silver',
        'Kapasitas': 7,
        'Transmisi': 'Matic',
        'Stok Unit': 3,
        'Harga':280000
    },
    {
        'Mobil':'Agyaa',
        'Warna': 'Silver',
        'Kapasitas': 7,
        'Transmisi': 'Matic',
        'Stok Unit': 2,
        'Harga':280000
    },
    {
        'Mobil':'Brioo',
        'Warna': 'Putih',
        'Kapasitas': 5,
        'Transmisi': 'Matic',
        'Stok Unit': 2,
        'Harga':270000
    },
    {
        'Mobil':'Ertiga',
        'Warna': 'Hitam',
        'Kapasitas': 7,
        'Transmisi': 'Matic',
        'Stok Unit': 4,
        'Harga':285000
    }
]

def car_spec():
    print('\n========== Spesifikasi Mobil ==========\n')
    print('\tCarID \t| Jenis Mobil  \t\t| Warna\t\t| Kursi\t| Transmisi')

    for car_id, dict in enumerate(car_all):
        print(f"\t{car_id} \t| {dict['Mobil']} \t\t| {dict['Warna']} \t| {dict['Kapasitas']} \t| {dict['Transmisi']}")

def car_available():
    print('\n========== DAFTAR MOBIL TERSEDIA ==========\n')
    print('\tCarID \t| Mobil  \t\t| Transmisi \t| Stok Unit\t| Harga Sewa per Hari')

    for car_id, dict in enumerate(car_all):
        print(f"\t{car_id} \t| {dict['Mobil']} \t\t| {dict['Transmisi']} \t| {dict['Stok Unit']} \t\t| {dict['Harga']}")

def add_carlist():
    car_available()
    while True:
        car_type = input('Masukkan Tipe Mobil: ').capitalize()
        car_color = input('Masukkan Warna Mobil: ').capitalize()

        try:
            car_cap = int(input('Masukkan Kapasitas Mobil: '))
        except ValueError:
            print('Input anda invalid, mohon masukan nilai yang sesuai!')
            continue
        else:
            pass

        while True:
            transmission = input('Masukkan jenis transmisi mobil (Manual/Matic): ').capitalize()
            if transmission == 'Manual':
                break
            elif transmission == 'Matic':
                break
            else:
                print('Masukkan jenis transmisi yang sesuai (Manual/Matic)')
                transmission
                continue

        try:
            stok_unit = int(input('Masukkan stok unit mobil: '))
        except ValueError:
            print('Input anda invalid, mohon masukan nilai yang sesuai!')
            continue
        else:
            pass

        try:
            price = int(input('Masukkan harga sewa unit mobil: '))
        except ValueError:
            print('Input anda invalid, mohon masukkan nilai yang sesuai!')
            continue
        else:
            break

    while True:
        user_input = input('\nKetik Y jika yakin dan N untuk membatalkan perintah: ').upper()

        if user_input == 'Y':
            break
        elif user_input == 'N':
            add_carlist()
        else:
            user_input

    for car in car_all:
        if (car['Mobil'] == car_type) and (car['Warna'] == car_color) and (car['Kapasitas'] == car_cap) and (car['Transmisi'] == transmission) and (car['Harga'] == price):
            car['Stok Unit'] += stok_unit
            break
    else:
        car_all.append({
            'Mobil': car_type,
            'Warna': car_color,
            'Kapasitas': car_cap,
            'Transmisi': transmission,
            'Stok Unit': stok_unit,
            'Harga': price
        })
    car_available()
    main_menu()

def remove_carlist():
    car_available()

    while True:
        car_id = input('Masukkan CarID yang ingin dihapus: ')
        if car_id.isdigit() == False:
            print('Harap masukkan carID yang sesuai')
            car_id
        else:
            car_id = int(car_id)

            if car_id not in range(len(car_all)):
                print(f'CarID {car_id} yang anda masukkan tidak ada dalam database. Mohon masukkan CarID yang sesuai.')
            else:
                print('\Apakah anda ingin menghapus stok unit?')

                while True:
                    user_input = input('\nSilahkan ketik: \n"Y" untuk menghapus stok unit \n"N" untuk batal \nSilahkan Input: ').upper()
                    if user_input == 'Y':
                        del car_all[car_id]
                        break
                    elif user_input == 'N':
                        read_sub3()
                    else:
                        print('Input tidak sesuai. Silahkan ulangi.')
                        user_input
                break

        break

    car_available()
    main_menu()

rent_cart = [] ## keranjang

def rent_car():
    car_available()

    while True:
        car_id = int(input('Masukkan CarID yang ingin di sewa: '))
        car_total = int(input('Masukkan jumlah mobil yang ingin di sewa: '))
        rent_day = int(input('Masukkan jumlah hari sewa: '))

        if car_total > car_all[car_id]['Stok Unit']:
            print(f"Stok mobil tidak mencukupi. Stok {car_all[car_id]['Mobil']} tersisa {car_all[car_id]['Stok Unit']} unit")
        else:
            rent_cart.append({
                'Mobil': car_all[car_id]['Mobil'],
                'Jumlah Mobil': car_total,
                'Jumlah Hari': rent_day,
                'Harga/hari': car_all[car_id]['Harga'],
                'CarID': car_id
            })

        for car in rent_cart:
            car_all[car['CarID']]['Stok Unit'] -= car['Jumlah Mobil']

            print('Isi Cart:')
            print(f'Mobil\t| Jumlah Mobil\t| Jumlah Hari\t| Harga per hari')

        for car in rent_cart:
            print(f"{car['Mobil']}\t| {car['Jumlah Mobil']}\t| {car['Jumlah Hari']}\t| {car['Harga/hari']}")
        break

    while True:
        check = input('Apakah anda ingin menyewa mobil lain ("Y": ya/"N": tidak): ').upper()

        if check == 'Y':
            car_id = int(input('Masukkan CarID yang ingin di sewa: '))
            print(f"Sisa stok {car_all['Mobil']} adalah {car_all[car_id]['Stok Unit']}.")

            if (car_all[car_id]['Stok Unit'] == 0):
                print('Mohon maaf unit mobil sedang kosong. Silahkan memilih unit mobil yang lain')
                continue
            else:
                add_car_total = int(input('Masukkan jumlah mobil yang ingin disewa: '))
                add_rent_day = int(input('Masukkan jumlah hari sewa: '))
                rent_cart.append([car_all[car_id]['Mobil'], add_car_total, add_rent_day, car_all[car_id]['Harga'], car_id])
                car_all[car_id]['Stok Unit'] -= add_car_total

                if (add_car_total > car_all[car_id]['Stok Unit']):
                    print('Unit mobil tidak cukup.')
                else:
                    rent_cart.append([car_all[car_id]['Mobil'], add_car_total, add_rent_day, car_all[car_id]['Harga'], car_id])
                    car_all[car_id]['Stok Unit'] -= add_car_total
        else:
            break

    print('\n========== DAFTAR SEWA MOBIL ==========\n')
    print('Mobil  \t| Jumlah Mobil \t| Jumlah Hari\t| Harga Sewa per Hari\t| Total Harga')

    total_price = 0

    for car in rent_cart:
        print(f"{car['Mobil']}\t\t| {car['Jumlah Mobil']}\t\t| {car['Jumlah Hari']}\t\t| {car['Harga/hari']}\t| {car['Jumlah Mobil'] * car['Jumlah Hari'] * car['Harga/hari']}")
        total_price += car['Harga/hari'] * car['Jumlah Mobil'] * car['Jumlah Hari']
        break

    while True:
        print(f'Total yang harus dibayar = {total_price}')
        if total_price != 0:
            total_money = int(input('Masukkan Jumlah Uang: '))

            if total_money > total_price:
                change = total_money - total_price
                print(f'Uang Kembalian: {change}')
                rent_cart.clear()
                break

            elif total_money < total_price:
                minus = total_price - total_money
                print(f'Maaf, uang kurang sebesar {minus}')

            elif total_money == total_price:
                print(f'Terima kasih telah percaya dengan jasa sewa kami')
                rent_cart.clear()
                break

        else:
            print(f'Terima kasih atas kunjungannya')
            break

    main_menu()

def read_sub1():
    while True:
        print('''
        ========== TREVO CAR RENTAL =========

        1. Spesifikasi daftar mobil
        2. Daftar mobil yang tersedia
        3. Kembali ke menu utama
        ''')
        try:
            menu_input = int(input('Masukan nomor menu utama: '))
        except ValueError:
            print('Input anda invalid, mohon hanya masukan angka!')
            continue

        if menu_input == 1:
           car_spec() ## daftar semua

        elif menu_input ==2:
            car_available()
        elif menu_input == 3:
            main_menu()
        else:
            print('Mohon hanya input nomor menu 1 sampai 3!')

def read_sub2():
    while True:
        print('''
        ========== TREVO CAR RENTAL =========

        1. Menambah Daftar Mobil
        2. Kembali ke menu utama
        ''')
        try:
            menu_input = int(input('Masukan nomor menu utama: '))
        except ValueError:
            print('Input anda invalid, mohon hanya masukan angka!')
            continue

        if menu_input == 1:
           add_carlist()

        elif menu_input == 2:
            main_menu()
        else:
            print('Mohon hanya input nomor menu 1 sampai 2!')

def read_sub3():
    while True:
        print('''
        ========== TREVO CAR RENTAL =========

        1. Menghapus Daftar Mobil
        2. Kembali ke menu utama
        ''')
        try:
            menu_input = int(input('Masukan nomor menu utama: '))
        except ValueError:
            print('Input anda invalid, mohon hanya masukan angka!')
            continue

        if menu_input == 1:
            remove_carlist()

        elif menu_input == 2:
            main_menu()

        else:
            print('Mohon hanya input nomor menu 1 sampai 2!')

def read_sub4():
    while True:
        print('''
        ========== TREVO CAR RENTAL =========

        1. Menyewa Mobil
        2. Kembali ke menu utama
        ''')
        try:
            menu_input = int(input('Masukan nomor menu utama: '))
        except ValueError:
            print('Input anda invalid, mohon hanya masukan angka!')
            continue

        if menu_input == 1:
            rent_car()

        elif menu_input ==2:
            main_menu()

        else:
            print('Mohon hanya input nomor menu 1 sampai 2!')

def main_menu():
    time = datetime.datetime.now()

    print('\n\tSELAMAT DATANG DI TREVO CAR RENTAL!\n')
    print('\tJasa Rental Mobil Termudah dan Terpecaya\n')
    print(f'\tTanggal dan Waktu: {time}')

    while True:
        print('''
        ========== TREVO CAR RENTAL =========

        1. Daftar Mobil
        2. Menambahkan daftar mobil
        3. Menghapus daftar mobil
        4. Menyewa Mobil
        5. Keluar program
        ''')

        try:
            menu_input = int(input('Masukan nomor menu utama: '))
        except ValueError:
            print('Input anda invalid, mohon hanya masukan angka!')
            continue

        if menu_input == 1:
            read_sub1()

        elif menu_input == 2:
            read_sub2()

        elif menu_input == 3:
            read_sub3()

        elif menu_input == 4:
            read_sub4()

        elif menu_input == 5:
            break

        else:
            print('Mohon hanya input nomor menu 1 sampai 3!')

main_menu()


from datetime import datetime
from typing import Optional
import aiohttp
from sqlalchemy import select, text
from sqlalchemy.ext.asyncio import AsyncSession

from db_config import async_session
from models import QuizQuestion
from cache import cache

async def get_answer(questions_num: int) -> list[dict]:  # make request to third party API
    url = f'https://jservice.io/api/random?count={questions_num}'
    async with aiohttp.ClientSession() as client:
        async with client.get(url) as response:
            return await response.json()

async def exist_check(question_collection: list[dict], session: AsyncSession, unique_question: Optional[list[dict] | None] = None) -> set[int]:
    collections_for_check_unique = {question['id'] for question in question_collection}  # set id's collection for check unique constraint
    repeatable_prev_response = None
    if unique_question:
        unique_collection = {question['id'] for question in unique_question}  # set id's collection for unique from prev response
        repeatable_prev_response = unique_collection.intersection(collections_for_check_unique)  # check current response for unique relatively prev response
        collections_for_check_unique = collections_for_check_unique.difference(repeatable_prev_response)  # get unique id's set compair with prev response
    query = select(QuizQuestion.id).where(QuizQuestion.id.in_(collections_for_check_unique))  # check for unique questions compair db
    repeatable_question = await session.scalars(query)  # get iterator of not unique questions
    repeatable_set = {id for id in repeatable_question}  # get not unique id's set
    if repeatable_prev_response:  # if is repeats compared prev response
        repeatable_set = repeatable_set.union(repeatable_prev_response)  # get union of repeats in db and repeats in prev response
    return repeatable_set  # return set of repeats

async def create_question(questions_collection: list[dict]) -> list[dict]:  # make DB insertion and return previous saved entries
    previous_question = cache.get_cache  # get last added entries
    async with async_session() as session:
        async with session.begin():  # run transaction
            await session.execute(text("LOCK TABLE quiz_question IN ROW EXCLUSIVE MODE;"))  # set lock on table including insertion
            repeatable_set = await exist_check(questions_collection, session)  # find repeat questions in db
            unique_question = [question for question in questions_collection if question['id'] not in repeatable_set]  # set the DB's unique questions
            repeatable_response = None
            while repeatable_set:  # while id's not unique in db or not unique db and responses
                repeatable_response = await get_answer(len(repeatable_set))  # keep send request to 3rd party API in size of repeatable set
                repeatable_set = await exist_check(repeatable_response, session, unique_question)  # keep check response for unique constraint in db in repeat requests
            if repeatable_response:
                unique_question.extend(repeatable_response)  # extend unique questions list with unique questions from repeatable request
            objects_for_insert = []  # prepare iterable for bulk insertion
            for question in unique_question:
                obj = QuizQuestion(
                    id=question['id'],
                    question=question['question'],
                    answer=question['answer'],
                    created_at=datetime.strptime((question['created_at']), "%Y-%m-%dT%H:%M:%S.%f%z")
                )
                objects_for_insert.append(obj)
                session.add_all(objects_for_insert)
    cache.set_cache((objects_for_insert[-1]).to_json())
    return previous_question


import os
import sys
import time
import pickle
import shutil
import logging
import argparse
import traceback
import numpy as np
import pandas as pd
import random
import cv2
import scipy.io as sio
from skimage import io

# from utils import *
from multiprocessing import Pool, Process
from multiprocessing_logging import install_mp_handler
from datetime import datetime

import torch
import torchvision.transforms as transforms
from torch.utils.data import Dataset, DataLoader
import torch.backends.cudnn as cudnn

from emonet.emonet.models import EmoNet

## facial expression
class EmonetDataset(Dataset):
    def __init__(self, root_path, transform_image=None, suffix=None):
        self.root_path = root_path
        self.transform_image = transform_image
        self.suffix = suffix

    def __len__(self):
        files = os.listdir(self.root_path)
        length = sum(['.jpg' in x for x in files])
        return length if self.suffix is None else int(length / 2)

    def __getitem__(self, index):
        image_files = os.listdir(self.root_path)
        image_files = [x for x in image_files if x.split('.')[1] == 'jpg']
        if self.suffix is not None:
            image_files = list(filter(lambda x: self.suffix in x, image_files))
        image_files = sorted(image_files, key=lambda x: int(x.split('_')[0].replace('frame', '')))
        image = io.imread(os.path.join(self.root_path, image_files[index]))
        if self.transform_image is not None:
            image = self.transform_image(image)
        if self.suffix is None:
            return dict(id=image_files[index].split('.')[0], image=image)
        else:
            return dict(id=image_files[index].split('.')[0].replace(f'_{self.suffix}', ''), image=image)

def facial_expression_analysis(align_dir, compo_dir, score_dir, fer_model=['ferplus', 'emonet'], dev='cpu'):
    '''
    :param align_dir: directory containing aligned face images.
    :param compo_dir: directory containing composite face images.
    :param fer_model: facial expression analysis model, 'ferplus' only classifies emotions, 'emonet' classifies emotions and scores valence-arousal.
    :return: no return
    '''
    ## aligned face images
    align_files = os.listdir(align_dir)
    align_files = [x for x in align_files if '.jpg' in x]
    align_files = sorted(align_files, key=lambda x: int(x.split('_')[0].replace('frame', '')))
    align_ids = [x.split('.')[0] for x in align_files]
    if len(align_ids) == 0:
        print(f"[{datetime.today().strftime('%Y-%m-%d %H:%M:%S')}] Video {align_dir.split('/')[-1]} has no target faces, will not analyze emotions")
        logging.info(f"Video {align_dir.split('/')[-1]} has no target faces, will not analyze emotions")
        return

    # 'ferplus': cpu, multiprocess
    if fer_model == 'ferplus':

        # save as a dict with 4 keys
        scores = dict.fromkeys(['align_emos', 'left_emos', 'right_emos', 'FAsy_emos'])

        emotions = ['Neutral', 'Happy', 'Surprise', 'Sad',
                    'Anger', 'Disgust', 'Fear', 'Contempt']
        # dnn emotion model
        ferplus_model = './emotion_ferplus/model.onnx'
        ferplus_net = cv2.dnn.readNetFromONNX(ferplus_model)

        for idx, dir_suf in enumerate(zip([align_dir, compo_dir, compo_dir], ['', '_left', '_right'])):
            image_files = [os.path.join(dir_suf[0], f'{k}{dir_suf[1]}.jpg') for k in align_ids]
            # print(image_files[:3])
            processed_faces = []
            for _, image_file in enumerate(image_files):
                image = cv2.imread(image_file)
                # color to gray, resize & reshape
                gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
                resized_face = cv2.resize(gray, (64, 64))
                processed_faces.append(resized_face[np.newaxis,np.newaxis,:,:])
            processed_faces = np.concatenate(processed_faces, axis=0) # np.ndarray (N, 1, 64, 64)
            # score emotions
            ferplus_net.setInput(processed_faces)
            Output = ferplus_net.forward() # (N, 8)
            # softmax
            exp = np.exp(Output)
            prob = exp / np.sum(exp, axis=1)[:, None] # (N, 8)
            score_df = pd.DataFrame(prob, columns=emotions) # (N, 8)
            score_df.insert(0, 'Frame_ID', align_ids) # (N, 9)
            # sort dataframes
            sort_index = score_df['Frame_ID'].apply(lambda x: x.split('_')[0].replace('frame','')).astype(int).argsort()
            score_df = score_df.iloc[sort_index].reset_index(drop=True)
            # record
            if idx == 0:
                scores['align_emos'] = score_df
            elif idx == 1:
                scores['left_emos'] = score_df
            else:
                scores['right_emos'] = score_df
        score_df = scores['left_emos'].copy()
        score_df.iloc[:, 1:] = scores['left_emos'].iloc[:, 1:] - scores['right_emos'].iloc[:, 1:]
        scores['FAsy_emos'] = score_df

    # 'emonet': gpu, serialization, batch
    elif fer_model == 'emonet':

        # save as a dict with 8 keys
        scores = dict.fromkeys(['align_emos', 'left_emos', 'right_emos', 'FAsy_emos',
                               'align_vai', 'left_vai', 'right_vai', 'FAsy_vai'])

        emotions_emonet = {0: 'neutral', 1: 'happy', 2: 'sad', 3: 'surprise',
                           4: 'fear', 5: 'disgust', 6: 'anger', 7: 'contempt'}
        n_expression = 8
        batch_size = 64 if dev != 'cpu' else len(align_ids)
        n_workers = min(8, os.cpu_count()//len(device)+1) if dev != 'cpu' else 0 # num_workers set to 4 or 8 fastest
        if dev != 'cpu':
            cudnn.benchmark = True

        # load and transform images
        trans_emonet = transforms.ToTensor()

        # load emonet
        emonet_state_path = './emonet/pretrained/emonet_8.pth'
        emonet_state_dict = torch.load(str(emonet_state_path), map_location='cpu')
        emonet_state_dict = {k.replace('module.', ''): v for k, v in emonet_state_dict.items()}
        emonet = EmoNet(n_expression=n_expression).to(dev)
        emonet.load_state_dict(emonet_state_dict, strict=False)
        emonet.eval()

        for idx, dir_suf in enumerate(zip([align_dir, compo_dir, compo_dir], [None, '_left', '_right'])):

            emonet_dataset = EmonetDataset(root_path=dir_suf[0], transform_image=trans_emonet, suffix=dir_suf[1])
            emonet_dataloader = DataLoader(emonet_dataset, batch_size=batch_size, shuffle=False, num_workers=n_workers)
            # forward emonet
            emotions, vais = [], []
            for _, data in enumerate(emonet_dataloader):

                images = data['image'].to(dev)
                with torch.no_grad():
                    out = emonet(images)
                emos = torch.softmax(out['expression'], axis=1).cpu().detach().numpy()  # (bs, 8)
                emos_df = pd.DataFrame(emos, columns=list(emotions_emonet.keys()))
                emos_df.insert(0, 'Frame_ID', data['id'])
                emotions.append(emos_df)
                vals = out['valence'].cpu().detach().numpy()  # (bs,)
                arous = out['arousal'].cpu().detach().numpy()  # (bs,)
                intens = np.sqrt(vals ** 2 + arous ** 2)
                vai_df = pd.DataFrame({'Frame_ID': data['id'], 'Valence': vals, 'Arousal': arous, 'Intensity': intens})
                vais.append(vai_df)
            if dev != 'cpu':
                torch.cuda.empty_cache()
            emotions_df = pd.concat(emotions)
            vais_df = pd.concat(vais)
            # sort dataframes
            sort_index = emotions_df['Frame_ID'].apply(lambda x: x.split('_')[0].replace('frame','')).astype(int).argsort()
            emotions_df = emotions_df.iloc[sort_index].reset_index(drop=True)
            sort_index = vais_df['Frame_ID'].apply(lambda x: x.split('_')[0].replace('frame','')).astype(int).argsort()
            vai_df = vais_df.iloc[sort_index].reset_index(drop=True)
            # record
            if idx == 0:
                scores['align_emos'] = emotions_df
                scores['align_vai'] = vais_df
            elif idx == 1:
                scores['left_emos'] = emotions_df
                scores['left_vai'] = vais_df
            else:
                scores['right_emos'] = emotions_df
                scores['right_vai'] = vais_df
        score_df = scores['left_emos'].copy()
        score_df.iloc[:, 1:] = scores['left_emos'].iloc[:, 1:] - scores['right_emos'].iloc[:, 1:]
        scores['FAsy_emos'] = score_df
        score_df = scores['left_vai'].copy()
        score_df.iloc[:, 1:] = scores['left_vai'].iloc[:, 1:] - scores['right_vai'].iloc[:, 1:]
        scores['FAsy_vai'] = score_df

    # save scores
    with open(os.path.join(score_dir, f'scores_{fer_model}.pkl'), 'wb') as f:
        pickle.dump(scores, f)
    print(f"[{datetime.today().strftime('%Y-%m-%d %H:%M:%S')}] Facial expression scored video {align_dir.split('/')[-1]}")
    logging.info(f"Facial expression scored video {align_dir.split('/')[-1]} (PID: {os.getpid()})")

def multi_facial_expression_analysis(video_ids_chunk, dev):
    for idx, vid_id in enumerate(video_ids_chunk):
        align_dir = os.path.join(align_root, vid_id)
        compo_dir = os.path.join(compo_root, vid_id)
        score_dir = os.path.join(score_root, vid_id)
        try:
            facial_expression_analysis(align_dir, compo_dir, score_dir, fer_model, dev)
        except Exception as e:
            logging.error(e)

if __name__ == '__main__':

    ## parse arguments
    parser = argparse.ArgumentParser()
    # directories
    parser.add_argument('--video-root', type=str, help='Specify the directory containing the videos.')
    parser.add_argument('--align-root', type=str, help='Specify the directory containing the aligned face images.')
    parser.add_argument('--compo-root', type=str, help='Specify the directory containing the composite face images.')
    parser.add_argument('--score-root', type=str, help='Specify the directory for saving information and scores.')
    # global setting
    parser.add_argument('--gpus', type=str, help='Specify the gpu indices for EmoNet.')
    # facial expression analysis
    parser.add_argument('--fer-model', type=str, choices=['ferplus', 'emonet'], help="Specify the model for emotion analysis. Available model: 'ferplus' and 'emonet'.")

    args = parser.parse_args()

    if args.gpus is not None:
        device = args.gpus.split(',') # a list of gpus
        device_name = f"gpu{args.gpus.replace(',','')}"
    else:
        device = 'cpu'
        device_name = 'cpu'
    video_root = args.video_root
    align_root = args.align_root
    compo_root = args.compo_root
    score_root = args.score_root
    fer_model = args.fer_model
    if fer_model == 'ferplus':
        cv2.setNumThreads(1)

    if not os.path.exists('logs'):
        os.makedirs('logs')
    log_file = f"logs/{align_root.split('/')[1].split('_')[0]}_emotion_{fer_model}_{device_name}.log"
    open(log_file, 'w').close()
    logging.basicConfig(filename=log_file,
                        level=logging.INFO,
                        format='[%(asctime)s] [%(levelname)s] %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S',
                        filemode='a+')
    install_mp_handler()
    logging.info(f'Devices: {device}, Emotion model: {fer_model}')
    print(f'Devices: {device}, Emotion model: {fer_model}')

    # check device and model
    if fer_model == 'ferplus':
        assert device == 'cpu', "Ferplus should run with cpu!"

    video_ids = os.listdir(video_root)
    video_ids = sorted([x.split('.')[0] for x in video_ids])

    ## 4. facial expression
    fer_model = args.fer_model
    if device == 'cpu':
        inp_args = []
        for idx, vid_id in enumerate(video_ids):
            align_dir = os.path.join(align_root, vid_id)
            compo_dir = os.path.join(compo_root, vid_id)
            score_dir = os.path.join(score_root, vid_id)
            # if os.path.exists(os.path.join(score_dir, f'scores_{fer_model}.pkl')):
            #     continue
            inp_args.append((align_dir, compo_dir, score_dir, fer_model, device))
        if len(inp_args) > 0:
            try:
                with Pool(os.cpu_count()) as pool:
                    pool.starmap(facial_expression_analysis, inp_args)
                    pool.close()
                    pool.join()
            except Exception as e:
                print(traceback.format_exc())
                logging.error(e)
    else:
        if len(device) == 1:
            list_video_ids = [video_ids]
        else:
            chunk_size = len(video_ids) // len(device) + 1
            list_video_ids = [video_ids[i:i+chunk_size] for i in range(0, len(video_ids), chunk_size)]
        procs = []
        for idx, video_ids_chunk in enumerate(list_video_ids):
            dev = f'cuda:{device[idx]}'
            print(f'{dev} will process {len(video_ids_chunk)} videos.')
            p = Process(target=multi_facial_expression_analysis, args=(video_ids_chunk, dev))
            procs.append(p)
            p.start()

        for p in procs:
            p.join()

    print(f"[{datetime.today().strftime('%Y-%m-%d %H:%M:%S')}] ********** Finish step 4: analyze facial expression **********")
    logging.info('********** Finish step 4: analyze facial expression **********')


from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer
from scipy import sparse
import os
import nltk
import re
import operator
import math
import numpy as np

porter_stemmer = PorterStemmer()
stop_words_table = stopwords.words('english')  # 创建英文默认倒排表

TRAIN_DATA_PATH = "./data/origin/train.txt"
TRAIN_changed_DATA_PATH="./data/train.txt"
TEST_DATA_PATH  = "./data/origin/test.txt"
TEST_changed_DATA_PATH="./data/test.txt"
TRAIN_SIZE = 6400
TEST_SIZE = 1600

#target_names = ['Cause-Effect', 'Component-Whole', 'Entity-Destination', 'Product-Producer', 'Entity-Origin',
  #              'Member-Collection', 'Message-Topic', 'Content-Container', 'Instrument-Agency', 'Other']

target_names=['1']

def build_stop_words_table():
    # print(stop_words_table)
    for w in ['!', ',', '.', '?', ':', ';', '<', '>', '@', '[', ']', '(', ')', '-', '\'\'', '--', '*', '$',
              '...', '=', '\'s', '\'t', '|', '%', '..', '&', '#', '`', '``',]:
        stop_words_table.append(w)

# handle train data
def handle_train_data():
    count=0
    with open(TRAIN_DATA_PATH) as f:
        with open(TRAIN_changed_DATA_PATH,"w") as write_f:
            while count < TRAIN_SIZE:
                sentence = []#句子分词结果
                prop=[]#词性标注
                label=[]#序列标注

                temp=f.readline().split('"')[1]
                temp=nltk.word_tokenize(temp)#分词

                for word in temp:
                    #w = porter_stemmer.stem(word)  # 词根化处理
                    if word not in stop_words_table:#去停用词
                        sentence.append(word)

                temp1=re.split(r'[(,)]',f.readline())
                relation=temp1[0]
                #e1=[porter_stemmer.stem(word) for word in temp1[1].split(" ")]
                e1=temp1[1].split(" ")
                #e2=[porter_stemmer.stem(word) for word in temp1[2].split(" ")]
                e2=temp1[2].split(" ")
                length=len(sentence)
                i=0
                while i < length:
                    if sentence[i] in e1:
                        #label.append("B-e1-"+relation)
                        label.append(("B-1"))
                        i+=1
                        while i<length and sentence[i] in e1:
                            #label.append("I-e1-"+relation)
                            label.append("I-1")
                            i+=1
                    elif sentence[i] in e2:
                        #label.append("B-e2-"+relation)
                        label.append("B-1")
                        i+=1
                        while i<length and sentence[i] in e2:
                            #label.append("I-e2-"+relation)
                            label.append(("I-1"))
                            i+=1
                    else:
                        label.append("O")
                        i+=1

                prop=nltk.pos_tag(sentence)#词性标注

                if(len(sentence)!=len(prop)):
                    exit("prop error")
                elif(len(sentence)!=len(label)):
                    exit("label error")

                for word in sentence:
                    write_f.write(word+" ")
                write_f.write("\n")
                for pro in prop:
                    write_f.write(pro[1]+" ")
                write_f.write("\n")
                for lab in label:
                    write_f.write(lab+" ")
                write_f.write("\n")

                count+=1
            #train_data.append()
            #train_target.append(target_names.index(f.readline().split('(')[0]))

#handle test data
def handle_test_data():
    count=0
    with open(TEST_DATA_PATH) as f:
        with open(TEST_changed_DATA_PATH,"w") as write_f:
            while count < TEST_SIZE:
                sentence = []#句子分词结果
                prop=[]#词性标注
                temp=f.readline().split('"')[1]
                temp=nltk.word_tokenize(temp)#分词

                for word in temp:
                    #w = porter_stemmer.stem(word)  # 词根化处理
                    if word not in stop_words_table:#去停用词
                        sentence.append(word)

                prop=nltk.pos_tag(sentence)#词性标注

                for word in sentence:
                    write_f.write(word+" ")
                write_f.write("\n")
                for pro in prop:
                    write_f.write(pro[1]+" ")
                write_f.write("\n")
                for i in range(0,len(sentence)):
                    write_f.write("O ")
                write_f.write("\n")
                count+=1

def main():
    build_stop_words_table()
    handle_train_data()
    handle_test_data()

if __name__=='__main__':
    main()


"""Solution for day 5"""

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

# Part 1

# Read line coordinates
with open('lines.txt', 'r', encoding='UTF-8') as f:
    lines_list = f.read().splitlines()

# Prepare clean dataframe (one column per coordinate)
line_coords_df = pd.DataFrame(lines_list)[0].str.split(' -> ', expand=True)
# Because we have just two columns to split, do it explicitly
line_coords_df[['x0', 'y0']] = line_coords_df[0].str.split(',', expand=True).astype(int)
line_coords_df[['x1', 'y1']] = line_coords_df[1].str.split(',', expand=True).astype(int)
# Drop temporary columns
line_coords_df.drop(columns=[0, 1], inplace=True)

# Get line equations and type (h - horizontal, v - vertical, d - diagonal)
line_coords_df['a'] = line_coords_df['y0'] - line_coords_df['y1']
line_coords_df['b'] = line_coords_df['x1'] - line_coords_df['x0']
line_coords_df['c'] = -line_coords_df['b'] * line_coords_df['y0'] - line_coords_df['a'] * line_coords_df['x0']
line_coords_df['type'] = np.select([line_coords_df['a'] == 0, line_coords_df['b'] == 0], ['h', 'v'], default='d')

# Get maximum coordinates to set the grid size
x_max = line_coords_df[['x0', 'x1']].max().max() + 1
y_max = line_coords_df[['y0', 'y1']].max().max() + 1

# Prepare numpy grid
grid_np = np.zeros([x_max, y_max], dtype=int)

# For part 1 we only need horizontal or vertical lines
for i, row in line_coords_df.iterrows():
    if row['type'] == 'h':
        points_x = np.linspace(row['x0'], row['x1'], num=abs(row['x0'] - row['x1']) + 1, dtype=int)
        points_y = np.ones_like(points_x) * row['y0']
        grid_np[points_y, points_x] += 1
    elif row['type'] == 'v':
        points_y = np.linspace(row['y0'], row['y1'], num=abs(row['y0'] - row['y1']) + 1, dtype=int)
        points_x = np.ones_like(points_y) * row['x0']
        grid_np[points_y, points_x] += 1

# Calculate number of dangerous areas
intersects = np.count_nonzero(grid_np >= 2)
print(grid_np)
print(f"Number of dangerous areas: {intersects}")

# Add a heatmap just for fun
fig, ax = plt.subplots()
x, y = np.meshgrid(np.arange(x_max),np.arange(y_max))
ax.pcolormesh(x, y, grid_np, cmap='inferno', shading='nearest')
fig.savefig('plots/heatmap_hv.png', bbox_inches='tight', pad_inches=0.1)

# Part 2
print('\nAdd diagonals')

# Iterate again over the data and add diagonal lines
for i, row in line_coords_df.iterrows():
    if row['type'] == 'd':
        points_x = np.linspace(row['x0'], row['x1'], num=abs(row['x0'] - row['x1']) + 1, dtype=int)
        points_y = np.linspace(row['y0'], row['y1'], num=abs(row['y0'] - row['y1']) + 1, dtype=int)
        grid_np[points_y, points_x] += 1

# Calculate number of dangerous areas
intersects = np.count_nonzero(grid_np >= 2)
print(grid_np)
print(f"Number of dangerous areas: {intersects}")

# Add a heatmap just for fun
ax.pcolormesh(x, y, grid_np, cmap='inferno', shading='nearest')
fig.savefig('plots/heatmap_hvd.png', bbox_inches='tight', pad_inches=0.1)


import nltk
import string
from nltk.tokenize import word_tokenize
import numpy as np

class Text_kit:
    '''Toolkit for processing texts'''
    def __init__(self, text_array = ['sampleText1', 'sampleText1']):
        ''' Initialize text_toolkit '''
        self.text_array = text_array

    def clean_text(self, text):
        '''
        1. Lowercase Text
        2. Remove punctuation
        3. remove non-alpha chars
        4. filter stop words
        '''
        tokens = word_tokenize(text)
        # convert to lower case
        tokens = [w.lower() for w in tokens]
        # remove punctuation from each word
        import string
        table = str.maketrans('', '', string.punctuation)
        stripped = [w.translate(table) for w in tokens]
        # remove remaining tokens that are not alphabetic
        words = [word for word in stripped if word.isalpha()]
        # filter out stop words
        from nltk.corpus import stopwords
        stop_words = set(stopwords.words('english'))
        words = [w for w in words if not w in stop_words]
        return words

    def get_wordcloud(self, text_array):
        ''' return text in format required for wordcloud'''
        word_dict = []
        cleaned_sentences = map(self.clean_text, text_array)
        tokens = []
        for sentence in list(cleaned_sentences):
            tokens.extend(sentence)
        words, word_counts = np.unique(tokens, return_counts=True)
        for word, word_count in zip(words, word_counts):
            word_dict.append(
            { "text" : str(word),
            "size" : int(word_count)}
            )
        return word_dict


import requests
import json, ast

sdn_con_ip = '172.18.0.2'
sdn_con_port = '8080'
sdn_con_url = 'http://'+sdn_con_ip+':'+sdn_con_port
switch_url = '/wm/core/controller/switches/json'

json_list = []

response = requests.get(sdn_con_url + switch_url,
                         auth=('user', 'password'))
switch_data = response.json()

for unique_json in switch_data:
	flow_table = {}
	uid = (unique_json['switchDPID'])
	response = requests.get(sdn_con_url + '/wm/staticflowpusher/list/'+uid+'/json',auth=('user', 'password'))
	data = response.json()
	if str(data) != '[None]':
		draft_json = (ast.literal_eval(json.dumps(data)))
		if len(draft_json[uid]) > 0:
			for flow in draft_json[uid]:
				for key in flow.keys():
					flow_name = key
				# print(flow[flow_name]['outPort'])
				flow_table['Switch'] = uid
				flow_table['Name'] = flow_name
				flow_table['outPort'] = flow[flow_name]['outPort']
				flow_table['outGroup'] = flow[flow_name]['outGroup']
				flow_table['idleTimeoutSec'] = int(flow[flow_name]['idleTimeoutSec'])
				flow_table['command'] = flow[flow_name]['command']
				flow_table['priority'] = int(flow[flow_name]['priority'])
				flow_table['cookieMask'] = int(flow[flow_name]['cookieMask'])
				flow_table['version'] = flow[flow_name]['version']
				flow_table['flags'] = int(flow[flow_name]['flags'])
				flow_table['hardTimeoutSec'] = flow[flow_name]['hardTimeoutSec']
				flow_table['cookie'] = int(flow[flow_name]['cookie'])
				flow_table['tableId'] = flow[flow_name]['tableId']
				if "eth_dst" in str(flow[flow_name]):
					flow_table['eth_dst'] = flow[flow_name]['match']['eth_dst']
				if "actions" in str(flow[flow_name]):
					flow_table['actions'] = flow[flow_name]['instructions']['instruction_apply_actions']['actions']

				flow_json = (ast.literal_eval(json.dumps(flow_table)))
				json_list.append(flow_json)
		else:
			flow_table['Switch'] = uid
			flow_table['Name'] = "ERROR: No Flows Found"
			flow_table['Data'] = 1
			flow_json = (ast.literal_eval(json.dumps(flow_table)))
			json_list.append(flow_json)
json_list = str(json_list).replace("'",'"')
print(json_list)


import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torchvision import models, datasets, transforms
import numpy as np
import random
import time
from pathlib import Path
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader, Subset, TensorDataset

use_cuda = True
device = torch.device("cuda" if use_cuda else "cpu")
batch_size = 64

np.random.seed(42)
torch.manual_seed(42)

def pgd_untargeted(model, x, labels, k, eps, eps_step):
    adv_x = x.clone().detach().requires_grad_()
    L = nn.CrossEntropyLoss()
    for i in range(k):
        adv_x_ = adv_x.clone().detach().requires_grad_()
        loss = L(model(adv_x_), labels)
        loss.backward()
        adv_x = adv_x + eps_step * torch.sign(adv_x_.grad)
        adv_x = torch.clamp(adv_x, x - eps, x + eps)
        adv_x = torch.clamp(adv_x, 0, 1)
    return adv_x

def adversarial_training(model, num_epochs):
    learning_rate = 1e-4

    opt = optim.Adam(params=model.parameters(), lr=learning_rate)

    ce_loss = torch.nn.CrossEntropyLoss()

    tot_steps = 0

    for epoch in range(1,num_epochs+1):
        t1 = time.time()
        for batch_idx, (x_batch, y_batch) in enumerate(train_loader):
            x_batch, y_batch = x_batch.to(device), y_batch.to(device)
            model.eval()
            x_batch = pgd_untargeted(model, x_batch, y_batch, k, eps, eps_step)
            model.train()
            tot_steps += 1
            opt.zero_grad()
            out = model(x_batch)
            batch_loss = ce_loss(out, y_batch)
            batch_loss.backward()
            opt.step()

        tot_test, tot_acc = 0.0, 0.0
        for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
            x_batch, y_batch = x_batch.to(device), y_batch.to(device)
            out = model(x_batch)
            pred = torch.max(out, dim=1)[1]
            acc = pred.eq(y_batch).sum().item()
            tot_acc += acc
            tot_test += x_batch.size()[0]
        t2 = time.time()

        print('Epoch %d: Accuracy %.5lf [%.2lf seconds]' % (epoch, tot_acc/tot_test, t2-t1))

def get_loaders():
    data_root = Path("/coconut/yifeng-data/surgical-finetuning/Datasets")
    image_dir_clean = data_root / "ImageNet" / "val"
    dataset_clean = ImageFolder(image_dir_clean, transform=transforms.Compose([transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor()]))
    num_train = 1000
    indices = random.sample(list(range(len(dataset_clean))), num_train)
    np.random.shuffle(indices)
    split = int(np.floor(0.2 * num_train))
    train_idx, test_idx = indices[split:], indices[:split]

    tr_dataset = Subset(dataset_clean, train_idx)
    te_dataset = Subset(dataset_clean, test_idx)

    trainloader = DataLoader(
        tr_dataset,
        batch_size=batch_size,
        shuffle=True
    )

    testloader = DataLoader(
        te_dataset,
        batch_size=batch_size,
        shuffle=True
    )

    return trainloader, testloader

if __name__ == '__main__':
    ## Dataloaders
    train_loader, test_loader = get_loaders()

    model = models.resnet18(pretrained=True)
    model = model.to(device)

    # test original accuracy
    model.eval()
    tot_test, tot_acc = 0.0, 0.0
    t1 = time.time()
    for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
        x_batch, y_batch = x_batch.to(device), y_batch.to(device)
        out = model(x_batch)
        pred = torch.max(out, dim=1)[1]
        acc = pred.eq(y_batch).sum().item()
        tot_acc += acc
        tot_test += x_batch.size()[0]
    t2 = time.time()
    print('Original Accuracy: %.5lf [%.2lf seconds]' % (tot_acc/tot_test, t2-t1))

    k = 50
    for eps in [0.001, 0.01, 0.1]:
        print("epsilon:", eps)
        eps_step = 1e-4
        num_epochs = 5

        # Test PGD attack on original model
        tot_test, tot_acc = 0.0, 0.0
        t1 = time.time()
        for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
            x_batch, y_batch = x_batch.to(device), y_batch.to(device)
            x_batch = pgd_untargeted(model, x_batch, y_batch, k, eps, eps_step)
            out = model(x_batch)
            pred = torch.max(out, dim=1)[1]
            acc = pred.eq(y_batch).sum().item()
            tot_acc += acc
            tot_test += x_batch.size()[0]
        t2 = time.time()
        print('PGD on original model: Accuracy %.5lf [%.2lf seconds]' % (tot_acc/tot_test, t2-t1))

        # start adversarial training
        model.train()
        print("PGD adversarial training:")
        adversarial_training(model, num_epochs)
        model.eval()
        # Test PGD attack on enhanced model
        tot_test, tot_acc = 0.0, 0.0
        t1 = time.time()
        for batch_idx, (x_batch, y_batch) in enumerate(test_loader):
            x_batch, y_batch = x_batch.to(device), y_batch.to(device)
            x_batch = pgd_untargeted(model, x_batch, y_batch, k, eps, eps_step)
            out = model(x_batch)
            pred = torch.max(out, dim=1)[1]
            acc = pred.eq(y_batch).sum().item()
            tot_acc += acc
            tot_test += x_batch.size()[0]
        t2 = time.time()
        print('PGD on enhanced model: Accuracy %.5lf [%.2lf seconds]' % (tot_acc/tot_test, t2-t1))
        torch.save(model, "models/resnet18_pgd_enhanced_eps_" + str(eps) + ".pt")


import torch

from torch.utils.data import Dataset

class CustomDataset(Dataset):
    def __init__(self, dataframe, tokenizer, source_len, summ_len):
        self.tokenizer = tokenizer
        self.data = dataframe
        self.source_len = source_len
        self.summ_len = summ_len
        self.sentence = self.data.sentence

    def __len__(self):
        return len(self.sentence)

    def __getitem__(self, index):
        sentence = str(self.sentence[index])
        sentence = " ".join(sentence.split())

        source = self.tokenizer.batch_encode_plus(
            [sentence],
            max_length=self.source_len,
            padding="max_length",
            truncation=True,
            return_tensors="pt",
        )

        source_ids = source["input_ids"].squeeze()
        source_mask = source["attention_mask"].squeeze()

        return {
            "source_ids": source_ids.to(dtype=torch.long),
            "source_mask": source_mask.to(dtype=torch.long),
        }


# This script gives some insight in to the benefits of PPR. Specifically, on a
# small dataset a NN may not have enough examples for gradient descent to run
# to completion, whereas for the cost of a little extra training time PPR can
# fully utilize few examples. As a result PPR's training error is always less
# than that for NN, and its testing error is usually less. Occasionally PPR's
# fit is irregular, and its testing error is worse; this can probably be
# remedied by adding regularization to the loss function and rederiving the
# optimization process to accommodate it.

import numpy
import time
import sys
sys.path.append("..")
from skpp import ProjectionPursuitRegressor
from sklearn.neural_network import MLPRegressor
from sklearn.datasets import load_boston

mlp = MLPRegressor()
ppr = ProjectionPursuitRegressor()

X, Y = load_boston(return_X_y=True)

n = int(X.shape[0]*0.8)
temp = numpy.arange(n)
numpy.random.shuffle(temp)
training = temp[0:int(n*0.8)]
testing = temp[int(n*0.8):]

a = time.time()
mlp.fit(X[training], Y[training])
b = time.time()
ppr.fit(X[training], Y[training])
c = time.time()

Ymlp = mlp.predict(X[training])
Yppr = ppr.predict(X[training])

print("Average squared error per training example for MLPRegressor:",
	numpy.sum((Y[training] - Ymlp)**2)/Ymlp.size)
print("Average squared error per training example for ProjectionPursuitRegressor:",
	numpy.sum((Y[training] - Yppr)**2)/Yppr.size)

Ymlp = mlp.predict(X[testing])
Yppr = ppr.predict(X[testing])

print("Average squared error per testing example for MLPRegressor:",
	numpy.sum((Y[testing] - Ymlp)**2)/Ymlp.size)
print("Average squared error per testing example for ProjectionPursuitRegressor:",
	numpy.sum((Y[testing] - Yppr)**2)/Yppr.size)

print("Training time for MLPRegressor:", b-a)
print("Training time for ProjectionPursuitRegressor:", c-b)


from sqlite3 import Cursor
from typing import List, Optional

from .utils import data_to_city
from ..models import Prefecture, City
from dataclasses import replace

def fetch_cities_from_prefecture(cursor: Cursor, prefecture: Prefecture) -> List[City]:
    cursor.execute(f"SELECT * FROM cities WHERE prefecture_id=?", (prefecture.id,))
    data = cursor.fetchall()
    cities = []
    for v in data:
        cities.append(replace(City(*v), prefecture=prefecture))

    return cities

def fetch_city_by_id(cursor: Cursor, id_: int) -> Optional[City]:
    cursor.execute((f"SELECT p.*, c.* FROM cities AS c "
                    f"JOIN prefectures AS p ON p.id = c.prefecture_id "
                    f"WHERE c.id=?"), (id_,))
    data = cursor.fetchone()
    return data_to_city(data) if data else None

def search_cities(cursor: Cursor, query: str, prefecture: Prefecture, type_: str) -> List[City]:
    query = [f"SELECT p.*, c.* FROM cities AS c "
             f"JOIN prefectures AS p ON p.id = c.prefecture_id "
             f"WHERE c.{type_} LIKE '%{query}%'", []]
    if prefecture:
        query[0] += ' AND p.id = ?'
        query[1].append(prefecture.id)

    cursor.execute(*query)
    data = cursor.fetchall()

    return [data_to_city(v) for v in data]


import collections
def findRedundantDirectedConnection(edges):
        # 是否出现入度为2的结点
        outs = [edge[1] for edge in edges]
        count = collections.Counter(outs)
        for key in count.keys():
            if count[key] == 2:
                redundance = [edge for edge in edges if edge[1]==key]
                edges = [edge for edge in edges if edge[1]!=key] + redundance
                break

        dic = {}
        n = len(edges)
        def find(p):
            while p!=dic[p]:
                p = dic[p]
            return p
        def union(p,q):
            root1,root2 = find(p),find(q)
            if root1==root2:
                return True
            else:
                dic[root1] = root2
                return False
        for i in range(1,n+1):
            dic[i] = i
        for edge in edges:
            if union(edge[0],edge[1]):
                return edge
edges=input()
edge=findRedundantDirectedConnection(edges)
print(edge)


import smtplib
import pickle
import datetime

from email_parameters import EmailParameters
from email.mime.text import MIMEText

def main():
    file = open('DO_NOT_COMMIT_email_parameters.dat','rb')
    param = pickle.loads(file.read())
    file.close()
    smtpserver = smtplib.SMTP(param.host,param.port)
    smtpserver.ehlo()
    smtpserver.starttls()
    smtpserver.ehlo
    smtpserver.login(param.sender, param.passwd)

    msg = MIMEText('Fill me in!')
    msg['Subject'] = 'GnuCash Envelope Status:' + datetime.datetime.now().strftime('%Y-%m-%d')
    msg['From'] = param.sender
    msg['To'] =  ';'.join([str(x) for x in param.receivers])

    print(param)
    print(msg)
    smtpserver.sendmail(param.sender, param.receivers, msg.as_string())
    smtpserver.quit()

if __name__ == "__main__":
    # execute only if run as a script
    main()