import matplotlib as mpl
mpl.use('Agg')
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
from glob import glob
import os
from shutil import copyfile
from time import sleep, strftime, localtime
from Loader import Loader
from configuration import config
from Utils import *
from layers import *
from net import FrontEnd, Unet, LSTM, BiLSTM, Dilated, BackEnd, Confidence
import librosa
import time
import openpyxl
from scipy import signal
from loss import loss_fn, loss_alpha
from build_graph import build_graph
import nsml

tf.reset_default_graph()
np.set_printoptions(precision=4)

class MDPHD(object):
    def __init__(self, name):
        self.name = name
        self.loader = Loader([config.train_root], config.batch_size)

    def build(self, reuse=False):
        self.reuse = reuse
        with tf.variable_scope(self.name, reuse=self.reuse):
            # train set
            self.mix = self.loader.mix_queue # mixture batch
            self.source = self.loader.source_queue # source batch
            self.other = self.mix - self.source

            # test samples (placeholder)
            self.mix_test = tf.placeholder(tf.float32, shape=(1,16384), name="mixture_sample")
            self.source_test = tf.placeholder(tf.float32, shape=(1,16384), name="source_sample")

            # define network
            self.FE = FrontEnd()
            if config.network == "unet":
                self.SEP = Unet(model=config.model)
            elif config.network == "lstm":
                self.SEP = LSTM(model=config.model)
            elif config.network == "bilstm":
                self.SEP = BiLSTM(model=config.model)
            elif config.network == "dilated":
                self.SEP = Dilated(model=config.model)
            else:
                print("No model chosen")
                raise ValueError
            self.BE = BackEnd()
            functions = [self.FE, self.SEP, self.BE]

            if config.network2 == "unet":
                self.SEP2 = Unet(model=config.model2)
                self.CON = Confidence(model=config.confidence)
                functions = functions + [self.SEP2, self.CON]
            elif config.network2 == "lstm":
                self.SEP2 = LSTM(model=config.model2)
                self.CON = Confidence(model=config.confidence)
                functions = functions + [self.SEP2, self.CON]
            elif config.network2 == "bilstm":
                self.SEP2 = BiLSTM(model=config.model2)
                self.CON = Confidence(model=config.confidence)
                functions = functions + [self.SEP2, self.CON]
            elif config.network2 == "dilated":
                self.SEP2 = Dilated(model=config.model2)
                self.CON = Confidence(model=config.confidence)
                functions = functions + [self.SEP2, self.CON]
            else:
                print("No model chosen")

            input = {"mix": self.mix, "source": self.source}
            input_test = {"mix": self.mix_test, "source": self.source_test}

            # draw graph
            self.graph = build_graph()
            self.graph_test = build_graph(train=False)

            self.graph(input, functions)
            self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
            self.graph_test(input_test, functions)

            # variable count
            self.variable_size = np.sum(
                np.array([np.prod(np.array(v.get_shape().as_list())) for v in tf.trainable_variables()]))
            print("\n total varialbes : {}".format(self.variable_size), "\n")
            for v in tf.trainable_variables():
                print(v, "{}".format(np.prod(np.array(v.get_shape().as_list()))))

            # Define loss and summarize
            # self.loss_pre = (loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
            #                     self.source, self.other, loss_type=config.loss) + \
            #                 loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
            #                         self.source, self.other, loss_type=config.loss))/2

            if config.hybrid:
                if config.loss_seq == 'mid':
                    self.loss = loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss) + \
                                loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss)
                elif config.loss_seq == 'end':
                    self.loss = loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss) + \
                                loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss)
                elif config.loss_seq == 'both':
                    self.loss = (loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss) +
                                loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss) +
                                loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss) +
                                loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss))/2
                elif config.loss_seq == 'two':
                    self.loss_pre = loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss) + \
                                loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss)

                    self.loss = (loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
                                         self.source, self.other, loss_type=config.loss) +
                                 loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
                                         self.source, self.other, loss_type=config.loss) +
                                 loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
                                         self.source, self.other, loss_type=config.loss) +
                                 loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
                                         self.source, self.other, loss_type=config.loss)) / 2
                elif config.loss_seq == 'first':
                    self.loss_pre = loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss)

                    self.loss = loss_fn(self.graph.masked_spec1, self.graph.estimated1, self.graph.source_spec,
                                         self.source, self.other, loss_type=config.loss) + \
                                 loss_fn(self.graph.masked_spec1_mid, self.graph.estimated1_mid, self.graph.source_spec,
                                         self.source, self.other, loss_type=config.loss)
                    self.graph.estimated = self.graph.estimated1
                    self.graph_test.estimated = self.graph_test.estimated1
                elif config.loss_seq == 'second':
                    self.loss_pre = loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
                                        self.source, self.other, loss_type=config.loss)

                    self.loss = loss_fn(self.graph.masked_spec2, self.graph.estimated2, self.graph.source_spec,
                                         self.source, self.other, loss_type=config.loss) + \
                                 loss_fn(self.graph.masked_spec2_mid, self.graph.estimated2_mid, self.graph.source_spec,
                                         self.source, self.other, loss_type=config.loss)
                    self.graph.estimated = self.graph.estimated2
                    self.graph_test.estimated = self.graph_test.estimated2
                else:
                    raise AssertionError("wrong config.loss_seq !!")
            else:
                self.loss = loss_fn(self.graph.masked_spec, self.graph.estimated, self.graph.source_spec,
                                    self.source, self.other, loss_type=config.loss)

            self.loss_summary = tf.summary.scalar("loss", self.loss)
            self.alpha_summary = tf.summary.scalar("alpha", tf.reduce_mean(self.graph.alpha))
            self.summary = tf.summary.merge_all()
            print("\n loss type : %s \n" % config.loss)

    def train(self):
        with tf.control_dependencies(self.update_ops):
            step = tf.Variable(0, trainable=False)
            lr_ = config.lr
            lr = tf.placeholder(tf.float32)
            optimizer = tf.train.AdamOptimizer(lr, beta1=config.Beta1, beta2=config.Beta2)
            if (config.loss_seq in ('two', 'first', 'second')) & (config.hybrid == True):
                self.trainer_pre = optimizer.minimize(
                    self.loss_pre, var_list=tf.trainable_variables(), global_step=step)
            self.trainer = optimizer.minimize(
                self.loss, var_list=tf.trainable_variables(), global_step=step)

        NUM_THREADS = 3
        configuration = tf.ConfigProto(inter_op_parallelism_threads=NUM_THREADS, \
                                       intra_op_parallelism_threads=NUM_THREADS, \
                                       allow_soft_placement=True, \
                                       device_count={'CPU': 3}, \
                                       )

        with tf.Session(config=configuration) as sess:

            path = os.path.join(os.getcwd(), 'Results', 'Spectrograms')
            os.makedirs(path, exist_ok=True)
            path = os.path.join(os.getcwd(), 'Results', 'Test')
            os.makedirs(path, exist_ok=True)
            path = os.path.join(os.getcwd(), 'Results', 'Simulation')
            os.makedirs(path, exist_ok=True)
            path = os.path.join(os.getcwd(), 'Results', 'CheckPoint')
            os.makedirs(path, exist_ok=True)
            path = os.path.join(os.getcwd(), 'Results', 'Logs_Training')
            os.makedirs(path, exist_ok=True)

            tf.global_variables_initializer().run()

            coord = tf.train.Coordinator()
            threads = tf.train.start_queue_runners(sess, coord=coord)

            saver = tf.train.Saver(max_to_keep=10)
            writer = tf.summary.FileWriter('./Results/Logs_Training', sess.graph)

            """
            Restore
                Restore checkpoint only when the user specify the 'config.restore' to be True
            """
            if config.restore:
                ckpt = tf.train.get_checkpoint_state(checkpoint_dir="./Results/CheckPoint")
                try:
                    if ckpt and ckpt.model_checkpoint_path:
                        print("check point path : ", ckpt.model_checkpoint_path)
                        saver.restore(sess, ckpt.model_checkpoint_path)
                        print('Restored!')
                except AttributeError:
                    print("No checkpoint")
            else:
                print("No model restoration")

            training_loss = 0
            training_alpha_sum = 0
            training_alpha_list = []
            f = open(os.path.join(os.getcwd(), 'Results', 'Logs_Training', 'training_loss.txt'), 'a+')
            f.write("Variable Size : {}\n".format(self.variable_size))
            f.close()

            for iter in range(config.iter + 1):
                if coord.should_stop():
                    print("break data pipeline")
                    break

                if (config.loss_seq in ('two', 'first', 'second')) & (config.hybrid == True):
                    if iter <= config.loss_thereshold:
                        _, loss_, alpha_ = sess.run([self.trainer_pre, self.loss_pre, self.graph.alpha], feed_dict={lr: lr_})
                    else:
                        _, loss_, alpha_ = sess.run([self.trainer, self.loss, self.graph.alpha], feed_dict={lr: lr_})
                else:
                    _, loss_, alpha_ = sess.run([self.trainer, self.loss, self.graph.alpha], feed_dict={lr: lr_})

                training_loss += loss_
                training_alpha_sum += np.mean(alpha_)
                training_alpha_list.append(np.squeeze(alpha_))

                if iter % 10 == 0:
                    print("iter : {} (loss : {:.3f}, alpha : {:.3f}/ {:.3f})".format(iter, loss_, np.mean(alpha_), np.std(alpha_)))
                    summary_ = sess.run(self.summary)
                    writer.add_summary(summary_, iter)

                # print spectrograms
                if iter % 5000 == 0:
                    spec_mix_, spec_tgt_, spec_est_ = sess.run(
                        [self.graph.input_spec, self.graph.source_spec, self.graph.masked_spec])

                    mag_mix_ = np.sqrt(np.square(spec_mix_[:, :, :, 0]) + np.square(spec_mix_[:, :, :, 1]))
                    mag_tgt_ = np.sqrt(np.square(spec_tgt_[:, :, :, 0]) + np.square(spec_tgt_[:, :, :, 1]))
                    mag_est_ = np.sqrt(np.square(spec_est_[:, :, :, 0]) + np.square(spec_est_[:, :, :, 1]))

                    mag_image1 = np.log10(np.concatenate((mag_mix_[0], mag_mix_[1], mag_mix_[2]), axis=1) + 1e-02)
                    mag_image2 = np.log10(np.concatenate((mag_tgt_[0], mag_tgt_[1], mag_tgt_[2]), axis=1) + 1e-02)
                    mag_image3 = np.log10(np.concatenate((mag_est_[0], mag_est_[1], mag_est_[2]), axis=1) + 1e-02)

                    mag_image4 = np.log10(
                        np.concatenate((mag_mix_[-3], mag_mix_[-2], mag_mix_[-1]), axis=1) + 1e-02)
                    mag_image5 = np.log10(
                        np.concatenate((mag_tgt_[-3], mag_tgt_[-2], mag_tgt_[-1]), axis=1) + 1e-02)
                    mag_image6 = np.log10(
                        np.concatenate((mag_est_[-3], mag_est_[-2], mag_est_[-1]), axis=1) + 1e-02)

                    fig = plt.figure(figsize=(15, 15))
                    fig.suptitle('Spectrograms', fontsize=20, family='serif')

                    font = {'family': 'serif',
                            'color': 'darkred',
                            'weight': 'normal',
                            'size': 15,
                            }

                    ax1 = fig.add_subplot(1, 3, 1)
                    ax1.set_title('Mixture', fontdict=font)
                    ax1.imshow(mag_image1, interpolation='nearest', aspect='auto', cmap='jet')
                    ax1.xaxis.set_tick_params(labelsize=15)
                    ax1.yaxis.set_tick_params(labelsize=15)
                    plt.gca().invert_yaxis()

                    ax2 = fig.add_subplot(1, 3, 2)
                    ax2.set_title('True Vocal', fontdict=font)
                    ax2.imshow(mag_image2, interpolation='nearest', aspect='auto', cmap='jet')
                    ax2.xaxis.set_tick_params(labelsize=15)
                    ax2.yaxis.set_tick_params(labelsize=15)
                    plt.gca().invert_yaxis()

                    ax3 = fig.add_subplot(1, 3, 3)
                    ax3.set_title('Estimated Vocal', fontdict=font)
                    ax3.imshow(mag_image3, interpolation='nearest', aspect='auto', cmap='jet')
                    ax3.xaxis.set_tick_params(labelsize=15)
                    ax3.yaxis.set_tick_params(labelsize=15)
                    plt.gca().invert_yaxis()

                    plt.tight_layout(pad=0.4, w_pad=1, h_pad=1.0)
                    plt.subplots_adjust(top=0.85)
                    fig.savefig("./Results/Spectrograms/Spectrogram{}.png".format(iter))
                    plt.close(fig)

                    fig = plt.figure(figsize=(15, 15))
                    fig.suptitle('NoiseSpectrograms', fontsize=20, family='serif')

                    font = {'family': 'serif',
                            'color': 'darkred',
                            'weight': 'normal',
                            'size': 15,
                            }

                    ax1 = fig.add_subplot(1, 3, 1)
                    ax1.set_title('Noise', fontdict=font)
                    ax1.imshow(mag_image4, interpolation='nearest', aspect='auto', cmap='jet')
                    ax1.xaxis.set_tick_params(labelsize=15)
                    ax1.yaxis.set_tick_params(labelsize=15)
                    plt.gca().invert_yaxis()

                    ax2 = fig.add_subplot(1, 3, 2)
                    ax2.set_title('True Zeros', fontdict=font)
                    ax2.imshow(mag_image5, interpolation='nearest', aspect='auto', cmap='jet')
                    ax2.xaxis.set_tick_params(labelsize=15)
                    ax2.yaxis.set_tick_params(labelsize=15)
                    plt.gca().invert_yaxis()

                    ax3 = fig.add_subplot(1, 3, 3)
                    ax3.set_title('Estimated Zeros', fontdict=font)
                    ax3.imshow(mag_image6, interpolation='nearest', aspect='auto', cmap='jet')
                    ax3.xaxis.set_tick_params(labelsize=15)
                    ax3.yaxis.set_tick_params(labelsize=15)
                    plt.gca().invert_yaxis()

                    plt.tight_layout(pad=0.4, w_pad=1, h_pad=1.0)
                    plt.subplots_adjust(top=0.85)
                    fig.savefig(
                        "./Results/Spectrograms/Noise_input_Spectrogram{}.png".format(iter))
                    plt.close(fig)

                if (iter) % 10000 == 0:
                    training_alpha_std = np.std(np.array(training_alpha_list))
                    f = open(os.path.join(os.getcwd(), 'Results', 'Logs_Training', 'training_loss.txt'), 'a+')
                    f.write("average {} loss : {:.4f} (iter {} ~ {})\n".format(config.loss, training_loss / 10000,
                                                                               iter - 9999, iter + 1))
                    f.close()
                    f = open(os.path.join(os.getcwd(), 'Results', 'Logs_Training', 'training_alpha.txt'), 'a+')
                    f.write("average alpha : {:.4f} / std : {:.4f} (iter {} ~ {})\n".format(training_alpha_sum / 10000,
                                                                                training_alpha_std ,iter - 9999, iter + 1))
                    f.close()

                    nsml.report(summary=True,
                                iter=iter,
                                train__loss=float(training_loss / 10000),
                                alpha=float(training_alpha_sum / 10000),
                                alpha_std=float(training_alpha_std),
                                lr=float(lr_)*10000
                                )
                    training_loss = 0
                    training_alpha_sum = 0
                    training_alpha_list = []

                if (iter+1) % config.lr_decay_iter == 0:
                    if config.lr_decay:
                        lr_ = lr_/config.lr_decay_rate
                        print("learning rate is decayed : {}".format(lr_))

                # test songs
                if (iter + 1) % config.model_save_iter == 0:
                    moving_rate = config.moving_rate
                    hop = int(config.audio_size / moving_rate)  # This has to be an interger without any rounding
                    mask = (signal.get_window('hanning', config.audio_size)) / (moving_rate / 2)
                    # mask = (1 / moving_rate)

                    for folder, root in [('Test', config.test_sample_root), ('Simulation', config.simul_sample_root)]:
                        test_sample_path = glob(os.path.join(root, "noisy/*.wav"))
                        print("{} audios for test".format(len(test_sample_path)))

                        SNR_list = []
                        infer_time = []

                        wb = openpyxl.Workbook()
                        ws = wb.active

                        for k, song_path in enumerate(sorted(test_sample_path)):
                            print("\n %d th song" % k)

                            # Load song
                            name = os.path.basename(song_path)
                            print("Loading {}...".format(name))
                            noisy, _ = librosa.core.load(song_path, sr=16000)
                            clean, _ = librosa.core.load(os.path.join(root, "spk", name[:8]+".wav"),
                                                         sr=16000)
                            noise = noisy - clean

                            print("song shape : ", noisy.shape)

                            song_length = noisy.shape[0]
                            num_frame = int(song_length / config.audio_size) + 1

                            # pad zeros to song to make the length of it be the multiple of config.audio_size
                            noisy_pad = np.lib.pad(noisy, (
                                config.audio_size, num_frame * config.audio_size - song_length + config.audio_size),
                                                   'constant')
                            clean_pad = np.lib.pad(clean, (
                                config.audio_size, num_frame * config.audio_size - song_length + config.audio_size),
                                                   'constant')
                            padded_song_shape = noisy_pad.shape[0]

                            # Slice mixture and source
                            noisy_slice_list = []
                            clean_slice_list = []

                            hop = int(
                                config.audio_size / config.moving_rate)  # This has to be an interger without any rounding
                            num_to_move = int((padded_song_shape - config.audio_size) / hop) + 1
                            for i in range(num_to_move):
                                start_point = int(i * hop)
                                end_point = int(start_point + config.audio_size)
                                noisy_slice = noisy_pad[start_point:end_point]  # ex: (1,16384)
                                noisy_slice_list.append(noisy_slice)
                                clean_slice = clean_pad[start_point:end_point]
                                clean_slice_list.append(clean_slice)

                            num_slice = len(noisy_slice_list)
                            noisy_stacked = np.array(noisy_slice_list)
                            clean_stacked = np.array(clean_slice_list)

                            # Separation
                            segments = []
                            spec_mix_list = []
                            spec_tgt_list = []
                            spec_est_list = []
                            spec_est1_list = []
                            spec_est2_list = []
                            alpha_list = []

                            for n, i in enumerate(range(num_slice)):
                                start_time = time.time()
                                print("processing {}: {}/{}".format(name, n, num_slice))
                                estimated_sample, spec_mix_, spec_tgt_, spec_est_, spec_est1_,  spec_est2_, alpha_ \
                                    = sess.run([self.graph_test.estimated, self.graph_test.input_spec,
                                                self.graph_test.source_spec, self.graph_test.masked_spec,
                                                self.graph_test.masked_spec1,  self.graph_test.masked_spec2,
                                                self.graph_test.alpha],
                                               feed_dict={self.mix_test: noisy_stacked[i:i + 1],
                                                          self.source_test: clean_stacked[i:i + 1]})
                                infer_time.append(time.time() - start_time)
                                masked_sample = np.expand_dims(mask, axis=0) * estimated_sample
                                segments.append(masked_sample)
                                alpha_list.append(np.mean(alpha_))

                                if (n+1) % 4 == 0:
                                    spec_mix_list.append(spec_mix_)
                                    spec_tgt_list.append(spec_tgt_)
                                    spec_est_list.append(spec_est_)
                                    spec_est1_list.append(spec_est1_)
                                    spec_est2_list.append(spec_est2_)

                            spec_mix_ = np.concatenate(spec_mix_list, axis=0)
                            spec_tgt_ = np.concatenate(spec_tgt_list, axis=0)
                            spec_est_ = np.concatenate(spec_est_list, axis=0)
                            spec_est1_ = np.concatenate(spec_est1_list, axis=0)
                            spec_est2_ = np.concatenate(spec_est2_list, axis=0)

                            # spectrograms
                            mag_mix_ = np.sqrt(np.square(spec_mix_[:, :, :, 0]) + np.square(spec_mix_[:, :, :, 1]))
                            mag_tgt_ = np.sqrt(np.square(spec_tgt_[:, :, :, 0]) + np.square(spec_tgt_[:, :, :, 1]))
                            mag_est_ = np.sqrt(np.square(spec_est_[:, :, :, 0]) + np.square(spec_est_[:, :, :, 1]))
                            mag_est1_ = np.sqrt(np.square(spec_est1_[:, :, :, 0]) + np.square(spec_est1_[:, :, :, 1]))
                            mag_est2_ = np.sqrt(np.square(spec_est2_[:, :, :, 0]) + np.square(spec_est2_[:, :, :, 1]))

                            mag_image1 = np.log10(np.concatenate((mag_mix_[0], mag_mix_[1], mag_mix_[2]), axis=1) + 1e-02)
                            mag_image2 = np.log10(np.concatenate((mag_tgt_[0], mag_tgt_[1], mag_tgt_[2]), axis=1) + 1e-02)
                            mag_image3 = np.log10(np.concatenate((mag_est_[0], mag_est_[1], mag_est_[2]), axis=1) + 1e-02)
                            mag_image4 = np.log10(np.concatenate((mag_est1_[0], mag_est1_[1], mag_est1_[2]), axis=1) + 1e-02)
                            mag_image5 = np.log10(np.concatenate((mag_est2_[0], mag_est1_[1], mag_est2_[2]), axis=1) + 1e-02)

                            fig = plt.figure(figsize=(25, 15))
                            fig.suptitle('Spectrograms', fontsize=20, family='serif')

                            font = {'family': 'serif',
                                    'color': 'darkred',
                                    'weight': 'normal',
                                    'size': 15,
                                    }

                            ax1 = fig.add_subplot(1, 5, 1)
                            ax1.set_title('Mixture', fontdict=font)
                            ax1.imshow(mag_image1, interpolation='nearest', aspect='auto', cmap='jet')
                            ax1.xaxis.set_tick_params(labelsize=15)
                            ax1.yaxis.set_tick_params(labelsize=15)
                            plt.gca().invert_yaxis()

                            ax2 = fig.add_subplot(1, 5, 2)
                            ax2.set_title('True Vocal', fontdict=font)
                            ax2.imshow(mag_image2, interpolation='nearest', aspect='auto', cmap='jet')
                            ax2.xaxis.set_tick_params(labelsize=15)
                            ax2.yaxis.set_tick_params(labelsize=15)
                            plt.gca().invert_yaxis()

                            ax3 = fig.add_subplot(1, 5, 3)
                            ax3.set_title('Estimated Vocal {:.3f}'.format(np.mean(alpha_)), fontdict=font)
                            ax3.imshow(mag_image3, interpolation='nearest', aspect='auto', cmap='jet')
                            ax3.xaxis.set_tick_params(labelsize=15)
                            ax3.yaxis.set_tick_params(labelsize=15)
                            plt.gca().invert_yaxis()

                            ax3 = fig.add_subplot(1, 5, 4)
                            ax3.set_title('Estimated-1 Vocal', fontdict=font)
                            ax3.imshow(mag_image4, interpolation='nearest', aspect='auto', cmap='jet')
                            ax3.xaxis.set_tick_params(labelsize=15)
                            ax3.yaxis.set_tick_params(labelsize=15)
                            plt.gca().invert_yaxis()

                            ax3 = fig.add_subplot(1, 5, 5)
                            ax3.set_title('Estimated-2 Vocal', fontdict=font)
                            ax3.imshow(mag_image5, interpolation='nearest', aspect='auto', cmap='jet')
                            ax3.xaxis.set_tick_params(labelsize=15)
                            ax3.yaxis.set_tick_params(labelsize=15)
                            plt.gca().invert_yaxis()

                            plt.tight_layout(pad=0.4, w_pad=1, h_pad=1.0)
                            plt.subplots_adjust(top=0.85)
                            fig.savefig("./Results/{}/{}_{}.png".format(folder, iter, os.path.splitext(name)[0]))
                            plt.close(fig)

                            # Post-processing(triangle mask)
                            # num_to_pad = int((config.audio_size/2)*(num_slice-1))
                            # temp = np.zeros(shape=(config.audio_size + num_to_pad))
                            temp = np.zeros(shape=(padded_song_shape))
                            for i in range(len(segments)):
                                start_point = int(i * (config.audio_size / config.moving_rate))
                                end_point = int(start_point + config.audio_size)
                                temp[start_point:end_point] = temp[start_point:end_point] + segments[i]

                            # Match the original song length
                            estimation = np.squeeze(temp[config.audio_size:config.audio_size + song_length])
                            # print("estimation shape: {}".format(estimation.shape))

                            # save separated source as audio
                            OUTPUT_FILENAME = "./Results/{}/{}_{}_{}_{}.wav".format(folder, iter, os.path.splitext(name)[0],
                                                                                config.network, config.network2)
                            librosa.output.write_wav(OUTPUT_FILENAME, estimation, 16000)

                            s_target = clean * np.sum(estimation * clean) / np.sum(clean ** 2)
                            error = estimation - s_target
                            snr = 10 * np.log10(sum(s_target ** 2) / sum(error ** 2))
                            SNR_list.append(snr)

                            noise_type = os.path.splitext(name)[0].split("_")[2]
                            noise_snr = os.path.splitext(name)[0].split("_")[3]

                            ws.cell(row= 3, column= k % 3 + 2).value = noise_snr
                            ws.cell(row= k//3 + 4, column= k % 3 + 2).value = snr
                            ws.cell(row= k//3 + 4, column= 1).value = noise_type
                            ws.cell(row= 3, column= k % 3 + 6).value = noise_snr
                            ws.cell(row= k//3 + 4, column= k % 3 + 6).value = np.mean(np.array(alpha_list))

                        snr_mean = np.mean(np.array(SNR_list))
                        snr_std = np.std(np.array(SNR_list))
                        print("snr mean : {}, std : {}".format(snr_mean, snr_std))
                        ws.cell(row=1, column=2).value = "mean"
                        ws.cell(row=1, column=3).value = "std"
                        ws.cell(row=1, column=4).value = "infer_time per frame"
                        ws.cell(row=2, column=2).value = snr_mean
                        ws.cell(row=2, column=3).value = snr_std
                        ws.cell(row=2, column=4).value = np.mean(np.array(infer_time))
                        ws.cell(row=2, column=6).value = "alpha"

                        wb.save('./Results/{}/snr_{}_{}_{}.xlsx'.format(folder, iter, config.network, config.network2))
                        wb.close()

                    print("Save model for epoch {}".format(iter))
                    saver.save(sess, "./Results/CheckPoint/model.ckpt", global_step=(iter+1))
                    print("-------------------Model saved-------------------")

            coord.request_stop()
            coord.join(threads)

    def test(self, model_path, model_number):
        """
        Make Saving Directories
        """

        NUM_THREADS = 3
        configuration = tf.ConfigProto(inter_op_parallelism_threads=NUM_THREADS, \
                                       intra_op_parallelism_threads=NUM_THREADS, \
                                       allow_soft_placement=True, \
                                       device_count={'CPU': 3}, \
                                       )

        with tf.Session(config=configuration) as sess:
            path = os.path.join(os.getcwd(), 'Test', 'Model_{}'.format(config.model_number))
            os.makedirs(path, exist_ok=True)
            path = os.path.join(os.getcwd(), 'Test', 'Model_{}'.format(config.model_number), 'Audio')
            os.makedirs(path, exist_ok=True)

            saver = tf.train.Saver()
            saver.restore(sess, model_path+model_number)
            print("model is restored!!", model_path+model_number)

            # test songs
            moving_rate = config.moving_rate
            hop = int(config.audio_size / moving_rate)  # This has to be an interger without any rounding

            # mask = (signal.get_window('hanning', config.audio_size)) / (moving_rate / 2)
            mask = (1 / moving_rate)

            test_sample_path = glob(os.path.join(config.test_root, "*.wav"))
            print("{} audios for test".format(len(test_sample_path)))

            for k, song_path in enumerate(sorted(test_sample_path)):
                print("\n %d th song" % k)

                # Load song
                name = os.path.basename(song_path)
                print("Loading {}...".format(name))
                noisy, _ = librosa.core.load(song_path, sr=16000)

                print("song shape : ", noisy.shape)

                song_length = noisy.shape[0]
                num_frame = int(song_length / config.audio_size) + 1

                # pad zeros to song to make the length of it be the multiple of config.audio_size
                noisy_pad = np.lib.pad(noisy, (
                    config.audio_size, num_frame * config.audio_size - song_length + config.audio_size),
                                       'constant')
                padded_song_shape = noisy_pad.shape[0]

                # Slice mixture and source
                noisy_slice_list = []

                hop = int(
                    config.audio_size / config.moving_rate)  # This has to be an interger without any rounding
                num_to_move = int((padded_song_shape - config.audio_size) / hop) + 1
                for i in range(num_to_move):
                    start_point = int(i * hop)
                    end_point = int(start_point + config.audio_size)
                    noisy_slice = noisy_pad[start_point:end_point]  # ex: (1,16384)
                    noisy_slice_list.append(noisy_slice)

                num_slice = len(noisy_slice_list)
                noisy_stacked = np.array(noisy_slice_list)

                # Separation
                segments = []

                start_time = time.time()
                for n, i in enumerate(range(num_slice)):
                    print("processing {}: {}/{}".format(name, n, num_slice))
                    estimated_sample = sess.run(self.estimated,
                                   feed_dict={self.mix_test: noisy_stacked[i:i + 1]})
                    masked_sample = np.expand_dims(mask, axis=0) * estimated_sample
                    segments.append(masked_sample)
                print("processed time : {:.3f} ({:.3f} per 1s)".format(time.time()-start_time,(time.time()-start_time)/num_slice))

                # Post-processing(triangle mask)
                # num_to_pad = int((config.audio_size/2)*(num_slice-1))
                # temp = np.zeros(shape=(config.audio_size + num_to_pad))
                temp = np.zeros(shape=(padded_song_shape))
                for i in range(len(segments)):
                    start_point = int(i * (config.audio_size / config.moving_rate))
                    end_point = int(start_point + config.audio_size)
                    temp[start_point:end_point] = temp[start_point:end_point] + segments[i]

                # Match the original song length
                estimation = np.squeeze(temp[config.audio_size:config.audio_size + song_length])

                # save separated source as audio
                OUTPUT_FILENAME = os.path.join(path, "{}.wav".format(os.path.splitext(name)[0]))
                librosa.output.write_wav(OUTPUT_FILENAME, estimation, 16000)


import streamlit as st
import pandas as pd
import plotly.express as px
from PIL import Image
import hashlib
import json
import sqlite3
from pathlib import Path
import time
import datetime

st.set_page_config(page_title = "GTE Certification System", page_icon = ":writing_hand:", layout = "wide")

st.write()

# DB Management

conn = sqlite3.connect("gte_course.db")
c = conn.cursor()

# DB Functions for create table
def create_course_table():
    c.execute(
        "CREATE TABLE IF NOT EXISTS course_table(grade TEXT, course_name TEXT, version TEXT,NO TEXT UNIQUE,question TEXT, c1 TEXT,c2 TEXT,c3 TEXT,c4 TEXT, answer TEXT)"
    )

# Insert or replace the data into table
def add_course(grade, course_name, version,NO,question, c1,c2,c3 ,c4 , answer):
    c.execute(
        "INSERT OR REPLACE INTO course_table(grade, course_name, version,NO,question, c1,c2,c3 ,c4 , answer) VALUES (?,?,?,?,?,?,?,?,?,?)",
        (grade, course_name, version,NO,question, c1,c2,c3 ,c4 , answer),
    )
    conn.commit()

# t_item  fetch  试题查询
def search_exam(grade,course_name, version):
    c.execute(
        "SELECT * FROM course_table WHERE grade =? AND course_name =? AND version =?", (grade,course_name,version)
    )
    exam_data = c.fetchall()
    return exam_data

def view_all_course():
    c.execute("SELECT * FROM course_table")
    data = c.fetchall()
    return data

def get_test_item():
    # 获取DB中的考题名称，选项，和答案
    c.execute("SELECT t_itemquestion, t_item_c1, t_item_c2, t_item_c3, t_item_c4, t_itemanswer FROM t_itemtable")
    test_item = c.fetchall()
    return test_item

def view_exam_course():
    c.execute("SELECT grade,course_name,version FROM course_table")
    data = c.fetchall()
    return data

def view_grade_course(grade):
    c.execute("SELECT course_name,version FROM course_table WHERE grade = ?",grade)
    data = c.fetchall()
    return data

def get_examlist():
        course_result = view_exam_course()
        course_db = pd.DataFrame( course_result, columns=["grade", "course_name", "version"])
        # st.dataframe(course_db, use_container_width=True)

        grade_list= sorted(set(course_db.iloc[:,0].to_list()),reverse=False)
        course_name_list= sorted(set(course_db.iloc[:,1].to_list()),reverse=True)
        course_version_list= sorted(set(course_db.iloc[:,2].to_list()),reverse=True)

        return grade_list,course_name_list,course_version_list

# DB Functions for create table
def create_score_table():
    c.execute(
        "CREATE TABLE IF NOT EXISTS score_table(grade TEXT, course_name TEXT, version TEXT,user_name TEXT,user_mail TEXT, user_dept TEXT, user_score TEXT, score_time TEXT )"
    )

# Insert or replace the data into table
def add_score(grade, course_name, version,user_name,user_mail, user_dept,user_score,score_time):
    c.execute(
        "INSERT OR REPLACE INTO score_table(grade, course_name, version,user_name,user_mail, user_dept,user_score,score_time) VALUES (?,?,?,?,?,?,?,?)",
        (grade, course_name, version,user_name,user_mail, user_dept,user_score,score_time),
    )
    conn.commit()

def view_all_score():
    c.execute("SELECT * FROM score_table")
    data = c.fetchall()
    return data

def main():
    st.write("# Welcome to GTE Certification System! 👋")
    st.divider()
    # 获取课程列表
    course_grade,course_name,course_version = get_examlist()
    # grade 和 course 要联动

    if 'search_result' not in st.session_state:
        st.session_state.search_result = False

    if 'exam_finish' not in st.session_state:
        st.session_state.exam_finish = False

    if 'exam_start' not in st.session_state:
        st.session_state.exam_start = False

    with st.form("user_form"):
        st.markdown("**:red[Please input user information and select course to start test]**")
        col1, col2, col3 = st.columns(3)
        with col1:
            user_name = st.text_input('Name', '')
            exam_grade = st.selectbox('Select grade?',course_grade)

        with col2:
            user_mail = st.text_input('E-mail', 'xxx@lenovo.com')
            exam_name = st.selectbox('Select course name?',course_name)

        with col3:
            user_dept = st.selectbox('Select dept?',('GE-GTE', 'LSSC-NB-ENG', 'LSSC-DT-ENG','LSSC-SVR-ENG','TJSC-NB-ENG'))
            exam_version = st.selectbox('Select course version?',course_version)

    # Every form must have a submit button.
        submitted = st.form_submit_button("Couse search")
        if submitted:
            if user_name == "":  # if user name empty then show the warnings
                st.warning("Inavlid user name")
            elif user_mail == "":  # if user name empty then show the warnings
                st.warning("Inavlid user mail")
            elif exam_name == "":  # if email empty then show the warnings
                st.warning("Invalid couse")
            else:
                st.success("course search success ")
                st.session_state.search_result = True
                st.session_state.exam_start = False
                st.session_state.exam_finish = False

    placeholder = st.empty()
    with placeholder.container():

        if st.session_state.search_result == True and st.session_state.exam_finish == False :
            with st.form('user_exam_form'):
                st.markdown("**:red[Please confirm the information to start test]**")
                st.write('Hello, ',user_name,'from',user_dept, 'with e-mail',user_mail,' ',':sunglasses:')
                st.write('You choose','Grade  ',exam_grade, 'Name ',exam_name, 'verison',exam_version,'for exam! :ledger:')
                st.write('Press following button to proceed. Good Luck to get 	:100: ')

            #  st.markdown("**:blue[Show exam table]**")
            #  exam_list = search_exam(exam_grade,exam_name,exam_version)
            #  show_exam_list = pd.DataFrame( exam_list, columns=["grade", "course_name", "version",'NO','question','c1','c2','c3','C4','Answer'])
            #  st.dataframe(show_exam_list,use_container_width=True)

            #st.session_state.user_agreement = st.checkbox('Confirm above information')
                st.divider()
                st.session_state.exam_start = True
                st.markdown("**:blue[Start Test ]**")
                exam_list=search_exam(exam_grade,exam_name,exam_version)
                exam_pd = pd.DataFrame( exam_list, columns=["grade", "course_name", "version",'NO','question','c1','c2','c3','C4','Answer'])
                exam_answer_list =exam_pd['Answer']
                user_answer_list =[]
                question_list = exam_pd['question']
                for row_index,row in exam_pd.iterrows():
                    exam_question = st.write('NO',str(row_index+1),',',row[4],':question:')
                    col1, col2 = st.columns([3,1])
                    with col1:
                        st.write('A,',row[5])
                        st.write('B,',row[6])
                        st.write('C,',row[7])
                        st.write('D,',row[8])
                    with col2:
                        exam_answer = st.radio('Please choose',('A','B','C','D'),label_visibility='visible',key=row_index,horizontal= True)
                        rate = str(format((row_index+1)/len(exam_pd),'.0%'))
                        st.write('Total',str(len(exam_pd)),',     :blue[Finish rate] =',rate )
                    user_answer_list.append(exam_answer)
                    st.divider()

                exam_finish = st.checkbox('Ready to submit')
                submitted_exam = st.form_submit_button("Submit the answer")
                if submitted_exam and exam_finish :
                    st.success("**:blue[Finished]**, Close exam in 3 seconds")
                    st.session_state.exam_finish = True
                   # Get score
                    st.write(user_answer_list)
                    st.write(exam_answer_list)

                    total, correct = 0, 0
                    for i in range(len(user_answer_list)):
                        if exam_answer_list[i] == user_answer_list[i]:
                            st.write('NO',str(i),'......',question_list[i], ':o:','......Answer is',user_answer_list[i])
                            correct += 1
                        else:
                            st.write('NO',str(i),'......',question_list[i],':x:', '......Coccrect answer is',exam_answer_list[i], '......You choose',user_answer_list[i])
                        total += 1

                    score = int(correct / total * 100)
                    st.write('Your Score is ',str(score))

                    # 写入score数据库
                    create_score_table()
                    grade=exam_grade
                    course_name=exam_name
                    version = exam_version
                    user_score = str(score)
                    score_time = datetime.datetime.now().strftime("%Y%m%d%H%M")
                    add_score(grade, course_name, version,user_name,user_mail,user_dept,user_score,score_time)

                    # 显示score DB
                    score_result = view_all_score()
                    score_db = pd.DataFrame( score_result, columns=["grade", "course_name", "version","user_name","user_mail","user_dept","user_score","score_time"])
                    st.dataframe(score_db, use_container_width=True)

                    time.sleep(10)
                    placeholder.empty()

if __name__ == "__main__":
    main()


from odoo import models
from odoo.addons.component.core import Component
import logging

_logger = logging.getLogger(__name__)

class VersionKeyModel(models.TransientModel):
    # In actual connector version is mandatory use a model
    _name = '_prestashop.version.key'

class VersionKey(Component):
    _name = '_prestashop.version.key'
    _apply_on = '_prestashop.version.key'
    _usage = 'prestashop.version.key'

    keys = {
        'messages': 'customer_messages',
    }

    def get_key(self, key):
        output_key = self.keys.get(key) or key
        _logger.info("Output key changed from %s to %s" % (key, output_key))
        return output_key

class VersionKey1609(Component):
    _name = '_prestashop.version.key.1.6.0.9'
    _inherit = '_prestashop.version.key'
    _usage = 'prestashop.version.key.1.6.0.9'

    keys = {
        'product_option_value': 'product_option_values',
        'category': 'categories',
        'order_slip': 'order_slips',
        'order_slip_detail': 'order_slip_details',
        'group': 'groups',
        'order_row': 'order_rows',
        'tax': 'taxes',
        'image': 'images',
        'combinations': 'combinations',
        'tag': 'tags',
        'manufacturers': 'manufacturers',
    }

class VersionKey1612(Component):
    _name = '_prestashop.version.key.1.6.1.2'
    _inherit = '_prestashop.version.key'
    _usage = 'prestashop.version.key.1.6.1.2'

    # keys checked in 1.6.1.9:
    # * customer_messages
    # * order_slip
    # * order_slip_detail

    keys = {
        'product_option_value': 'product_option_value',
        'category': 'category',
        'image': 'image',
        'order_slip': 'order_slip',
        'order_slip_detail': 'order_slip_detail',
        'group': 'group',
        'order_row': 'order_row',
        'tax': 'taxes',
        'combinations': 'combination',
        'product_features': 'product_feature',
        'tag': 'tag',
        'messages': 'customer_messages',
        'manufacturers': 'manufacturers',
    }

class VersionKey1619(Component):
    _name = '_prestashop.version.key.1.6.1.9'
    _inherit = '_prestashop.version.key'
    _usage = 'prestashop.version.key.1.6.1.9'

    # keys checked in 1.6.1.9:
    # * customer_messages
    # * order_slip
    # * order_slip_detail

    keys = {
        'product_option_value': 'product_option_value',
        'category': 'category',
        'image': 'image',
        'order_slip': 'order_slip',
        'order_slip_detail': 'order_slip_detail',
        'group': 'group',
        'order_row': 'order_row',
        'tax': 'tax',
        'combinations': 'combination',
        'product_features': 'product_feature',
        'tag': 'tag',
        'messages': 'customer_messages',
        'manufacturers': 'manufacturers',
    }

class VersionKey1750(Component):
    _name = '_prestashop.version.key.1.7.5.0'
    _inherit = '_prestashop.version.key'
    _usage = 'prestashop.version.key.1.7.5.0'

    # TODO: Don't forget to adpt your PS server due to this regression :
    # https://www.prestashop.com/forums/topic/890277-r%C3%A9cup%C3%A9rer-le-d%C3%A9tail-des-r%C3%A9ductions-sur-commande-via-lapi/?tab=comments#comment-2955986
    keys = {
        'product_option_value': 'product_option_value',
        'category': 'categories',
        'image': 'image',
        'order_slip': 'order_slip',
        'order_slip_detail': 'order_slip_details',
        'order_row': 'order_row',
        'group': 'groups',
        'tax': 'tax',
        'product_features': 'product_features',
        'combinations': 'combination',
        'tag': 'tags',
        'messages': 'customer_messages',
        'manufacturers': 'manufacturers',
    }


from math import sqrt
from vector import VectorObject2D, obj as v

class Transform():
  def __init__(
    self,
    position: VectorObject2D,
    scale: VectorObject2D
  ) -> None:
    self.position: VectorObject2D = position
    self.scale: VectorObject2D = scale
    pass

  def distance_to(self, other: 'Transform') -> float:
    dx: float = float(abs(other.position.x - self.position.x))
    dy: float = float(abs(other.position.y - self.position.y))

    sum_squares = dx**2 + dy**2

    return sqrt(sum_squares)

  def is_overalpping(self, other: 'Transform') -> bool:
    parent_scale: VectorObject2D = self.scale
    other_scale: VectorObject2D = other.scale

    distance_x: float = float(abs(self.position.x - other.position.x))
    distance_y: float = float(abs(self.position.y - other.position.y))

    is_overlapping_x: bool = distance_x <= float(parent_scale.x / 2) + float(other_scale.x / 2)
    is_overlapping_y: bool = distance_y <= float(parent_scale.y / 2) + float(other_scale.y / 2)

    return is_overlapping_x and is_overlapping_y

  def move(self, direction: str, amount: float) -> None:
    if direction == "up":
      self.position.y += amount
    elif direction == "down":
      self.position.y -= amount
    elif direction == "left":
      self.position.x -= amount
    elif direction == "right":
      self.position.x += amount

  def relocate(self, x: float, y: float) -> None:
    self.position.x = x
    self.position.y = y

  def rescale(self, x: float, y: float) -> None:
    self.scale.x = x
    self.scale.y = y


import re

from configparser import ConfigParser
from uuid import uuid4

from fantastic.exercises.fill.fill import Fill
from fantastic.exercises.utils import text_to_html

class RemplirClavierCadre(Fill):
    """exercises consisting of finding verbes between parenthesis
    and adding a frame above the sentence with the verb"""

    def __init__(self, json_path: str, config: ConfigParser) -> None:
        Fill.__init__(
            self,
            json_path,
            config,
            template_name="fill",
            output_folder_name="remplir_clavier_cadre",
        )

    def adapt_guideline(self):
        """should adapt the guideline for some exercises"""

        guideline = super().find_guideline()
        guideline = guideline = guideline.replace("entre parenthèses ", "")
        guideline = guideline.replace(" entre parenthèses", "")

        return guideline

    def convert_to_html(self):
        """get the raw text and output an html version of it readty to be adapted"""

        # we get the data from json
        guideline = self.adapt_guideline()
        # find additional guideline returns a tuple
        additional_guideline = "".join(super().find_additional_guideline())
        # list of sentences from exercise_text
        sentences = super().find_sentences()

        # text to html convert each word, but we need to manke changes before text to html
        # so we need to encapsulate what we want to give to the fill box in only one word : hex hash
        # replacement dict contains all change to be made after the conversion to html
        replacement, sentences_with_hash = ({}, [])
        for sentence in sentences:
            # we initialize sentence_id in case of bad extracted xml
            sentence_id = 0
            # getting the verbs
            verbs = re.findall(r"\(.+?\)", sentence)
            sentence_with_hash = ""
            sentence_without_verb = sentence

            for verb in verbs:
                # verb contains parenthesis
                sentence_without_verb = sentence_without_verb.replace(f"{verb}", "…")

                # we generate an unique id for the sentence, that we will not found in the text
                # because after the conversion to html, we'll replace the unique id by html code
                sentence_id = uuid4().hex
                replacement[
                    str(sentence_id)
                ] = f"{verb[1:-1]}"
                sentence_with_hash += str(sentence_id) + " "

            if sentence_id == 0:
                # no verb found
                sentences_with_hash.append(sentence)
            else:
                # verb found
                sentences_with_hash.append(
                    sentence_with_hash + "
 " + sentence_without_verb
                )

        # we conver to html the text of guideline, additional guideline and exercise_text
        guideline = text_to_html([guideline])
        # we want it to be empty if there are no additional guideline
        additional_guideline = (
            text_to_html([additional_guideline]) if additional_guideline != "" else ""
        )
        exercise_text = text_to_html(sentences_with_hash, True)

        return guideline, additional_guideline, exercise_text, replacement

    def adapt(self):
        """specific adaptation for this type of exercise"""
        super().adapt_html()
        return self


import cv2
from imageai import Detection
import os
import pathlib
class Model:
    def __init__(self, model_type):
        # Init model here
        self.model_type = model_type # Type of model to init
        self._init_model()
        pass

    def train(self, params):
        # Add your implementation
        # Maybe dont need, for stretch goals only
        return {}

    def predict(self,input):
        # Maybe dont need, for stretch goals only
        raise NotImplementedError

    def run(self, frame, threshold, class_filter):
        """This is the main model to run model inference
            return input image + annotated image + dictionary of prediction params + class_filter

            class_filter: a dict of 80 classes with value "valid" or "invalid"

            returns
            frame: input image
            image: annotated image
            params: [{'name': 'person',
                    'percentage_probability': 76.65538787841797,
                    'box_points': [223, 300, 458, 499]}]
        """
        frame = self._resize(frame) # Test code only change as required
        image, detection_results = self.model.detectCustomObjectsFromImage(custom_objects=class_filter,
                        input_image=frame,
                        input_type="array",
                        output_type="array",
                        minimum_percentage_probability=threshold,
                        display_percentage_probability=True,
                        display_object_name=True)

        return frame, image, detection_results

    def _resize(self, image):
        # Resize
        return image

    def _init_model(self):
        HERE = os.path.dirname(__file__)
        model_type = self.model_type
        if model_type == 'YOLOv3':
            self.model = Detection.ObjectDetection()
            self.model.setModelTypeAsYOLOv3()
            self.model.setModelPath(pathlib.Path(os.path.join(HERE, "../models/yolo.h5")))
            self.model.loadModel(detection_speed="flash")
        elif model_type == "RetinaNet":
            self.model = Detection.ObjectDetection()
            self.model.setModelTypeAsRetinaNet()
            self.model.setModelPath(pathlib.Path(os.path.join(HERE, "../models/resnet50_coco_best_v2.1.0.h5")))
            self.model.loadModel()


import json
import pysher
import logging

class cpusher(pysher.Pusher, object):
    IsConnected = False
    ChannelsMap = []
    SessionId = None
    def __init__(self, appkey, host="", port="6001", secure=False, secret="", user_data=None, log_level=logging.NOTSET, reconnect_interval=10):
        super().__init__(appkey, secure = secure, custom_host=host, secret=secret, user_data = user_data, log_level = log_level, port = port, reconnect_interval=reconnect_interval)

    def Connect(self):
        self.connection.bind('pusher:connection_established',self.connect_handler)
        self.connect()

    def Disconnect(self):
        self.disconnect(1)

    def connect_handler(self, data):
        self.IsConnected = True
        obj = json.loads(data)
        self.SessionId = obj['socket_id']
        for x in self.ChannelsMap:
            channel = super().subscribe(x[0])
            channel.bind(x[1], x[2])

    def Subscribe(self, channel, event, callback):
        self.ChannelsMap.append([channel, event, callback])
        if self.IsConnected :
            channel = super().subscribe(channel)
            channel.bind(event, callback)
            return channel
        return None


from dataclasses import dataclass
from datetime import datetime
import re
from os import path
from typing import Any, cast, Type, TypeVar

from jsons import (
    dump,
    dumps,
    NoneType,
    load,
    loads,
    JsonsError,
    set_deserializer,
    set_serializer,
    set_validator,
    suppress_warnings,
    default_nonetype_deserializer,
)

S = TypeVar("S", bound="Serializable")
_T = TypeVar("_T")

JSON_FILE_EXTENSION: str = ".json"
WRITE: str = "w"
READ: str = "r"
JSON_PARSE_ERROR = '{"error": "Object could not be parsed due to json issue"}'
# TODO Issue #??: Jsons library incorrectly dumps class method
KEYS_TO_REMOVE = ["from_json", "from_json_file", "from_json_object"]

@dataclass
class Serializable:
    """
    Serializable class with methods to convert to json
    """

    def to_json(self, strip_privates: bool = True) -> str:
        """
        Serialize to json string
        :param strip_privates: strip private variables
        :return: the json string representation of this object
        """
        return write_json(self, strip_privates)

    def to_json_object(self, strip_privates: bool = True) -> Any:
        """
        Serialize to json object
        :param strip_privates: strip private variables
        :return: the json representation of this object
        """
        return write_json_object(self, strip_privates)

    def to_json_file(
        self, file_name: str, file_path: str = "", strip_privates: bool = True
    ) -> None:
        """
        Serialize an object to a json file
        :param file_name: File name
        :param file_path: File path
        :param strip_privates: Strip private variables
        """
        write_json_file(self, file_name, file_path, strip_privates)

    @classmethod
    def from_json(cls: Type[S], data: str) -> S:
        """
        Deserialize the provided data string into the specified instance
        :param data: JSON string
        """
        return read_json(data, cls)

    @classmethod
    def from_json_object(cls: Type[S], data: object) -> S:
        """
        Deserialize the provided data object into the specified instance
        :param data: JSON object
        """
        return read_json_object(data, cls)

    @classmethod
    def from_json_file(cls: Type[S], file_name: str, file_path: str = "") -> S:
        """
        Deserialize the provided file into the specified instance
        :param file_name: File name
        :param file_path: File path
        """
        return read_json_file(cls, file_name, file_path)

def _remove_key(obj: Any, key_to_remove: str) -> Any:
    """
    Remove key from object recursively
    :param obj: Any object
    :param key_to_remove: key to remove
    """
    if isinstance(obj, dict):
        for key in list(obj.keys()):
            if key == key_to_remove:
                del obj[key]
            else:
                _remove_key(obj[key], key_to_remove)
    elif isinstance(obj, list):
        for i in reversed(range(len(obj))):
            if obj[i] == key_to_remove:
                del obj[i]
            else:
                _remove_key(obj[i], key_to_remove)

def write_json(object_to_write: object, strip_privates: bool = True) -> str:
    """
    Serialize to json string
    :param object_to_write: object to write to json
    :param strip_privates: strip private variables
    :return: the json string representation of this object
    """
    set_serializers()
    suppress_warnings()
    try:
        json_object = write_json_object(object_to_write, strip_privates)
        json_string = cast(
            str, dumps(json_object, strip_privates=strip_privates, strip_nulls=True)
        )
        return json_string
    except JsonsError:
        return JSON_PARSE_ERROR

def write_json_object(object_to_write: object, strip_privates: bool = True) -> object:
    """
    Serialize to json object
    :param object_to_write: object to write to json
    :param strip_privates: strip private variables
    :return: the json representation of this object
    """
    set_serializers()
    suppress_warnings()
    try:
        json_object = dump(
            object_to_write, strip_privates=strip_privates, strip_nulls=True
        )
        for key in KEYS_TO_REMOVE:
            _remove_key(json_object, key)
        return json_object
    except JsonsError:
        return JSON_PARSE_ERROR

def write_json_file(
    object_to_write: object,
    file_name: str,
    file_path: str = "",
    strip_privates: bool = True,
) -> None:
    """
    Serialize json data string to json file
    :param object_to_write: object to write to json
    :param file_name: File name
    :param file_path: File path
    :param strip_privates: strip private variables
    """
    json_file_path: str = path.join(file_path, file_name + JSON_FILE_EXTENSION)
    with open(json_file_path, WRITE) as json_file:
        json_file.write(write_json(object_to_write, strip_privates))

def read_json(data: Any, class_out: Type[_T]) -> _T:
    """
    Deserialize json file to object
    :param data: Json file data
    :param class_out: Object type
    :return: Deserialized object
    """
    set_deserializers()
    return cast(_T, loads(data, class_out))

def read_json_object(data: Any, class_out: Type[_T]) -> _T:
    """
    Deserialize json file to object
    :param data: Json file data
    :param class_out: Object type
    :return: Deserialized object
    """
    set_deserializers()
    return cast(_T, load(data, class_out))

def read_json_file(class_out: Type[_T], file_name: str, file_path: str = "") -> _T:
    """
    Deserialize json file to object
    :param class_out: Object type
    :param file_name: File name
    :param file_path: File path
    :return: Deserialized object
    """
    set_deserializers()
    json_file_path: str = path.join(file_path, file_name + JSON_FILE_EXTENSION)
    with open(json_file_path, READ) as json_file:
        data = json_file.read()
        target: _T = read_json(data, class_out)
    return target

def set_serializers() -> None:
    """Set serializers for jsons to use to cast specific classes"""

    # Local import to minimize jsons usage across files
    from .group import ElementModP, ElementModQ

    set_serializer(lambda p, **_: str(p), ElementModP)
    set_serializer(lambda q, **_: str(q), ElementModQ)
    set_serializer(lambda dt, **_: dt.isoformat(), datetime)

def set_deserializers() -> None:
    """Set deserializers and validators for json to use to cast specific classes"""

    # Local import to minimize jsons usage across files
    from .group import ElementModP, ElementModQ, int_to_p_unchecked, int_to_q_unchecked

    set_deserializer(
        lambda p_as_int, cls, **_: int_to_p_unchecked(p_as_int), ElementModP
    )
    set_validator(lambda p: p.is_in_bounds(), ElementModP)

    set_deserializer(
        lambda q_as_int, cls, **_: int_to_q_unchecked(q_as_int), ElementModQ
    )
    set_validator(lambda q: q.is_in_bounds(), ElementModQ)

    set_deserializer(
        lambda none, cls, **_: None
        if none == "None"
        else default_nonetype_deserializer(none),
        NoneType,
    )

    set_deserializer(lambda dt, cls, **_: _deserialize_datetime(dt), datetime)

def _deserialize_datetime(value: str) -> datetime:
    """
    The `fromisoformat` function doesn't recognize the Z (Zulu) suffix
    to indicate UTC.  For compatibility with more external clients, we
    should allow it.
    """
    tz_corrected = re.sub("Z$", "+00:00", value)
    return datetime.fromisoformat(tz_corrected)


#역순으로 저장된 연결 리스트의 숫자를 더할 것.
#입력) (2-> 4-> 3) + (5-> 6-> 4) 출력) 7-> 0-> 8

#풀이방법: 모든 수를 역순한 후, 더하면 값이 나오는데 그 값도 역순으로 출력하면 된다.

#풀이1) 자료형 변환
from typing import List

class Solution:
    def reverseList(self, head: ListNode) -> ListNode:
        node,prev = head, None

        while node:
            next, node.next = node.next, prev
            prev, node = node, next

        return prev

    def toList(self,node: ListNode)-> List:
        list: List=[]
        while node:
            list.append(node.val)
            node = node.next
        return List


from connector.dockerhub_connector import DockerHubConnector
from connector.ecr_connector import EcrConnector
import yaml
import sys

def main():
    try:
        with open("./conf/config.yaml", "r") as f:
            config = yaml.load(f, Loader=yaml.FullLoader)
            repository_statements = config['repositories']
    except FileNotFoundError as e:
        print(e)
        sys.exit(1)
    except yaml.YAMLError as e:
        print(f"Invalid yaml format! {e}")
        sys.exit(1)

    for repository_statement in repository_statements:
        if repository_statement['repository_type'] == "DOCKER_HUB":
            connector = DockerHubConnector(repository_statement)
        elif repository_statement['repository_type'] == "ECR":
            connector = EcrConnector(repository_statement)
        else:
            print('Invalid Repository type. (DOCKER_HUB|ECR)')
            sys.exit(1)

        images = connector.list_images()
        old_tags_by_image = {}
        for image in images:
            old_tags_by_image[image] = connector.list_old_tags(image)

        for image in old_tags_by_image:
            if old_tags_by_image[image]:
                for tag in old_tags_by_image[image]:
                    connector.delete(image,tag)

if __name__ == "__main__":
    main()


from pyspark import SparkContext, SparkConf
from pyspark.mllib.linalg import Vectors
import numpy as np
import time

# Define function to calculate cosine similarity score
def cosine_similarity(query, embedding):
    dot_product = np.dot(query, embedding)
    norm_query = np.linalg.norm(query)
    norm_embedding = np.linalg.norm(embedding)
    return dot_product / (norm_query * norm_embedding)

# Define query and query embedding as RDDs
conf = SparkConf().setAppName("Cosine Similarity")
sc = SparkContext.getOrCreate(conf)
query = sc.parallelize([np.random.rand(786)])
embeddings = sc.parallelize(np.random.rand(1500, 786))

# Define list of number of partitions to try
num_partitions_list = [20]

# Loop over number of partitions and calculate cosine similarity scores
for num_partitions in num_partitions_list:
    # Repartition the embedding RDD with num_partitions partitions
    embeddings_partitioned = embeddings.repartition(num_partitions)

    # Calculate cosine similarity score using RDD zipWithIndex and map functions
    start_time = time.time()
    scores = embeddings_partitioned.zipWithIndex().map(lambda x: (x[1], cosine_similarity(query.collect()[0], x[0]))).collect()
    scores.sort(key=lambda x: x[0])
    scores = [x[1] for x in scores]
    end_time = time.time()

    print(f"Time taken with {num_partitions} partitions:", end_time - start_time, "seconds")
    print("---------------------------------------------------------------------------")

# Stop SparkContext
sc.stop()


from collections.abc import Sequence

from django.core import paginator
from django.db.models import QuerySet
from django.utils.timezone import now
from django.utils.translation import get_language
from elasticsearch_dsl import InnerDoc, AttrList
from marshmallow import pre_dump
from marshmallow.schema import SchemaOpts
from querystring_parser import builder

from mcod.core.api import schemas, fields
from mcod.core.serializers import SerializerRegistry

object_attrs_registry = SerializerRegistry()

class RelationshipData(schemas.ExtSchema):
    id = fields.String(required=True)
    _type = fields.String(required=True, data_key='type')

class RelationshipLinks(schemas.ExtSchema):
    related = fields.String(required=True)

class RelationshipMeta(schemas.ExtSchema):
    count = fields.Integer()

class Relationship(schemas.ExtSchema):
    data = fields.Nested(RelationshipData)
    links = fields.Nested(RelationshipLinks, required=True, many=False)
    meta = fields.Nested(RelationshipMeta, many=False)

    @pre_dump
    def prepare_data(self, data):
        object_url = self.context.get('object_url', None) or self.api_url
        url_template = self.context.get('url_template') or None
        res = {}
        if isinstance(data, (Sequence, QuerySet, AttrList)):
            if url_template:
                related_url = url_template.format(
                    api_url=self.api_url, object_url=object_url
                )
                res['links'] = {
                    'related': related_url
                }
            res['meta'] = {
                'count': len(data)
            }
        else:
            ident = getattr(data, 'id', None) or getattr(data.meta, 'id')
            slug = getattr(data, 'slug', None)
            if isinstance(slug, InnerDoc):
                lang = get_language()
                slug = slug[lang]
            if slug:
                ident = '{},{}'.format(ident, slug)
            if url_template:
                related_url = url_template.format(
                    api_url=self.api_url, object_url=object_url, ident=ident
                )
                res['links'] = {
                    'related': related_url
                }
            res['data'] = {
                'id': ident,
                '_type': self.context['_type']
            }

        return res

class Relationships(schemas.ExtSchema):
    def __init__(
            self, only=None, exclude=(), many=False, context=None,
            load_only=(), dump_only=(), partial=False, unknown=None,
    ):
        super().__init__(only=only, exclude=exclude, many=many, context=context,
                         load_only=load_only, dump_only=dump_only,
                         partial=partial, unknown=unknown)

        for field_name, field in self.fields.items():
            field._schema.context = dict(self.context)
            field._schema.context.update(field.metadata)
            field.many = False

class ObjectAttrsMeta(schemas.ExtSchema):
    pass

class ObjectAttrsOpts(SchemaOpts):
    def __init__(self, meta, **kwargs):
        SchemaOpts.__init__(self, meta, **kwargs)
        self.relationships_schema = getattr(meta, 'relationships_schema', None)

        if self.relationships_schema and not issubclass(self.relationships_schema, Relationships):
            raise Exception("{} must be a subclass of Relationships".format(self.relationships_schema))

        self.meta_schema = getattr(meta, 'meta_schema', None)

        if self.meta_schema and not issubclass(self.meta_schema, ObjectAttrsMeta):
            raise Exception("{} must be a subclass of ObjectAttrsMeta".format(self.meta_schema))

        self.object_type = getattr(meta, 'object_type', None) or 'undefined'
        self.model_name = getattr(meta, 'model', None) or None
        self.url_template = getattr(meta, 'url_template', None) or '{api_url}'

class ObjectAttrs(schemas.ExtSchema):
    OPTIONS_CLASS = ObjectAttrsOpts

    @classmethod
    def prepare(cls):
        object_attrs_registry.register(cls)

class ObjectLinks(schemas.ExtSchema):
    self = fields.String(required=True)

class ObjectOpts(SchemaOpts):
    def __init__(self, meta, **kwargs):
        SchemaOpts.__init__(self, meta, **kwargs)
        self.attrs_schema = getattr(meta, 'attrs_schema', None)
        if self.attrs_schema and not issubclass(self.attrs_schema, ObjectAttrs):
            raise Exception("{} must be a subclass of ObjectAttrs".format(self.attrs_schema))

class Object(schemas.ExtSchema):
    OPTIONS_CLASS = ObjectOpts
    id = fields.Str(required=True)
    links = fields.Nested(ObjectLinks, name='links')
    _type = fields.String(required=True, data_key='type')

    # meta = fields.Nested(ObjectMeta)

    def __init__(
            self, only=None, exclude=(), many=False, context=None,
            load_only=(), dump_only=(), partial=False, unknown=None,
    ):

        self._declared_fields['attributes'] = fields.Nested(
            self.opts.attrs_schema, name='attributes', many=False
        )

        relationships_schema = getattr(self.opts.attrs_schema.opts, 'relationships_schema', None)

        if relationships_schema:
            self._declared_fields['relationships'] = fields.Nested(relationships_schema, many=False,
                                                                   name='relationships')

        meta_schema = getattr(self.opts.attrs_schema.opts, 'meta_schema', None)

        if meta_schema:
            self._declared_fields['meta'] = fields.Nested(meta_schema, many=False,
                                                          name='meta')

        super().__init__(only=only, exclude=exclude, many=many, context=context,
                         load_only=load_only, dump_only=dump_only,
                         partial=partial, unknown=unknown)

    @pre_dump(pass_many=False)
    def prepare_data(self, data):
        ident = getattr(data, 'id', None) or getattr(data.meta, 'id')
        slug = getattr(data, 'slug', None)
        if isinstance(slug, InnerDoc):
            lang = get_language()
            slug = slug[lang]
        if slug:
            ident = '{},{}'.format(ident, slug)

        res = dict(
            attributes=data,
            id=str(ident),
            _type=self.opts.attrs_schema.opts.object_type
        )
        object_url = self.opts.attrs_schema.opts.url_template.format(
            api_url=self.api_url,
            ident=ident,
            data=data
        )
        res['links'] = {
            'self': object_url
        }

        if 'meta' in self._declared_fields:
            res['meta'] = data

        if 'relationships' in self.fields:
            relationships = {}
            for name, field in self.fields['relationships'].schema.fields.items():
                _name = field.attribute or name
                field._schema.context.update(object_url=object_url)
                value = getattr(data, _name, None)
                if value:
                    relationships[_name] = value
            if relationships:
                res['relationships'] = relationships
        return res

class TopLevelMeta(schemas.ExtSchema):
    language = fields.String()
    params = fields.Raw()
    path = fields.String()
    count = fields.Integer()
    relative_uri = fields.String()
    aggregations = fields.Raw()
    server_time = fields.DateTime(default=now)

class TopLevelLinks(schemas.ExtSchema):
    self = fields.String()
    first = fields.String()
    last = fields.String()
    prev = fields.String()
    next = fields.String()

class TopLevelOpts(SchemaOpts):
    def __init__(self, meta, **kwargs):
        SchemaOpts.__init__(self, meta, **kwargs)
        self.attrs_schema = getattr(meta, 'attrs_schema', None)
        self.aggs_schema = getattr(meta, 'aggs_schema', None)

        # self.errors_schema = getattr(meta, 'errors_schema', ResponseErrors)
        self.meta_schema = getattr(meta, 'meta_schema', TopLevelMeta)
        self.links_schema = getattr(meta, 'links_schema', TopLevelLinks)

        self.max_items_num = getattr(meta, 'max_items_num', 10000)

        if self.attrs_schema and not issubclass(self.attrs_schema, ObjectAttrs):
            raise Exception("{} must be a subclass of ObjectAttrs".format(self.attrs_schema))
        if self.meta_schema and not issubclass(self.meta_schema, TopLevelMeta):
            raise Exception("{} must be a subclass of Meta".format(self.meta_schema))
        if self.aggs_schema:
            if not issubclass(self.aggs_schema, schemas.ExtSchema):
                raise Exception("{} must be a subclass of ExtSchema".format(self.aggs_schema))
            self.meta_schema.aggregations = fields.Nested(self.aggs_schema, many=False)
        if self.links_schema and not issubclass(self.links_schema, TopLevelLinks):
            raise Exception("{} must be a subclass of Links".format(self.links_schema))

class Aggregation(schemas.ExtSchema):
    id = fields.String(attribute='key')
    title = fields.String()
    doc_count = fields.Integer()

    @pre_dump(pass_many=True)
    def prepare_data(self, data, many):
        if many:
            for item in data:
                item['title'] = str(item.key).upper()
            return data

class TopLevel(schemas.ExtSchema):
    included = fields.Raw()
    jsonapi = fields.Raw(default={'version': '1.0'})
    errors = fields.Raw()

    OPTIONS_CLASS = TopLevelOpts

    def __init__(
            self, only=None, exclude=(), many=False, context=None,
            load_only=(), dump_only=(), partial=False, unknown=None,
    ):
        data_cls = type(
            '{}Data'.format(self.__class__.__name__),
            (Object,), {}
        )
        setattr(data_cls.opts, 'attrs_schema', self.opts.attrs_schema)

        self._declared_fields['data'] = fields.Nested(data_cls, name='data', many=many)

        if self.opts.meta_schema:
            if self.opts.aggs_schema:
                self.opts.meta_schema._declared_fields['aggregations'] = fields.Nested(self.opts.aggs_schema,
                                                                                many=False)
            self._declared_fields['meta'] = fields.Nested(
                self.opts.meta_schema, name='meta', many=False
            )

        if self.opts.links_schema:
            self._declared_fields['links'] = fields.Nested(
                self.opts.links_schema, name='links', many=False
            )
        context = context or {}
        context['is_listing'] = many

        super().__init__(only=only, exclude=exclude, many=False, context=context,
                         load_only=load_only, dump_only=dump_only,
                         partial=partial, unknown=unknown)

    @pre_dump
    def prepare_top_level(self, c):
        def _get_page_link(page_number):
            cleaned_data['page'] = page_number
            return '{}{}?{}'.format(request.prefix, request.path, builder.build(cleaned_data))

        c.data = getattr(c, 'data', {})
        request = self.context['request']
        c.meta = getattr(c, 'meta', {})
        c.links = getattr(c, 'links', {})
        cleaned_data = dict(request.context.cleaned_data)

        c.meta.update({
            'language': request.language,
            'params': request.params,
            'path': request.path,
            'relative_uri': request.relative_uri,
        })

        c.links['self'] = request.uri

        if self.context['is_listing']:
            c.meta['aggregations'] = getattr(c.data, 'aggregations', {}) or {}
            if isinstance(c.data, paginator.Page):
                items_count = c.data.paginator.count
            else:
                items_count = c.data.hits.total \
                    if hasattr(c.data, 'hits') else 0
            c.meta['count'] = items_count
            page, per_page = cleaned_data.get('page', 1), cleaned_data.get('per_page', 20)
            c.links['self'] = _get_page_link(page)
            if page > 1:
                c.links['first'] = _get_page_link(1)
                c.links['prev'] = _get_page_link(page - 1)
            if items_count:
                max_count = min(items_count, self.opts.max_items_num)
                off = 1 if max_count % per_page else 0
                last_page = max_count // per_page + off
                if last_page > 1:
                    c.links['last'] = _get_page_link(last_page)
                if page * per_page < max_count:
                    c.links['next'] = _get_page_link(page + 1)
        return c