import os
import glob
import torch
import librosa
import argparse

import pyaudio, queue, wave, webrtcvad, logging, collections
from scipy import signal
import numpy as np

from utils.audio import Audio
from utils.hparams import HParam
from model.model import VoiceFilter
from model.embedder import SpeechEmbedder

class voice(object):
    """Streams raw audio from microphone. Data is received in a separate thread, and stored in a buffer, to be read from."""

    FORMAT = pyaudio.paInt16
    # Network/VAD rate-space
    RATE_PROCESS = 16000
    CHANNELS = 1
    BLOCKS_PER_SECOND = 50

    def __init__(self, callback=None, device=None, input_rate=RATE_PROCESS, file=None):
        def proxy_callback(in_data, frame_count, time_info, status):
            #pylint: disable=unused-argument
            if self.chunk is not None:
                in_data = self.wf.readframes(self.chunk)
            callback(in_data)
            return (None, pyaudio.paContinue)
        if callback is None: callback = lambda in_data: self.buffer_queue.put(in_data)
        self.buffer_queue = queue.Queue()
        self.device = device
        self.input_rate = input_rate
        self.sample_rate = self.RATE_PROCESS
        self.block_size = int(self.RATE_PROCESS / float(self.BLOCKS_PER_SECOND))
        self.block_size_input = int(self.input_rate / float(self.BLOCKS_PER_SECOND))
        self.pa = pyaudio.PyAudio()

        kwargs = {
            'format': self.FORMAT,
            'channels': self.CHANNELS,
            'rate': self.input_rate,
            'input': True,
            'frames_per_buffer': self.block_size_input,
            'stream_callback': proxy_callback,
        }

        self.chunk = None
        # if not default device
        if self.device:
            kwargs['input_device_index'] = self.device
        elif file is not None:
            self.chunk = 320
            self.wf = wave.open(file, 'rb')

        self.stream = self.pa.open(**kwargs)
        self.stream.start_stream()

    def resample(self, data, input_rate):
        """
        Microphone may not support our native processing sampling rate, so
        resample from input_rate to RATE_PROCESS here for webrtcvad and
        deepspeech
        Args:
            data (binary): Input audio stream
            input_rate (int): Input audio rate to resample from
        """
        data16 = np.fromstring(string=data, dtype=np.int16)
        resample_size = int(len(data16) / self.input_rate * self.RATE_PROCESS)
        resample = signal.resample(data16, resample_size)
        resample16 = np.array(resample, dtype=np.int16)
        return resample16.tostring()

    def read_resampled(self):
        """Return a block of audio data resampled to 16000hz, blocking if necessary."""
        return self.resample(data=self.buffer_queue.get(),
                             input_rate=self.input_rate)

    def read(self):
        """Return a block of audio data, blocking if necessary."""
        return self.buffer_queue.get()

    def destroy(self):
        self.stream.stop_stream()
        self.stream.close()
        self.pa.terminate()

    frame_duration_ms = property(lambda self: 1000 * self.block_size // self.sample_rate)

    def write_wav(self, filename, data):
        logging.info("write wav %s", filename)
        wf = wave.open(filename, 'wb')
        wf.setnchannels(self.CHANNELS)
        # wf.setsampwidth(self.pa.get_sample_size(FORMAT))
        assert self.FORMAT == pyaudio.paInt16
        wf.setsampwidth(2)
        wf.setframerate(self.sample_rate)
        wf.writeframes(data)
        wf.close()

class VADAudio(voice):
    """Filter & segment audio with voice activity detection."""

    def __init__(self, aggressiveness=3, device=None, input_rate=None, file=None):
        super().__init__(device=device, input_rate=input_rate, file=file)
        self.vad = webrtcvad.Vad(aggressiveness)

    def frame_generator(self):
        """Generator that yields all audio frames from microphone."""
        if self.input_rate == self.RATE_PROCESS:
            while True:
                yield self.read()
        else:
            while True:
                yield self.read_resampled()

    def vad_collector(self, padding_ms=300, ratio=0.75, frames=None):
        """Generator that yields series of consecutive audio frames comprising each utterence, separated by yielding a single None.
            Determines voice activity by ratio of frames in padding_ms. Uses a buffer to include padding_ms prior to being triggered.
            Example: (frame, ..., frame, None, frame, ..., frame, None, ...)
                      |---utterence---|        |---utterence---|
        """
        if frames is None: frames = self.frame_generator()
        num_padding_frames = padding_ms // self.frame_duration_ms
        ring_buffer = collections.deque(maxlen=num_padding_frames)
        triggered = False

        for frame in frames:
            if len(frame) < 640:
                return

            is_speech = self.vad.is_speech(frame, self.sample_rate)

            if not triggered:
                ring_buffer.append((frame, is_speech))
                num_voiced = len([f for f, speech in ring_buffer if speech])
                if num_voiced > ratio * ring_buffer.maxlen:
                    triggered = True
                    for f, s in ring_buffer:
                        yield f
                    ring_buffer.clear()

            else:
                yield frame
                ring_buffer.append((frame, is_speech))
                num_unvoiced = len([f for f, speech in ring_buffer if not speech])
                if num_unvoiced > ratio * ring_buffer.maxlen:
                    triggered = False
                    yield None
                    ring_buffer.clear()

def main(args, hp):
    with torch.no_grad():

        model = VoiceFilter(hp).cuda()
        chkpt_model = torch.load(args.checkpoint_path)['model']
        model.load_state_dict(chkpt_model, strict=False)
        model.eval()

        embedder = SpeechEmbedder(hp).cuda()
        chkpt_embed = torch.load(args.embedder_path)
        embedder.load_state_dict(chkpt_embed)
        embedder.eval()

        audio = Audio(hp)
        dvec_wav, _ = librosa.load(args.reference_file, sr=16000)
        dvec_mel = audio.get_mel(dvec_wav)
        dvec_mel = torch.from_numpy(dvec_mel).float().cuda()
        dvec = embedder(dvec_mel)
        dvec = dvec.unsqueeze(0)

        if args.mixed_file:
            mixed_wav, _ = librosa.load(args.mixed_file, sr=16000)
            mag, phase = audio.wav2spec(mixed_wav)
            mag = torch.from_numpy(mag).float().cuda()
        elif args.device is not None:
            vad_audio = VADAudio(aggressiveness=args.vad_aggressiveness,
                         device=args.device,
                         input_rate=args.rate,
                         file=args.mixed_file)
            print("Listening (ctrl-C to exit)...")
            frames = vad_audio.vad_collector()
            mag, phase = audio.wav2spec(frames)
            mag = torch.from_numpy(mag).float().cuda()
        else:
            print("error")

        mag = mag.unsqueeze(0)
        mask = model(mag, dvec)
        est_mag = mag * mask

        est_mag = est_mag[0].cpu().detach().numpy()
        est_wav = audio.spec2wav(est_mag, phase)

        os.makedirs(args.out_dir, exist_ok=True)
        out_path = os.path.join(args.out_dir, 'result.wav')
        librosa.output.write_wav(out_path, est_wav, sr=16000)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-c', '--config', type=str, required=True,
                        help="yaml file for configuration")
    parser.add_argument('-e', '--embedder_path', type=str, required=True,
                        help="path of embedder model pt file")
    parser.add_argument('--checkpoint_path', type=str, default=None,
                        help="path of checkpoint pt file")
    parser.add_argument('-m', '--mixed_file', type=str, default=None,
                        help='path of mixed wav file')
    parser.add_argument('-r', '--reference_file', type=str, required=True,
                        help='path of reference wav file')
    parser.add_argument('-o', '--out_dir', type=str, required=True,
                        help='directory of output')

    #VAD
    DEFAULT_SAMPLE_RATE = 16000
    parser.add_argument('-v', '--vad_aggressiveness', type=int, default=3,
                        help="Set aggressiveness of VAD: an integer between 0 and 3, 0 being the least aggressive about filtering out non-speech, 3 the most aggressive. Default: 3")
    parser.add_argument('-d', '--device', type=int, default=None,
                        help="Device input index (Int) as listed by pyaudio.PyAudio.get_device_info_by_index(). If not provided, falls back to PyAudio.get_default_device().")
    parser.add_argument('--rate', type=int, default=DEFAULT_SAMPLE_RATE,
                        help=f"Input device sample rate. Default: {DEFAULT_SAMPLE_RATE}. Your device may require 44100.")

    args = parser.parse_args()

    hp = HParam(args.config)

    main(args, hp)


# -*- coding: utf-8 -*-
# @Time    : 2020/12/29 17:19
# @Author  : Zeqi@@
# @FileName: preprocess_yolo_boxes.py
# @Software: PyCharm

import os
import numpy as np
from PIL import Image
from Preprocess.utils import Data_augmentation

def get_classes(classes_path):
    '''loads the classes'''
    with open(classes_path) as f:
        class_names = f.readlines()
    class_names = [c.strip() for c in class_names]
    return class_names

def get_anchors(anchors_path):
    '''loads the anchors from a file'''
    with open(anchors_path) as f:
        anchors = f.readline()
    anchors = [float(x) for x in anchors.split(',')]
    return np.array(anchors).reshape(-1, 2)

def preprocess_true_boxes(true_boxes, input_shape, anchors, num_classes):
    assert (true_boxes[..., 4]<num_classes).all(), 'class id must be less than num_classes'
    # 一共有三个特征层数
    num_layers = len(anchors)//3 # 9//3
    # 先验框
    # 678为 142,110,  192,243,  459,401
    # 345为 36,75,  76,55,  72,146
    # 012为 12,16,  19,36,  40,28
    anchor_mask = [[6,7,8], [3,4,5], [0,1,2]] if num_layers==3 else [[3,4,5], [1,2,3]]
    # print('Anchor mask: {}'.format(anchor_mask))

    true_boxes = np.array(true_boxes, dtype='float32')  # N, 100, 5
    input_shape = np.array(input_shape, dtype='int32')  # 160, 180

    # 读出xy轴，读出长宽
    # 中心点(m,n,2)
    boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2  # 中心点
    boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2]         # 长宽
    # 计算比例
    # print('input_shape[::-1] ', input_shape[::-1])
    true_boxes[..., 0:2] = boxes_xy/input_shape[::-1] # 是对字符串的截取操作
    true_boxes[..., 2:4] = boxes_wh/input_shape[::-1]

    # m张图
    m = true_boxes.shape[0]
    # 得到网格的shape为13,13;26,26;52,52 (406,406) || 5,15;10,30;20,60 (160, 480)
    grid_shapes = [input_shape//{0:32, 1:16, 2:8}[l] for l in range(num_layers)]
    # y_true的格式为(m,13,13,3,85)(m,26,26,3,85)(m,52,52,3,85) || (m,5,15,3,13)(m,10,30,3,13)(m,20,60,3,13)
    # grid_h, grid_w, layer_anchor_number, 5+num_classes
    y_true = [np.zeros((m,grid_shapes[l][0], grid_shapes[l][1], len(anchor_mask[l]), 5+num_classes),dtype='float32') for l in range(num_layers)]
    # for true in y_true:
    #     print('true: ', np.shape(true))
    # [1,9,2]
    anchors = np.expand_dims(anchors, 0)
    anchor_maxes = anchors / 2.
    anchor_mins = -anchor_maxes
    # print('anchor_maxes: \n', anchor_maxes)
    # print('anchor_mins: \n', anchor_mins)

    # 长宽要大于0才有效
    valid_mask = boxes_wh[..., 0]>0  # 426, 100
    # print('anchor_maxes: \n', np.shape(valid_mask))

    for b in range(m):
        # 对每一张图进行处理
        wh = boxes_wh[b, valid_mask[b]] # boxes_wh: (426, 100, 2) 部分wh有效

        if len(wh)==0: continue
        # [n,1,2]
        wh = np.expand_dims(wh, -2)
        box_maxes = wh / 2.
        box_mins = -box_maxes
        # 关于0中心化

        # 计算真实框和哪个先验框最契合
        intersect_mins = np.maximum(box_mins, anchor_mins)
        intersect_maxes = np.minimum(box_maxes, anchor_maxes)
        intersect_wh = np.maximum(intersect_maxes - intersect_mins, 0.)
        intersect_area = intersect_wh[..., 0] * intersect_wh[..., 1]
        box_area = wh[..., 0] * wh[..., 1]
        anchor_area = anchors[..., 0] * anchors[..., 1]
        iou = intersect_area / (box_area + anchor_area - intersect_area)
        best_anchor = np.argmax(iou, axis=-1)
        # print('best anchor: ', best_anchor)

        for t, n in enumerate(best_anchor):
            for l in range(num_layers):
                if n in anchor_mask[l]:
                    # floor用于向下取整
                    i = np.floor(true_boxes[b,t,0]*grid_shapes[l][1]).astype('int32') # w
                    j = np.floor(true_boxes[b,t,1]*grid_shapes[l][0]).astype('int32') # h
                    # 找到真实框在特征层l中第b副图像对应的位置
                    k = anchor_mask[l].index(n)
                    c = true_boxes[b, t, 4].astype('int32') # 目标类别
                    # 根据 best anchor 中心 设定 框 x_min, y_min, x_max, y_max
                    y_true[l][b, j, i, k, 0:4] = true_boxes[b, t, 0:4]
                    # 根据 best anchor，如果有物体，中心点为类别 1，无物体为 0
                    y_true[l][b, j, i, k, 4] = 1
                    # onehot vector 物体识别
                    y_true[l][b, j, i, k, 5+c] = 1

    return y_true

if __name__ == "__main__":
    annotation_path =  '../Preparation/data_txt'
    annotation_lines = open(os.path.join(annotation_path, 'kitti_ssd_obj_test.txt')).readlines()
    print(len(annotation_lines))

    data_aug = Data_augmentation(input_shape=(160, 480),
                                 max_boxes=100,
                                 jitter=.3,
                                 hue=.1,
                                 sat=1.5,
                                 val=1.5,
                                 visual=False)

    image_data, box_data = [], []
    for annotation_line in annotation_lines:
        line = annotation_line.split()
        image = Image.open(line[0])
        box = np.array([np.array(list(map(int, box.split(',')))) for box in line[1:]])
        aug_image, aug_box = data_aug.main(image, box)
        image_data.append(aug_image)
        box_data.append(aug_box)
    print('image_data: {}, box_data: {}'.format(np.shape(image_data), np.shape(box_data)))

    classes_path = '../Preparation/data_txt/kitti_classes.txt'
    class_names = get_classes(classes_path)
    num_classes = len(class_names)
    print(class_names, num_classes)

    anchors_path = '../Preparation/data_txt/kitti_yolov4_anchors.txt'
    yolo_anchors = get_anchors(anchors_path)
    print(yolo_anchors, np.shape(yolo_anchors))

    image_data = np.array(image_data)
    box_data = np.array(box_data)
    yolo_gt = preprocess_true_boxes(box_data, (160, 480), yolo_anchors, num_classes)
    for gt in yolo_gt:
        print('true: ', np.shape(gt))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

# ldig : Language Detector with Infinite-Gram
# This code is available under the MIT License.
# (c)2011 Nakatani Shuyo / Cybozu Labs Inc.

import os, sys, re, codecs, json
import optparse
import numpy
import html.entities as htmlentitydefs
import subprocess
import da

class ldig(object):
    def __init__(self, model_dir):
        self.features = os.path.join(model_dir, 'features')
        self.labels = os.path.join(model_dir, 'labels.json')
        self.param = os.path.join(model_dir, 'parameters.npy')
        self.doublearray = os.path.join(model_dir, 'doublearray.npz')

    def load_da(self):
        trie = da.DoubleArray()
        trie.load(self.doublearray)
        return trie

    def load_features(self):
        features = []
        with open(self.features, 'r') as f:
            pre_feature = ""
            for n, s in enumerate(f):
                m = re.match(r'(.+)\t([0-9]+)', s)
                if not m:
                    sys.exit("irregular feature : '%s' at %d" % (s, n + 1))
                if pre_feature >= m.group(1):
                    sys.exit("unordered feature : '%s' at %d" % (s, n + 1))
                pre_feature = m.group(1)
                features.append(m.groups())
        return features

    def load_labels(self):
        with open(self.labels, 'r') as f:
            return json.load(f)

    def init(self, temp_path, corpus_list, lbff, ngram_bound):
        """
        Extract features from corpus and generate TRIE(DoubleArray) data
        - load corpus
        - generate temporary file for maxsubst
        - generate double array and save it
        - parameter: lbff = lower bound of feature frequency
        """

        labels = []
        with open(temp_path, 'w') as f:
            for file in corpus_list:
                with open(file, 'r') as g:
                    for i, s in enumerate(g):
                        label, text, org_text =  normalize_text(s)
                        if label is None or label == "":
                            sys.stderr.write("no label data at %d in %s \n" % (i+1, file))
                            continue
                        if label not in labels:
                            labels.append(label)
                        f.write(text)
                        f.write("\n")

        labels.sort()
        print ("labels: %d" % len(labels))
        with open(self.labels, 'w') as f:
            f.write(json.dumps(labels))

        print ("generating max-substrings...")
        temp_features = self.features + ".temp"
        maxsubst = options.maxsubst
        if os.name == 'nt': maxsubst += ".exe"
        subprocess.call([maxsubst, temp_path, temp_features])

        # count features
        M = 0
        features = []
        r1 = re.compile(u'.\u0001.')
        r2 = re.compile(u'[A-Za-z\u00a1-\u00a3\u00bf-\u024f\u1e00-\u1eff\u0370-\u03ff\u0400-\u04ff]') # added greek (\u0370-\u03ff) and cyrillic (\u0400–\u04ff) alphabets
        with open(temp_features, 'r') as f:
            for line in f:
                i = line.index('\t')
                st = line[0:i]
                c = int(line[i+1:-1])
                if c >= lbff and len(st) <= ngram_bound and (not r1.search(st)) and r2.search(st) and (st[0] != u'\u0001' or st[-1] != u'\u0001'):
                    M += 1
                    features.append((st, line))
        print ("# of features = %d" % M)

        features.sort()
        with open(self.features, 'w') as f:
            for s in features:
                f.write(s[1])

        generate_doublearray(self.doublearray, [s[0] for s in features])

        numpy.save(self.param, numpy.zeros((M, len(labels))))

    def shrink(self):
        features = self.load_features()
        param = numpy.load(self.param)

        list = (numpy.abs(param).sum(1) > 0.0000001)
        new_param = param[list]
        print ("# of features : %d => %d" % (param.shape[0], new_param.shape[0]))

        numpy.save(self.param, new_param)
        new_features = []
        with open(self.features, 'w') as f:
            for i, x in enumerate(list):
                if x:
                    f.write("%s\t%s\n" % features[i])
                    new_features.append(features[i][0])

        generate_doublearray(self.doublearray, new_features)

    def debug(self, args):
        features = self.load_features()
        trie = self.load_da()
        labels = self.load_labels()
        param = numpy.load(self.param)

        for st in args:
            label, text, org_text = normalize_text(st)
            events = trie.extract_features(u"\u0001" + text + u"\u0001")
            print ("orig: '%s'" % st)
            print ("norm: '%s'" % text)
            sum = numpy.zeros(len(labels))
            print ("id\tfeat\tfreq\t%s" % "\t".join(labels))
            for id in sorted(events, key=lambda id:features[id][0]):
                phi = param[id,]
                sum += phi * events[id]
                print ("%d\t%s\t%d\t%s" % (id,features[id][0], events[id], "\t".join(["%0.2f" % x for x in phi])))
            exp_w = numpy.exp(sum - sum.max())
            prob = exp_w / exp_w.sum()
            print ("\t\t\t%s" % "\t".join(["%0.2f" % x for x in sum]))
            print ("\t\t\t%s" % "\t".join(["%0.1f%%" % (x * 100) for x in prob]))

    def learn(self, options, args):
        trie = self.load_da()
        param = numpy.load(self.param)
        labels = self.load_labels()

        import time
        print ("loading corpus... " + time.strftime("%H:%M:%S", time.localtime()))
        corpus, idlist = load_corpus(args, labels)
        print ("inference... " + time.strftime("%H:%M:%S", time.localtime()))
        inference(param, labels, corpus, idlist, trie, options)
        print ("finish... " + time.strftime("%H:%M:%S", time.localtime()))
        numpy.save(self.param, param)

    def detect(self, options, args):
        trie = self.load_da()
        param = numpy.load(self.param)
        labels = self.load_labels()

        log_likely = likelihood(param, labels, trie, args, options)

# from http://www.programming-magic.com/20080820002254/
reference_regex = re.compile(u'&(#x?[0-9a-f]+|[a-z]+);', re.IGNORECASE)
num16_regex = re.compile(u'#x\d+', re.IGNORECASE)
num10_regex = re.compile(u'#\d+', re.IGNORECASE)
def htmlentity2unicode(text):
    result = u''
    i = 0
    while True:
        match = reference_regex.search(text, i)
        if match is None:
            result += text[i:]
            break

        result += text[i:match.start()]
        i = match.end()
        name = match.group(1)

        if name in htmlentitydefs.name2codepoint.keys():
            result += unichr(htmlentitydefs.name2codepoint[name])
        elif num16_regex.match(name):
            result += unichr(int(u'0'+name[1:], 16))
        elif num10_regex.match(name):
            result += unichr(int(name[1:]))
    return result

def normalize_twitter(text):
    """normalization for twitter"""
    text = re.sub(r'(@|#|https?:\/\/)[^ ]+', '', text)
    text = re.sub(r'(^| )[:;x]-?[\(\)dop]($| )', ' ', text)  # facemark
    text = re.sub(r'(^| )(rt[ :]+)*', ' ', text)
    text = re.sub(r'([hj])+([aieo])+(\1+\2+){1,}', r'\1\2\1\2', text, re.IGNORECASE)  # laugh
    text = re.sub(r' +(via|live on) *$', '', text)
    return text

re_ignore_i = re.compile(r'[^Iİ]')
re_turkish_alphabet = re.compile(u'[\u011e\u011f\u0130\u0131]')
vietnamese_norm = {
	u'\u0041\u0300':u'\u00C0', u'\u0045\u0300':u'\u00C8', u'\u0049\u0300':u'\u00CC', u'\u004F\u0300':u'\u00D2',
	u'\u0055\u0300':u'\u00D9', u'\u0059\u0300':u'\u1EF2', u'\u0061\u0300':u'\u00E0', u'\u0065\u0300':u'\u00E8',
	u'\u0069\u0300':u'\u00EC', u'\u006F\u0300':u'\u00F2', u'\u0075\u0300':u'\u00F9', u'\u0079\u0300':u'\u1EF3',
	u'\u00C2\u0300':u'\u1EA6', u'\u00CA\u0300':u'\u1EC0', u'\u00D4\u0300':u'\u1ED2', u'\u00E2\u0300':u'\u1EA7',
	u'\u00EA\u0300':u'\u1EC1', u'\u00F4\u0300':u'\u1ED3', u'\u0102\u0300':u'\u1EB0', u'\u0103\u0300':u'\u1EB1',
	u'\u01A0\u0300':u'\u1EDC', u'\u01A1\u0300':u'\u1EDD', u'\u01AF\u0300':u'\u1EEA', u'\u01B0\u0300':u'\u1EEB',

	u'\u0041\u0301':u'\u00C1', u'\u0045\u0301':u'\u00C9', u'\u0049\u0301':u'\u00CD', u'\u004F\u0301':u'\u00D3',
	u'\u0055\u0301':u'\u00DA', u'\u0059\u0301':u'\u00DD', u'\u0061\u0301':u'\u00E1', u'\u0065\u0301':u'\u00E9',
	u'\u0069\u0301':u'\u00ED', u'\u006F\u0301':u'\u00F3', u'\u0075\u0301':u'\u00FA', u'\u0079\u0301':u'\u00FD',
	u'\u00C2\u0301':u'\u1EA4', u'\u00CA\u0301':u'\u1EBE', u'\u00D4\u0301':u'\u1ED0', u'\u00E2\u0301':u'\u1EA5',
	u'\u00EA\u0301':u'\u1EBF', u'\u00F4\u0301':u'\u1ED1', u'\u0102\u0301':u'\u1EAE', u'\u0103\u0301':u'\u1EAF',
	u'\u01A0\u0301':u'\u1EDA', u'\u01A1\u0301':u'\u1EDB', u'\u01AF\u0301':u'\u1EE8', u'\u01B0\u0301':u'\u1EE9',

	u'\u0041\u0303':u'\u00C3', u'\u0045\u0303':u'\u1EBC', u'\u0049\u0303':u'\u0128', u'\u004F\u0303':u'\u00D5',
	u'\u0055\u0303':u'\u0168', u'\u0059\u0303':u'\u1EF8', u'\u0061\u0303':u'\u00E3', u'\u0065\u0303':u'\u1EBD',
	u'\u0069\u0303':u'\u0129', u'\u006F\u0303':u'\u00F5', u'\u0075\u0303':u'\u0169', u'\u0079\u0303':u'\u1EF9',
	u'\u00C2\u0303':u'\u1EAA', u'\u00CA\u0303':u'\u1EC4', u'\u00D4\u0303':u'\u1ED6', u'\u00E2\u0303':u'\u1EAB',
	u'\u00EA\u0303':u'\u1EC5', u'\u00F4\u0303':u'\u1ED7', u'\u0102\u0303':u'\u1EB4', u'\u0103\u0303':u'\u1EB5',
	u'\u01A0\u0303':u'\u1EE0', u'\u01A1\u0303':u'\u1EE1', u'\u01AF\u0303':u'\u1EEE', u'\u01B0\u0303':u'\u1EEF',

	u'\u0041\u0309':u'\u1EA2', u'\u0045\u0309':u'\u1EBA', u'\u0049\u0309':u'\u1EC8', u'\u004F\u0309':u'\u1ECE',
	u'\u0055\u0309':u'\u1EE6', u'\u0059\u0309':u'\u1EF6', u'\u0061\u0309':u'\u1EA3', u'\u0065\u0309':u'\u1EBB',
	u'\u0069\u0309':u'\u1EC9', u'\u006F\u0309':u'\u1ECF', u'\u0075\u0309':u'\u1EE7', u'\u0079\u0309':u'\u1EF7',
	u'\u00C2\u0309':u'\u1EA8', u'\u00CA\u0309':u'\u1EC2', u'\u00D4\u0309':u'\u1ED4', u'\u00E2\u0309':u'\u1EA9',
	u'\u00EA\u0309':u'\u1EC3', u'\u00F4\u0309':u'\u1ED5', u'\u0102\u0309':u'\u1EB2', u'\u0103\u0309':u'\u1EB3',
	u'\u01A0\u0309':u'\u1EDE', u'\u01A1\u0309':u'\u1EDF', u'\u01AF\u0309':u'\u1EEC', u'\u01B0\u0309':u'\u1EED',

	u'\u0041\u0323':u'\u1EA0', u'\u0045\u0323':u'\u1EB8', u'\u0049\u0323':u'\u1ECA', u'\u004F\u0323':u'\u1ECC',
	u'\u0055\u0323':u'\u1EE4', u'\u0059\u0323':u'\u1EF4', u'\u0061\u0323':u'\u1EA1', u'\u0065\u0323':u'\u1EB9',
	u'\u0069\u0323':u'\u1ECB', u'\u006F\u0323':u'\u1ECD', u'\u0075\u0323':u'\u1EE5', u'\u0079\u0323':u'\u1EF5',
	u'\u00C2\u0323':u'\u1EAC', u'\u00CA\u0323':u'\u1EC6', u'\u00D4\u0323':u'\u1ED8', u'\u00E2\u0323':u'\u1EAD',
	u'\u00EA\u0323':u'\u1EC7', u'\u00F4\u0323':u'\u1ED9', u'\u0102\u0323':u'\u1EB6', u'\u0103\u0323':u'\u1EB7',
	u'\u01A0\u0323':u'\u1EE2', u'\u01A1\u0323':u'\u1EE3', u'\u01AF\u0323':u'\u1EF0', u'\u01B0\u0323':u'\u1EF1',
}
re_vietnamese = re.compile(u'[AEIOUYaeiouy\u00C2\u00CA\u00D4\u00E2\u00EA\u00F4\u0102\u0103\u01A0\u01A1\u01AF\u01B0][\u0300\u0301\u0303\u0309\u0323]')
re_latin_cont = re.compile(u'([a-z\u00e0-\u024f])\\1{2,}')
re_symbol_cont = re.compile(u'([^a-z\u00e0-\u024f])\\1{1,}')
def normalize_text(org):
    m = re.match(r'([-A-Za-z]+)\t(.+)', org)
    if m:
        label, org = m.groups()
    else:
        label = ""
    m = re.search(r'\t([^\t]+)$', org)
    if m:
        s = m.group(0)
    else:
        s = org
    s = htmlentity2unicode(s)
    s = re.sub(u'[\u2010-\u2015]', '-', s)
    s = re.sub(u'[0-9]+', '0', s)
    s = re.sub(u'[^\u0020-\u007e\u00a1-\u024f\u0300-\u036f\u0370-\u03FF\u0400-\u04ff\u1e00-\u1eff]+', ' ', s) # added greek (\u0370-\u03ff) and cyrillic (\u0400–\u04ff) alphabets
    s = re.sub(u'  +', ' ', s)

    # vietnamese normalization
    s = re_vietnamese.sub(lambda x:vietnamese_norm[x.group(0)], s)

    # lower case with Turkish
    s = re_ignore_i.sub(lambda x:x.group(0).lower(), s)
    s = s.replace(u'İ', u'\u0069')  # This is to manage this question : https://bugs.python.org/issue17252
    #if re_turkish_alphabet.search(s):
        #s = s.replace(u'I', u'\u0131')
    #s = s.lower()

    # Romanian normalization
    s = s.replace(u'\u0219', u'\u015f').replace(u'\u021b', u'\u0163')

    s = normalize_twitter(s)
    s = re_latin_cont.sub(r'\1\1', s)
    s = re_symbol_cont.sub(r'\1', s)

    return label, s.strip(), org

# load courpus
def load_corpus(filelist, labels):
    idlist = dict((x, []) for x in labels)
    corpus = []
    for filename in filelist:
        f = open(filename, 'r')
        for i, s in enumerate(f):
            label, text, org_text = normalize_text(s)
            if label not in labels:
                sys.exit("unknown label '%s' at %d in %s " % (label, i+1, filename))
            idlist[label].append(len(corpus))
            corpus.append((label, text, org_text))
        f.close()
    return corpus, idlist

def shuffle(idlist):
    n = max(len(idlist[lang]) for lang in idlist)
    list = []
    for lang in idlist:
        text_ids = idlist[lang]
        n_text = len(text_ids)
        list += text_ids * (n // n_text)
        numpy.random.shuffle(text_ids)
        list += text_ids[:n % n_text]
    numpy.random.shuffle(list)
    return list

# prediction probability
# this code comes from https://github.com/Nift/ldig/commit/75fc547dc1c3412d216fba83d759b38b414e528a in order to port it to Python 3
def predict(param, events):
    names = ['id', 'data']
    formats = ['f8', 'f8']
    dtype = dict(names=names, formats=formats)
    res1 = list(events.keys())
    arr = numpy.asarray(res1)  # numpy.fromiter(res1.iteritems(), dtype=dtype, count=len(res1))
    array = arr.astype(int)
    res2 = list(events.values())
    valuesA = numpy.asarray(res2)  # numpy.fromiter(res2.iteritems(), dtype=dtype, count=len(res2))

    sum_w = numpy.dot(param[array,].T, valuesA)
    exp_w = numpy.exp(sum_w - sum_w.max())
    return exp_w / exp_w.sum()

# inference and learning
def inference(param, labels, corpus, idlist, trie, options):
    K = len(labels)
    M = param.shape[0]

    list = shuffle(idlist)
    N = len(list)
    WHOLE_REG_INT = (N / options.n_whole_reg) + 1

    # learning rate
    eta = options.eta
    if options.reg_const:
        penalties = numpy.zeros_like(param)
        alpha = pow(0.9, -1.0 / N)
        uk = 0

    corrects = numpy.zeros(K, dtype=int)
    counts = numpy.zeros(K, dtype=int)
    for m, target in enumerate(list):
        label, text, org_text = corpus[target]
        events = trie.extract_features(u"\u0001" + text + u"\u0001")
        label_k = labels.index(label)

        y = predict(param, events)
        predict_k = y.argmax()
        counts[label_k] += 1
        if label_k == predict_k:
            corrects[label_k] += 1

        # learning
        if options.reg_const:
            eta *= alpha
            uk += options.reg_const * eta / N
        y[label_k] -= 1
        y *= eta

        if options.reg_const:
            indexes = events
            if (N - m) % WHOLE_REG_INT == 1:
                print ("full regularization: %d / %d" % (m, N))
                indexes = range(M)
            for id in indexes:
                prm = param[id]
                pnl = penalties[id]
                if id in events: prm -= y * events[id]

                for j in range(K):
                    w = prm[j]
                    if w > 0:
                        w1 = w - uk - pnl[j]
                        if w1 > 0:
                            prm[j] = w1
                            pnl[j] += w1 - w
                        else:
                            prm[j] = 0
                            pnl[j] -= w
                    elif w < 0:
                        w1 = w + uk - pnl[j]
                        if w1 < 0:
                            prm[j] = w1
                            pnl[j] += w1 - w
                        else:
                            prm[j] = 0
                            pnl[j] -= w
        else:
            for id, freq in events.items():
                param[id,] -= y * freq

    for lbl, crct, cnt in zip(labels, corrects, counts):
        if cnt > 0:
            print (">    %s = %d / %d = %.2f" % (lbl, crct, cnt, 100.0 * crct / cnt))
    print ("> total = %d / %d = %.2f" % (corrects.sum(), N, 100.0 * corrects.sum() / N))
    list = (numpy.abs(param).sum(1) > 0.0000001)
    print ("> # of relevant features = %d / %d" % (list.sum(), M))

def likelihood(param, labels, trie, filelist, options):
    K = len(labels)
    corrects = numpy.zeros(K, dtype=int)
    counts = numpy.zeros(K, dtype=int)

    label_map = dict((x, i) for i, x in enumerate(labels))

    n_available_data = 0
    log_likely = 0.0
    for filename in filelist:
        f = open(filename, 'r')
        for i, s in enumerate(f):
            label, text, org_text = normalize_text(s)

            if label not in label_map:
                sys.stderr.write("WARNING : unknown label '%s' at %d in %s (ignore the later same labels)\n" % (label, i+1, filename))
                label_map[label] = -1
            label_k = label_map[label]

            events = trie.extract_features(u"\u0001" + text + u"\u0001")
            y = predict(param, events)
            predict_k = y.argmax()

            if label_k >= 0:
                if y[label_k]==0: # lkevers: added in order to avoid warning because of log(0) below
                    y[label_k]=0.0000000001
                log_likely -= numpy.log(y[label_k])
                n_available_data += 1
                counts[label_k] += 1
                if label_k == predict_k and y[predict_k] >= 0.6:
                    corrects[predict_k] += 1

            predict_lang = labels[predict_k]
            if y[predict_k] < 0.6: predict_lang = ""
            print ("%s\t%s\t%s" % (label, predict_lang, org_text))
        f.close()

    if n_available_data > 0:
        log_likely /= n_available_data

        for lbl, crct, cnt in zip(labels, corrects, counts):
            if cnt > 0:
                print (">    %s = %d / %d = %.2f" % (lbl, crct, cnt, 100.0 * crct / cnt))
        print ("> total = %d / %d = %.2f" % (corrects.sum(), n_available_data, 100.0 * corrects.sum() / n_available_data))
        print ("> average negative log likelihood = %.3f" % log_likely)

    return log_likely

def generate_doublearray(file, features):
    trie = da.DoubleArray()
    trie.initialize(features)
    trie.save(file)

if __name__ == '__main__':
    #sys.stdout = codecs.getwriter('utf-8')(sys.stdout)

    parser = optparse.OptionParser()
    parser.add_option("-m", dest="model", help="model directory")
    parser.add_option("--init", dest="init", help="initialize model", action="store_true")
    parser.add_option("--learning", dest="learning", help="learn model", action="store_true")
    parser.add_option("--shrink", dest="shrink", help="remove irrevant features", action="store_true")
    parser.add_option("--debug", dest="debug", help="detect command line text for debug", action="store_true")

    # for initialization
    parser.add_option("--ff", dest="bound_feature_freq", help="threshold of feature frequency (for initialization)", type="int", default=8)
    parser.add_option("-n", dest="ngram_bound", help="n-gram upper bound (for initialization)", type="int", default=99999)
    parser.add_option("-x", dest="maxsubst", help="max substring extractor", default="./maxsubst")

    # for learning
    parser.add_option("-e", "--eta", dest="eta", help="learning rate", type="float", default=0.1)
    parser.add_option("-r", "--regularity", dest="reg_const", help="regularization constant", type="float")
    parser.add_option("--wr", dest="n_whole_reg", help="number of whole regularizations", type="int", default=2)

    (options, args) = parser.parse_args()
    if not options.model: parser.error("need model directory (-m)")

    detector = ldig(options.model)
    if options.init:
        if not os.path.exists(options.model):
            os.mkdir(options.model)
        if len(args) == 0:
            parser.error("need corpus")
    else:
        if not os.path.exists(detector.features):
            parser.error("features file doesn't exist")
        if not os.path.exists(detector.labels):
            parser.error("labels file doesn't exist")
        if not os.path.exists(detector.param):
            parser.error("parameters file doesn't exist")

    if options.init:
        temp_path = os.path.join(options.model, 'temp')
        detector.init(temp_path, args, options.bound_feature_freq, options.ngram_bound)

    elif options.debug:
        detector.debug(args)

    elif options.shrink:
        detector.shrink()

    elif options.learning:
        detector.learn(options, args)

    else:
        detector.detect(options, args)
        #import cProfile
        #cProfile.runctx('detector.detect(options, args)', globals(), locals(), 'ldig.profile')


# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Modified from espnet(https://github.com/espnet/espnet)
import math
from typing import Any
from typing import Dict
from typing import List
from typing import Optional

import numpy as np
import paddle
from paddle import nn

from paddlespeech.t2s.modules.activation import get_activation
from paddlespeech.t2s.modules.nets_utils import initialize
from paddlespeech.t2s.modules.residual_block import WaveNetResidualBlock as ResidualBlock
from paddlespeech.t2s.modules.upsample import ConvInUpsampleNet

class PWGGenerator(nn.Layer):
    """Wave Generator for Parallel WaveGAN

    Parameters
    ----------
    in_channels : int, optional
        Number of channels of the input waveform, by default 1
    out_channels : int, optional
        Number of channels of the output waveform, by default 1
    kernel_size : int, optional
        Kernel size of the residual blocks inside, by default 3
    layers : int, optional
        Number of residual blocks inside, by default 30
    stacks : int, optional
        The number of groups to split the residual blocks into, by default 3
        Within each group, the dilation of the residual block grows
        exponentially.
    residual_channels : int, optional
        Residual channel of the residual blocks, by default 64
    gate_channels : int, optional
        Gate channel of the residual blocks, by default 128
    skip_channels : int, optional
        Skip channel of the residual blocks, by default 64
    aux_channels : int, optional
        Auxiliary channel of the residual blocks, by default 80
    aux_context_window : int, optional
        The context window size of the first convolution applied to the
        auxiliary input, by default 2
    dropout : float, optional
        Dropout of the residual blocks, by default 0.
    bias : bool, optional
        Whether to use bias in residual blocks, by default True
    use_weight_norm : bool, optional
        Whether to use weight norm in all convolutions, by default True
    use_causal_conv : bool, optional
        Whether to use causal padding in the upsample network and residual
        blocks, by default False
    upsample_scales : List[int], optional
        Upsample scales of the upsample network, by default [4, 4, 4, 4]
    nonlinear_activation : Optional[str], optional
        Non linear activation in upsample network, by default None
    nonlinear_activation_params : Dict[str, Any], optional
        Parameters passed to the linear activation in the upsample network,
        by default {}
    interpolate_mode : str, optional
        Interpolation mode of the upsample network, by default "nearest"
    freq_axis_kernel_size : int, optional
        Kernel size along the frequency axis of the upsample network, by default 1
    """

    def __init__(
            self,
            in_channels: int=1,
            out_channels: int=1,
            kernel_size: int=3,
            layers: int=30,
            stacks: int=3,
            residual_channels: int=64,
            gate_channels: int=128,
            skip_channels: int=64,
            aux_channels: int=80,
            aux_context_window: int=2,
            dropout: float=0.,
            bias: bool=True,
            use_weight_norm: bool=True,
            use_causal_conv: bool=False,
            upsample_scales: List[int]=[4, 4, 4, 4],
            nonlinear_activation: Optional[str]=None,
            nonlinear_activation_params: Dict[str, Any]={},
            interpolate_mode: str="nearest",
            freq_axis_kernel_size: int=1,
            init_type: str="xavier_uniform", ):
        super().__init__()

        # initialize parameters
        initialize(self, init_type)

        # for compatibility
        if nonlinear_activation:
            nonlinear_activation = nonlinear_activation.lower()

        self.in_channels = in_channels
        self.out_channels = out_channels
        self.aux_channels = aux_channels
        self.aux_context_window = aux_context_window
        self.layers = layers
        self.stacks = stacks
        self.kernel_size = kernel_size

        assert layers % stacks == 0
        layers_per_stack = layers // stacks

        self.first_conv = nn.Conv1D(
            in_channels, residual_channels, 1, bias_attr=True)
        self.upsample_net = ConvInUpsampleNet(
            upsample_scales=upsample_scales,
            nonlinear_activation=nonlinear_activation,
            nonlinear_activation_params=nonlinear_activation_params,
            interpolate_mode=interpolate_mode,
            freq_axis_kernel_size=freq_axis_kernel_size,
            aux_channels=aux_channels,
            aux_context_window=aux_context_window,
            use_causal_conv=use_causal_conv)
        self.upsample_factor = np.prod(upsample_scales)

        self.conv_layers = nn.LayerList()
        for layer in range(layers):
            dilation = 2**(layer % layers_per_stack)
            conv = ResidualBlock(
                kernel_size=kernel_size,
                residual_channels=residual_channels,
                gate_channels=gate_channels,
                skip_channels=skip_channels,
                aux_channels=aux_channels,
                dilation=dilation,
                dropout=dropout,
                bias=bias,
                use_causal_conv=use_causal_conv)
            self.conv_layers.append(conv)

        self.last_conv_layers = nn.Sequential(nn.ReLU(),
                                              nn.Conv1D(
                                                  skip_channels,
                                                  skip_channels,
                                                  1,
                                                  bias_attr=True),
                                              nn.ReLU(),
                                              nn.Conv1D(
                                                  skip_channels,
                                                  out_channels,
                                                  1,
                                                  bias_attr=True))

        if use_weight_norm:
            self.apply_weight_norm()

    def forward(self, x, c):
        """Generate waveform.

        Parameters
        ----------
        x : Tensor
            Shape (N, C_in, T), The input waveform.
        c : Tensor
            Shape (N, C_aux, T'). The auxiliary input (e.g. spectrogram). It
            is upsampled to match the time resolution of the input.

        Returns
        -------
        Tensor
            Shape (N, C_out, T), the generated waveform.
        """
        c = self.upsample_net(c)
        assert c.shape[-1] == x.shape[-1]

        x = self.first_conv(x)
        skips = 0
        for f in self.conv_layers:
            x, s = f(x, c)
            skips += s
        skips *= math.sqrt(1.0 / len(self.conv_layers))

        x = self.last_conv_layers(skips)
        return x

    def apply_weight_norm(self):
        """Recursively apply weight normalization to all the Convolution layers
        in the sublayers.
        """

        def _apply_weight_norm(layer):
            if isinstance(layer, (nn.Conv1D, nn.Conv2D)):
                nn.utils.weight_norm(layer)

        self.apply(_apply_weight_norm)

    def remove_weight_norm(self):
        """Recursively remove weight normalization from all the Convolution
        layers in the sublayers.
        """

        def _remove_weight_norm(layer):
            try:
                nn.utils.remove_weight_norm(layer)
            except ValueError:
                pass

        self.apply(_remove_weight_norm)

    def inference(self, c=None):
        """Waveform generation. This function is used for single instance
        inference.
        Parameters
        ----------
        c : Tensor, optional
            Shape (T', C_aux), the auxiliary input, by default None
        x : Tensor, optional
            Shape (T, C_in), the noise waveform, by default None
            If not provided, a sample is drawn from a gaussian distribution.
        Returns
        -------
        Tensor
            Shape (T, C_out), the generated waveform
        """
        # when to static, can not input x, see https://github.com/PaddlePaddle/Parakeet/pull/132/files
        x = paddle.randn(
            [1, self.in_channels, paddle.shape(c)[0] * self.upsample_factor])
        c = paddle.transpose(c, [1, 0]).unsqueeze(0)  # pseudo batch
        c = nn.Pad1D(self.aux_context_window, mode='replicate')(c)
        out = self(x, c).squeeze(0).transpose([1, 0])
        return out

class PWGDiscriminator(nn.Layer):
    """A convolutional discriminator for audio.

    Parameters
    ----------
    in_channels : int, optional
        Number of channels of the input audio, by default 1
    out_channels : int, optional
        Output feature size, by default 1
    kernel_size : int, optional
        Kernel size of convolutional sublayers, by default 3
    layers : int, optional
        Number of layers, by default 10
    conv_channels : int, optional
        Feature size of the convolutional sublayers, by default 64
    dilation_factor : int, optional
        The factor with which dilation of each convolutional sublayers grows
        exponentially if it is greater than 1, else the dilation of each
        convolutional sublayers grows linearly, by default 1
    nonlinear_activation : str, optional
        The activation after each convolutional sublayer, by default "leakyrelu"
    nonlinear_activation_params : Dict[str, Any], optional
        The parameters passed to the activation's initializer, by default
        {"negative_slope": 0.2}
    bias : bool, optional
        Whether to use bias in convolutional sublayers, by default True
    use_weight_norm : bool, optional
        Whether to use weight normalization at all convolutional sublayers,
        by default True
    """

    def __init__(
            self,
            in_channels: int=1,
            out_channels: int=1,
            kernel_size: int=3,
            layers: int=10,
            conv_channels: int=64,
            dilation_factor: int=1,
            nonlinear_activation: str="leakyrelu",
            nonlinear_activation_params: Dict[str, Any]={"negative_slope": 0.2},
            bias: bool=True,
            use_weight_norm: bool=True,
            init_type: str="xavier_uniform", ):
        super().__init__()

        # initialize parameters
        initialize(self, init_type)
        # for compatibility
        if nonlinear_activation:
            nonlinear_activation = nonlinear_activation.lower()

        assert kernel_size % 2 == 1
        assert dilation_factor > 0
        conv_layers = []
        conv_in_channels = in_channels
        for i in range(layers - 1):
            if i == 0:
                dilation = 1
            else:
                dilation = i if dilation_factor == 1 else dilation_factor**i
                conv_in_channels = conv_channels
            padding = (kernel_size - 1) // 2 * dilation
            conv_layer = nn.Conv1D(
                conv_in_channels,
                conv_channels,
                kernel_size,
                padding=padding,
                dilation=dilation,
                bias_attr=bias)
            nonlinear = get_activation(nonlinear_activation,
                                       **nonlinear_activation_params)
            conv_layers.append(conv_layer)
            conv_layers.append(nonlinear)
        padding = (kernel_size - 1) // 2
        last_conv = nn.Conv1D(
            conv_in_channels,
            out_channels,
            kernel_size,
            padding=padding,
            bias_attr=bias)
        conv_layers.append(last_conv)
        self.conv_layers = nn.Sequential(*conv_layers)

        if use_weight_norm:
            self.apply_weight_norm()

    def forward(self, x):
        """
        Parameters
        ----------
        x : Tensor
            Shape (N, in_channels, num_samples), the input audio.

        Returns
        -------
        Tensor
            Shape (N, out_channels, num_samples), the predicted logits.
        """
        return self.conv_layers(x)

    def apply_weight_norm(self):
        def _apply_weight_norm(layer):
            if isinstance(layer, (nn.Conv1D, nn.Conv2D)):
                nn.utils.weight_norm(layer)

        self.apply(_apply_weight_norm)

    def remove_weight_norm(self):
        def _remove_weight_norm(layer):
            try:
                nn.utils.remove_weight_norm(layer)
            except ValueError:
                pass

        self.apply(_remove_weight_norm)

class ResidualPWGDiscriminator(nn.Layer):
    """A wavenet-style discriminator for audio.

    Parameters
    ----------
    in_channels : int, optional
        Number of channels of the input audio, by default 1
    out_channels : int, optional
        Output feature size, by default 1
    kernel_size : int, optional
        Kernel size of residual blocks, by default 3
    layers : int, optional
        Number of residual blocks, by default 30
    stacks : int, optional
        Number of groups of residual blocks, within which the dilation
        of each residual blocks grows exponentially, by default 3
    residual_channels : int, optional
        Residual channels of residual blocks, by default 64
    gate_channels : int, optional
        Gate channels of residual blocks, by default 128
    skip_channels : int, optional
        Skip channels of residual blocks, by default 64
    dropout : float, optional
        Dropout probability of residual blocks, by default 0.
    bias : bool, optional
        Whether to use bias in residual blocks, by default True
    use_weight_norm : bool, optional
        Whether to use weight normalization in all convolutional layers,
        by default True
    use_causal_conv : bool, optional
        Whether to use causal convolution in residual blocks, by default False
    nonlinear_activation : str, optional
        Activation after convolutions other than those in residual blocks,
        by default "leakyrelu"
    nonlinear_activation_params : Dict[str, Any], optional
        Parameters to pass to the activation, by default {"negative_slope": 0.2}
    """

    def __init__(
            self,
            in_channels: int=1,
            out_channels: int=1,
            kernel_size: int=3,
            layers: int=30,
            stacks: int=3,
            residual_channels: int=64,
            gate_channels: int=128,
            skip_channels: int=64,
            dropout: float=0.,
            bias: bool=True,
            use_weight_norm: bool=True,
            use_causal_conv: bool=False,
            nonlinear_activation: str="leakyrelu",
            nonlinear_activation_params: Dict[str, Any]={"negative_slope": 0.2},
            init_type: str="xavier_uniform", ):
        super().__init__()

        # initialize parameters
        initialize(self, init_type)

        # for compatibility
        if nonlinear_activation:
            nonlinear_activation = nonlinear_activation.lower()

        assert kernel_size % 2 == 1
        self.in_channels = in_channels
        self.out_channels = out_channels
        self.layers = layers
        self.stacks = stacks
        self.kernel_size = kernel_size

        assert layers % stacks == 0
        layers_per_stack = layers // stacks

        self.first_conv = nn.Sequential(
            nn.Conv1D(in_channels, residual_channels, 1, bias_attr=True),
            get_activation(nonlinear_activation, **nonlinear_activation_params))

        self.conv_layers = nn.LayerList()
        for layer in range(layers):
            dilation = 2**(layer % layers_per_stack)
            conv = ResidualBlock(
                kernel_size=kernel_size,
                residual_channels=residual_channels,
                gate_channels=gate_channels,
                skip_channels=skip_channels,
                aux_channels=None,  # no auxiliary input
                dropout=dropout,
                dilation=dilation,
                bias=bias,
                use_causal_conv=use_causal_conv)
            self.conv_layers.append(conv)

        self.last_conv_layers = nn.Sequential(
            get_activation(nonlinear_activation, **nonlinear_activation_params),
            nn.Conv1D(skip_channels, skip_channels, 1, bias_attr=True),
            get_activation(nonlinear_activation, **nonlinear_activation_params),
            nn.Conv1D(skip_channels, out_channels, 1, bias_attr=True))

        if use_weight_norm:
            self.apply_weight_norm()

    def forward(self, x):
        """
        Parameters
        ----------
        x : Tensor
            Shape (N, in_channels, num_samples), the input audio.

        Returns
        -------
        Tensor
            Shape (N, out_channels, num_samples), the predicted logits.
        """
        x = self.first_conv(x)
        skip = 0
        for f in self.conv_layers:
            x, h = f(x, None)
            skip += h
        skip *= math.sqrt(1 / len(self.conv_layers))

        x = skip
        x = self.last_conv_layers(x)
        return x

    def apply_weight_norm(self):
        def _apply_weight_norm(layer):
            if isinstance(layer, (nn.Conv1D, nn.Conv2D)):
                nn.utils.weight_norm(layer)

        self.apply(_apply_weight_norm)

    def remove_weight_norm(self):
        def _remove_weight_norm(layer):
            try:
                nn.utils.remove_weight_norm(layer)
            except ValueError:
                pass

        self.apply(_remove_weight_norm)

class PWGInference(nn.Layer):
    def __init__(self, normalizer, pwg_generator):
        super().__init__()
        self.normalizer = normalizer
        self.pwg_generator = pwg_generator

    def forward(self, logmel):
        normalized_mel = self.normalizer(logmel)
        wav = self.pwg_generator.inference(normalized_mel)
        return wav


import numpy as np
from keras.models import load_model
import pandas as pd
import glob
from config import TestConfig

config = TestConfig()

data_path = config.data_path
data_name = config.data_name
val_part = config.val_part
model_path = config.model_path

test_path = "{}/X_test_round2_{}.npy".format(data_path, data_name)
test_path_phase1 = "{}/X_test_round1_{}.npy".format(data_path, data_name)
out_name = model_path.replace("checkpoints", "submissions").replace("hdf5", "txt")
out_name_phase1 = model_path.replace("checkpoints", "submissions").replace(".hdf5", "_phase1.txt")

print "Loading model..."
model = load_model(config.model_path)

print "Loading training data mean..."
X_train_mean = np.load(config.X_train_mean_path)

print "PHASE 2"
print "Reading test data..."
X_test = np.load(test_path)
X_test = X_test.astype('float32')
X_test -= X_train_mean
X_test /= 255.0

print "Predicting..."
preds = model.predict(X_test)
preds = preds[:, 0]

dummy_preds = np.repeat(config.angle_train_mean, config.num_channels)
preds = np.concatenate((dummy_preds, preds))

# join predictions with frame_ids
filenames = glob.glob("{}/round2/test/center/*.jpg".format(data_path))
filenames = sorted(filenames)
frame_ids = [f.replace(".jpg", "").replace("{}/round2/test/center/".format(data_path), "") for f in filenames]

print "Writing predictions..."
pd.DataFrame({"frame_id": frame_ids, "steering_angle": preds}).to_csv(out_name, index=False, header=True)

print "PHASE 1"
print "Reading phase1 test data..."
X_test = np.load(test_path_phase1)
X_test = X_test.astype('float32')
X_test -= X_train_mean
X_test /= 255.0

print "Predicting..."
preds = model.predict(X_test)
preds = preds[:, 0]

dummy_preds = np.repeat(config.angle_train_mean, config.num_channels)
preds = np.concatenate((dummy_preds, preds))

# join predictions with frame_ids
filenames = glob.glob("{}/round1/test/center/*.png".format(data_path))
filenames = sorted(filenames)
frame_ids = [f.replace(".png", "").replace("{}/round1/test/center/".format(data_path), "") for f in filenames]

print "Writing predictions..."
pd.DataFrame({"frame_id": frame_ids, "steering_angle": preds}).to_csv(out_name_phase1, index=False, header=True)

print "Done!"


import argparse
import logging
import time
import math
import cv2
import numpy as np
import click
from tf_pose.estimator import TfPoseEstimator
from tf_pose.networks import get_graph_path, model_wh

from tf_pose.estimator import initialize_variables
#import matplotlib
#matplotlib.use('Agg')
#import matplotlib.pyplot as plt
import requests
import json
import config
from sklearn import metrics
import random
from random import randint
import os
import tensorflow as tf
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

logger = logging.getLogger('TfPoseEstimator-WebCam')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

fps_time = 0
url = config.serverPath
featureName = config.featureName
headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}

@click.command()
@click.argument('streamingurl')
@click.argument('camid')
@click.argument('devicename')

def main(streamingurl, camid, devicename):
    resize='0x0'
    resize_out_ratio=4.0
    model = 'mobilenet_thin'
    show_process=False
    fps_time = 0
    (sess, accuracy, pred, optimizer) = initialize_variables()
    #print(pred)

    #logger.debug('initialization %s : %s' % (args.model, get_graph_path(args.model)))
    w, h = model_wh(resize)
    if w > 0 and h > 0:
        e = TfPoseEstimator(get_graph_path(model), target_size=(w, h))
    else:
        e = TfPoseEstimator(get_graph_path(model), target_size=(432, 368))
    if streamingurl.find("webcam") == 0:
        streamingurl = int(streamingurl[-1:])
    cam = cv2.VideoCapture(streamingurl)
    print("@@@@@@",streamingurl)
    ret_val, image = cam.read()
    frameRate = cam.get(5)
    print(frameRate)
    frameRate=math.floor(frameRate/8)
    #frameRate = 100

    frameId = 0
    frame_number=0
    sequence_arr=[]
    while True:
        ret_val, image = cam.read()
        if ret_val == True:
            if(frameId % frameRate ==0):
                #logger.debug('image process+')
                humans = e.inference(image, resize_to_default=(w > 0 and h > 0), upsample_size=resize_out_ratio)

                #logger.debug('postprocess+')
                sequence_arr,image, label = TfPoseEstimator.draw_humans(image, humans, frame_number,sequence_arr,imgcopy=False)

                try:
                    if label =='':
                        label="No Activity Detected"
                    #logger.debug("########", camid, devicename)
                    r = requests.post(url, data=json.dumps({"featureName":featureName,"label":label, "camId": camid, "deviceName": devicename}), headers=headers)
                except:
                    print("Check connection with the node server")

                frame_number= frame_number+1
                print("#",label)

                #logger.debug('show+')
                cv2.putText(image,
                    "FPS: %f" % (1.0 / (time.time() - fps_time)),
                    (10, 10),  cv2.FONT_HERSHEY_SIMPLEX, 0.5,
                    (0, 255, 0), 2)
                cv2.imshow('tf-pose-estimation result', image)
                fps_time = time.time()
                if cv2.waitKey(1) == 27:
                    break
                #logger.debug('finished+')
            frameId = frameId + 1
        else:
            cam = cv2.VideoCapture(streamingurl)

    cv2.destroyAllWindows()

if __name__ == '__main__':
     print("___________in wecam run__________")

     main()


from torch import nn
import torch
import numpy as np

class SemiCRFShiftRelay(nn.Module):
    """
    该模块是一个decoder，但当前不支持含有tag的decode。

    """
    def __init__(self, L):
        """

        :param L: 不包含relay的长度
        """
        if L<2:
            raise RuntimeError()
        super().__init__()
        self.L = L

    def forward(self, logits, relay_logits, relay_target, relay_mask, end_seg_mask, seq_len):
        """
        relay node是接下来L个字都不是它的结束。relay的状态是往后滑动1个位置

        :param logits: batch_size x max_len x L, 当前位置往左边L个segment的分数，最后一维的0是长度为1的segment(即本身)
        :param relay_logits: batch_size x max_len, 当前位置是接下来L-1个位置都不是终点的分数
        :param relay_target: batch_size x max_len 每个位置他的segment在哪里开始的。如果超过L，则一直保持为L-1。比如长度为
            5的词，L=3, [0, 1, 2, 2, 2]
        :param relay_mask: batch_size x max_len, 在需要relay的地方为1, 长度为5的词, L=3时，为[1, 1, 1, 0, 0]
        :param end_seg_mask: batch_size x max_len, segment结束的地方为1。
        :param seq_len: batch_size, 句子的长度
        :return: loss: batch_size,
        """
        batch_size, max_len, L = logits.size()

        # 当前时刻为relay node的分数是多少
        relay_scores = logits.new_zeros(batch_size, max_len)
        # 当前时刻结束的分数是多少
        scores = logits.new_zeros(batch_size, max_len+1)
        # golden的分数
        gold_scores = relay_logits[:, 0].masked_fill(relay_mask[:, 0].eq(0), 0) + \
                                logits[:, 0, 0].masked_fill(end_seg_mask[:, 0].eq(0), 0)
        # 初始化
        scores[:, 1] = logits[:, 0, 0]
        batch_i = torch.arange(batch_size).to(logits.device).long()
        relay_scores[:, 0] = relay_logits[:, 0]
        last_relay_index = max_len - self.L
        for t in range(1, max_len):
            real_L = min(t+1, L)
            flip_logits_t = logits[:, t, :real_L].flip(dims=[1])  # flip之后低0个位置为real_L-1的segment
            # 计算relay_scores的更新
            if t<last_relay_index:
                #   (1) 从正常位置跳转
                tmp1 = relay_logits[:, t] + scores[:, t] # batch_size
                #   (2) 从relay跳转
                tmp2 = relay_logits[:, t] + relay_scores[:, t-1] # batch_size
                tmp1 = torch.stack([tmp1, tmp2], dim=0)
                relay_scores[:, t] = torch.logsumexp(tmp1, dim=0)
            # 计算scores的更新
            #  (1)从之前的位置跳转过来的
            tmp1 = scores[:, t-real_L+1:t+1] + flip_logits_t  # batch_size x L
            if t>self.L-1:
                #  (2)从relay跳转过来的
                tmp2 = relay_scores[:, t-self.L] # batch_size
                tmp2 = tmp2 + flip_logits_t[:, 0] # batch_size
                tmp1 = torch.cat([tmp1, tmp2.unsqueeze(-1)], dim=-1)
            scores[:, t+1] = torch.logsumexp(tmp1, dim=-1) # 更新当前时刻的分数

            # 计算golden
            seg_i = relay_target[:, t] # batch_size
            gold_segment_scores = logits[:, t][(batch_i, seg_i)].masked_fill(end_seg_mask[:, t].eq(0), 0) # batch_size, 后向从0到L长度的segment的分数
            relay_score = relay_logits[:, t].masked_fill(relay_mask[:, t].eq(0), 0)
            gold_scores = gold_scores + relay_score + gold_segment_scores
        all_scores = scores.gather(dim=1, index=seq_len.unsqueeze(1)).squeeze(1) # batch_size
        return all_scores - gold_scores

    def predict(self, logits, relay_logits, seq_len):
        """
        relay node是接下来L个字都不是它的结束。relay的状态是往后滑动L-1个位置

        :param logits: batch_size x max_len x L, 当前位置左边L个segment的分数，最后一维的0是长度为1的segment(即本身)
        :param relay_logits: batch_size x max_len, 当前位置是接下来L-1个位置都不是终点的分数
        :param seq_len: batch_size, 句子的长度
        :return: pred: batch_size x max_len以该点开始的segment的(长度-1); pred_mask为1的地方预测有segment开始
        """
        batch_size, max_len, L = logits.size()
        # 当前时刻为relay node的分数是多少
        max_relay_scores = logits.new_zeros(batch_size, max_len)
        relay_bt = seq_len.new_zeros(batch_size, max_len)  # 当前结果是否来自于relay的结果
        # 当前时刻结束的分数是多少
        max_scores = logits.new_zeros(batch_size, max_len+1)
        bt = seq_len.new_zeros(batch_size, max_len)
        # 初始化
        max_scores[:, 1] = logits[:, 0, 0]
        max_relay_scores[:, 0] = relay_logits[:, 0]
        last_relay_index = max_len - self.L
        for t in range(1, max_len):
            real_L = min(t+1, L)
            flip_logits_t = logits[:, t, :real_L].flip(dims=[1])  # flip之后低0个位置为real_L-1的segment
            # 计算relay_scores的更新
            if t<last_relay_index:
                #  (1) 从正常位置跳转
                tmp1 = relay_logits[:, t] + max_scores[:, t]
                #   (2) 从relay跳转
                tmp2 = relay_logits[:, t] + max_relay_scores[:, t-1] # batch_size
                # 每个sample的倒数L位不能是relay了
                tmp2 = tmp2.masked_fill(seq_len.le(t+L), float('-inf'))
                mask_i = tmp1.lt(tmp2) # 为1的位置为relay跳转
                relay_bt[:, t].masked_fill_(mask_i, 1)
                max_relay_scores[:, t] = torch.max(tmp1, tmp2)

            # 计算scores的更新
            #  (1)从之前的位置跳转过来的
            tmp1 = max_scores[:, t-real_L+1:t+1] + flip_logits_t  # batch_size x L
            tmp1 = tmp1.flip(dims=[1]) # 0的位置代表长度为1的segment
            if self.L-1<t:
                #  (2)从relay跳转过来的
                tmp2 = max_relay_scores[:, t-self.L] # batch_size
                tmp2 = tmp2 + flip_logits_t[:, 0]
                tmp1 = torch.cat([tmp1, tmp2.unsqueeze(-1)], dim=-1)
            # 看哪个更大
            max_score, pt = torch.max(tmp1, dim=1)
            max_scores[:, t+1] = max_score
            # mask_i = pt.ge(self.L)
            bt[:, t] = pt # 假设L=3, 那么对于0,1,2,3分别代表的是[t, t], [t-1, t], [t-2, t], [t-self.L(relay), t]
        # 需要把结果decode出来
        pred = np.zeros((batch_size, max_len), dtype=int)
        pred_mask = np.zeros((batch_size, max_len), dtype=int)
        seq_len = seq_len.tolist()
        bt = bt.tolist()
        relay_bt = relay_bt.tolist()
        for b in range(batch_size):
            seq_len_i = seq_len[b]
            bt_i = bt[b][:seq_len_i]
            relay_bt_i = relay_bt[b][:seq_len_i]
            j = seq_len_i - 1
            assert relay_bt_i[j]!=1
            while j>-1:
                if bt_i[j]==self.L:
                    seg_start_pos = j
                    j = j-self.L
                    while relay_bt_i[j]!=0 and j>-1:
                        j = j - 1
                    pred[b, j] = seg_start_pos - j
                    pred_mask[b, j] = 1
                else:
                    length = bt_i[j]
                    j = j - bt_i[j]
                    pred_mask[b, j] = 1
                    pred[b, j] = length
                j = j - 1

        return torch.LongTensor(pred).to(logits.device), torch.LongTensor(pred_mask).to(logits.device)

class FeatureFunMax(nn.Module):
    def __init__(self, hidden_size:int, L:int):
        """
        用于计算semi-CRF特征的函数。给定batch_size x max_len x hidden_size形状的输入，输出为batch_size x max_len x L的
        分数，以及batch_size x max_len的relay的分数。两者的区别参考论文 TODO 补充

        :param hidden_size: 输入特征的维度大小
        :param L: 不包含relay node的segment的长度大小。
        """
        super().__init__()

        self.end_fc = nn.Linear(hidden_size, 1, bias=False)
        self.whole_w = nn.Parameter(torch.randn(L, hidden_size))
        self.relay_fc = nn.Linear(hidden_size, 1)
        self.length_bias = nn.Parameter(torch.randn(L))
        self.L = L
    def forward(self, logits):
        """

        :param logits: batch_size x max_len x hidden_size
        :return: batch_size x max_len x L # 最后一维为左边segment的分数，0处为长度为1的segment
                 batch_size x max_len, # 当前位置是接下来L-1个位置都不是终点的分数

        """
        batch_size, max_len, hidden_size = logits.size()
        # start_scores = self.start_fc(logits) # batch_size x max_len x 1 # 每个位置作为start的分数
        tmp = logits.new_zeros(batch_size, max_len+self.L-1, hidden_size)
        tmp[:, -max_len:] = logits
        # batch_size x max_len x hidden_size x (self.L) -> batch_size x max_len x (self.L) x hidden_size
        start_logits = tmp.unfold(dimension=1, size=self.L, step=1).transpose(2, 3).flip(dims=[2])
        end_scores = self.end_fc(logits) # batch_size x max_len x 1
        # 计算relay的特征
        relay_tmp = logits.new_zeros(batch_size, max_len, hidden_size)
        relay_tmp[:, :-self.L] = logits[:, self.L:]
        # batch_size x max_len x hidden_size
        relay_logits_max = torch.max(relay_tmp, logits) # end - start
        logits_max = torch.max(logits.unsqueeze(2), start_logits) # batch_size x max_len x L x hidden_size
        whole_scores = (logits_max*self.whole_w).sum(dim=-1) # batch_size x max_len x self.L
        # whole_scores = self.whole_fc().squeeze(-1) # bz x max_len x self.L
        # batch_size x max_len
        relay_scores = self.relay_fc(relay_logits_max).squeeze(-1)
        return whole_scores+end_scores+self.length_bias.view(1, 1, -1), relay_scores


import os
import logging
from PIL import Image
from model import FaceCascade, FindFaces, Blur, yolo_v2_model, Blur_yolo
from load_data import load_pictures_haar, load_pictures_yolo
import matplotlib.pyplot as plt
from optparse import OptionParser
import cv2
import numpy as np
from utils_yolo_v2 import WeightReader, decode_netout, draw_boxes

logging.basicConfig(level=logging.INFO)
#logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger("logger-app.py")

parser = OptionParser()
parser.allow_interspersed_args = True
parser.add_option("-i", "--infile",default="/home/ubuntu/Insight/data/raw/face/test/187b5572-391b-41a0-817f-542322b81f00___DSC06733A.jpg",
                  dest="INFILE", help="input picture in JPG format (full path)")
parser.add_option("-o", "--outfile",default="/home/ubuntu/Insight/data/raw/face/187b5572-391b-41a0-817f-542322b81f00___DSC06733A_blur_Haar.jpg",
                  dest="OUTFILE", help="output picture (full path)")
parser.add_option("-b", "--blur", default="medianBlur",
                  dest="BLUR", help="blur type (medianBlur, GaussianBlur, bilateralFilter, blur)")
parser.add_option("-m", "--model", default="CascadeClassifier",
                  dest="MODEL", help="detection model (CascadeClassifier)")

(options,args) = parser.parse_args()
imagePath = options.INFILE
outPath = options.OUTFILE
blurMode = options.BLUR
Architecture = options.MODEL

logger.info('HideFaces.AI')
logger.info('Options:')
logger.info('--infile = %s', imagePath)
logger.info('--outfile = %s', outPath)

if blurMode in ['medianBlur', 'GaussianBlur', 'bilateralFilter', 'blur', 'none']:
    blur_mode = blurMode
    logger.info('--blur = %s', blurMode)
else:
    blur_mode = 'medianBlur'
    logger.info('--blur = %s', blurMode)

if Architecture in ['haar', 'yolov2']:
    architecture = Architecture
    logger.info('--model = %s', Architecture)
else:
    architecture = 'haar'
    logger.info('--model = %s', Architecture)

if architecture == 'haar':
    try:
        img, img_arr, gray, img_shape = load_pictures_haar(imagePath)
        logger.info('picture loaded')
    except cv2.error as e:
        logger.error('Cannot load pictures from = %s; error = %s', imagePath, e)
        raise

    model, dir_haar_face = FaceCascade()
    logger.info('Haar Cascade Classifier loaded with weights %s', dir_haar_face)

    faces = FindFaces(gray, model)
    logger.info('Haar Cascade Classifier found %s faces', len(faces))

    final_picture = Blur(img_arr, faces, blur_mode)
    logger.info('Face blurring finished')
    plt.imshow(final_picture)
    plt.savefig(outPath)

if architecture == 'yolov2':
    try:
        image, input_image = load_pictures_yolo(imagePath)
    except cv2.error as e:
        logger.error('Cannot load pictures from = %s; error = %s', imagePath, e)
        raise

    model, wt_path, model_png_path  = yolo_v2_model()
    logger.info('Yolo model loaded')
    logger.info('Yolo weights from %s are used', wt_path)
    logger.info('Yolo model stored as png in %s', model_png_path)

    dummy_array = np.zeros((1,1,1,1,50,4))

    plt.figure(figsize=(10,10))
    netout = model.predict([input_image, dummy_array])
    boxes = decode_netout(netout[0],
                      obj_threshold=0.3,
                      nms_threshold=0.3,
                      anchors=[0.57273, 0.677385, 1.87446, 2.06253, 3.33843, 5.47434, 7.88282, 3.52778, 9.77052, 9.16828],
                      nb_class=1)
    logger.info('Yolo found %s faces', len(boxes))
    LABELS = ['face']
    image = Blur_yolo(image, boxes, blur_mode, labels=LABELS)

    logger.info('Face blurring finished')
    plt.imshow(image[:,:,::-1]);
    plt.savefig(outPath)


import datetime
import glob
import os
import sys
import time
import random
import traceback
import torchvision.models as models
import cv2
import keyboard
import skimage.color
import torch
from matplotlib import pyplot as plt
from torch import nn, optim
from keyboard import press, release

from DeadExp import DeadExp as DE, DeadExp
from men_handler import MemHandler
from models.cnn import alexnet
from video_handler import VideoHandler
from window_loader import WindowLoader
import numpy as np
from numpy import log as ln

def rgb2gray(rgb):
    return skimage.color.rgb2gray(rgb)

class ProjectTouhou:

    def __init__(self,load = False):
        self.TouHou = WindowLoader()
        self.data = self.TouHou.get_window()
        self.video_handler = VideoHandler(640,480,4)
        self.net = models.resnet18(pretrained=True)
        self.net.fc = nn.Linear(self.net.fc.in_features, 5)
        self.moves = (['shift', 'up', 'z'], ['shift', 'down', 'z'], ['shift', 'left', 'z'], ['shift', 'right', 'z'], ['z'])
        self.optimizer = optim.SGD(self.net.parameters(), lr=0.001)
        self.frame = []
        self.frame_inv = []
        self.move_sequels = []
        self.death_count = 0

        self.last_action = []
        self.hp = 10
        self.mem_handler = MemHandler()
        self.success = 0
        self.power = 0
        self.dg = 0

        if 'run' not in os.listdir():
            os.mkdir('run')
        self.path = 'run/exp'+str(len(glob.glob('run/exp*'))) + '/'
        os.mkdir('run/exp'+str(len(glob.glob('run/exp*'))))
        os.mkdir(self.path+'death')
        torch.save(self.net.state_dict(), self.path + 'model@' + str(datetime.datetime.now()).replace(' ','-').replace('-','_').replace(':','_'))

        if load:
            self.net.load_state_dict(torch.load('model'))

    def get_frame(self):
        return cv2.cvtColor(next(self.data).reshape(480, 640, 4), cv2.COLOR_BGRA2BGR)

    def set_input(self, inputs):
        if inputs is not None:
            for i in inputs:
                if i is not None:
                    press(i)

    def release_input(self, inputs):
        if inputs is not None:
            for i in inputs:
                if i is not None:
                    release(i)
    def loss(self,output, target):
        return torch.mean(torch.abs(output - target))

    def carrot(self, action):
        ret = [None]*len(action)
        _, tmp = torch.max(action, 1)
        for i in range(len(action)):
            tmper = [0] * len(action[0])
            tmper[tmp[i]] = 1
            ret[i] = tmper
        print('🥕', action)
        #return None
        return torch.Tensor(ret)

    def stick(self, action):
        ret = [None]*len(action)
        conf, tmp = torch.max(action, 1)
        #print(conf, tmp,action)
        for i in range(len(action)):
            mov = [float(x) for x in action[i]]
            tmper = [float(conf[i]) / (len(action[i]) - 1)] * len(action[0])
            tmper[tmp[i]] = 0
            #print(mov,tmper)

            ret[i] = np.array(mov) + np.array(tmper)
            ret[i][tmp[i]]=0
            #print(tmp[i],ret[i])
        print('🏒', action)
        return torch.Tensor(ret)

    def soso(self, action):
        print('😶', action)
        return None

    def get_incentive(self, action):

        hp = self.mem_handler.get_mem(offsets=[0x4A57f4])
        bm = self.mem_handler.get_mem(offsets=[0x4A5800])
        dg = self.mem_handler.get_mem(offsets=[0x4A57C0])
        score = self.mem_handler.get_mem(offsets=[0x4A57B0])
        power = self.mem_handler.get_mem(offsets=[0x4A57E4])

        if hp < self.hp:
            self.hp = hp
            return self.stick(action)
        self.hp = hp
        success = ln(score)

        _, tmper = torch.max(action, 1)
        tmper = tmper[0]
        if not self.last_action == []:
            if self.last_action[-1] == tmper:
                if len(self.last_action) > 3:
                    success = self.success - 1
            else:
                self.last_action = []
        self.last_action.append(tmper)

        print(self.moves[tmper], hp, bm, dg, score, ln(score), power)

        if hp > 12:
            raise DeadExp
        if tmper == 5 and bm == 0:
            return self.stick(action)
        if power > self.power:
            self.power = power
            #return selfZ.carrot(action)
        if dg > self.dg:
            self.dg = dg
            #return self.carrot(action)
        self.dg = dg
        if success > self.success:
            self.success = success
            #return self.carrot(action)
        elif success == self.success:
            self.success = success
            #return self.soso(action)
        else:
            self.success = success
            return self.stick(action)

    def backward(self, reward):
        for i in reversed(self.move_sequels):
            if reward:
                loss = self.loss(i, self.stick(i))
            else:
                loss = self.loss(i, self.carrot(i))
            loss.backward()
            self.optimizer.step()

    def save_model(self):
        torch.save(self.net.state_dict(), self.path + 'model@' + str(datetime.datetime.now()).replace(' ','-').replace('-','_').replace(':','_'))

    def save_death_frame(self):
        plt.imsave(self.death_path + 'death@' + str(datetime.datetime.now()).replace(' ', '-').replace('-', '_').replace(':', '_') + '.png', self.frame, cmap=plt.cm.gray)

    def GameOn(self):
        self.start = time.time()
        move = None
        self.optimizer.zero_grad()
        frame = self.get_frame()

        tmp_frame = np.array(rgb2gray(frame))
        print(tmp_frame.shape)
        for x in range(3):
            self.frame.append(tmp_frame)

        print(np.array(self.frame).shape)

        action = self.net(torch.Tensor([self.frame]) / 255)

        while(True):
            if keyboard.is_pressed('/'):  # if key 'q' is pressed
                print('You Pressed A Key!')
                self.release_input(move)
                self.video_handler.close()
                break  # finishing the loop\
            #self.video_handler.write_frame(frame)
            #print('outputs -> ',outputs)

            try:
                frame = self.get_frame()
                target = self.get_incentive(action)
            except DeadExp:
                if self.death_count == 0:
                    self.save_death_frame()
                if self.death_count == 10:
                    break
                print('Dead :3')
                #e = sys.exc_info()[0]
                #print(e)
                #traceback.print_exc()

                self.release_input(move)
                self.start = time.time()
                self.death_count += 1
                try:
                    pass
                    #loss = self.loss(action, self.stick(action))
                    #loss.backward()
                    #self.optimizer.step()
                except:
                    pass
                #self.mem_handler = MemHandler()
                time.sleep(max(1 / 4 - (time.time() - self.start), 0))
                move = self.moves[4]
                self.set_input(move)
                print('=>', move)
                self.start = time.time()
                time.sleep(max(1 / 4 - (time.time() - self.start), 0))
                continue
            except:
                e = sys.exc_info()[0]
                print(e)
                traceback.print_exc()
                target = self.stick(action)
                flag = False
                print('exp and saved')
                self.release_input(move)
                break
            self.death_count = 0

            if target is not None:
                self.optimizer.zero_grad()
                loss = self.loss(action, target)
                loss.backward()
                self.optimizer.step()

            self.frame.pop(0)
            self.frame.append(rgb2gray(frame))

            action = self.net(torch.Tensor([self.frame]) / 255)
            #print(action)
            self.release_input(move)
            conf, move = torch.max(action, 1)
            move = self.moves[move[0]]
            #print(round(float(conf[0]),2), move, action)
            self.set_input(move)

            time.sleep(max(1 / 10 - (time.time() - self.start), 0))
            print(1 / (time.time() - self.start))
            self.start = time.time()
        self.save_model()
        torch.save(self.net.state_dict(), self.path + 'model')

if __name__ == "__main__":
    tmp = ProjectTouhou(load=False)
    #tmp = ProjectTouhou()
    tmp.GameOn()


'''
Modified by mengtianjian
Adapted from https://github.com/tornadomeet/ResNet/blob/master/symbol_resnet.py
Original author Wei Wu

Implemented the following paper:

Ke Sun, Yang Zhao, Borui Jiang, Tianheng Cheng, Bin Xiao, Dong Liu, Yadong Mu, Xinggang Wang,
Wenyu Liu, Jingdong Wang. 'High-Resolution Representations for Labeling Pixels and Regions'

'''
import numpy as np
import mxnet as mx

def residual_unit(data,
                  output_channels,
                  stride,
                  dim_match,
                  name,
                  bottle_neck=True,
                  bn_mom=0.9,
                  workspace=256):
    '''Return ResNet Unit symbol for building ResNet
    Parameters
    ----------
    data : str
        Input data
    output_channels : int
        Number of output channels
    bnf : int
        Bottle neck channels factor with regard to output_channels
    stride : tuple
        Stride used in convolution
    dim_match : Boolean
        True means channel number between input and output is the same, otherwise means differ
    name : str
        Base name of the operators
    workspace : int
        Workspace used in convolution operator
    '''
    if bottle_neck:
        output_channels_act1 = int(output_channels * 0.25)
        conv1 = mx.sym.Convolution(
            data=data,
            num_filter=output_channels_act1,
            kernel=(1, 1),
            stride=(1, 1),
            pad=(0, 0),
            no_bias=True,
            workspace=workspace,
            name=name + '_conv1')
        bn1 = mx.sym.BatchNorm(
            data=conv1,
            fix_gamma=False,
            eps=2e-5,
            momentum=bn_mom,
            name=name + '_bn1')
        act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1')
        output_channels_act2 = int(output_channels * 0.25)
        conv2 = mx.sym.Convolution(
            data=act1,
            num_filter=output_channels_act2,
            kernel=(3, 3),
            stride=stride,
            pad=(1, 1),
            no_bias=True,
            workspace=workspace,
            name=name + '_conv2')
        bn2 = mx.sym.BatchNorm(
            data=conv2,
            fix_gamma=False,
            eps=2e-5,
            momentum=bn_mom,
            name=name + '_bn2')
        act2 = mx.sym.Activation(data=bn2, act_type='relu', name=name + '_relu2')
        conv3 = mx.sym.Convolution(
            data=act2,
            num_filter=output_channels,
            kernel=(1, 1),
            stride=(1, 1),
            pad=(0, 0),
            no_bias=True,
            workspace=workspace,
            name=name + '_conv3')
        bn3 = mx.sym.BatchNorm(
            data=conv3,
            fix_gamma=False,
            eps=2e-5,
            momentum=bn_mom,
            name=name + '_bn3')
        if dim_match:
            shortcut = data
        else:
            shortcut = mx.sym.Convolution(
                data=data,
                num_filter=output_channels,
                kernel=(1, 1),
                stride=stride,
                no_bias=True,
                workspace=workspace,
                name=name + '_sc')
            shortcut = mx.sym.BatchNorm(
                data=shortcut,
                fix_gamma=False,
                eps=2e-5,
                momentum=bn_mom,
                name=name + '_sc_bn')
        out = mx.sym.Activation(data=bn3+shortcut, act_type='relu', name=name + '_relu3')
        return out
    else:
        conv1 = mx.sym.Convolution(
            data=data,
            num_filter=output_channels,
            kernel=(3, 3),
            stride=stride,
            pad=(1, 1),
            no_bias=True,
            workspace=workspace,
            name=name + '_conv1')
        bn1 = mx.sym.BatchNorm(
            data=conv1,
            fix_gamma=False,
            momentum=bn_mom,
            eps=2e-5,
            name=name + '_bn1')
        act1 = mx.sym.Activation(data=bn1, act_type='relu', name=name + '_relu1')
        conv2 = mx.sym.Convolution(
            data=act1,
            num_filter=output_channels,
            kernel=(3, 3),
            stride=stride,
            pad=(1, 1),
            no_bias=True,
            workspace=workspace,
            name=name + '_conv2')
        bn2 = mx.sym.BatchNorm(
            data=conv2,
            fix_gamma=False,
            momentum=bn_mom,
            eps=2e-5,
            name=name + '_bn2')
        if dim_match:
            shortcut = data
        else:
            shortcut = mx.sym.Convolution(
                data=data,
                num_filter=output_channels,
                kernel=(1, 1),
                stride=stride,
                no_bias=True,
                workspace=workspace,
                name=name + '_sc')
            shortcut = mx.sym.BatchNorm(
                data=shortcut,
                fix_gamma=False,
                eps=2e-5,
                momentum=bn_mom,
                name=name + '_sc_bn')
        out = mx.sym.Activation(data=bn2+shortcut, act_type='relu', name=name + '_relu2')
        return out

def conv3x3(data,
            output_channel,
            stride,
            name,
            act=True,
            bn_mom=0.9,
            workspace=256):
    out = mx.sym.Convolution(
        data=data,
        num_filter=output_channel,
        kernel=(3, 3),
        stride=stride,
        pad=(1, 1),
        no_bias=True,
        name=name+'_conv',
        workspace=workspace)
    out = mx.sym.BatchNorm(
        data=out,
        fix_gamma=False,
        eps=2e-5,
        momentum=bn_mom,
        name=name+'_bn')
    if act:
        out = mx.sym.Activation(data=out, act_type='relu', name=name+'_relu')
    return out

def conv1x1(data,
            output_channel,
            stride,
            name,
            act=True,
            bn_mom=0.9,
            workspace=256):
    out = mx.sym.Convolution(
        data=data,
        num_filter=output_channel,
        kernel=(1, 1),
        stride=stride,
        pad=(0, 0),
        no_bias=True,
        name=name+'_conv',
        workspace=workspace)
    out = mx.sym.BatchNorm(
        data=out,
        fix_gamma=False,
        eps=2e-5,
        momentum=bn_mom,
        name=name+'_bn')
    if act:
        out = mx.sym.Activation(data=out, act_type='relu', name=name+'_relu')
    return out

def hr_unit(data,
            num_branches,
            input_channels,
            output_channels,
            num_blocks,
            bottleneck,
            name,
            multi_scale_output):
    branches = []
    expansion = 4 if bottleneck else 1
    for i in range(num_branches):
        branch = residual_unit(
            data[i],
            output_channels[i] * expansion,
            (1, 1),
            input_channels[i] == output_channels[i] * expansion,
            name=name+"_branch%d_block%d" % (i + 1, 1),
            bottle_neck=bottleneck
        )
        input_channels[i] = output_channels[i] * expansion
        for j in range(1, num_blocks[i]):
            branch = residual_unit(
                branch,
                output_channels[i] * expansion,
                (1, 1),
                True,
                name=name+"_branch%d_block%d" % (i + 1, j + 1),
                bottle_neck=bottleneck
            )
        branches.append(branch)

    output = fuse_layer(branches,
                        input_channels,
                        num_branches,
                        multi_scale_output,
                        name)

    return output

def fuse_layer(data,
               input_channels,
               num_branches,
               multi_scale_output,
               name):
    outputs = []

    for i in range(num_branches if multi_scale_output else 1):
        output = data[0]
        if i != 0:
            for k in range(i):
                if k == i - 1:
                    output = conv3x3(output,
                                     input_channels[i],
                                     (2, 2),
                                     name+'_branch%d_down%d'%(i+1, k+1),
                                     False)
                else:
                    output = conv3x3(output,
                                     input_channels[0],
                                     (2, 2),
                                     name+'_branch%d_down%d'%(i+1, k+1),
                                     True)
        for j in range(1, num_branches):
            if i == j:
                output = output + data[j]
            elif i < j:
                output = output + mx.sym.UpSampling(conv1x1(data[j],
                                                            input_channels[i],
                                                            (1, 1),
                                                            name+'_branch%d_up%d'%(i+1, j+1)),
                                                    scale=2 ** (j - i),
                                                    sample_type='nearest')
            else:
                conv = data[j]
                for k in range(i - j):
                    if k == i - j - 1:
                        conv = conv3x3(conv,
                                       input_channels[i],
                                       (2, 2),
                                       name+'_branch%d_down%d_conv%d'%(i+1, j+1, k+1),
                                       False)
                    else:
                        conv = conv3x3(conv,
                                       input_channels[j],
                                       (2, 2),
                                       name+'_branch%d_down%d_conv%d'%(i+1, j+1, k+1),
                                       True)
                output = output + conv
        output = mx.sym.Activation(data=output, act_type='relu', name=name+'_branch%d_relu'%(i+1))
        outputs.append(output)

    return outputs, input_channels

def hrnet(config, dtype):
    data = mx.sym.Variable(name='data')
    if dtype == 'float32':
        data = mx.sym.identity(data=data, name='id')
    elif dtype == 'float16':
        data = mx.sym.Cast(data=data, dtype=np.float16)

    stem1 = conv3x3(data, 64, (2, 2), 'stem1', True)
    stem2 = conv3x3(stem1, 64, (2, 2), 'stem2', True)

    stage1_cfg = config['stage1']
    expansion = 4 if stage1_cfg['block'] == 'BOTTLENECK' else 1
    stage1_output_channels = stage1_cfg['num_channels'][0] * expansion
    layer1 = residual_unit(
        stem2,
        stage1_output_channels,
        (1, 1),
        stage1_output_channels == 64,
        name='layer1_unit1',
        bottle_neck=stage1_cfg['block'] == 'BOTTLENECK')
    for i in range(1, stage1_cfg['num_blocks'][0]):
        layer1 = residual_unit(
            layer1,
            stage1_output_channels,
            (1, 1),
            True,
            name='layer1_unit%d' % (i+1),
            bottle_neck=stage1_cfg['block'] == 'BOTTLENECK')
    pre_stage_channels = [stage1_output_channels]
    output = [layer1]# * config['stage'][1]['num_branches']

    for i in range(1, 4):
        stage_cfg = config['stage'+str(i+1)]
        expansion = 4 if stage_cfg['block'] == 'BOTTLENECK' else 1
        num_channels = [stage_cfg['num_channels'][j] * expansion for j in range(len(stage_cfg['num_channels']))]
        output = transition_layer(output, pre_stage_channels, num_channels, 'layer%d_trans'%(i+1))
        output, pre_stage_channels = hr_stage(output, stage_cfg, num_channels, 'layer%d'%(i+1))

    return output, pre_stage_channels

def transition_layer(data, num_channels_pre_layer, num_channels_cur_layer, name):
    num_branches_cur = len(num_channels_cur_layer)
    num_branches_pre = len(num_channels_pre_layer)

    outputs = []
    for i in range(num_branches_cur):
        if i < num_branches_pre:
            if num_channels_cur_layer[i] != num_channels_pre_layer[i]:
                outputs.append(conv3x3(data[-1],
                                       num_channels_cur_layer[i],
                                       (1, 1),
                                       name+'_match_branch%d'%(i+1),
                                       True))
            else:
                outputs.append(data[i])
        else:
            output = data[-1]
            for j in range(i + 1 - num_branches_pre):
                if j == i - num_branches_pre:
                    output_channels = num_channels_cur_layer[i]
                else:
                    output_channels = num_channels_pre_layer[-1]
                output = conv3x3(output,
                                 output_channels,
                                 (2, 2),
                                 name+'_down%d_branch%d'%(j+1, i+1),
                                 True)
            outputs.append(output)
    return outputs

def hr_stage(data, config, input_channels, name, multi_scale_output=True):
    num_modules = config['num_modules']
    output = data
    for i in range(num_modules):
        if not multi_scale_output and i == num_modules - 1:
            reset_multi_scale_output = False
        else:
            reset_multi_scale_output = True

        output, input_channels = hr_unit(output,
                                         config['num_branches'],
                                         input_channels,
                                         config['num_channels'],
                                         config['num_blocks'],
                                         config['block'] == 'BOTTLENECK',
                                         name+'_unit%d'%(i+1),
                                         reset_multi_scale_output)
    return output, input_channels

def get_symbol(config,
               image_shape,
               conv_workspace=256,
               dtype='float32',
               **kwargs):
    '''
      Adapted from https://github.com/tornadomeet/ResNet/blob/master/train_resnet.py
      Original author Wei Wu
      '''
    image_shape = [int(l) for l in image_shape.split(',')
                   ] if type(image_shape) is str else image_shape
    (channels, height, width) = image_shape

    if config == 'w18':
        extra=dict(
            stage1=dict(
                num_modules=1,
                num_branches=1,
                block='BOTTLENECK',
                num_blocks=(4,),
                num_channels=(64,),
                fuse_method='SUM'),
            stage2=dict(
                num_modules=1,
                num_branches=2,
                block='BASIC',
                num_blocks=(4, 4),
                num_channels=(18, 36),
                fuse_method='SUM'),
            stage3=dict(
                num_modules=4,
                num_branches=3,
                block='BASIC',
                num_blocks=(4, 4, 4),
                num_channels=(18, 36, 72),
                fuse_method='SUM'),
            stage4=dict(
                num_modules=3,
                num_branches=4,
                block='BASIC',
                num_blocks=(4, 4, 4, 4),
                num_channels=(18, 36, 72, 144),
                fuse_method='SUM'))
    elif config == 'w32':
        extra=dict(
            stage1=dict(
                num_modules=1,
                num_branches=1,
                block='BOTTLENECK',
                num_blocks=(4,),
                num_channels=(64,),
                fuse_method='SUM'),
            stage2=dict(
                num_modules=1,
                num_branches=2,
                block='BASIC',
                num_blocks=(4, 4),
                num_channels=(32, 64),
                fuse_method='SUM'),
            stage3=dict(
                num_modules=4,
                num_branches=3,
                block='BASIC',
                num_blocks=(4, 4, 4),
                num_channels=(32, 64, 128),
                fuse_method='SUM'),
            stage4=dict(
                num_modules=3,
                num_branches=4,
                block='BASIC',
                num_blocks=(4, 4, 4, 4),
                num_channels=(32, 64, 128, 256),
                fuse_method='SUM'))

    return hrnet(extra, dtype)

if __name__ == '__main__':
    sym = get_symbol('w18', '3,224,224')
    vis = mx.viz.plot_network(sym)
    vis.render('hrnet_example')


from flask import jsonify, request
from flask_restful import Resource
import os, librosa, soundfile, pickle
import noisereduce as nr
import numpy as np

# Load the data and extract features
def load_recording(filename):
    x = []
    feature = extract_feature(filename ,mfcc = True, chroma = True, mel = True)
    x.append(feature)
    return np.array(x)

# loading model from pickle
model = pickle.load(open('./resources/speech_emotion/model_speech.pickle','rb'))

# API ENDPOINT ACCESS HERE
class SpeechSentiment(Resource):
    def post(self):
        f = request.files['speech']
        if f.filename != '':
            f.save(os.path.join('static', f.filename))
            prediction = load_recording("static/" + f.filename)
            res = model.predict(prediction)
            os.remove("./static/" + f.filename)
            return {'success': True, 'speech_sentiment': res[0]}
        else:
            return {'success': False, 'speech_sentiment': "False, some error occured."}