import json
import boto3
import google.auth
from google.cloud import bigquery
from collections import defaultdict

# Switch for turn on debugging messages.
debug = False

s3 = boto3.client('s3')
a = s3.get_object(
                  Bucket='config-lambda',
                  Key='layers/google-cloud-storage/gabinete-compartilhado.json')
open('/tmp/key.json', 'w').write(a['Body'].read().decode('utf-8'))

# Create credentials with Drive & BigQuery API scopes
# Both APIs must be enabled for your project before running this code
credentials, project = google.auth.default(scopes=[
    'https://www.googleapis.com/auth/drive',
    'https://www.googleapis.com/auth/bigquery',
])
bq = bigquery.Client(credentials=credentials, project=project)

def query_bigquery(query):

    result = bq.query(
        query,
        # Location must match that of the dataset(s) referenced in the query.
        location="US",
    )  # API request - starts the query

    result = [dict(r.items()) for r in result]

    return result

def translate_to_slack(datum, casa):
    """
    For a single entry in the query results, create a dict with the relevant information.
    """

    if casa == 'camara':

        datum['url_proposicao'] = 'http://www.camara.leg.br/proposicoesWeb/fichadetramitacao?idProposicao=' + str(datum['id'])

    elif casa == 'senado':

        datum['url_proposicao'] = 'https://www25.senado.leg.br/web/atividade/materias/-/materia/' + str(datum['id'])

    if casa == 'camara' or casa == 'senado':
        return dict(
            url_proposicao=   datum['url_proposicao'],
            tipo=             datum['sigla_tipo'],
            numero=           datum['numero'],
            ano=              datum['ano'],
            autor=            datum['nome_autor'],
            partido_autor=    datum['sigla_partido_autor'],
            relator=          datum['relator'],
            partido_relator=  datum['partido_relator'],
            ementa=           datum['ementa'],
            url_inteiro_teor= datum['url'],
            status=           datum['descricao_tramitacao'],
            sequencia=        datum['sequencia'],
            regime=           datum['regime'],
            situacao=         datum['decricao_situacao'],
            orgao=            datum['sigla_orgao'],
            despacho=         datum['despacho'])
    elif casa == 'dou':
        if datum['ementa'] == 'None':
            resumo_tipo0 = 'Excerto'
            resumo0 = datum['resumo']
        else:
            resumo_tipo0 = 'Ementa'
            resumo0 = datum['ementa']
        return dict(
            identifica=      datum['identifica'],
            orgao=           datum['orgao'],
            assina=          datum['assina'],
            cargo=           datum['cargo'],
            pub_date=        datum['pub_date'],
            edicao=          datum['edicao'],
            secao=           datum['secao'],
            pagina=          datum['pagina'],
            url=             datum['url'],
            url_certificado= datum['url_certificado'],
            resumo_tipo=     resumo_tipo0,
            resumo=          resumo0)

    else:
        return datum

def to_slack_format(bot_info, result):
    """
    Takes the bigQuery result and transforms it into a new structure that
    contains metadata about the media destination.
    """
    print('first result', len(result))

    # Remove double quotes from bigQuery results (list of dicts which are new tramitações)
    # to avoid bugs:
    result = map(lambda x: {k: str(v).replace('"', '') for k, v in x.items()}, result)

    # Add default dict to avoid error if chosen key is inexistent
    # (in this case, return ''):
    result = list(map(lambda x: defaultdict(lambda: '', x), result))

    # Reorganize and rename dict:
    result = map(lambda x: translate_to_slack(x, bot_info['casa']), result)

    # Create a dict with metadata and the data as a list inside a key:
    payload = dict(
        casa = bot_info['casa'],
        descricao_post= bot_info['media']['description'],
        media={'type': bot_info['media']['type'],
                'channel': bot_info['media']['channel']},
        data=list(result))

    # Create a json from the structure above:
    return json.dumps(payload, ensure_ascii=False)

def post_to_sns(bot_info, payload):

    print('Sending to SNS')
    sns = boto3.client('sns')
    # Create a topic in the SNS if it does not exist:
    res = sns.create_topic(Name=bot_info['sns-topic'])
    # Subscribe the Lambda function to the topic created above:
    sns.subscribe(TopicArn=res['TopicArn'],
                  Protocol='lambda',
                  Endpoint='arn:aws:lambda:us-east-1:085250262607:function:post-to-slack:JustLambda')

    # Send the payload to the topic. Everyone subscribed (in this case, the Lambda
    # function 'post-to-slack') will receive the Message:
    sns.publish(TopicArn=res['TopicArn'],
                Message=payload)

def get_relevant_results(bot_info, results):
    """
    Filter the query result 'results' that gets the last 30 minutes new stuff
    in the database.
    -- results: list of dicionaries. Each entry in list (a dict) is a row in the query results,
       and each key in dict is a column.

    # About bot_info['filters']:
    # This is a list of filters (dict), each filter has the entries:
    # 1 FILTER: column_name, positive_filter, negative_filter.
    #           -- The value for column_name is the column to check.
    #           -- The positive and negative_filter are a list of keywords that are combined with OR;
    #           -- The positive and negative filters are combined with AND;
    # The filters are combined with AND;
    # To combine filters with OR, create a new bot_info with a different list of filters.
    """

    filters = bot_info['filters']

    if len(filters):
        print('name:', bot_info['nome'])
        print('len results', len(results))

        for f in filters:

            if debug:
                print('f', f)

            # Delete None
            # Select all desired columns that actually exist in the bigquery results:
            results = list(filter(lambda x: x[f['column_name']] is not None, results))

            if 'positive_filter' in f.keys():
                results = list(filter(lambda x:
                    any([var.lower() in x[f['column_name']].lower()
                        for var in f['positive_filter']]), results))

            # Changed on 2019-05-21 from elif to if:
            if 'negative_filter' in f.keys():
                results = list(filter(lambda x:
                    all([var.lower() not in x[f['column_name']].lower()
                        for var in f['negative_filter']]), results))

            print('len results', len(results))

    return results

def data_to_sns(bot_info, results):
    """
    Takes the dicionary with search keywords 'bot_info' and all new data in BigQuery
    'results', selects the relevant entries, format to slack style and send
    it to Amazon's SNS.
    """
    # Filter the results (select what we want to track):
    partial_results = get_relevant_results(bot_info, results)
    # Format to slack structure:
    payload = to_slack_format(bot_info, partial_results)
    # Send the post to Amazon's SNS (notification system):
    post_to_sns(bot_info, payload)

# Functions to load Gabi filters:

def csvrow_to_list(csvrow):
    """
    Takes a string 'csvrow' that has substrings separated by commas and
    returns a list of substrings.
    """
    if csvrow != None:
        return list(map(lambda s: s.strip(), csvrow.split(';')))
    else:
        return None

def check_for_ambiguity(filter_group, filter_ids, key):
    """
    Check if all filters numbered the same way have the same tag 'key'
    (nome, casa, channel, description). Returns an error if it
    doesn't. This is needed to group filters under the same
    filter set.
    -- filter_group: pandas grouby object, grouped by filter number.
    -- key: nome, casa, channel, description
    """
    unique_key = all([[f[key] == filter_group[i][0][key] for f in filter_group[i]] for i in filter_ids])
    if not unique_key:
        raise Exception('Found multiple entries of \''+key+'\' for same filter.')

def get_filter_par(filter_group, filter_ids, key):
    """
    Given a pandas groupby object 'filter_group' that group the
    filters by filter_number (giving a filter set) and a key,
    get the first value for that key in each group.
    """
    return [filter_group[i][0][key] for i in filter_ids]

def filterset_gen(filters_raw, fnumber):
    """
    Group all filters in table 'filter_raw' in a filter set according to filter_number 'fnumber'.
    Then, transforms comma separated filter keywords into a list.
    It also deals with missing values in filter entries / filter entry at all.
    """
    # If there is no filter, return an empty list:
    flist = [f for f in filters_raw if f['filter_number'] == fnumber]
    if len(flist) == 1 and flist[0]['column_name'] == None:
            return []
    else:
        # If there are filters, group them:
        return [{k:(f[k] if k=='column_name' else csvrow_to_list(f[k]))
                 for k in ('column_name', 'positive_filter', 'negative_filter') if f[k]!=None}
                for f in filters_raw if f['filter_number'] == fnumber]

def format_filters(filters_raw):
    """
    Format the filters loaded from Google sheets (a table 'filters_raw')
    into the dict used by João Carabetta:
    """
    # Group list of filters by filter_number, to form a filter set:
    filter_group = defaultdict(list)
    for f in filters_raw:
        filter_group[f['filter_number']].append(f)
    # Get filter set ids:
    filter_ids = list(filter_group.keys())
    # List of filter set tags:
    filter_id_keys = ['nome', 'casa', 'channel', 'description']

    # Check if every filter set has unique tags:
    dump = [check_for_ambiguity(filter_group, filter_ids, key) for key in filter_id_keys]
    # Get filter set tags:
    filter_tag = {key: get_filter_par(filter_group, filter_ids, key) for key in filter_id_keys}

    # Organize filters in a filter set:
    filter_set = [filterset_gen(filters_raw, fnumber) for fnumber in filter_ids]

    # Put all filter sets (with tags) into a list:
    event = [{'nome': filter_tag['nome'][i],
              'casa': filter_tag['casa'][i],
              'media': {'type': 'slack',
                        'channel': filter_tag['channel'][i],
                        'description': filter_tag['description'][i]},
              #'sns-topic': 'slack-test-DEV', # For debugging.
              'sns-topic': 'slack-test',
              'filters': filter_set[i]}
             for i in range(len(filter_ids))]

    return event

def lambda_handler(event, context):

    print('Starting bigquery-to-sns...')

    # Load filters information from Google Sheets:
    #filters_raw = query_bigquery('SELECT * FROM `gabinete-compartilhado.gabi_bot.gabi_filters`')
    filters_raw = query_bigquery("SELECT * FROM `gabinete-compartilhado.gabi_bot.gabi_filters` WHERE casa != 'dou'")
    # Each entry in the list below is a bot_info:
    event = format_filters(filters_raw)
    print('Loaded {} filters from Google sheets.'.format(len(event)))

    # Will get every new entry in BigQuery that appeared in the last 30min:
    queries_metadata = [
        {'casa': 'camara',
        'query': "SELECT * FROM `gabinete-compartilhado.gabi_bot.camara_tramitacao_last30minutos`"},
        #'query': "SELECT * FROM `gabinete-compartilhado.gabi_bot.teste`"},
        {'casa': 'senado',
        'query': "SELECT * FROM `gabinete-compartilhado.gabi_bot.senado_tramitacao_last30minutos`"}#,
        #{'casa': 'dou',
        #'query': "SELECT * FROM `gabinete-compartilhado.gabi_bot.artigos_dou_last30minutos`"}
        ]

    for metadata in queries_metadata:
        metadata['results'] = query_bigquery(metadata['query'])

    # LOOP over the different selection criteria:
    for bot_info in event:

        if 'dou' in bot_info['casa']:
            data_to_sns(bot_info, queries_metadata[2]['results'])

        elif 'senado' in bot_info['casa']:
            data_to_sns(bot_info, queries_metadata[1]['results'])

        elif 'camara' in bot_info['casa']:
            data_to_sns(bot_info, queries_metadata[0]['results'])


from models.defineConst import TOPLOGY_NAME_PRE
from tools.counter import counter
from tools.functions import option
from lib.router import Router
from models.toplogyMaps import routerIDMap, cabelIDMap
from lib.cabel import Cabel

class Toplogy:

    def __init__(self, **kwargs):
        self.__id = counter.generateTopID()
        self.__isActivate = False
        self.__name = TOPLOGY_NAME_PRE + str(self.__id)
        self.__conf = ""
        self.__routers = []
        self.__cabels = []
        if 'conf' in kwargs:
            self.__initByFile__(kwargs['conf'])

    def __initByFile__(self, conf):
        self.__name = option(self.__name, conf['name'])
        minRouterId = float('inf')
        for routerConf in conf['routers']:
            router = Router(conf=routerConf)
            minRouterId = min(minRouterId, router.getID())
            self.addRouter(router)
        for cabelConf in conf['cabels']:
            cabel = Cabel(conf=cabelConf, minId=minRouterId)
            self.addCabel(cabel)
        self.__conf = {"name": self.__name,
                       "routers": list(map(lambda r: r.toJsonFile(), self.__routers)),
                       "cabels": list(map(lambda c: c.toJsonFile(), self.__cabels))}
        # self.__conf = conf

    def addRouter(self, router):
        self.__routers.append(router)
        routerIDMap[router.getID()] = router

    def addCabel(self, cabel):
        self.__cabels.append(cabel)
        cabelIDMap[cabel.getID()] = cabel

    def getID(self):
        return self.__id

    def getName(self):
        return self.__name

    def toSimpleJson(self):
        return {"id": self.__id,
                "name": self.__name,
                "isActive": self.__isActivate}

    def toJson(self):
        return {"id": self.__id,
                "name": self.__name,
                "isActive": self.__isActivate,
                "routerCount": len(self.__routers),
                "routers": list(map(lambda x: x.toSimpleJson(), self.__routers)),
                "cabelCount": len(self.__cabels),
                "cabels": list(map(lambda  x: x.toSimpleJson(), self.__cabels))}

    def toJsonFile(self):
        return self.__conf

    def isActivate(self):
        return self.__isActivate

    #激活拓扑图
    def activate(self):
        self.__isActivate = True
        for router in self.__routers:
            router.activate()

    #失效拓扑图
    def deActivate(self):
        self.__isActivate = False
        for router in self.__routers:
            router.enterConfigMode()
            router.deActivate()


from django.urls import path
from rest_framework import routers

from zinc import views

router = routers.DefaultRouter(trailing_slash=False)
router.register('policies', views.PolicyViewset, 'policy')
router.register('zones', views.ZoneViewset, 'zone')

urlpatterns = router.urls + [
    path('zones/<int:zone_id>/records/<str:record_id>',
        views.RecordDetail.as_view(), name='record-detail'),
    path('zones/<int:zone_id>/records',
        views.RecordCreate.as_view(), name='record-create'),
]


# create 0~9 figures' images from darknet format dataset to training set, separate front and last figures
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import re
import numpy as np
import matplotlib.image as mpimg
from matplotlib import pyplot as plt

def rgb2gray(rgb):
    return np.dot(rgb[..., :3], [0.2989, 0.5870, 0.1140])

dataset_path = "/home/neusoft/amy/AT-201/data/water_mechanical_190625/gray/"
output_path_front = "/home/neusoft/amy/AT-201/data/water_mechanical_190625/digits_front/"
output_path_end = "/home/neusoft/amy/AT-201/data/water_mechanical_190625/digits_end/"

# create folder if not exists
if not os.path.exists(output_path_front):
    os.makedirs(output_path_front)
for i in range(10):
    if not os.path.exists(output_path_front + str(i)):
        os.makedirs(output_path_front + str(i))
# create folder if not exists
if not os.path.exists(output_path_end):
    os.makedirs(output_path_end)
for i in range(10):
    if not os.path.exists(output_path_end + str(i)):
        os.makedirs(output_path_end + str(i))

g = os.walk(dataset_path)
for root, dir, files in g:
    for file in files:
        if '.jpg' in file:
            filename, file_extension = os.path.splitext(file)
            txt = open(root + filename + '.txt')
            img = mpimg.imread(root + file)
            #print(filename)
            #plt.imshow(img, interpolation='nearest')
            #plt.show()
            index = 0
            readings = []
            locations = []
            imgs = []
            while(1):
                line = txt.readline()
                #print(line)
                if len(line) == 0:
                    break
                location = []
                reading = str.split(line)[0]
                readings.append(reading)
                #precision = (float)(str.split(line)[5])
                #print("reading:", reading)

                location.append((int)(img.shape[1]*(float)(str.split(line)[1]) - img.shape[1]*(float)(str.split(line)[3])/2))
                location.append((int)(img.shape[0]*(float)(str.split(line)[2]) - img.shape[0]*(float)(str.split(line)[4])/2))
                location.append((int)(img.shape[1]*(float)(str.split(line)[1]) + img.shape[1]*(float)(str.split(line)[3])/2))
                location.append((int)(img.shape[0]*(float)(str.split(line)[2]) + img.shape[0]*(float)(str.split(line)[4])/2))
                locations.append(location)

                img_figure = img[location[1]:location[3], location[0]:location[2]]
                #img_figure = rgb2gray(img_figure)
                imgs.append(img_figure)
                #plt.imshow(img_figure, cmap='gray', vmin=0, vmax=255)
                #plt.show()
            max = locations[-1][0]
            end_index = len(locations)-1
            for i in range(len(locations)-1):
                if locations[i][0] > max:
                    max = locations[i][0]
                    end_index = i
            if end_index != 4:
                print(end_index)
                print(filename)
            for i in range(len(locations)):
                if i != end_index:
                    savepath = output_path_front + readings[i] + '/' + filename + '_' + str(index) + '.jpg'
                    #print(savepath)
                    #print(i)
                    mpimg.imsave(savepath, imgs[i], cmap='gray')
                    index += 1
            savepath = output_path_end + readings[end_index] + '/' + filename + '_' + str(index) + '.jpg'
            mpimg.imsave(savepath, imgs[end_index], cmap='gray')
            #print(savepath)


#coding:utf-8
from __future__ import (print_function, unicode_literals)
from Crypto.Cipher import DES3, DES

DEBUG_DES = 1

class MyDES(object):

    skey = ''
    mkey = ''

    @classmethod
    def encrypt(self, data):
        '''3DES ECB mode.

        '''
        D = data

        '''在明文M后附加0x80，然后在右端填充最少的0x00，使得填充后消息M = (M||80||00||00||…||00)的长度为8的整数倍'''
        D += b'\x80'
        if len(D)%8:
            D += b'\x00' * (8 - len(D)%8)

        cipher = DES3.new(self.skey, DES3.MODE_ECB)
        EData = cipher.encrypt(D)
        return EData

    @classmethod
    def decrypt(self, EData):

        cipher = DES3.new(self.skey, DES3.MODE_ECB)
        D = cipher.decrypt(EData)
        return D

    @classmethod
    def CalcMAC(self, data):
        D = data + b'\x80'
        if len(D)%8:
            D += b'\x00' * (8 - len(D)%8)

        #CBC模式加密
        Ecipher = DES.new(self.mkey[:8], DES.MODE_CBC, IV=b'\x00'*8)
        C = Ecipher.encrypt(D)

        #使用最后一块数据计算消息认证码
        Cn = C[-8:]
        Dcipher = DES.new(self.mkey[8:], DES.MODE_ECB)
        Dn = Dcipher.decrypt(Cn)

        Ecipher = DES.new(self.mkey[:8], DES.MODE_ECB)
        Mac = Ecipher.encrypt(Dn)

        return Mac


import click
import requests
import json
import yaml

link = "localhost"

__author__ = "Anshu Jain"

@click.group()
def main():
    """
    Simple CLI for controlling Raibu tool for application deployment.
    """
    pass

@main.command()
@click.argument('query')
def provision(query):
    """This will provision number of EC2 VMs on AWS cloud."""

    url_format = "http://" + link + ":5000/Launch"

    data=json.loads(yamlToJson(query))

    response = requests.post(url_format, json = data)
    rp=json.dumps(response.json(), sort_keys=True,indent=4,separators=(',',':'))
    click.echo(rp)

@main.command()
@click.argument('query')
def deploy(query):
    """This will deploy client's application on number of EC2 VMs on AWS cloud."""

    url_format = "http://" + link + ":5001/deploy"

    data=json.loads(yamlToJson(query))

    response = requests.post(url_format, json = data)
    rp=json.dumps(response.json(), sort_keys=True,indent=4,separators=(',',':'))
    click.echo(rp)

def yamlToJson(YamlFile):
    with open(YamlFile) as f:
        dataMap = yaml.safe_load(f)

    return (json.dumps(dataMap))

if __name__ == "__main__":
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Oct  5 11:20:02 2018

@author: kranthiyanamandra
"""

import matplotlib.pyplot as plt
import numpy as np
import madmom
#import librosa
import operator
import yam
import pyloudnorm

class TonalityDetector:

    @staticmethod
    def tonalityDetection(y, sr, specWhiten = False, loudNorm = False, plots = False):

        # Load the audio
        #y, sr = librosa.load(filePath, sr = None)

        # Take the specified length of audio
        #y = y[0 : length * sr]

        if specWhiten == True:

            # Apply spectral whitening
            y = yam.correlate.spectral_whitening(data = y, sr = sr, smooth = 256, filter = [10, 10000])

        if loudNorm == True:

            # Create loudness meter and calculate loudness
            meter = pyloudnorm.Meter(sr)
            loudness = meter.integrated_loudness(y)

            # Loudness normalise the audio to -6 dB
            y = pyloudnorm.normalize.loudness(y, loudness, -6)

        # Deep chroma processing
        dcp = madmom.audio.DeepChromaProcessor()

        chroma_ = dcp(y)

        chroma = chroma_.T

        # Summing all chroma energies across time
        chromaEnergy = chroma.sum(axis = 0)

        # Normalising chroma energies
        chromaNormalised = chromaEnergy / max(chromaEnergy)

        # Calculate duration of audio
        duration = len(y) / sr

        # Calculate tonality score as sum of normalised chroma over the duration of the audio
        tonality = np.sum(chromaNormalised) / duration

        # Calculate the amount of energy of each pitch class
        pitchClassEnergy = chroma.sum(axis = 1)
        rootNoteEnergy = max(pitchClassEnergy)

        # Full spectrum reference chroma calculations
#        referenceChroma = np.ones((12, chroma.shape[1]))
#        refChromaEnergy = referenceChroma.sum(axis=0)
#        refChromaNorm = refChromaEnergy / max(refChromaEnergy)
#        refTonality = np.sum(refChromaNorm) / duration

        # Scaling the tonality value to lie between 0 and 1
        scaledTonality = TonalityDetector.scaleValue(tonality, 0, 5, 0, 0.5)

        threshold = 20
        rootNote = None
        if rootNoteEnergy >= threshold:

            index, value = max(enumerate(pitchClassEnergy), key = operator.itemgetter(1))

            #finding first three indices of pitch classes that have the max energy
            #indices = np.argpartition(pitchClassEnergy, -3)[-3:]

            #sort
            #indices = indices[np.argsort(pitchClassEnergy[indices])]

            if index == 0:
                rootNote = 'C'

            elif index == 1:
                rootNote = 'C#'

            elif index == 2:
                rootNote = 'D'

            elif index == 3:
                rootNote = 'D#'

            elif index == 4:
                rootNote = 'E'

            elif index == 5:
                rootNote = 'F'

            elif index == 6:
                rootNote = 'F#'

            elif index == 7:
                rootNote = 'G'

            elif index == 8:
                rootNote = 'G#'

            elif index == 9:
                rootNote = 'A'

            elif index == 10:
                rootNote = 'A#'

            elif index == 11:
                rootNote = 'B'

            elif index == 12:
                rootNote = 'B#'

        elif rootNoteEnergy < threshold:

            print('You need ear training')

        if plots == True:

            # Plotting the chromagram and chroma energies
            plt.subplot(2,1,1)
            plt.imshow(chroma_.T, origin = 'lower', aspect = 'auto')

            plt.subplot(2,1,2)
            plt.plot(np.arange(0, duration, duration / len(chromaNormalised)), chromaEnergy)

            plt.xlabel('Time (seconds)')
            plt.ylabel('Chroma energy')
            plt.xticks(np.arange(0, duration, step = 10))

            plt.yticks(np.arange(0, 4, step = 0.5))

            plt.axis([0, duration, 0, 4])
            plt.show()

            plt.tight_layout()

        return scaledTonality

    @staticmethod
    def scaleValue(value, leftMin, leftMax, rightMin, rightMax):

        # Figure out how 'wide' each range is
        leftSpan = leftMax - leftMin
        rightSpan = rightMax - rightMin

        # Convert the left range into a 0-1 range (float)
        valueScaled = float(value - leftMin) / float(leftSpan)

        # Convert the 0-1 range into a value in the right range.
        return rightMin + (valueScaled * rightSpan)


from itertools import combinations

N = int(input())
numbers = [list(map(int, input().split())) for _ in range(N)]
result = 0
maxi = 0

for i in range(N):
    comb = list(combinations(numbers[i], 3))
    temp = 0
    for num in comb:
        temp = max(temp, sum(num) % 10)
    if temp >= maxi:
        result = i + 1
        maxi = temp
print(result)


from datetime import datetime, timedelta

import requests
import re
import pandas as pd
import pathlib

savePath = str(pathlib.Path(__file__).parent.resolve())

base_url = 'https://jira.goodt.me/rest/api/2'
headers = {'Content-type': 'application/json'}

log = 'g.danilov'
pas = 'qazxsw2!'
maxres = '1'

# log = input("login : ")
# pas = input("password : ")
# maxres = str(input("max results : "))

current_date = datetime.now()
date_format = '%Y-%m-%d'
cd_str = current_date.strftime(date_format)
cd_str_minus_day = (current_date - timedelta(days=3)).strftime(date_format)

base_jql = 'project = ABCHR AND sprint=631 and status != Cancelled'

testers = ["m.druzhinin@goodt.me", "f.shabalkin@goodt.me", "a.kolganov@goodt.me", "a.svalova@goodt.me", "e.voronina@goodt.me", "e.kazova@goodt.me"]

def correct_url(base_jql):
    URL = base_url + '/search?jql=' + base_jql + '&maxResults=' + maxres
    return URL

def json_returner(URL):
    get_req = requests.get(URL, auth=(log, pas))
    return get_req.json()

def extract_issue_from_json(json):
    return json['issues']

def list_self_links(d, testers):
    list_self = []
    values = []
    keys = []
    for i in d:
        list_self.append(i['self'])
        comm_url = i["self"] + "/comment"
        comm_req = requests.get(comm_url, auth=(log, pas))
        comm_data = comm_req.json()
        key = i["key"]

        for comment in comm_data["comments"]:

            value = None

            #print(comment["body"])
            #print(comment["author"]["name"])
            text = comment["body"]
            mail = comment["author"]["emailAddress"]
            if mail in testers:

                if "t: " in str(comment["body"]):
                    search_in = text.partition("t:")[2]
                    value = re.search(r'\d+', search_in).group()

                elif "t - " in str(comment["body"]):
                    search_in = text.partition("t -")[2]
                    value = re.search(r'\d+', search_in).group()

                elif "=" in str(comment["body"]):
                    search_in = text.partition("=")[2]
                    value = re.search(r'\d+', search_in).group()

            if value != None:
                values.append(value)
                keys.append(key)

    output = {
        "Ключ задачи" : keys,
        "Сложность" : values
    }
    df = pd.DataFrame(output, columns = ["Ключ задачи", "Сложность"])
    df.to_csv(savePath + "\difficulty.csv",index=False)

    print('Проверка размера списка ссылок селф на задачи', len(list_self), sep='/t')
    return list_self

#######################
# При запросе ["fields"] реквест не возвращает ["comment"] с содержимым по неизвестной причине

def extract_comment_from_issue(issue_json):
    comm_self = []
    for i in issue_json:
        print(i['self'])
        data = json_returner(i['self'])
        comm_self.append(data["fields"]["comment"])
    print(comm_self)

######################

class Comment:

    def __init__(self, issue_id, author, body, updatedDate):
        self.issue_id = issue_id
        self.author = author
        self.body = body
        self.updatedDate = updatedDate

    def get_id(self):
        return self.issue_id

    def get_author(self):
        return self.author

    def get_body(self):
        return self.body

    def get_updated(self):
        return self.updatedDate

full_jql_url = correct_url(base_jql)
links = list_self_links(extract_issue_from_json(json_returner(full_jql_url)), testers)
extract_comment_from_issue(extract_issue_from_json(json_returner(full_jql_url)))


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
import math
import pywt
from PyEMD import EMD
from scipy import signal
from Processing import parse_motion_file, lowpass_filter
from vmdpy import VMD
import emd as emd2
from tqdm import tqdm, trange
from scipy import signal
import scipy.interpolate as interpolate
import scipy.io

def stft_decomp(data):
    sampling_freq = 200
    window_size = 2048
    overlap = 2047

    # s, f, t, im = plt.specgram(data, Fs=sampling_freq, NFFT=window_size, noverlap=overlap, cmap='rainbow')
    # print(f[:20])
    # plt.ylim(0, 20)

    f, t, Zxx = signal.stft(data, fs=sampling_freq, nfft=window_size, nperseg=window_size, noverlap=overlap)
    return f, t, Zxx

def wavelet_decomp(data):
    fs = 200
    wavelet = 'cmor1.5-1'
    frequencies = np.array(np.arange(start=0.01, stop=20.01, step=0.05)) / fs
    scales = pywt.frequency2scale(wavelet, frequencies)
    print(len(scales))

    coef, freqs = pywt.cwt(data, scales, wavelet)

    return coef, freqs
    # print(coef.shape)
    # print(len(freqs))

    # graph
    # plt.imshow(abs(coef), extent=[0, 7510, 20, 0] , aspect='auto', cmap='rainbow')
    # plt.gca().invert_yaxis()
    # plt.xticks(ticks=np.arange(0, coef.shape[1], step=1000), labels=np.arange(0, coef.shape[1], step=1000)/fs)

def emd_decomp(data):
    imf = emd2.sift.sift(data)

    # instantaneous frequency, phase and amplitude
    IP, IF, IA = emd2.spectra.frequency_transform(imf, 200, 'hilbert')
    return imf

def graphEMDModeSpectrums(data, filename):
    imf = emd_decomp(data)
    with PdfPages(filename) as pdf:
        for i in trange(imf.shape[1]):
            fig, ax = plt.subplots(figsize=(8, 6))
            fft_result = np.fft.fft(imf[:, i])
            frequencies = np.fft.fftfreq(len(fft_result), 1/200)
            ax.set_title(f'IMF {i+1} Spectrum')
            ax.semilogy(frequencies, np.abs(fft_result))
            ax.set_xlabel('Frequency (Hz)')
            ax.set_ylabel('Amplitude')
            ax.grid(True)
            ax.set_xlim(0, frequencies.max())
            pdf.savefig(fig)
            plt.close(fig)

def vmd_decomp(data):
    alpha = 2000       # moderate bandwidth constraint
    tau = 0            # noise-tolerance (no strict fidelity enforcement)
    K = 8              # modes
    DC = 0             # no DC part imposed
    init = 1           # initialize omegas uniformly
    tol = 1e-7

    u, u_hat, omega = VMD(data, alpha, tau, K, DC, init, tol)
    return u, omega[-1:]

def graphVMDModeSpectrums(data, filename):
    u, u_hat, omega, K = vmd_decomp(data)

    with PdfPages(filename) as pdf:
        for i in trange(K):
            fig, ax = plt.subplots(figsize=(8, 6))
            magnitude_spectrum = np.abs(u_hat[:, i])
            freqs = np.fft.fftshift(np.fft.fftfreq(len(magnitude_spectrum)))

            ax.semilogy(freqs, magnitude_spectrum)

            # Set labels and title
            ax.set_xlabel('Frequency (Hz)')
            ax.set_ylabel('Magnitude Spectrum')
            ax.set_title(f'Mode {i + 1} Spectrum')
            ax.set_xlim(0, freqs.max())
            pdf.savefig(fig)
            plt.close(fig)

    print(omega[-1:])

def comparePerturbation(data, time, n_perturb):
    imfs, target_freqs = vmd_decomp(data)
    target_freqs = target_freqs[0]
    freq_segments = []

    plt.figure()
    for i in range(imfs.shape[0]):
        plt.plot(imfs[i])

    # create ranges for each VMD IMF frequency
    for i in range(1, len(target_freqs)-1):
        freq_segments.append(((target_freqs[i] + target_freqs[i-1])/2, (target_freqs[i] + target_freqs[i+1])/2))

    freq_segments.insert(0, (0, freq_segments[0][0]))
    freq_segments.append((freq_segments[-1][1], target_freqs[-1] + (target_freqs[-1] - freq_segments[-1][1])))

    # create warpings for each range
    warping = np.zeros([n_perturb, 8, 7510])
    for j in range(n_perturb):
        for i in range(8):
            warping[j, i, :] = MagnitudeWarp(time, 8)

    plotWarping(warping[0])
    show_buckets(freq_segments, data)

    plt.figure()
    # # warp VMD IMFS
    new_vmd_imfs = np.zeros(imfs.shape)
    new_VMDs = np.zeros([n_perturb, len(data)])
    for j in trange(n_perturb):
        for i in range(8):
            new_vmd_imfs[i, :] = imfs[i, :] * warping[j, i, :]
        new_VMD = np.sum(new_vmd_imfs, axis=0)
        new_VMDs[j] = new_VMD
        plt.plot(new_VMD, alpha=0.7)
    plt.plot(data, color='k', label='original')
    plt.title('VMD Perturbed Trajectories')

    m_VMD, u_VMD, l_VMD = calculateMeanStdDev(new_VMDs)
    plt.plot(m_VMD, color=(1, 0, 1), label='mean')
    plt.legend()

    # # # warp STFT
    f, t, Zxx = stft_decomp(data)
    new_Zxx = np.zeros(Zxx.shape)
    new_STFTS = np.zeros([n_perturb, len(data)])

    plt.figure()
    show_stft_buckets(freq_segments, f, t, Zxx)

    plt.figure()
    for k in trange(n_perturb):
        for i in range(len(f)):
            for j in range(len(freq_segments)):
                if (freq_segments[j])[0] <= f[i] < (freq_segments[j])[1]:
                    new_Zxx[i][:7510] = Zxx[i][:7510] * warping[k, j, :]
        new_STFT = signal.istft(new_Zxx, fs=200, nperseg=2048, nfft=2048, noverlap=2047)
        new_STFTS[k, :] = new_STFT[1]
        plt.plot(new_STFT[1], alpha=0.7)
    plt.plot(data, color='k', label='original')
    plt.title('STFT Perturbed Trajectories')

    m_STFT, u_STFT, l_STFT = calculateMeanStdDev(new_STFTS)
    plt.plot(m_STFT, color=(1, 0, 1), label='mean')
    plt.legend()

    scipy.io.savemat('data.mat', mdict={'data': data})
    scipy.io.savemat('warpings.mat', mdict={'warpings': warping})

    # # # Warp EMD
    imf = emd_decomp(data)

    # plt.figure()
    new_emd_imfs = np.zeros(imf.shape)
    new_EMDs = np.zeros([n_perturb, len(data)])
    max_frequencies = np.zeros(8)

    for i in range(imf.shape[1]):
        fft_result = np.fft.fft(imf[:, i])
        frequencies = np.fft.fftfreq(len(fft_result), 1/200)
        magnitude = np.abs(fft_result)
        max_amplitude_index = np.argmax(magnitude)
        frequency_with_max_amplitude = frequencies[max_amplitude_index]
        max_frequencies[i] = frequency_with_max_amplitude

    show_emd_buckets(freq_segments, imf, max_frequencies)

    plt.figure()
    for k in trange(n_perturb):
        for i in range(imf.shape[1]):
            for j in range(len(freq_segments)):
                if (freq_segments[j])[0] <= np.abs(max_frequencies[i]) < (freq_segments[j])[1]:
                    new_emd_imfs[:, i] = imf[:, i] * warping[k, j, :]
        new_IMF = np.sum(new_emd_imfs.T, axis=0)
        new_EMDs[k, :] = new_IMF
        plt.plot(new_IMF, alpha=0.7)

    plt.plot(data, color='k', label='original')
    plt.title('EMD Perturbed Trajectories')
    m_EMD, u_EMD, l_EMD = calculateMeanStdDev(new_EMDs)
    plt.plot(m_EMD, color=(1, 0, 1), label='mean')
    plt.legend()

    plt.figure()
    plt.title('Different Method Trajectories Overview')
    plt.plot(m_VMD, color='r', label='VMD')
    plt.fill_between(range(len(m_VMD)), u_VMD, l_VMD, color='r', alpha = 0.2)
    plt.plot(m_STFT, color='b', label='STFT')
    plt.fill_between(range(len(m_STFT)), u_STFT, l_STFT, color='b', alpha = 0.2)
    plt.plot(m_EMD, color='g', label='EMD')
    plt.fill_between(range(len(m_EMD)), u_EMD, l_EMD, color='g', alpha = 0.2)
    plt.plot(data, color='k')
    plt.legend()
    plt.show()

def calculateMeanStdDev(dataset):
    num_columns = dataset.shape[1]
    mean = np.zeros(num_columns)
    upper = np.zeros(num_columns)
    lower = np.zeros(num_columns)

    for i in range(num_columns):
        column_data = dataset[:, i]
        mean[i] = np.mean(column_data)
        std = np.std(column_data)
        upper[i] = mean[i] + std
        lower[i] = mean[i] - std

    return mean, upper, lower

def MagnitudeWarp(time, n_points):
    step = int(len(time) / n_points)
    time_indices = np.arange(0, len(time), step, dtype=int)
    time_points = time[time_indices]

    distortion = np.random.normal(loc=1, scale=0.2, size=len(time_indices))

    cs = interpolate.CubicSpline(time_points, distortion)
    return cs(time)

h, df, t = parse_motion_file("/Users/FranklinZhao/OpenSimProject/Simulation/Models/Rajapogal_2015/inverse_kinematics_data/SN001_0024_tug_01.mot")
original_data = df.to_numpy()
feature = original_data[:, 6]
t = np.array(t)

# imf = emd_decomp(feature)
# new_signal = np.sum(imf.T, axis=0)
# print(imf.shape)
# plt.plot(new_signal)
# plt.show()

# graphEMDModeSpectrums(feature, 'EMDModeSpectrums.pdf')
# graphVMDModeSpectrums(feature, 'VMDModeSpectrums.pdf')

def plotWarping(warping):
    colors = ['#FF3333', '#FF9933', '#E6EC38', '#BBFF33', '#33FFD4', '#3377FF', '#7733FF', '#F333FF']
    n_warp = warping.shape[0]
    fig, axs = plt.subplots(n_warp, 1)

    for i in range(n_warp):
        axs[i].axhline(y=1, color='b', linestyle='--')
        axs[i].plot(warping[i][:], color = colors[i])
        axs[i].set_title(f"Warping {i+1}")
        axs[i].set_ylim(0.25, 1.75)
    fig.suptitle("Warpings")

def show_buckets(buckets, signal):
    plt.figure()
    colors = ['#FF3333', '#FF9933', '#E6EC38', '#BBFF33', '#33FFD4', '#3377FF', '#7733FF', '#F333FF']

    fft_result = np.fft.fft(signal)
    frequencies = np.fft.fftfreq(len(signal), 1/200)
    plt.plot(frequencies[:len(frequencies)//2], np.abs(fft_result[:len(frequencies)//2]))
    for i in range(len(buckets)):
        plt.axvspan(buckets[i][0], buckets[i][1], alpha=0.5, color=colors[i])
    plt.title("Frequency-Amplitude of Signal with Warping Segments")
    plt.xlim(0, 3)
    plt.title("Buckets vs Original Signal FFT")

def show_stft_buckets(buckets, f, t, Zxx):
    plt.figure()
    colors = ['#FF3333', '#FF9933', '#E6EC38', '#BBFF33', '#33FFD4', '#3377FF', '#7733FF', '#F333FF']
    spectrogram = np.abs(Zxx)**2
    plt.pcolormesh(t, f, 10 * np.log10(spectrogram), shading='gouraud', cmap='gray')  # Using logarithmic scale for better visualization
    plt.ylabel('Frequency (Hz)')
    plt.ylim(0, 3)
    plt.title("Spectrogram of SIgnal with Warping Segments")
    for i in range(len(buckets)):
        plt.axhspan(buckets[i][0], buckets[i][1], alpha=0.5, color=colors[i])
    plt.title("Buckets vs STFT Spectrogram")

def show_emd_buckets(buckets, imf, max_frequency):
    imf = imf.T
    fig, axs = plt.subplots(imf.shape[0], 1)
    colors = ['#FF3333', '#FF9933', '#E6EC38', '#BBFF33', '#33FFD4', '#3377FF', '#7733FF', '#F333FF']

    for i in range(imf.shape[0]):
        for j in range(len(buckets)):
            if (buckets[j])[0] <= np.abs(max_frequency[i]) < (buckets[j])[1]:
                axs[i].plot(imf[i], color=colors[j])
            else:
                axs[i].plot(imf[i], color='black')
            axs[i].set_title(f"imf {i}")
    fig.suptitle("Buckets vs EMD IMFs")

# comparePerturbation(feature, t, 1)

imfs, target_freqs = vmd_decomp(feature)
target_freqs = target_freqs[0]
freq_segments = []

plt.figure()
for i in range(imfs.shape[0]):
    plt.plot(imfs[i])
plt.show()


import inspect
from typing import Callable, Any, Type, Optional, Sequence, List
from inspect import stack
import warnings
from fuse.utils.file_io.file_io import load_pickle, save_pickle_safe
import os
from fuse.utils.cpu_profiling import Timer
import hashlib

def get_function_call_str(
    func: Callable,
    *_args: list,
    _ignore_kwargs_names: List = None,
    _include_code: bool = True,
    **_kwargs: dict,
) -> str:
    """
    Converts a function and its kwargs into a hash value which can be used for caching.
    NOTE:
    1. This is far from being bulletproof, the op might call another function which is not covered and is changed,
    which will make the caching processing be unaware.
    2. This is a mechanism that helps to spot SOME of such issues, NOT ALL
    3. Only a specific subset of arg types contribute to the caching, mainly simple native python types.
    see 'value_to_string' for more details.
    For example, if an arg is an entire numpy array, it will not contribute to the total hash.
    The reason is that it will make the cache calculation too slow, and might

    _ignore_kwargs_names: args to ignore when generating the string representation. This is useful for kwargs
        that you don't want to have an effect on the hash, for example, verbose flag.
        example usage: ignore_kwargs_names=['verbose', 'cpu_cores_num']

    _include_code: would code be included in the string representation. Note, it will only include code found *directly*
        in the provided function. Set to False if you don't want it to influence the generated string.
        Default is True.

    """
    if _ignore_kwargs_names is None:
        _ignore_kwargs_names = []
    kwargs = convert_func_call_into_kwargs_only(func, *_args, **_kwargs)

    args_flat_str = func.__name__ + "@"
    use_keys = [k for k in sorted(kwargs.keys()) if k not in _ignore_kwargs_names]
    # ignore_kwargs_names
    args_flat_str += "@".join(
        ["{}@{}".format(str(k), value_to_string(kwargs[k])) for k in use_keys]
    )
    args_flat_str += "@" + str(
        inspect.getmodule(func)
    )  # adding full (including scope) name of the function, for the case of multiple functions with the same name
    if _include_code:
        args_flat_str += "@" + inspect.getsource(
            func
        )  # considering the source code (first level of it...)

    return args_flat_str

def value_to_string(val: Any, warn_on_types: Optional[Sequence] = None) -> str:
    """
    Used by default in several caching related hash builders.
    Ignores <...> string as they usually change between different runs
    (for example, due to pointing to a specific memory address)
    """
    if warn_on_types is not None:
        if isinstance(val, tuple(list(warn_on_types))):
            warnings.warn(
                f"type {type(val)} is possibly participating in hashing, this is usually not optimal performance wise."
            )
    ans = str(val)
    if ans.startswith("<"):
        return ""
    return str(val)

def convert_func_call_into_kwargs_only(
    func: Callable, *args: list, **kwargs: dict
) -> dict:
    """
    considers positional and kwargs (including their default values !)
    and converts into ONLY kwargs
    """
    signature = inspect.signature(func)

    my_kwargs = {
        k: v.default
        for k, v in signature.parameters.items()
        if v.default is not inspect.Parameter.empty
    }

    # convert positional args into kwargs
    # uses the fact that zip stops on the smallest length ( so only as much as len(args))
    for curr_pos_arg, pos_arg_name in zip(args, inspect.getfullargspec(func).args):
        my_kwargs[pos_arg_name] = curr_pos_arg

    my_kwargs.update(kwargs)

    return my_kwargs

def get_callers_string_description(
    max_look_up: int,
    expected_class: Type,
    expected_function_name: str,
    value_to_string_func: Callable = value_to_string,
    ignore_first_frames: int = 3,  # one for this function, one for HashableCallable, and one for OpBase
) -> str:
    """
    iterates on the callstack, and accumulates a string representation of the callers args.
    Used in OpBase to "record" the __init__ args, to be used in the string representation of an Op,
    which is used for building a hash value for samples caching in SamplesCacher

    example call:

    class A:
        def __init__(self):
            text = get_callers_string_description(4, A,

    class B(A):
        def __init__(self, blah, blah2):
            super().__init__()
            #... some logic

    :param max_look_up: how many stack frames to look up
    :param expected_class: what class is the method expected to be,
        stack frames in a different class will be skipped.
        pass None for not requiring any class
    :param expected_function_name: what is the name of the function to allow,
        stack frames in a different function name will be skipped,
        pass None for not requiring anything
    :param value_to_string_func: allows to provide a custom function for converting values to strings
    :param
    """

    str_desc = ""
    try:
        curr_stack = stack()
        curr_locals = None
        # note: frame 0 is this function, frame 1 is whoever called this (and wanted to know about its callers),
        # so both frames 0+1 are skipped.
        for i in range(
            ignore_first_frames, min(len(curr_stack), max_look_up + ignore_first_frames)
        ):
            curr_locals = curr_stack[i].frame.f_locals
            if expected_class is not None:
                if "self" not in curr_locals:
                    continue
                if not isinstance(curr_locals["self"], expected_class):
                    continue

            if expected_function_name is not None:
                if expected_function_name != str(curr_stack[i].function):
                    continue

            curr_str = ".".join(
                [
                    str(
                        curr_locals["self"].__module__
                    ),  # module is probably not needed as class already contains it
                    str(curr_locals["self"].__class__),
                    str(curr_stack[i].function),
                ]
            )

            curr_str += inspect.getsource(curr_stack[i].frame)
            for k, d in curr_stack[i].frame.f_locals.items():
                if "self" == k:
                    continue
                if k.startswith("__"):
                    continue
                curr_str += "@" + str(k) + "@" + value_to_string_func(d)

            str_desc += curr_str

    finally:
        del curr_locals
        del curr_stack

    return str_desc

# TODO: consider adding "ignore list" of args that should not participate in cache value calculation (for example - "verbose")
def run_cached_func(cache_dir: str, func: Callable, *args: list, **kwargs: dict) -> Any:
    """
    Will cache the function output in the first time that
     it is executed, and will load from cache on the next times.

    The cache hash value will be based on the function name, the args, and the code of the function.

    Args:

    :param cache_dir: the directory into which caches will be stored/loaded
    :param func: the function to run
    :param *args: positional args to provide to the function
    :param **kwargs: kwargs to provide to the function

    """
    os.makedirs(cache_dir, exist_ok=True)
    call_str = get_function_call_str(func, *args, **kwargs)
    call_hash = hashlib.md5(call_str.encode("utf-8")).hexdigest()

    cache_full_file_path = os.path.join(cache_dir, call_hash + ".pkl.gz")
    print(f"cache_full_file_path={cache_full_file_path}")

    if os.path.isfile(cache_full_file_path):
        with Timer(f"loading {cache_full_file_path}"):
            ans = load_pickle(cache_full_file_path)
        return ans

    with Timer("run_cached_func::running func ..."):
        ans = func(*args, **kwargs)

    with Timer(f"saving {cache_full_file_path}"):
        save_pickle_safe(ans, cache_full_file_path, compress=True)

    return ans


from django.shortcuts import render, get_object_or_404
from django.contrib.auth.decorators import login_required, permission_required
from .models import *
from taggit.models import Tag
from django.db.models import Count

from django.contrib.contenttypes.models import ContentType

def courselist(request, tag_slug=None):
    #course = Course.objects.all()
    course = Course.objects.filter(id__in=[1, 2, 3])
    free_course = Course.objects.filter(course_type="FREE")
    tag = None
    if tag_slug:
        tag = get_object_or_404(Tag, slug=tag_slug)
        course = course.filter(tags__in=[tag])

    context = {
              "course": course,
              'free':free_course,
              'tag': tag,

            }
    return render (request, 'mycourse/course_list.html',context)

def course_detail(request, slug):
    course = Course.objects.filter(course_slug=slug).first()
    section = Section.objects.filter(course=course)
    lecture = Lecture.objects.filter(course=course)

    course_tags_ids =course.tags.values_list('id', flat=True)
    similar_course = Course.objects.filter(tags__in=course_tags_ids).exclude(id=course.id)
    similar_course = similar_course.annotate(same_tags=Count('tags')).order_by('-same_tags')[:4]

    context ={

        "course":course,
        "section":section,
        "lecture":lecture,
        'similar_course':similar_course,

    }

    return render(request, 'mycourse/course_detail.html', context)


import json
import os
import logging
import signal
import subprocess
import time

import psutil

from app_context_lock import AppContextLock
from FifoFile import FifoFile
from skills.timer import Timer

class Player:
    """
    Base class of a player
    """
    def resume(self):
        pass

    def stop(self):
        pass

    def get_duration(self):
        pass

    def load(self, track, seek):
        pass

    def has_track_completed(self):
        pass

class MPlayerProcess(Player):
    """
    This object controls the actual mplayer process and fifo stack
    """
    def __init__(self, mplayer_config_path, fifo_directory):
        super().__init__()
        logging.debug("Creating MPlayerProcess object")
        self._init( mplayer_config_path, fifo_directory)

    def _init(self, mplayer_config_path, fifo_directory):
        self.audio_config = None
        self._load_config(mplayer_config_path)
        self.mplayer_command = self.audio_config["shell"]
        self.process = None
        self.timer = None
        self.fifo_file_prefix = "mplayer"
        self.fifo = FifoFile(fifo_directory, self.fifo_file_prefix)
        self.has_started = False

    def _load_config(self, file_path):
        """Set the mplayer config which is the same for all instances"""
        with open(file_path, "r") as config_fh:
            self.audio_config = json.load(config_fh)

    def resume(self):
        self.fifo.write(self.audio_config["commands"]["continue"])
        self.timer.resume()

    def stop(self):
        if self._is_alive():
            self.fifo.write(self.audio_config["commands"]["pause"])
        self.timer.stop()

    def play(self, track, seek):
        if self._is_alive():
            self.load(track, seek)
        else:
            self._start(track, seek)

    def get_duration(self):
        return self.timer.elapsedTime()

    def load(self, track, seek=0):
        """
        From mplayer slave commands:
        loadfile <file|url> <append>
        Load the given file/URL, stopping playback of the current file/URL.
            If <append> is nonzero playback continues and the file/URL is
            appended to the current playlist instead.
        """
        load_command = self.audio_config["commands"]["load"] + " \"{}\" 0".format(track)
        self.fifo.write(load_command)
        self._seek(seek)
        self.timer = Timer(seek)

    def has_track_completed(self):
        if self.has_started:
            return not self._is_alive()
        return False

    def _seek(self, value):
        if value > 0:
            seek_command = self.audio_config["commands"]["seek"]
            seek_command += " {} 2".format(value)
            self.fifo.write(seek_command)

    def _wait_for_mplayer_to_load(self):
        delay = 5
        if "start_delay" in self.audio_config:
            delay = self.audio_config["start_delay"]
        time.sleep(delay)

    def _start(self, track, seek=0):
        app_context_lock = AppContextLock()
        shell_cmd = self.mplayer_command
        shell_cmd = shell_cmd.replace("%fifo%", self.fifo.full_path)
        shell_cmd = shell_cmd.replace("%track%", track)
        if seek > 0:
            shell_cmd = shell_cmd.replace("%seek%", "-ss {}".format(int(seek)))
        else:
            shell_cmd = shell_cmd.replace("%seek%", "")
        logging.debug(shell_cmd)

        # if user wanted output to be diverted to a log file then do so otherwise do not
        # as its quite a lot of stuff not much use to anyone
        logfile = ""
        if "logfile" in self.audio_config:
            if self.audio_config["logfile"]:
                logfile = " > {} 2>&1".format(self.audio_config["logfile"])
        self.process = subprocess.Popen("exec {} {}".format(shell_cmd, logfile),
                                        shell=True,
                                        stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)
        logging.info("PID of mplayer process:" + str(self.process.pid))
        self.timer = Timer(seek)
        self._wait_for_mplayer_to_load()
        self.has_started = True

    def _is_alive(self):
        """
        This is quite confusing.  We poll and this will clean up any zombie process that might still be around that
        will throw off our signal to the process.  We then send the 0 signal to our process and if it fails it will
        trigger an exception and we know that the process is dead.
        :return:
        """
        if self.process is None:
            if self.timer:
                self.timer.stop()
            return False
        try:
            self.process.poll()
            parent = psutil.Process(self.process.pid)
            for child in parent.children(recursive=True):
                os.kill(child.pid, 0)
            os.kill(parent.pid, 0)
            return True
        except:
            if self.timer:
                self.timer.stop()
            return False

    def _reset(self):
        # must close these otherwise terminal disappears when we quit
        # I think its because mplayer uses lots of escape sequences in output
        # which messes with the terminal
        os.system('stty sane')
        self.process = None
        self.fifo = None

    def _check_if_dead_via_shell_script(self):
        # we don't trust this so we are going to do a hacky shell check as well
        # we are looking for any mplayer processes that have our fifo file attached
        logging.debug("checking for any running mplayer processes with our fifo")
        cmd = "ps aux | grep mplayer | grep '{}' | grep -v grep | awk '{{print $2}}'".format(self.fifo)
        output = subprocess.check_output(cmd, shell=True)
        pids = output.decode("utf-8").split("\n")
        if len(pids) > 0:
            for pid in pids:
                if pid != "":
                    logging.debug("killing process {} as it should be dead".format(pid))
                    os.kill(int(pid), signal.SIGKILL)

    def _kill(self):
        try:
            self.process.poll()  # this will cause an exception is process is not available
            self.process.stderr.close()
            self.process.stdout.close()
            self.process.send_signal(signal.SIGINT)
            self.process.wait()
            self._check_if_dead_via_shell_script()
        except Exception as e:
            logging.debug(e)
        finally:
            self._reset()
        logging.info("Process should be dead")

    def __del__(self):
        logging.debug("Destroying MPlayerProcess object")
        self._kill()


from aqt.qt import *
import aqt
from aqt.utils import showInfo, getOnlyText, shortcut, restoreGeom, saveGeom
from anki.hooks import addHook, remHook

class StudyDeck(QDialog):
    def __init__(self, mw, names=None, accept=None, title=None,
                 help="studydeck", current=None, cancel=True,
                 parent=None, dyn=False, buttons=[], geomKey="default"):
        QDialog.__init__(self, parent or mw)
        self.mw = mw
        self.form = aqt.forms.studydeck.Ui_Dialog()
        self.form.setupUi(self)
        self.form.filter.installEventFilter(self)
        self.cancel = cancel
        addHook('reset', self.onReset)
        self.geomKey = "studyDeck-"+geomKey
        restoreGeom(self, self.geomKey)

        if title:
            self.setWindowTitle(title)
        if not names:
            names = sorted(self.mw.col.decks.allNames(dyn=dyn))
            self.nameFunc = None
            self.origNames = names
        else:
            self.nameFunc = names
            self.origNames = names()
        self.name = None

        self.form.buttonBox.setStandardButtons(QDialogButtonBox.Cancel)
        self.ok = self.form.buttonBox.addButton(
            accept or _("Review"), QDialogButtonBox.AcceptRole)
        self.setWindowModality(Qt.WindowModal)
        self.form.filter.textEdited.connect(self.redraw)

        self.show()
        # redraw after show so position at center correct
        self.redraw("", current)
        self.exec_()

    def eventFilter(self, obj, evt):
        if evt.type() == QEvent.KeyPress:
            if evt.key() == Qt.Key_Up:
                c = self.form.list.count()
                row = self.form.list.currentRow() - 1
                if row < 0:
                    row = c - 1
                self.form.list.setCurrentRow(row)
                return True
            elif evt.key() == Qt.Key_Down:
                c = self.form.list.count()
                row = self.form.list.currentRow() + 1
                if row == c:
                    row = 0
                self.form.list.setCurrentRow(row)
                return True
        return False

    def redraw(self, filt, focus=None):
        self.filt = filt
        self.focus = focus
        self.names = [n for n in self.origNames if self._matches(n, filt)]
        l = self.form.list
        l.setSelectionMode(2) #2=MultiSelection, 3=ExtendedSelection
        l.clear()
        l.addItems(self.names)
        if focus in self.names:
            idx = self.names.index(focus)
        else:
            idx = 0
        l.setCurrentRow(idx)
        l.scrollToItem(l.item(idx), QAbstractItemView.PositionAtCenter)

    def _matches(self, name, filt):
        name = name.lower()
        filt = filt.lower()
        if not filt:
            return True
        for word in filt.split(" "):
            if word not in name:
                return False
        return True

    def onReset(self):
        # model updated?
        if self.nameFunc:
            self.origNames = self.nameFunc()
        self.redraw(self.filt, self.focus)

    def accept(self):
        saveGeom(self, self.geomKey)
        remHook('reset', self.onReset)
        row = self.form.list.currentRow()
        if row < 0:
            showInfo(_("Please select something."))
            return
        self.name = self.names[self.form.list.currentRow()]
        QDialog.accept(self)

    def reject(self):
        self.name = None
        saveGeom(self, self.geomKey)
        remHook('reset', self.onReset)
        QDialog.reject(self)

    def getSelectedDids(self):
        d=[]
        for sel in self.form.list.selectedItems():
            name=sel.text()
            deck=self.mw.col.decks.byName(name)
            d.append(deck['id'])
        return d


import argparse
import os.path as osp
from typing import Union

import numpy as np
import torch
import torch.nn.functional as F
from yacs.config import CfgNode as CN

UTILS_DIR = osp.dirname(osp.abspath(__file__))
DRL_DIR = osp.join(osp.dirname(UTILS_DIR))
CONFIGS_DIR = osp.join(DRL_DIR, "configs")
DEFAULT_CONFIG_FILE = osp.join(CONFIGS_DIR, "default.yaml")

def get_default_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser()
    parser.add_argument("-o", "--opts", nargs="*", default=[])
    return parser

def construct_config(config_path=None, opts=None):
    if config_path is None:
        config_path = DEFAULT_CONFIG_FILE
    if opts is None:
        opts = []
    config = CN()
    config.set_new_allowed(True)
    config.merge_from_file(config_path)
    config.merge_from_list(opts)

    return config

@torch.jit.script
def mse_loss(tensor1, tensor2):
    """Do F.mse_loss but ensure shapes match"""
    assert tensor1.shape == tensor2.shape, f"{tensor1.shape} != {tensor2.shape}"
    return F.mse_loss(tensor1, tensor2)

def get_num_reward_terms(envs, num_envs):
    # Count the amount of reward terms by doing a dummy step
    actions = torch.as_tensor(envs.action_space.sample())
    if actions.dim() == 1:
        actions = actions.repeat(num_envs, 1)
    _, _, _, infos = envs.step(actions)
    assert "reward_terms" in infos, "Key 'reward_terms' must be in info for EPPO!"
    num_reward_terms = infos["reward_terms"].shape[1]
    return num_reward_terms

def reparameterize_action(mu, sigma, actions):
    eps = (actions - mu) / sigma
    reparam_action = mu + sigma * eps.detach()
    return reparam_action

class MeanReturns:
    def __init__(self, window_size=50):
        self.window_size = window_size
        self.total_returns = None
        self.curr_returns = None

    def update(
        self,
        rewards: Union[np.ndarray, torch.Tensor],
        dones: Union[np.ndarray, torch.Tensor],
    ):
        rewards_tensor = torch.as_tensor(rewards).flatten()
        if self.curr_returns is None:
            self.curr_returns = torch.zeros_like(rewards_tensor)

        dones_mask = torch.as_tensor(dones).bool().flatten()

        # Update buffer of last window_size total returns
        if self.total_returns is None:
            self.total_returns = self.curr_returns[dones_mask]
        else:
            self.total_returns = torch.cat(
                [self.total_returns, self.curr_returns[dones_mask]]
            )
        self.total_returns = self.total_returns[-self.window_size :]

        self.curr_returns[dones_mask] = 0
        self.curr_returns += rewards_tensor

    def mean(self):
        if self.total_returns is None or self.total_returns.nelement() == 0:
            return 0
        return self.total_returns.mean().item()


from keras.models import Sequential
from keras.layers.core import Dense
from keras.layers import Flatten, Convolution2D
import glob, re
import matplotlib.image as mpimg
import numpy as np
from sklearn.model_selection import train_test_split
from keras.callbacks import EarlyStopping

d = {"Excuse":0, "Goodby": 1, "Hello": 2, "How": 3, "Nice": 4, "Seeyou": 5, "Sorry": 6, "Thank": 7, "Thanks":7, "Time" : 8, "Welcome": 9}

def get_class(filename):
    """Given a filename return a hot encoded vector of the class"""

    pattern = re.compile(".*/([A-z]*).*")

    phrase = pattern.match(filename).groups()[0]

    if "_smooth" in phrase:
        phrase = phrase[:-7]

    cl = [0]*10
    cl[d[phrase]] = 1

    return cl

def rgb2gray(rgb):
    return np.dot(rgb[...,:3], [0.299, 0.587, 0.114])

def create_plot_conv():
    model = Sequential()
    model.add(Convolution2D(32, 2, 2, input_shape=(640, 480, 1)))
    model.add(Flatten())
    model.add(Dense(10, activation="softmax"))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=["categorical_accuracy"])
    return model

def train_and_evaluate(model, X_train, y_train, X_test, y_test):
    early_stopping = EarlyStopping(monitor='val_loss', patience=2)
    model.fit(X_train, y_train, nb_epoch=25, batch_size=10, verbose=1, validation_split=0.1, callbacks=[early_stopping])
    results = model.evaluate(X_test, y_test, batch_size=20, verbose=1)
    print(results)

def load_plot_images():
    X = []
    y = []

    image_files = glob.glob("videos/*/*/*.png")

    for i, image in enumerate(image_files):
        print(" {0:.2f}".format(float(i)*100/len(image_files)), end="\r")

        img = mpimg.imread(image)
        img = rgb2gray(img)
        img = img.reshape(640, 480, 1)

        X.append(img)
        y.append(get_class(image))

    return np.asarray(X), np.asarray(y)

if __name__ == '__main__':

    print("Loading data...")
    X,y = load_plot_images()
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42)

    print("Creating Model...")
    model = create_plot_conv()
    model.summary()

    train_and_evaluate(model, X_train, y_train, X_test, y_test)


from http import HTTPStatus

from pydantic import BaseModel, ValidationError

from nrlf.core.errors import NRLF_TO_SPINE_4XX_ERROR
from nrlf.core.nhsd_codings import (
    META,
    IssueSeverity,
    IssueType,
    NrlfCoding,
    SpineCoding,
)
from nrlf.core.validators import json_loads
from nrlf.producer.fhir.r4.model import (
    CodeableConcept,
    Coding,
    OperationOutcome,
    OperationOutcomeIssue,
)

HTTP_STATUS_CODE_MAPPING = {
    SpineCoding.RESOURCE_NOT_FOUND: HTTPStatus.NOT_FOUND,
    SpineCoding.VALIDATION_ERROR: HTTPStatus.BAD_REQUEST,
    SpineCoding.NOT_ACCEPTABLE: HTTPStatus.FORBIDDEN,
    SpineCoding.MISSING_HEADER: HTTPStatus.BAD_REQUEST,
    SpineCoding.MISSING_VALUE: HTTPStatus.BAD_REQUEST,
    SpineCoding.ACCESS_DENIED_LEVEL: HTTPStatus.FORBIDDEN,
    SpineCoding.SERVICE_ERROR: HTTPStatus.INTERNAL_SERVER_ERROR,
    SpineCoding.INVALID_RESOURCE_ID: HTTPStatus.BAD_REQUEST,
    SpineCoding.INVALID_VALUE: HTTPStatus.CONFLICT,
}

class Response(BaseModel):
    status_code: int
    result: dict

def _get_error_from_exception(exception: Exception) -> tuple[int, SpineCoding]:
    coding: SpineCoding = NRLF_TO_SPINE_4XX_ERROR.get(
        exception.__class__, SpineCoding.SERVICE_ERROR
    )
    http_status: HTTPStatus = HTTP_STATUS_CODE_MAPPING[coding]
    return http_status, coding

def _operation_outcome(
    id: str,
    issue_type: IssueType,
    coding: SpineCoding,
    severity: IssueSeverity,
    diagnostics: str,
) -> dict:
    issue_details = CodeableConcept(coding=[coding.value])
    issue = [
        OperationOutcomeIssue(
            code=issue_type.value,
            severity=severity.value,
            diagnostics=diagnostics,
            details=issue_details,
        )
    ]
    return OperationOutcome(
        resourceType=OperationOutcome.__name__, id=id, meta=META, issue=issue
    ).dict(exclude_none=True)

def get_error_message(exception: Exception) -> str:
    return (
        _format_validation_error_message(exception=exception)
        if type(exception) is ValidationError
        else str(
            exception
        )  ### TODO: better error message, e.g. error type in the message?
    )

def _format_validation_error_message(exception: ValidationError) -> str:
    errors = json_loads(exception.json())
    first_error = errors[0]
    model_name = (
        exception.model.public_alias()
        if exception.model.__dict__.get("public_alias")
        else exception.model.__name__
    )
    error_msg_prefix = f"{model_name} validation failure"
    first_bad_property = f"Invalid {first_error['loc'][0]}"
    message_components = [error_msg_prefix, first_bad_property]
    if first_error["type"] == "value_error":
        message_components.append(first_error["msg"])
    return " - ".join(message_components)

def operation_outcome_ok(transaction_id: str, coding: NrlfCoding) -> dict:
    _coding: Coding = coding.value
    operation_outcome = _operation_outcome(
        id=transaction_id,
        coding=coding,
        diagnostics=_coding.display,
        issue_type=IssueType.INFORMATIONAL,
        severity=IssueSeverity.INFORMATION,
    )
    return operation_outcome

def operation_outcome_not_ok(
    transaction_id: str, exception: Exception
) -> tuple[int, dict]:
    status_code, coding = _get_error_from_exception(exception=exception)
    diagnostics = get_error_message(exception=exception)
    operation_outcome = _operation_outcome(
        id=transaction_id,
        coding=coding,
        diagnostics=diagnostics,
        issue_type=IssueType.PROCESSING,
        severity=IssueSeverity.ERROR,
    )
    return status_code, operation_outcome


import pandas as pd
import numpy as np
import datetime
import matplotlib.pyplot as plt
from pandas.plotting import radviz
from utils import code_to_word

"""
    构建一个具有1个隐藏层的神经网络，隐层的大小为10
    输入层为4个特征，输出层为3个分类
    (1,0,0)为第一类，(0,1,0)为第二类，(0,0,1)为第三类
"""

# 1.初始化参数
def initialize_parameters(n_x, n_h, n_y):
    np.random.seed(2)

    # 权重和偏置矩阵
    w1 = np.random.randn(n_h, n_x) * 0.01
    b1 = np.zeros(shape=(n_h, 1))
    w2 = np.random.randn(n_y, n_h) * 0.01
    b2 = np.zeros(shape=(n_y, 1))

    # 通过字典存储参数
    parameters = {"w1": w1, "b1": b1, "w2": w2, "b2": b2}

    return parameters

# 2.前向传播
def forward_propagation(X, parameters):
    w1 = parameters["w1"]
    b1 = parameters["b1"]
    w2 = parameters["w2"]
    b2 = parameters["b2"]

    # 通过前向传播来计算a2
    z1 = np.dot(w1, X) + b1  # 这个地方需注意矩阵加法：虽然(w1*X)和b1的维度不同，但可以相加
    a1 = np.tanh(z1)  # 使用tanh作为第一层的激活函数
    z2 = np.dot(w2, a1) + b2
    a2 = 1 / (1 + np.exp(-z2))  # 使用sigmoid作为第二层的激活函数

    # 通过字典存储参数
    cache = {"z1": z1, "a1": a1, "z2": z2, "a2": a2}

    return a2, cache

# 3.计算代价函数
def compute_cost(a2, Y, parameters):
    m = Y.shape[1]  # Y的列数即为总的样本数

    # 采用交叉熵（cross-entropy）作为代价函数
    logprobs = np.multiply(np.log(a2), Y) + np.multiply((1 - Y), np.log(1 - a2))
    cost = -np.sum(logprobs) / m

    return cost

# 4.反向传播（计算代价函数的导数）
def backward_propagation(parameters, cache, X, Y):
    m = Y.shape[1]

    w2 = parameters["w2"]

    a1 = cache["a1"]
    a2 = cache["a2"]

    # 反向传播，计算dw1、db1、dw2、db2
    dz2 = a2 - Y
    dw2 = (1 / m) * np.dot(dz2, a1.T)
    db2 = (1 / m) * np.sum(dz2, axis=1, keepdims=True)
    dz1 = np.multiply(np.dot(w2.T, dz2), 1 - np.power(a1, 2))
    dw1 = (1 / m) * np.dot(dz1, X.T)
    db1 = (1 / m) * np.sum(dz1, axis=1, keepdims=True)

    grads = {"dw1": dw1, "db1": db1, "dw2": dw2, "db2": db2}

    return grads

# 5.更新参数
def update_parameters(parameters, grads, learning_rate=0.1):
    w1 = parameters["w1"]
    b1 = parameters["b1"]
    w2 = parameters["w2"]
    b2 = parameters["b2"]

    dw1 = grads["dw1"]
    db1 = grads["db1"]
    dw2 = grads["dw2"]
    db2 = grads["db2"]

    # 更新参数
    w1 = w1 - dw1 * learning_rate
    b1 = b1 - db1 * learning_rate
    w2 = w2 - dw2 * learning_rate
    b2 = b2 - db2 * learning_rate

    parameters = {"w1": w1, "b1": b1, "w2": w2, "b2": b2}

    return parameters

# 6.模型评估
def predict(parameters, x_test, y_test):
    w1 = parameters["w1"]
    b1 = parameters["b1"]
    w2 = parameters["w2"]
    b2 = parameters["b2"]

    z1 = np.dot(w1, x_test) + b1
    a1 = np.tanh(z1)
    z2 = np.dot(w2, a1) + b2
    a2 = 1 / (1 + np.exp(-z2))

    # 结果的维度
    n_rows = y_test.shape[0]
    n_cols = y_test.shape[1]

    # 预测值结果存储
    output = np.empty(shape=(n_rows, n_cols), dtype=int)
    print(a2)

    for i in range(n_rows):
        for j in range(n_cols):
            if a2[i][j] > 0.5:
                output[i][j] = 1
            else:
                output[i][j] = 0

    print("预测结果：")
    print(code_to_word(output.T))
    print("真实结果：")
    print(code_to_word(y_test.T))

    print("准确率：", accuracy_calculate(output.T, y_test.T))
    return output.T

def accuracy_calculate(predict, real):
    """计算结果准确率"""
    # 结果的维度
    n_rows = predict.shape[0]
    n_cols = predict.shape[1]
    accuracy = 0.00
    for i in range(n_rows):
        count = 0
        for j in range(n_cols):
            if predict[i][j] == real[i][j]:
                count += 1
        print(round(count / n_cols, 2))
        accuracy += round(count / n_cols, 2)
    return round(accuracy / n_rows, 2)

# 建立神经网络并进行训练
def nn_model(X, Y, n_h, n_input, n_output, num_iterations=20000, print_cost=False):
    np.random.seed(3)

    n_x = n_input  # 输入层节点数
    n_y = n_output  # 输出层节点数

    # 1.初始化参数
    parameters = initialize_parameters(n_x, n_h, n_y)
    learning_rate = 0.1
    # 梯度下降循环
    for i in range(0, num_iterations):
        # 2.前向传播
        a2, cache = forward_propagation(X, parameters)
        # 3.计算代价函数
        cost = compute_cost(a2, Y, parameters)
        # 4.反向传播
        grads = backward_propagation(parameters, cache, X, Y)
        # 5.更新参数
        parameters = update_parameters(parameters, grads, learning_rate)

        # 每10000次迭代，输出一次代价函数
        if print_cost and i % 10000 == 0:
            print("迭代第%i次，代价函数为：%f" % (i, cost), "学习率：" + str(learning_rate))
            # 学习率梯度下降
            learning_rate = 0.9 * learning_rate

    return parameters

if __name__ == "__main__":
    # 读取数据
    data_set = pd.read_csv("samples/gift_training.csv", header=None)

    # 第1种取数据方法：
    X = data_set.iloc[:, 0:13].values.T  # 前十三列是特征，T表示转置
    Y = data_set.iloc[:, 13:].values.T  # 后十六列是标签

    Y = Y.astype("uint8")

    # 开始训练
    start_time = datetime.datetime.now()
    # 输入4个节点，隐层10个节点，输出3个节点，迭代200000次
    parameters = nn_model(
        X, Y, n_h=20, n_input=13, n_output=16, num_iterations=200000, print_cost=True
    )
    end_time = datetime.datetime.now()
    print(
        "用时："
        + str((end_time - start_time).seconds)
        + "s"
        + str(round((end_time - start_time).microseconds / 1000))
        + "ms"
    )

    # 对模型进行测试
    data_test = pd.read_csv("samples/gift_test.csv", header=None)
    x_test = data_test.iloc[:, 0:13].values.T
    y_test = data_test.iloc[:, 13:].values.T
    y_test = y_test.astype("uint8")

    result = predict(parameters, x_test, y_test)


import torch
from   torchmetrics import Metric
class MyAccuracy(Metric):
    def __init__( self):
        super().__init__()
        self.add_state("correct" , default = torch.zeros(3,3), dist_reduce_fx = "sum")
        self.add_state("total", default = torch.tensor(0), dist_reduce_fx = "sum")
    def update(self, precision):
        self.correct += precision
        self.total   += 1

    def compute(self):
        return self.correct / self.total

def top_precision(pred, target, L_left, L_right = None ):
    precisions = {}
    T,T        = target.shape
    mask_mat   = torch.ones(T,T)
    if(L_right == None):L_right = T

    L_preds    = pred.tril(L_right - 1).triu(L_left)
    mask_mat   = mask_mat.tril(L_right - 1).triu(L_left)
    L_preds    = torch.squeeze(L_preds.view(1,-1))
    target     = torch.squeeze(target.view(1,-1))

    true_L = torch.tensor(T)
    for k in [1,2,5]:
        top_k          = torch.ceil(true_L/k)
        values, indics = torch.topk(L_preds, k = int(top_k))
        right_count    = torch.gather(target, -1 , indics)
        pred_count = torch.gather(L_preds, -1, indics)
        precision      = torch.sum(right_count) / top_k
        precisions[f'precision_L{k}']  = precision

    return precisions

def all_range_presicion(pred, target):
    pred   = torch.squeeze(pred)
    target  = torch.squeeze(target)
    T,T     = target.shape
    acc_mat = torch.zeros(3,3)
    sep     = [6,12,24,T]
    for index in range(3):
        precisions = top_precision( pred  = pred
                                  , target = target
                                  , L_left = sep[index]
                                  , L_right= sep[index +1 ])
        acc_mat[index][0] = precisions[f'precision_L1']
        acc_mat[index][1] = precisions[f'precision_L2']
        acc_mat[index][2] = precisions[f'precision_L5']

    return acc_mat

def negative_probalbility(contact_maps, predict_prob, k = 6):

    B,T,T     = contact_maps.size()

    mask_mat   = torch.ones(T,T)
    mask_mat   = mask_mat.triu(k)
    num = 2 * torch.count_nonzero(mask_mat)

    logsoftmax = torch.nn.LogSoftmax()
    log_pred = logsoftmax(predict_prob)

    mask_c0_up = log_pred[0][0].triu(k)
    mask_c0_down = log_pred[0][0].tril(-k)
    mask_c0 = mask_c0_up + mask_c0_down

    mask_c1_up = log_pred[0][1].triu(k)
    mask_c1_down = log_pred[0][1].tril(-k)
    mask_c1 = mask_c1_up + mask_c1_down

    mask_c0 = mask_c0.unsqueeze(0)
    mask_c0 = mask_c0.unsqueeze(0)
    mask_c1 = mask_c1.unsqueeze(0)
    mask_c1 = mask_c1.unsqueeze(0)
    output = torch.cat((mask_c0, mask_c1), 1)
    target_up = torch.triu(contact_maps, diagonal = k)
    target_down = torch.tril(contact_maps, diagonal = -k)
    target = target_up + target_down

    nll = torch.nn.NLLLoss(reduction = 'sum')
    prob = nll(output, target) / num
    return prob

class MyLoss(Metric):
    def __init__(self):
        super().__init__()
        self.add_state("loss_sum"
                       , default        = torch.tensor(0, dtype=torch.float)
                       , dist_reduce_fx = "sum")
        self.add_state("total"
                       , default        = torch.tensor(0, dtype=torch.float)
                       , dist_reduce_fx = "sum")

    def update(self, loss: torch.tensor):
        self.loss_sum += loss
        self.total    += 1

    def compute(self):
        return self.loss_sum.float() / self.total


# In this script, we will create some basic nlp features.
import time
notebookstart= time.time()

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import os
import gc
print("Data:\n",os.listdir("../input"))

# Models Packages
from sklearn import metrics
from sklearn.metrics import mean_squared_error
from sklearn import feature_selection
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from sklearn.decomposition import NMF

# Gradient Boosting
import lightgbm as lgb
from sklearn.linear_model import Ridge
from sklearn.cross_validation import KFold

# Tf-Idf
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.pipeline import FeatureUnion
from scipy.sparse import hstack, csr_matrix
from nltk.corpus import stopwords
russian_stop = set(stopwords.words('russian'))
stop_2 = set([w for w in open("../tmp/russian_stopwords.txt", "r").readlines()])
russian_stop = russian_stop.union(stop_2)
import string
punctuations = string.punctuation

# Viz
import re
import string

# I/O
from utils import *

# others
from joblib import Parallel, delayed
from scipy.stats import skew, kurtosis, entropy
from scipy import sparse
from scipy.sparse.linalg import svds
import umap
import gensim
from gensim.models.doc2vec import LabeledSentence
from gensim import corpora, models

NFOLDS = 5
SEED = 42

print("\nData Load Stage")
training = pd.read_csv('../input/train_stemmed.csv', index_col = "item_id")
traindex = training.index
testing = pd.read_csv('../input/test_stemmed.csv', index_col = "item_id")
testdex = testing.index

ntrain = training.shape[0]
ntest = testing.shape[0]

y = pd.read_csv('../input/train.csv', usecols = ["deal_probability"]).deal_probability.values

print('Train shape: {} Rows, {} Columns'.format(*training.shape))
print('Test shape: {} Rows, {} Columns'.format(*testing.shape))

print("Combine Train and Test")
df = pd.concat([training,testing],axis=0)
df["text_all"] = df.description.fillna("") + " "  + df.title.fillna("")
del training, testing
gc.collect()
print('\nAll Data shape: {} Rows, {} Columns'.format(*df.shape))

# Define Functions
def count_upper(x): return len([w for w in x if w.isupper()])
def count_lower(x): return len([w for w in x if w.islower()])
def count_stops(x): return len([w for w in x if w in russian_stop])
def count_punctuations(x): return len([w for w in x if w in punctuations])
def count_numbers(x): return len([w for w in x if w.isnumeric()])
def count_words(x): return len(x.split())
def count_unique_words(x): return len(set(x.split()))
def get_std(x): return np.std(x.todense())
def get_skew(x): return skew(x.todense().T)[0]
def get_kur(x): return kurtosis(x.todense().T)[0]
def get_entropy(x): return entropy(x.todense().T)[0]

# textcols = ["description", "title", "text_all"]#"description"
# for col in textcols:
#     print("Creating features {}...".format(col))
#     df_target = df[[col]].fillna("")
#
#     print("Creating basic tfidf features...")
#     df_target = df[col].fillna("")
#     tfidf_para = {
#         "stop_words": russian_stop,
#         "analyzer": 'word',
#         "token_pattern": r'\w{1,}',
#         "sublinear_tf": True,
#         "dtype": np.float32,
#         "norm": 'l2',
#         "min_df":3,
#         "smooth_idf":True
#     }
#
#     vectorizer = TfidfVectorizer(ngram_range=(1, 2), max_features=17000, **tfidf_para)
#     vecs = vectorizer.fit_transform(df_target.values)

    # df_out = pd.DataFrame()
    # df_out[col + "_stemmed_tfidf_mean"] = Parallel(n_jobs=-1)([delayed(np.mean)(v) for v in vecs])
    # print("mean done")
    # df_out[col + "_stemmed_tfidf_max"] = Parallel(n_jobs=-1)([delayed(np.max)(v) for v in vecs])
    # print("max done")
    # df_out[col + "_stemmed_tfidf_min"] = Parallel(n_jobs=-1)([delayed(np.min)(v) for v in vecs])
    # print("min done")
    # df_out[col + "_stemmed_tfidf_std"] = Parallel(n_jobs=-1)([delayed(get_std)(v) for v in vecs])
    # print("std done")
    # df_out[col + "_stemmed_tfidf_skew"] = Parallel(n_jobs=-1)([delayed(get_skew)(v) for v in vecs])
    # print("skew done")
    # df_out[col + "_stemmed_tfidf_kur"] = Parallel(n_jobs=-1)([delayed(get_kur)(v) for v in vecs])
    # print("kurtoisis done")
    # df_out[col + "_stemmed_tfidf_entropy"] = Parallel(n_jobs=-1)([delayed(get_entropy)(v) for v in vecs])
    # print("entropy done")
    # df_out[col + "_stemmed_tfidf_sum"] = Parallel(n_jobs=-1)([delayed(np.sum)(v) for v in vecs])
    # print("sum done")
    # to_parquet(df_out.iloc[:ntrain, :], "../features/fe_tfidf_stemmed_basic_{}_train.parquet".format(col))
    # to_parquet(df_out.iloc[ntrain:, :], "../features/fe_tfidf_stemmed_basic_{}_test.parquet".format(col))
    # del df_out; gc.collect()
    #
    # oof_sgd(vecs[:ntrain,:],vecs[ntrain:,:],y,"tfidf_stemmed_{}".format(col))
    # oof_lgbm(vecs[:ntrain,:].astype(np.float32),vecs[ntrain:,:].astype(np.float32),y,"tfidf_stemmed_{}".format(col))
    # del vecs; gc.collect()

# for col in textcols:
#     df_target = df[col].fillna("")
#     tfidf_para = {
#         "stop_words": russian_stop,
#         "analyzer": 'word',
#         "token_pattern": r'\w{1,}',
#         "sublinear_tf": True,
#         "dtype": np.float32,
#         "norm": 'l2',
#         "min_df":3,
#         "smooth_idf":False
#     }
#     vectorizer = TfidfVectorizer(ngram_range=(1, 2), max_features=50000, **tfidf_para)
#
#     vecs = vectorizer.fit_transform(df_target.values)
#     # tfidf dimensionality reduction
#     print("Start dimensionality reduction")
#     ## SVD
#     U, S , _ = svds(vecs.tocsc(),k=3)
#     m_svd = [U[i] * S for i in range(U.shape[0])]
#     m_svd = np.array(m_svd)
#     train_svd = pd.DataFrame(m_svd[:ntrain, :], columns=["svd_stemmed_{}_1".format(col+"_tfidf"), "svd_stemmed_{}_2".format(col+"_tfidf"), "svd_stemmed_{}_3".format(col+"_tfidf")])
#     test_svd = pd.DataFrame(m_svd[ntrain:, :], columns=["svd_stemmed_{}_1".format(col+"_tfidf"), "svd_stemmed_{}_2".format(col+"_tfidf"), "svd_stemmed_{}_3".format(col+"_tfidf")])
#     to_parquet(train_svd, "../features/fe_tfidf_stemmed_svd_{}_train.parquet".format(col))
#     to_parquet(test_svd, "../features/fe_tfidf_stemmed_svd_{}_test.parquet".format(col))
#     del m_svd, train_svd, test_svd; gc.collect()
#
#     ## NMF
#     nmf = NMF(n_components=3)
#     X_nmf = nmf.fit_transform(vecs)
#     df_nmf = pd.DataFrame(X_nmf, columns=["nmf_stemmed_{}_1".format(col+"_tfidf"), "nmf_stemmed_{}_2".format(col+"_tfidf"), "nmf_stemmed_{}_3".format(col+"_tfidf")])
#     nmf_train = df_nmf.iloc[:ntrain,:]
#     nmf_test = df_nmf.iloc[ntrain:,:]
#     to_parquet(nmf_train, "../features/fe_tfidf_stemmed_nmf_{}_train.parquet".format(col))
#     to_parquet(nmf_test, "../features/fe_tfidf_stemmed_nmf_{}_test.parquet".format(col))
#     del df_nmf, nmf_train, nmf_test; gc.collect()

# Meta Text Features

kfolds = get_kfolds()

print("Creating Ridge Features...")
class SklearnWrapper(object):
    def __init__(self, clf, seed=0, params=None, seed_bool = True):
        if(seed_bool == True):
            params['random_state'] = seed
        self.clf = clf(**params)

    def train(self, x_train, y_train):
        self.clf.fit(x_train, y_train)

    def predict(self, x):
        return self.clf.predict(x)

def get_oof(clf, x_train, y, x_test):
    oof_train = np.zeros((ntrain,))
    oof_test = np.zeros((ntest,))
    oof_test_skf = np.empty((NFOLDS, ntest))

    for i, (train_index, test_index) in enumerate(kfolds):
        print('\nFold {}'.format(i))
        x_tr = x_train[train_index]
        y_tr = y[train_index]
        x_te = x_train[test_index]

        clf.train(x_tr, y_tr)

        oof_train[test_index] = clf.predict(x_te)
        oof_test_skf[i, :] = clf.predict(x_test)

    oof_test[:] = oof_test_skf.mean(axis=0)
    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)

def cleanName(text):
    try:
        textProc = text.lower()
        textProc = " ".join(map(str.strip, re.split('(\d+)',textProc)))
        regex = re.compile(u'[^[:alpha:]]')
        textProc = regex.sub(" ", textProc)
        textProc = " ".join(textProc.split())
        return textProc
    except:
        return "name error"

df['title'] = df['title'].apply(lambda x: cleanName(x)).fillna("")
df["description"]   = df["description"].apply(lambda x: cleanName(x)).fillna("")

# textfeats = ["description", "title"]
# tfidf_para = {
#     "stop_words": russian_stop,
#     "analyzer": 'word',
#     "token_pattern": r'\w{1,}',
#     "sublinear_tf": True,
#     "dtype": np.float32,
#     "norm": 'l2',
#     "min_df":3,
#     "smooth_idf":False
# }
#
# def get_col(col_name): return lambda x: x[col_name]
#
# vectorizer = FeatureUnion([
#         ('description',TfidfVectorizer(
#             ngram_range=(1, 2),
#             max_features=17000,
#             **tfidf_para,
#             preprocessor=get_col('description'))),
#         ('title',CountVectorizer(
#             ngram_range=(1, 2),
#             stop_words = russian_stop,
#             preprocessor=get_col('title')))
#     ])
#
# #Fit my vectorizer on the entire dataset instead of the training rows
# #Score improved by .0001
# vectorizer.fit(df.to_dict('records'))
#
# ready_df = vectorizer.transform(df.to_dict('records'))
# tfvocab = vectorizer.get_feature_names()
#
# # Drop Text Cols
# textfeats = ["description", "title"]
#
#
# from sklearn.metrics import mean_squared_error
# from math import sqrt
#
# ridge_params = {'alpha':20.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,
#                 'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}
#
# ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)
# ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y, ready_df[ntrain:])
# train_out = pd.DataFrame(ridge_oof_train,columns=["ridge_oof_base_stemmed_2"])
# test_out = pd.DataFrame(ridge_oof_test,columns=["ridge_oof_base_stemmed_2"])
#
# to_parquet(train_out, "../features/oof_ridge_tfidf_stemmed_2_train.parquet")
# to_parquet(test_out, "../features/oof_ridge_tfidf_stemmed_2_test.parquet")
#

tfidf_para = {
    "stop_words": russian_stop,
    "analyzer": 'word',
    "token_pattern": r'\w{1,}',
    "sublinear_tf": True,
    "dtype": np.float32,
    "norm": 'l2',
    "min_df":3,
    "smooth_idf":False
}

def get_col(col_name): return lambda x: x[col_name]

vectorizer = FeatureUnion([
        ('description',TfidfVectorizer(
            ngram_range=(1, 3),
            max_features=20000,
            **tfidf_para,
            preprocessor=get_col('description'))),
        ('title',CountVectorizer(
            ngram_range=(1, 3),
            stop_words = russian_stop,
            preprocessor=get_col('title')))
    ])

#Fit my vectorizer on the entire dataset instead of the training rows
#Score improved by .0001
vectorizer.fit(df.to_dict('records'))

ready_df = vectorizer.transform(df.to_dict('records'))
tfvocab = vectorizer.get_feature_names()

# Drop Text Cols
textfeats = ["description", "title"]
df.drop(textfeats, axis=1,inplace=True)

from sklearn.metrics import mean_squared_error
from math import sqrt

ridge_params = {'alpha':20.0, 'fit_intercept':True, 'normalize':False, 'copy_X':True,
                'max_iter':None, 'tol':0.001, 'solver':'auto', 'random_state':SEED}

ridge = SklearnWrapper(clf=Ridge, seed = SEED, params = ridge_params)
ridge_oof_train, ridge_oof_test = get_oof(ridge, ready_df[:ntrain], y, ready_df[ntrain:])
train_out = pd.DataFrame(ridge_oof_train,columns=["ridge_oof_base_stemmed_3"])
test_out = pd.DataFrame(ridge_oof_test,columns=["ridge_oof_base_stemmed_3"])

to_parquet(train_out, "../features/oof_ridge_tfidf_stemmed_3_train.parquet")
to_parquet(test_out, "../features/oof_ridge_tfidf_stemmed_3_test.parquet")